c2ea0b4704
We introduce a fork of internal/httpx, named internal/httpapi, where there is a clear split between the concept of an API endpoint (such as https://0.th.ooni.org/) and of an API descriptor (such as using `GET` to access /api/v1/test-list/url). Additionally, httpapi allows to create a SequenceCaller that tries to call a given API descriptor using multiple API endpoints. The SequenceCaller will stop once an endpoint works or when all the available endpoints have been tried unsuccessfully. The definition of "success" is the following: we consider "failure" any error that occurs during the HTTP round trip or when reading the response body. We DO NOT consider "failure" errors (1) when parsing the input URL; (2) when the server returns >= 400; (3) when the server returns a string that does not parse as valid JSON. The idea of this classification of failures is that we ONLY want to retry when we see what looks like a network error that may be caused by (collateral or targeted) censorship. We take advantage of the availability of this new package and we refactor web_connectivity@v0.4 and web_connectivity@v0.5 to use a SequenceCaller for calling the web connectivity TH API. This means that we will now try all the available THs advertised by the backend rather than just selecting and using the first one provided by the backend. Because this diff is designed to be backported to the `release/3.16` branch, we have omitted additional changes to always use httpapi where we are currently using httpx. Yet, to remind ourselves about the need to do that, we have deprecated the httpx package. We will rewrite all the code currently using httpx to use httpapi as part of future work. It is also worth noting that httpapi will allow us to refactor the backend code such that (1) we remove code to select a backend URL endpoint at the beginning and (2) we try several endpoints. The design of the code is such that we can add to the mix some endpoints using as `http.Client` a special client using a tunnel. This will allow us to automatically fallback backend queries. Closes https://github.com/ooni/probe/issues/2353. Related to https://github.com/ooni/probe/issues/1519.
93 lines
3.0 KiB
Go
93 lines
3.0 KiB
Go
package httpapi
|
|
|
|
//
|
|
// Sequentially call available API endpoints until one succeed
|
|
// or all of them fail. A future implementation of this code may
|
|
// (probably should?) take into account knowledge of what is
|
|
// working and what is not working to optimize the order with
|
|
// which to try different alternatives.
|
|
//
|
|
|
|
import (
|
|
"context"
|
|
"errors"
|
|
|
|
"github.com/ooni/probe-cli/v3/internal/multierror"
|
|
)
|
|
|
|
// SequenceCaller calls the API specified by |Descriptor| once for each of
|
|
// the available |Endpoints| until one of them succeeds.
|
|
//
|
|
// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
|
|
// the error originates in the HTTP round trip or while reading the body.
|
|
type SequenceCaller struct {
|
|
// Descriptor is the API |Descriptor|.
|
|
Descriptor *Descriptor
|
|
|
|
// Endpoints is the list of |Endpoint| to use.
|
|
Endpoints []*Endpoint
|
|
}
|
|
|
|
// NewSequenceCaller is a factory for creating a |SequenceCaller|.
|
|
func NewSequenceCaller(desc *Descriptor, endpoints ...*Endpoint) *SequenceCaller {
|
|
return &SequenceCaller{
|
|
Descriptor: desc,
|
|
Endpoints: endpoints,
|
|
}
|
|
}
|
|
|
|
// ErrAllEndpointsFailed indicates that all endpoints failed.
|
|
var ErrAllEndpointsFailed = errors.New("httpapi: all endpoints failed")
|
|
|
|
// shouldRetry returns true when we should try with another endpoint given the
|
|
// value of |err| which could (obviously) be nil in case of success.
|
|
func (sc *SequenceCaller) shouldRetry(err error) bool {
|
|
var kind *errMaybeCensorship
|
|
belongs := errors.As(err, &kind)
|
|
return belongs
|
|
}
|
|
|
|
// Call calls |Call| for each |Endpoint| and |Descriptor| until one endpoint succeeds. The
|
|
// return value is the response body and the selected endpoint index or the error.
|
|
//
|
|
// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
|
|
// the error originates in the HTTP round trip or while reading the body.
|
|
func (sc *SequenceCaller) Call(ctx context.Context) ([]byte, int, error) {
|
|
var selected int
|
|
merr := multierror.New(ErrAllEndpointsFailed)
|
|
for _, epnt := range sc.Endpoints {
|
|
respBody, err := Call(ctx, sc.Descriptor, epnt)
|
|
if sc.shouldRetry(err) {
|
|
merr.Add(err)
|
|
selected++
|
|
continue
|
|
}
|
|
// Note: some errors will lead us to return
|
|
// early as documented for this method
|
|
return respBody, selected, err
|
|
}
|
|
return nil, -1, merr
|
|
}
|
|
|
|
// CallWithJSONResponse is like |SequenceCaller.Call| except that it invokes the
|
|
// underlying |CallWithJSONResponse| rather than invoking |Call|.
|
|
//
|
|
// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when
|
|
// the error originates in the HTTP round trip or while reading the body.
|
|
func (sc *SequenceCaller) CallWithJSONResponse(ctx context.Context, response any) (int, error) {
|
|
var selected int
|
|
merr := multierror.New(ErrAllEndpointsFailed)
|
|
for _, epnt := range sc.Endpoints {
|
|
err := CallWithJSONResponse(ctx, sc.Descriptor, epnt, response)
|
|
if sc.shouldRetry(err) {
|
|
merr.Add(err)
|
|
selected++
|
|
continue
|
|
}
|
|
// Note: some errors will lead us to return
|
|
// early as documented for this method
|
|
return selected, err
|
|
}
|
|
return -1, merr
|
|
}
|