c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Spring cleanup: remove unused/unneded code (#761)

Browse Source 
* cleanup: remove the archival package

See https://github.com/ooni/probe/issues/2116

* cleanup: remove websteps fall 2021 edition

See https://github.com/ooni/probe/issues/2116

* cleanup: remove JavaScript based testing framework

https://github.com/ooni/probe/issues/2116

* cleanup: remove the unused ooapi package

See https://github.com/ooni/probe/issues/2116
This commit is contained in:
Simone Basso
2022-05-25 13:21:39 +02:00
committed by GitHub
parent 8b0815efab
commit 7a0a156aec
89 changed files with 2 additions and 18567 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/allexperiments.go
-13
View File
@@ -29,7 +29,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/vanillator"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
)
@@ -352,18 +351,6 @@ var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
}
},
"websteps": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, webstepsx.NewExperimentMeasurer(
*config.(*webstepsx.Config),
))
},
config: &webstepsx.Config{},
inputPolicy: InputOrQueryBackend,
}
},
"whatsapp": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
internal/engine/experiment/webstepsx/doc.go
-6
View File
@@ -1,6 +0,0 @@
// Package webstepsx contains a websteps implementation
// based on the internal/measurex package.
//
// This implementation does not follow any existing spec
// rather we are modeling the spec on this one.
package webstepsx
internal/engine/experiment/webstepsx/measurer.go
-223
View File
@@ -1,223 +0,0 @@
package webstepsx
//
// Measurer
//
// This file contains the client implementation.
//
import (
"context"
"errors"
"net/http"
"net/url"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/measurex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
const (
testName = "websteps"
testVersion = "0.0.3"
)
// Config contains the experiment config.
type Config struct{}
// TestKeys contains the experiment's test keys.
type TestKeys struct {
*measurex.ArchivalURLMeasurement
}
// Measurer performs the measurement.
type Measurer struct {
Config Config
}
var (
_ model.ExperimentMeasurer = &Measurer{}
_ model.ExperimentMeasurerAsync = &Measurer{}
)
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return &Measurer{Config: config}
}
// ExperimentName implements ExperimentMeasurer.ExperExperimentName.
func (mx *Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperExperimentVersion.
func (mx *Measurer) ExperimentVersion() string {
return testVersion
}
var (
// ErrNoAvailableTestHelpers is emitted when there are no available test helpers.
ErrNoAvailableTestHelpers = errors.New("no available helpers")
// ErrNoInput indicates that no input was provided.
ErrNoInput = errors.New("no input provided")
// ErrInputIsNotAnURL indicates that the input is not an URL.
ErrInputIsNotAnURL = errors.New("input is not an URL")
// ErrUnsupportedInput indicates that the input URL scheme is unsupported.
ErrUnsupportedInput = errors.New("unsupported input scheme")
)
// RunAsync implements ExperimentMeasurerAsync.RunAsync.
func (mx *Measurer) RunAsync(
ctx context.Context, sess model.ExperimentSession, input string,
callbacks model.ExperimentCallbacks) (<-chan *model.ExperimentAsyncTestKeys, error) {
// 1. Parse and verify URL
URL, err := url.Parse(input)
if err != nil {
return nil, ErrInputIsNotAnURL
}
if URL.Scheme != "http" && URL.Scheme != "https" {
return nil, ErrUnsupportedInput
}
// 2. Find the testhelper
testhelpers, _ := sess.GetTestHelpersByName("web-connectivity")
var testhelper *model.OOAPIService
for _, th := range testhelpers {
if th.Type == "https" {
testhelper = &th
break
}
}
if testhelper == nil {
return nil, ErrNoAvailableTestHelpers
}
testhelper.Address = "https://1.th.ooni.org/api/v1/websteps" // TODO(bassosimone): remove!
out := make(chan *model.ExperimentAsyncTestKeys)
go mx.runAsync(ctx, sess, input, testhelper, out)
return out, nil
}
var measurerResolvers = []*measurex.ResolverInfo{{
Network: "system",
Address: "",
}, {
Network: "udp",
Address: "8.8.4.4:53",
}}
func (mx *Measurer) runAsync(ctx context.Context, sess model.ExperimentSession,
URL string, th *model.OOAPIService, out chan<- *model.ExperimentAsyncTestKeys) {
defer close(out)
helper := &measurerMeasureURLHelper{
Clnt: sess.DefaultHTTPClient(),
Logger: sess.Logger(),
THURL: th.Address,
UserAgent: sess.UserAgent(),
}
mmx := &measurex.Measurer{
Begin: time.Now(),
HTTPClient: sess.DefaultHTTPClient(),
MeasureURLHelper: helper,
Logger: sess.Logger(),
Resolvers: measurerResolvers,
TLSHandshaker: netxlite.NewTLSHandshakerStdlib(sess.Logger()),
}
cookies := measurex.NewCookieJar()
const parallelism = 3
in := mmx.MeasureURLAndFollowRedirections(
ctx, parallelism, URL, measurex.NewHTTPRequestHeaderForMeasuring(), cookies)
for m := range in {
out <- &model.ExperimentAsyncTestKeys{
Extensions: map[string]int64{
archival.ExtHTTP.Name: archival.ExtHTTP.V,
archival.ExtDNS.Name: archival.ExtDNS.V,
archival.ExtNetevents.Name: archival.ExtNetevents.V,
archival.ExtTCPConnect.Name: archival.ExtTCPConnect.V,
archival.ExtTLSHandshake.Name: archival.ExtTLSHandshake.V,
},
Input: model.MeasurementTarget(m.URL),
MeasurementRuntime: m.TotalRuntime.Seconds(),
TestKeys: &TestKeys{
ArchivalURLMeasurement: measurex.NewArchivalURLMeasurement(m),
},
}
}
}
// measurerMeasureURLHelper injects the TH into the normal
// URL measurement flow implemented by measurex.
type measurerMeasureURLHelper struct {
// Clnt is the MANDATORY client to use
Clnt model.HTTPClient
// Logger is the MANDATORY Logger to use
Logger model.Logger
// THURL is the MANDATORY TH URL.
THURL string
// UserAgent is the OPTIONAL user-agent to use.
UserAgent string
}
func (mth *measurerMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
curEndpoints ...*measurex.HTTPEndpoint) (
[]*measurex.HTTPEndpoint, *measurex.THMeasurement, error) {
cc := &THClientCall{
Endpoints: measurex.HTTPEndpointsToEndpoints(curEndpoints),
HTTPClient: mth.Clnt,
Header: headers,
THURL: mth.THURL,
TargetURL: URL.String(),
UserAgent: mth.UserAgent,
}
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
ol := measurex.NewOperationLogger(
mth.Logger, "THClientCall %s", URL.String())
resp, err := cc.Call(ctx)
ol.Stop(err)
if err != nil {
return nil, resp, err
}
var out []*measurex.HTTPEndpoint
for _, epnt := range resp.Endpoints {
out = append(out, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),
Network: epnt.Network,
Address: epnt.Address,
SNI: URL.Hostname(),
ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network),
URL: URL,
Header: headers,
})
}
return out, resp, nil
}
// Run implements ExperimentMeasurer.Run.
func (mx *Measurer) Run(ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, callbacks model.ExperimentCallbacks) error {
return errors.New("sync run is not implemented")
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (mx *Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{}
return sk, nil
}
internal/engine/experiment/webstepsx/th.go
-372
View File
@@ -1,372 +0,0 @@
package webstepsx
//
// TH (Test Helper)
//
// This file contains an implementation of the
// (proposed) websteps test helper spec.
//
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io"
"net/http"
"net/url"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/measurex"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
//
// Messages exchanged by the TH client and server
//
// THClientRequest is the request received by the test helper.
type THClientRequest struct {
// Endpoints is a list of endpoints to measure.
Endpoints []*measurex.Endpoint
// URL is the URL we want to measure.
URL string
// HTTPRequestHeaders contains the request headers.
HTTPRequestHeaders http.Header
}
// THServerResponse is the response from the test helper.
type THServerResponse = measurex.THMeasurement
// thMaxAcceptableBodySize is the maximum acceptable body size by TH code.
const thMaxAcceptableBodySize = 1 << 20
//
// TH client implementation
//
// THClient is the high-level API to invoke the TH. This API
// should be used by command line clients.
type THClient struct {
// DNSServers is the MANDATORY list of DNS-over-UDP
// servers to use to discover endpoints locally.
DNServers []*measurex.ResolverInfo
// HTTPClient is the MANDATORY HTTP client to
// use for contacting the TH.
HTTPClient model.HTTPClient
// ServerURL is the MANDATORY URL of the TH HTTP endpoint.
ServerURL string
}
// Run calls the TH and returns the response or an error.
//
// Arguments:
//
// - ctx is the context with timeout/deadline/cancellation
//
// - URL is the URL the TH server should measure for us
//
// Algorithm:
//
// - use DNSServers to discover extra endpoints for the target URL
//
// - call the TH using the HTTPClient and the ServerURL
//
// - return response or error.
func (c *THClient) Run(ctx context.Context, URL string) (*THServerResponse, error) {
parsed, err := url.Parse(URL)
if err != nil {
return nil, err
}
mx := measurex.NewMeasurerWithDefaultSettings()
var dns []*measurex.DNSMeasurement
const parallelism = 3
for m := range mx.LookupURLHostParallel(ctx, parallelism, parsed, c.DNServers...) {
dns = append(dns, m)
}
endpoints, err := measurex.AllEndpointsForURL(parsed, dns...)
if err != nil {
return nil, err
}
return (&THClientCall{
Endpoints: endpoints,
HTTPClient: c.HTTPClient,
Header: measurex.NewHTTPRequestHeaderForMeasuring(),
THURL: c.ServerURL,
TargetURL: URL,
}).Call(ctx)
}
// THClientCall allows to perform a single TH client call. Make sure
// you fill all the fields marked as MANDATORY before use.
type THClientCall struct {
// Endpoints contains the MANDATORY endpoints we discovered.
Endpoints []*measurex.Endpoint
// HTTPClient is the MANDATORY HTTP client to
// use for contacting the TH.
HTTPClient model.HTTPClient
// Header contains the MANDATORY request headers.
Header http.Header
// THURL is the MANDATORY test helper URL.
THURL string
// TargetURL is the MANDATORY URL to measure.
TargetURL string
// UserAgent is the OPTIONAL user-agent to use.
UserAgent string
}
// Call performs the specified TH call and returns either a response or an error.
func (c *THClientCall) Call(ctx context.Context) (*THServerResponse, error) {
creq := &THClientRequest{
Endpoints: c.Endpoints,
URL: c.TargetURL,
HTTPRequestHeaders: c.Header,
}
reqBody, err := json.Marshal(creq)
if err != nil {
return nil, err
}
req, err := http.NewRequestWithContext(
ctx, "POST", c.THURL, bytes.NewReader(reqBody))
if err != nil {
return nil, err
}
req.Header.Set("User-Agent", c.UserAgent)
return c.httpClientDo(req)
}
// errTHRequestFailed is the error returned if the TH response is not 200 Ok.
var errTHRequestFailed = errors.New("th: request failed")
func (c *THClientCall) httpClientDo(req *http.Request) (*THServerResponse, error) {
resp, err := c.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 { // THHandler returns either 400 or 200
return nil, errTHRequestFailed
}
r := io.LimitReader(resp.Body, thMaxAcceptableBodySize)
respBody, err := netxlite.ReadAllContext(req.Context(), r)
if err != nil {
return nil, err
}
var sresp THServerResponse
if err := json.Unmarshal(respBody, &sresp); err != nil {
return nil, err
}
return &sresp, nil
}
//
// TH server implementation
//
// THHandler implements the test helper API.
//
// This handler exposes a unique HTTP endpoint that you need to
// mount to the desired path when creating the server.
//
// The canonical mount point for the HTTP endpoint is /api/v1/websteps.
//
// Accepted methods and request body:
//
// - we only accept POST;
//
// - we expect a THClientRequest as the body.
//
// Status code and response body:
//
// - on success, status is 200 and THServerResponse is the body;
//
// - on failure, status is 400 and there is no body.
//
type THHandler struct{}
// ServerHTTP implements http.Handler.ServeHTTP.
func (h *THHandler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
w.Header().Add("Server", fmt.Sprintf("oohelperd/%s", version.Version))
if req.Method != "POST" {
w.WriteHeader(400)
return
}
reader := io.LimitReader(req.Body, thMaxAcceptableBodySize)
data, err := netxlite.ReadAllContext(req.Context(), reader)
if err != nil {
w.WriteHeader(400)
return
}
var creq THClientRequest
if err := json.Unmarshal(data, &creq); err != nil {
w.WriteHeader(400)
return
}
cresp, err := h.singleStep(req.Context(), &creq)
if err != nil {
w.WriteHeader(400)
return
}
// We assume that the following call cannot fail because it's a
// clearly serializable data structure.
data, err = json.Marshal(cresp)
runtimex.PanicOnError(err, "json.Marshal failed")
w.Header().Add("Content-Type", "application/json")
w.Write(data)
}
// singleStep performs a singleStep measurement.
//
// The function name derives from the definition (we invented)
// of "web steps". Each redirection is a step. For each step you
// need to figure out the endpoints to use with the DNS. After
// that, you need to check all endpoints. Because here we do not
// perform redirection, this is just a single "step".
//
// The algorithm is the following:
//
// 1. parse the URL and return error if it does not parse or
// the scheme is neither HTTP nor HTTPS;
//
// 2. discover additional endpoints using a suitable DoH
// resolver and the URL's hostname as the domain;
//
// 3. measure each discovered endpoint.
//
// The return value is either a THServerResponse or an error.
func (h *THHandler) singleStep(
ctx context.Context, req *THClientRequest) (*THServerResponse, error) {
mx := measurex.NewMeasurerWithDefaultSettings()
mx.MeasureURLHelper = &thMeasureURLHelper{req.Endpoints}
mx.Resolvers = []*measurex.ResolverInfo{{
Network: measurex.ResolverForeign,
ForeignResolver: thResolver,
}}
jar := measurex.NewCookieJar()
const parallelism = 3
meas, err := mx.MeasureURL(ctx, parallelism, req.URL, req.HTTPRequestHeaders, jar)
if err != nil {
return nil, err
}
return &THServerResponse{
DNS: meas.DNS,
Endpoints: h.simplifyEndpoints(meas.Endpoints),
}, nil
}
func (h *THHandler) simplifyEndpoints(
in []*measurex.HTTPEndpointMeasurement) (out []*measurex.HTTPEndpointMeasurement) {
for _, epnt := range in {
out = append(out, &measurex.HTTPEndpointMeasurement{
URL: epnt.URL,
Network: epnt.Network,
Address: epnt.Address,
Measurement: h.simplifyMeasurement(epnt.Measurement),
})
}
return
}
func (h *THHandler) simplifyMeasurement(in *measurex.Measurement) (out *measurex.Measurement) {
out = &measurex.Measurement{
Connect: in.Connect,
TLSHandshake: h.simplifyHandshake(in.TLSHandshake),
QUICHandshake: h.simplifyHandshake(in.QUICHandshake),
LookupHost: in.LookupHost,
LookupHTTPSSvc: in.LookupHTTPSSvc,
HTTPRoundTrip: h.simplifyHTTPRoundTrip(in.HTTPRoundTrip),
}
return
}
func (h *THHandler) simplifyHandshake(
in []*measurex.QUICTLSHandshakeEvent) (out []*measurex.QUICTLSHandshakeEvent) {
for _, ev := range in {
out = append(out, &measurex.QUICTLSHandshakeEvent{
CipherSuite: ev.CipherSuite,
Failure: ev.Failure,
NegotiatedProto: ev.NegotiatedProto,
TLSVersion: ev.TLSVersion,
PeerCerts: nil,
Finished: 0,
RemoteAddr: ev.RemoteAddr,
SNI: ev.SNI,
ALPN: ev.ALPN,
SkipVerify: ev.SkipVerify,
Oddity: ev.Oddity,
Network: ev.Network,
Started: 0,
})
}
return
}
func (h *THHandler) simplifyHTTPRoundTrip(
in []*measurex.HTTPRoundTripEvent) (out []*measurex.HTTPRoundTripEvent) {
for _, ev := range in {
out = append(out, &measurex.HTTPRoundTripEvent{
Failure: ev.Failure,
Method: ev.Method,
URL: ev.URL,
RequestHeaders: ev.RequestHeaders,
StatusCode: ev.StatusCode,
ResponseHeaders: ev.ResponseHeaders,
ResponseBody: nil, // we don't transfer the body
ResponseBodyLength: ev.ResponseBodyLength,
ResponseBodyIsTruncated: ev.ResponseBodyIsTruncated,
ResponseBodyIsUTF8: ev.ResponseBodyIsUTF8,
Finished: ev.Finished,
Started: ev.Started,
Oddity: ev.Oddity,
})
}
return
}
type thMeasureURLHelper struct {
epnts []*measurex.Endpoint
}
func (thh *thMeasureURLHelper) LookupExtraHTTPEndpoints(
ctx context.Context, URL *url.URL, headers http.Header,
serverEpnts ...*measurex.HTTPEndpoint) (
epnts []*measurex.HTTPEndpoint, thMeaurement *measurex.THMeasurement, err error) {
for _, epnt := range thh.epnts {
epnts = append(epnts, &measurex.HTTPEndpoint{
Domain: URL.Hostname(),
Network: epnt.Network,
Address: epnt.Address,
SNI: URL.Hostname(),
ALPN: measurex.ALPNForHTTPEndpoint(epnt.Network),
URL: URL,
Header: headers, // but overriden later anyway
})
}
return
}
// thResolverURL is the DNS resolver URL used by the TH. We use an
// encrypted resolver to reduce the risk that there is DNS-over-UDP
// censorship in the place where we deploy the TH.
const thResolverURL = "https://dns.google/dns-query"
// thResolver is the DNS resolver used by the TH.
//
// Here we're using github.com/apex/log as the logger, which
// is fine because this is backend only code.
var thResolver = netxlite.WrapResolver(log.Log, netxlite.NewSerialResolver(
netxlite.NewDNSOverHTTPSTransport(http.DefaultClient, thResolverURL),
))