ba9151d4fa
This diff adds the prototype websteps implementation that used to live at https://github.com/ooni/probe-cli/pull/506. The code is reasonably good already and it's pointing to a roaming test helper that I've properly configured. You can run websteps with: ``` ./miniooni -n websteps ``` This will go over the test list for your country. At this stage the mechanics of the experiment is set, but we still need to have a conversation on the following topics: 1. whether we're okay with reusing the data format used by other OONI experiments, or we would like to use a more compact data format (which may either be a more compact JSON or we can choose to always submit compressed measurements for websteps); 2. the extent to which we would like to keep the measurement as a collection of "the experiment saw this" and "the test helper saw that" and let the pipeline choose an overall score: this is clearly an option, but there is also the opposite option to build a summary of the measurement on the probe. Compared to the previous prototype of websteps, the main architectural change we have here is that we are following the point of view of the probe and the test helper is much more dumb. Basically, the probe will choose which redirection to follow and ask the test helper every time it discovers a new URL to measure it w/o redirections. Reference issue: https://github.com/ooni/probe/issues/1733
76 lines
2.0 KiB
Go
76 lines
2.0 KiB
Go
// Command oohelperd contains the Web Connectivity test helper.
|
|
package main
|
|
|
|
import (
|
|
"context"
|
|
"flag"
|
|
"net/http"
|
|
"sync"
|
|
"time"
|
|
|
|
"github.com/apex/log"
|
|
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/webconnectivity"
|
|
"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal/websteps"
|
|
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webstepsx"
|
|
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
|
"github.com/ooni/probe-cli/v3/internal/runtimex"
|
|
)
|
|
|
|
const maxAcceptableBody = 1 << 24
|
|
|
|
var (
|
|
dialer netx.Dialer
|
|
endpoint = flag.String("endpoint", ":8080", "Endpoint where to listen")
|
|
httpx *http.Client
|
|
resolver netx.Resolver
|
|
srvcancel context.CancelFunc
|
|
srvctx context.Context
|
|
srvwg = new(sync.WaitGroup)
|
|
)
|
|
|
|
func init() {
|
|
srvctx, srvcancel = context.WithCancel(context.Background())
|
|
dialer = netx.NewDialer(netx.Config{Logger: log.Log})
|
|
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
|
|
httpx = &http.Client{Transport: txp}
|
|
// fix: use 8.8.8.8:53/udp so we pin to a specific resolver.
|
|
var err error
|
|
resolver, err = netx.NewDNSClient(netx.Config{Logger: log.Log}, "udp://8.8.8.8:53")
|
|
runtimex.PanicOnError(err, "NewDNSClient failed")
|
|
}
|
|
|
|
func shutdown(srv *http.Server) {
|
|
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
|
defer cancel()
|
|
srv.Shutdown(ctx)
|
|
}
|
|
|
|
func main() {
|
|
logmap := map[bool]log.Level{
|
|
true: log.DebugLevel,
|
|
false: log.InfoLevel,
|
|
}
|
|
debug := flag.Bool("debug", false, "Toggle debug mode")
|
|
flag.Parse()
|
|
log.SetLevel(logmap[*debug])
|
|
testableMain()
|
|
}
|
|
|
|
func testableMain() {
|
|
mux := http.NewServeMux()
|
|
mux.Handle("/api/unstable/websteps", &websteps.Handler{Config: &websteps.Config{}})
|
|
mux.Handle("/api/v1/websteps", &webstepsx.THHandler{})
|
|
mux.Handle("/", webconnectivity.Handler{
|
|
Client: httpx,
|
|
Dialer: dialer,
|
|
MaxAcceptableBody: maxAcceptableBody,
|
|
Resolver: resolver,
|
|
})
|
|
srv := &http.Server{Addr: *endpoint, Handler: mux}
|
|
srvwg.Add(1)
|
|
go srv.ListenAndServe()
|
|
<-srvctx.Done()
|
|
shutdown(srv)
|
|
srvwg.Done()
|
|
}
|