chore: merge probe-engine into probe-cli (#201)
This is how I did it: 1. `git clone https://github.com/ooni/probe-engine internal/engine` 2. ``` (cd internal/engine && git describe --tags) v0.23.0 ``` 3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod` 4. `rm -rf internal/.git internal/engine/go.{mod,sum}` 5. `git add internal/engine` 6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;` 7. `go build ./...` (passes) 8. `go test -race ./...` (temporary failure on RiseupVPN) 9. `go mod tidy` 10. this commit message Once this piece of work is done, we can build a new version of `ooniprobe` that is using `internal/engine` directly. We need to do more work to ensure all the other functionality in `probe-engine` (e.g. making mobile packages) are still WAI. Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
GitHub
parent
b1ce300c8d
commit
d57c78bc71
535 changed files with 66182 additions and 23 deletions
514
internal/engine/session.go
Normal file
514
internal/engine/session.go
Normal file
|
|
@ -0,0 +1,514 @@
|
|||
package engine
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"os"
|
||||
"path/filepath"
|
||||
"sync"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/platform"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/sessionresolver"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/tunnel"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/resources"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/version"
|
||||
)
|
||||
|
||||
// SessionConfig contains the Session config
|
||||
type SessionConfig struct {
|
||||
AssetsDir string
|
||||
AvailableProbeServices []model.Service
|
||||
KVStore KVStore
|
||||
Logger model.Logger
|
||||
ProxyURL *url.URL
|
||||
SoftwareName string
|
||||
SoftwareVersion string
|
||||
TempDir string
|
||||
TorArgs []string
|
||||
TorBinary string
|
||||
}
|
||||
|
||||
// Session is a measurement session
|
||||
type Session struct {
|
||||
assetsDir string
|
||||
availableProbeServices []model.Service
|
||||
availableTestHelpers map[string][]model.Service
|
||||
byteCounter *bytecounter.Counter
|
||||
httpDefaultTransport netx.HTTPRoundTripper
|
||||
kvStore model.KeyValueStore
|
||||
location *geolocate.Results
|
||||
logger model.Logger
|
||||
proxyURL *url.URL
|
||||
queryProbeServicesCount *atomicx.Int64
|
||||
resolver *sessionresolver.Resolver
|
||||
selectedProbeServiceHook func(*model.Service)
|
||||
selectedProbeService *model.Service
|
||||
softwareName string
|
||||
softwareVersion string
|
||||
tempDir string
|
||||
torArgs []string
|
||||
torBinary string
|
||||
tunnelMu sync.Mutex
|
||||
tunnelName string
|
||||
tunnel tunnel.Tunnel
|
||||
}
|
||||
|
||||
// NewSession creates a new session or returns an error
|
||||
func NewSession(config SessionConfig) (*Session, error) {
|
||||
if config.AssetsDir == "" {
|
||||
return nil, errors.New("AssetsDir is empty")
|
||||
}
|
||||
if config.Logger == nil {
|
||||
return nil, errors.New("Logger is empty")
|
||||
}
|
||||
if config.SoftwareName == "" {
|
||||
return nil, errors.New("SoftwareName is empty")
|
||||
}
|
||||
if config.SoftwareVersion == "" {
|
||||
return nil, errors.New("SoftwareVersion is empty")
|
||||
}
|
||||
if config.KVStore == nil {
|
||||
config.KVStore = kvstore.NewMemoryKeyValueStore()
|
||||
}
|
||||
// Implementation note: if config.TempDir is empty, then Go will
|
||||
// use the temporary directory on the current system. This should
|
||||
// work on Desktop. We tested that it did also work on iOS, but
|
||||
// we have also seen on 2020-06-10 that it does not work on Android.
|
||||
tempDir, err := ioutil.TempDir(config.TempDir, "ooniengine")
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
sess := &Session{
|
||||
assetsDir: config.AssetsDir,
|
||||
availableProbeServices: config.AvailableProbeServices,
|
||||
byteCounter: bytecounter.New(),
|
||||
kvStore: config.KVStore,
|
||||
logger: config.Logger,
|
||||
proxyURL: config.ProxyURL,
|
||||
queryProbeServicesCount: atomicx.NewInt64(),
|
||||
softwareName: config.SoftwareName,
|
||||
softwareVersion: config.SoftwareVersion,
|
||||
tempDir: tempDir,
|
||||
torArgs: config.TorArgs,
|
||||
torBinary: config.TorBinary,
|
||||
}
|
||||
httpConfig := netx.Config{
|
||||
ByteCounter: sess.byteCounter,
|
||||
BogonIsError: true,
|
||||
Logger: sess.logger,
|
||||
}
|
||||
sess.resolver = sessionresolver.New(httpConfig)
|
||||
httpConfig.FullResolver = sess.resolver
|
||||
httpConfig.ProxyURL = config.ProxyURL // no need to proxy the resolver
|
||||
sess.httpDefaultTransport = netx.NewHTTPTransport(httpConfig)
|
||||
return sess, nil
|
||||
}
|
||||
|
||||
// ASNDatabasePath returns the path where the ASN database path should
|
||||
// be if you have called s.FetchResourcesIdempotent.
|
||||
func (s *Session) ASNDatabasePath() string {
|
||||
return filepath.Join(s.assetsDir, resources.ASNDatabaseName)
|
||||
}
|
||||
|
||||
// KibiBytesReceived accounts for the KibiBytes received by the HTTP clients
|
||||
// managed by this session so far, including experiments.
|
||||
func (s *Session) KibiBytesReceived() float64 {
|
||||
return s.byteCounter.KibiBytesReceived()
|
||||
}
|
||||
|
||||
// KibiBytesSent is like KibiBytesReceived but for the bytes sent.
|
||||
func (s *Session) KibiBytesSent() float64 {
|
||||
return s.byteCounter.KibiBytesSent()
|
||||
}
|
||||
|
||||
// Close ensures that we close all the idle connections that the HTTP clients
|
||||
// we are currently using may have created. It will also remove the temp dir
|
||||
// that contains data from this session. Not calling this function may likely
|
||||
// cause memory leaks in your application because of open idle connections,
|
||||
// as well as excessive usage of disk space.
|
||||
func (s *Session) Close() error {
|
||||
s.httpDefaultTransport.CloseIdleConnections()
|
||||
s.resolver.CloseIdleConnections()
|
||||
s.logger.Infof("%s", s.resolver.Stats())
|
||||
if s.tunnel != nil {
|
||||
s.tunnel.Stop()
|
||||
}
|
||||
return os.RemoveAll(s.tempDir)
|
||||
}
|
||||
|
||||
// CountryDatabasePath is like ASNDatabasePath but for the country DB path.
|
||||
func (s *Session) CountryDatabasePath() string {
|
||||
return filepath.Join(s.assetsDir, resources.CountryDatabaseName)
|
||||
}
|
||||
|
||||
// GetTestHelpersByName returns the available test helpers that
|
||||
// use the specified name, or false if there's none.
|
||||
func (s *Session) GetTestHelpersByName(name string) ([]model.Service, bool) {
|
||||
services, ok := s.availableTestHelpers[name]
|
||||
return services, ok
|
||||
}
|
||||
|
||||
// DefaultHTTPClient returns the session's default HTTP client.
|
||||
func (s *Session) DefaultHTTPClient() *http.Client {
|
||||
return &http.Client{Transport: s.httpDefaultTransport}
|
||||
}
|
||||
|
||||
// KeyValueStore returns the configured key-value store.
|
||||
func (s *Session) KeyValueStore() model.KeyValueStore {
|
||||
return s.kvStore
|
||||
}
|
||||
|
||||
// Logger returns the logger used by the session.
|
||||
func (s *Session) Logger() model.Logger {
|
||||
return s.logger
|
||||
}
|
||||
|
||||
// MaybeLookupLocation is a caching location lookup call.
|
||||
func (s *Session) MaybeLookupLocation() error {
|
||||
return s.MaybeLookupLocationContext(context.Background())
|
||||
}
|
||||
|
||||
// MaybeLookupBackends is a caching OONI backends lookup call.
|
||||
func (s *Session) MaybeLookupBackends() error {
|
||||
return s.maybeLookupBackends(context.Background())
|
||||
}
|
||||
|
||||
// MaybeLookupBackendsContext is like MaybeLookupBackends but with context.
|
||||
func (s *Session) MaybeLookupBackendsContext(ctx context.Context) (err error) {
|
||||
return s.maybeLookupBackends(ctx)
|
||||
}
|
||||
|
||||
// ErrAlreadyUsingProxy indicates that we cannot create a tunnel with
|
||||
// a specific name because we already configured a proxy.
|
||||
var ErrAlreadyUsingProxy = errors.New(
|
||||
"session: cannot create a new tunnel of this kind: we are already using a proxy",
|
||||
)
|
||||
|
||||
// MaybeStartTunnel starts the requested tunnel.
|
||||
//
|
||||
// This function silently succeeds if we're already using a tunnel with
|
||||
// the same name or if the requested tunnel name is the empty string. This
|
||||
// function fails, tho, when we already have a proxy or a tunnel with
|
||||
// another name and we try to open a tunnel. This function of course also
|
||||
// fails if we cannot start the requested tunnel. All in all, if you request
|
||||
// for a tunnel name that is not the empty string and you get a nil error,
|
||||
// you can be confident that session.ProxyURL() gives you the tunnel URL.
|
||||
//
|
||||
// The tunnel will be closed by session.Close().
|
||||
func (s *Session) MaybeStartTunnel(ctx context.Context, name string) error {
|
||||
s.tunnelMu.Lock()
|
||||
defer s.tunnelMu.Unlock()
|
||||
if s.tunnel != nil && s.tunnelName == name {
|
||||
// We've been asked more than once to start the same tunnel.
|
||||
return nil
|
||||
}
|
||||
if s.proxyURL != nil && name == "" {
|
||||
// The user configured a proxy and here we're not actually trying
|
||||
// to start any tunnel since `name` is empty.
|
||||
return nil
|
||||
}
|
||||
if s.proxyURL != nil || s.tunnel != nil {
|
||||
// We already have a proxy or we have a different tunnel. Because a tunnel
|
||||
// sets a proxy, the second check for s.tunnel is for robustness.
|
||||
return ErrAlreadyUsingProxy
|
||||
}
|
||||
tunnel, err := tunnel.Start(ctx, tunnel.Config{
|
||||
Name: name,
|
||||
Session: s,
|
||||
})
|
||||
if err != nil {
|
||||
s.logger.Warnf("cannot start tunnel: %+v", err)
|
||||
return err
|
||||
}
|
||||
// Implementation note: tunnel _may_ be NIL here if name is ""
|
||||
if tunnel == nil {
|
||||
return nil
|
||||
}
|
||||
s.tunnelName = name
|
||||
s.tunnel = tunnel
|
||||
s.proxyURL = tunnel.SOCKS5ProxyURL()
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewExperimentBuilder returns a new experiment builder
|
||||
// for the experiment with the given name, or an error if
|
||||
// there's no such experiment with the given name
|
||||
func (s *Session) NewExperimentBuilder(name string) (*ExperimentBuilder, error) {
|
||||
return newExperimentBuilder(s, name)
|
||||
}
|
||||
|
||||
// NewProbeServicesClient creates a new client for talking with the
|
||||
// OONI probe services. This function will benchmark the available
|
||||
// probe services, and select the fastest. In case all probe services
|
||||
// seem to be down, we try again applying circumvention tactics.
|
||||
func (s *Session) NewProbeServicesClient(ctx context.Context) (*probeservices.Client, error) {
|
||||
if err := s.maybeLookupBackends(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := s.MaybeLookupLocationContext(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if s.selectedProbeServiceHook != nil {
|
||||
s.selectedProbeServiceHook(s.selectedProbeService)
|
||||
}
|
||||
return probeservices.NewClient(s, *s.selectedProbeService)
|
||||
}
|
||||
|
||||
// NewSubmitter creates a new submitter instance.
|
||||
func (s *Session) NewSubmitter(ctx context.Context) (Submitter, error) {
|
||||
psc, err := s.NewProbeServicesClient(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return probeservices.NewSubmitter(psc, s.Logger()), nil
|
||||
}
|
||||
|
||||
// NewOrchestraClient creates a new orchestra client. This client is registered
|
||||
// and logged in with the OONI orchestra. An error is returned on failure.
|
||||
func (s *Session) NewOrchestraClient(ctx context.Context) (model.ExperimentOrchestraClient, error) {
|
||||
clnt, err := s.NewProbeServicesClient(ctx)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return s.initOrchestraClient(ctx, clnt, clnt.MaybeLogin)
|
||||
}
|
||||
|
||||
// Platform returns the current platform. The platform is one of:
|
||||
//
|
||||
// - android
|
||||
// - ios
|
||||
// - linux
|
||||
// - macos
|
||||
// - windows
|
||||
// - unknown
|
||||
//
|
||||
// When running on the iOS simulator, the returned platform is
|
||||
// macos rather than ios if CGO is disabled. This is a known issue,
|
||||
// that however should have a very limited impact.
|
||||
func (s *Session) Platform() string {
|
||||
return platform.Name()
|
||||
}
|
||||
|
||||
// ProbeASNString returns the probe ASN as a string.
|
||||
func (s *Session) ProbeASNString() string {
|
||||
return fmt.Sprintf("AS%d", s.ProbeASN())
|
||||
}
|
||||
|
||||
// ProbeASN returns the probe ASN as an integer.
|
||||
func (s *Session) ProbeASN() uint {
|
||||
asn := geolocate.DefaultProbeASN
|
||||
if s.location != nil {
|
||||
asn = s.location.ASN
|
||||
}
|
||||
return asn
|
||||
}
|
||||
|
||||
// ProbeCC returns the probe CC.
|
||||
func (s *Session) ProbeCC() string {
|
||||
cc := geolocate.DefaultProbeCC
|
||||
if s.location != nil {
|
||||
cc = s.location.CountryCode
|
||||
}
|
||||
return cc
|
||||
}
|
||||
|
||||
// ProbeNetworkName returns the probe network name.
|
||||
func (s *Session) ProbeNetworkName() string {
|
||||
nn := geolocate.DefaultProbeNetworkName
|
||||
if s.location != nil {
|
||||
nn = s.location.NetworkName
|
||||
}
|
||||
return nn
|
||||
}
|
||||
|
||||
// ProbeIP returns the probe IP.
|
||||
func (s *Session) ProbeIP() string {
|
||||
ip := geolocate.DefaultProbeIP
|
||||
if s.location != nil {
|
||||
ip = s.location.ProbeIP
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// ProxyURL returns the Proxy URL, or nil if not set
|
||||
func (s *Session) ProxyURL() *url.URL {
|
||||
return s.proxyURL
|
||||
}
|
||||
|
||||
// ResolverASNString returns the resolver ASN as a string
|
||||
func (s *Session) ResolverASNString() string {
|
||||
return fmt.Sprintf("AS%d", s.ResolverASN())
|
||||
}
|
||||
|
||||
// ResolverASN returns the resolver ASN
|
||||
func (s *Session) ResolverASN() uint {
|
||||
asn := geolocate.DefaultResolverASN
|
||||
if s.location != nil {
|
||||
asn = s.location.ResolverASN
|
||||
}
|
||||
return asn
|
||||
}
|
||||
|
||||
// ResolverIP returns the resolver IP
|
||||
func (s *Session) ResolverIP() string {
|
||||
ip := geolocate.DefaultResolverIP
|
||||
if s.location != nil {
|
||||
ip = s.location.ResolverIP
|
||||
}
|
||||
return ip
|
||||
}
|
||||
|
||||
// ResolverNetworkName returns the resolver network name.
|
||||
func (s *Session) ResolverNetworkName() string {
|
||||
nn := geolocate.DefaultResolverNetworkName
|
||||
if s.location != nil {
|
||||
nn = s.location.ResolverNetworkName
|
||||
}
|
||||
return nn
|
||||
}
|
||||
|
||||
// SoftwareName returns the application name.
|
||||
func (s *Session) SoftwareName() string {
|
||||
return s.softwareName
|
||||
}
|
||||
|
||||
// SoftwareVersion returns the application version.
|
||||
func (s *Session) SoftwareVersion() string {
|
||||
return s.softwareVersion
|
||||
}
|
||||
|
||||
// TempDir returns the temporary directory.
|
||||
func (s *Session) TempDir() string {
|
||||
return s.tempDir
|
||||
}
|
||||
|
||||
// TorArgs returns the configured extra args for the tor binary. If not set
|
||||
// we will not pass in any extra arg. Applies to `-OTunnel=tor` mainly.
|
||||
func (s *Session) TorArgs() []string {
|
||||
return s.torArgs
|
||||
}
|
||||
|
||||
// TorBinary returns the configured path to the tor binary. If not set
|
||||
// we will attempt to use "tor". Applies to `-OTunnel=tor` mainly.
|
||||
func (s *Session) TorBinary() string {
|
||||
return s.torBinary
|
||||
}
|
||||
|
||||
// UserAgent constructs the user agent to be used in this session.
|
||||
func (s *Session) UserAgent() (useragent string) {
|
||||
useragent += s.softwareName + "/" + s.softwareVersion
|
||||
useragent += " ooniprobe-engine/" + version.Version
|
||||
return
|
||||
}
|
||||
|
||||
// MaybeUpdateResources updates the resources if needed.
|
||||
func (s *Session) MaybeUpdateResources(ctx context.Context) error {
|
||||
return (&resources.Client{
|
||||
HTTPClient: s.DefaultHTTPClient(),
|
||||
Logger: s.logger,
|
||||
UserAgent: s.UserAgent(),
|
||||
WorkDir: s.assetsDir,
|
||||
}).Ensure(ctx)
|
||||
}
|
||||
|
||||
func (s *Session) getAvailableProbeServices() []model.Service {
|
||||
if len(s.availableProbeServices) > 0 {
|
||||
return s.availableProbeServices
|
||||
}
|
||||
return probeservices.Default()
|
||||
}
|
||||
|
||||
func (s *Session) initOrchestraClient(
|
||||
ctx context.Context, clnt *probeservices.Client,
|
||||
maybeLogin func(ctx context.Context) error,
|
||||
) (*probeservices.Client, error) {
|
||||
// The original implementation has as its only use case that we
|
||||
// were registering and logging in for sending an update regarding
|
||||
// the probe whereabouts. Yet here in probe-engine, the orchestra
|
||||
// is currently only used to fetch inputs. For this purpose, we don't
|
||||
// need to communicate any specific information. The code that will
|
||||
// perform an update used to be responsible of doing that. Now, we
|
||||
// are not using orchestra for this purpose anymore.
|
||||
meta := probeservices.Metadata{
|
||||
Platform: "miniooni",
|
||||
ProbeASN: "AS0",
|
||||
ProbeCC: "ZZ",
|
||||
SoftwareName: "miniooni",
|
||||
SoftwareVersion: "0.1.0-dev",
|
||||
SupportedTests: []string{"web_connectivity"},
|
||||
}
|
||||
if err := clnt.MaybeRegister(ctx, meta); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err := maybeLogin(ctx); err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return clnt, nil
|
||||
}
|
||||
|
||||
// LookupASN maps an IP address to its ASN and network name. This method implements
|
||||
// LocationLookupASNLookupper.LookupASN.
|
||||
func (s *Session) LookupASN(dbPath, ip string) (uint, string, error) {
|
||||
return geolocate.LookupASN(dbPath, ip)
|
||||
}
|
||||
|
||||
// ErrAllProbeServicesFailed indicates all probe services failed.
|
||||
var ErrAllProbeServicesFailed = errors.New("all available probe services failed")
|
||||
|
||||
func (s *Session) maybeLookupBackends(ctx context.Context) error {
|
||||
// TODO(bassosimone): do we need a mutex here?
|
||||
if s.selectedProbeService != nil {
|
||||
return nil
|
||||
}
|
||||
s.queryProbeServicesCount.Add(1)
|
||||
candidates := probeservices.TryAll(ctx, s, s.getAvailableProbeServices())
|
||||
selected := probeservices.SelectBest(candidates)
|
||||
if selected == nil {
|
||||
return ErrAllProbeServicesFailed
|
||||
}
|
||||
s.logger.Infof("session: using probe services: %+v", selected.Endpoint)
|
||||
s.selectedProbeService = &selected.Endpoint
|
||||
s.availableTestHelpers = selected.TestHelpers
|
||||
return nil
|
||||
}
|
||||
|
||||
// LookupLocationContext performs a location lookup. If you want memoisation
|
||||
// of the results, you should use MaybeLookupLocationContext.
|
||||
func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results, error) {
|
||||
// Implementation note: we don't perform the lookup of the resolver IP
|
||||
// when we are using a proxy because that might leak information.
|
||||
task := geolocate.Must(geolocate.NewTask(geolocate.Config{
|
||||
EnableResolverLookup: s.proxyURL == nil,
|
||||
HTTPClient: s.DefaultHTTPClient(),
|
||||
Logger: s.Logger(),
|
||||
ResourcesManager: s,
|
||||
UserAgent: s.UserAgent(),
|
||||
}))
|
||||
return task.Run(ctx)
|
||||
}
|
||||
|
||||
// MaybeLookupLocationContext is like MaybeLookupLocation but with a context
|
||||
// that can be used to interrupt this long running operation.
|
||||
func (s *Session) MaybeLookupLocationContext(ctx context.Context) error {
|
||||
if s.location == nil {
|
||||
location, err := s.LookupLocationContext(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
s.location = location
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ model.ExperimentSession = &Session{}
|
||||
Loading…
Reference in a new issue