c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

refactor: redesign how we import assets (#260)

Browse source 
* fix(pkg.go.dev): import a subpackage containing the assets

We're trying to fix this issue that pkg.go.dev does not build.

Thanks to @hellais for this very neat idea! Let's keep our
fingers crossed and see whether it fixes!

* feat: use embedded geoip databases

Closes https://github.com/ooni/probe/issues/1372.

Work done as part of https://github.com/ooni/probe/issues/1369.

* fix(assetsx): add tests

* feat: simplify and just vendor uncompressed DBs

* remove tests that seems not necessary anymore

* fix: run go mod tidy

* Address https://github.com/ooni/probe-cli/pull/260/files#r605181364

* rewrite a test in a better way

* fix: gently cleanup the legacy assetsdir

Do not remove the whole directory with brute force. Just zap the
files whose name we know. Then attempt to delete the legacy directory
as well. If not empty, just fail. This is fine because it means the
user has stored other files inside the directory.

* fix: create .miniooni if missing
This commit is contained in:
Simone Basso 2021-04-01 16:57:31 +02:00 • committed by GitHub
commit 31e478b04e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
51 changed files with 243 additions and 808 deletions
Download patch file Download diff file Expand all files Collapse all files

43
internal/engine/session.go
View file

@ -8,7 +8,6 @@ import (
"net/http"
"net/url"
"os"
"path/filepath"
"sync"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
@ -21,14 +20,11 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/resources"
"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
"github.com/ooni/probe-cli/v3/internal/version"
)
// SessionConfig contains the Session config
type SessionConfig struct {
AssetsDir string
AvailableProbeServices []model.Service
KVStore KVStore
Logger model.Logger
@ -40,9 +36,11 @@ type SessionConfig struct {
TorBinary string
}
// Session is a measurement session.
// Session is a measurement session. It contains shared information
// required to run a measurement session, and it controls the lifecycle
// of such resources. It is not possible to reuse a Session. You MUST
// NOT attempt to use a Session again after Session.Close.
type Session struct {
assetsDir string
availableProbeServices []model.Service
availableTestHelpers map[string][]model.Service
byteCounter *bytecounter.Counter
@ -64,6 +62,9 @@ type Session struct {
tunnelName string
tunnel tunnel.Tunnel
// closeOnce allows us to call Close just once.
closeOnce sync.Once
// mu provides mutual exclusion.
mu sync.Mutex
@ -93,9 +94,6 @@ type sessionProbeServicesClientForCheckIn interface {
// NewSession creates a new session or returns an error
func NewSession(config SessionConfig) (*Session, error) {
if config.AssetsDir == "" {
return nil, errors.New("AssetsDir is empty")
}
if config.Logger == nil {
return nil, errors.New("Logger is empty")
}
@ -117,7 +115,6 @@ func NewSession(config SessionConfig) (*Session, error) {
return nil, err
}
sess := &Session{
assetsDir: config.AssetsDir,
availableProbeServices: config.AvailableProbeServices,
byteCounter: bytecounter.New(),
kvStore: config.KVStore,
@ -147,12 +144,6 @@ func NewSession(config SessionConfig) (*Session, error) {
return sess, nil
}
// ASNDatabasePath returns the path where the ASN database path should
// be if you have called s.FetchResourcesIdempotent.
func (s *Session) ASNDatabasePath() string {
return filepath.Join(s.assetsDir, resources.ASNDatabaseName)
}
// KibiBytesReceived accounts for the KibiBytes received by the HTTP clients
// managed by this session so far, including experiments.
func (s *Session) KibiBytesReceived() float64 {
@ -255,19 +246,19 @@ func (s *Session) newProbeServicesClientForCheckIn(
// cause memory leaks in your application because of open idle connections,
// as well as excessive usage of disk space.
func (s *Session) Close() error {
// TODO(bassosimone): introduce a sync.Once to make this method idempotent.
s.closeOnce.Do(s.doClose)
return nil
}
// doClose implements Close. This function is called just once.
func (s *Session) doClose() {
s.httpDefaultTransport.CloseIdleConnections()
s.resolver.CloseIdleConnections()
s.logger.Infof("%s", s.resolver.Stats())
if s.tunnel != nil {
s.tunnel.Stop()
}
return os.RemoveAll(s.tempDir)
}
// CountryDatabasePath is like ASNDatabasePath but for the country DB path.
func (s *Session) CountryDatabasePath() string {
return filepath.Join(s.assetsDir, resources.CountryDatabaseName)
_ = os.RemoveAll(s.tempDir)
}
// GetTestHelpersByName returns the available test helpers that
@ -546,11 +537,6 @@ func (s *Session) UserAgent() (useragent string) {
return
}
// MaybeUpdateResources updates the resources if needed.
func (s *Session) MaybeUpdateResources(ctx context.Context) error {
return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure()
}
// getAvailableProbeServicesUnlocked returns the available probe
// services. This function WILL NOT acquire the mu mutex, therefore,
// you MUST ensure you are using it from a locked context.
@ -629,7 +615,6 @@ func (s *Session) LookupLocationContext(ctx context.Context) (*geolocate.Results
EnableResolverLookup: s.proxyURL == nil,
Logger: s.Logger(),
Resolver: s.resolver,
ResourcesManager: s,
UserAgent: s.UserAgent(),
}))
return task.Run(ctx)