c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: miniooni should be outside of the engine (#206)

Browse Source 
* refactor: miniooni should be outside of the engine

This is part of https://github.com/ooni/probe/issues/1335. We also need
to think whether we wanna keep libminiooni and miniooni separated.

The previous use case for having a top-level libminiooni was that of
enabling others to integrate miniooni into other binaries.

This was usegul when studying internet censorship in Spain in May 2020.

I am wondering whether we should be keeping this complexity. I am not
sure about this and probably we should be killing it.

(In any case, reducing complexity is not the objective of this diff,
since I would like instead to move things around with minimal changes
and make sure we have a ~good repository organization here.)

* fix: import in libminiooni
This commit is contained in:
Simone Basso
2021-02-03 11:21:10 +01:00
committed by GitHub
parent 99b28c1d95
commit 6351d898d6
12 changed files with 16 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/cmd/miniooni/README.md
-8
View File
@@ -1,8 +0,0 @@
# miniooni
This directory contains the source code of a simple CLI client that we
use for research as well as for running QA scripts. We designed this tool
to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar
scripts that check whether these tools behave similarly.
See also libminiooni.
internal/engine/cmd/miniooni/main.go
-21
View File
@@ -1,21 +0,0 @@
// Command miniooni is a simple binary for research and QA purposes
// with a CLI interface similar to MK and OONI Probe v2.x.
//
// See also libminiooni, which is where we implement this CLI.
package main
import (
"fmt"
"os"
"github.com/ooni/probe-cli/v3/internal/engine/libminiooni"
)
func main() {
defer func() {
if s := recover(); s != nil {
fmt.Fprintf(os.Stderr, "%s", s)
}
}()
libminiooni.Main()
}
internal/engine/experiment/dash/dash.go
+1 -1
View File
@@ -15,7 +15,7 @@ import (
"time"
"github.com/montanaflynn/stats"
"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
internal/engine/experiment/ndt7/ndt7.go
+1 -1
View File
@@ -11,7 +11,7 @@ import (
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocatev2"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
internal/engine/internal/humanizex/humanizex.go → internal/engine/humanizex/humanizex.go
View File
internal/engine/internal/humanizex/humanizex_test.go → internal/engine/humanizex/humanizex_test.go
+1 -1
View File
@@ -3,7 +3,7 @@ package humanizex_test
import (
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/humanizex"
)
func TestGood(t *testing.T) {
internal/engine/libminiooni/.gitignore
-1
View File
@@ -1 +0,0 @@
/report.jsonl
internal/engine/libminiooni/README.md
-14
View File
@@ -1,14 +0,0 @@
# Package github.com/ooni/probe-engine/libminiooni
Package libminiooni implements the cmd/miniooni CLI. Miniooni is our
experimental client used for research and QA testing.
This CLI has CLI options that do not conflict with Measurement Kit
v0.10.x CLI options. There are some options conflict with the legacy
OONI Probe CLI options. Perfect backwards compatibility is not a
design goal for miniooni. Rather, we aim to have as little conflict
as possible such that we can run side by side QA checks.
We extracted this package from cmd/miniooni to allow us to further
integrate the miniooni CLI into other binaries (see for example the
code at github.com/bassosimone/aladdin).
internal/engine/libminiooni/libminiooni.go
-442
View File
@@ -1,442 +0,0 @@
// Package libminiooni implements the cmd/miniooni CLI. Miniooni is our
// experimental client used for research and QA testing.
//
// This CLI has CLI options that do not conflict with Measurement Kit
// v0.10.x CLI options. There are some options conflict with the legacy
// OONI Probe CLI options. Perfect backwards compatibility is not a
// design goal for miniooni. Rather, we aim to have as little conflict
// as possible such that we can run side by side QA checks.
//
// We extracted this package from cmd/miniooni to allow us to further
// integrate the miniooni CLI into other binaries (see for example the
// code at github.com/bassosimone/aladdin).
package libminiooni
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"math/rand"
"net/url"
"os"
"path"
"path/filepath"
"runtime"
"strings"
"time"
"github.com/apex/log"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
"github.com/ooni/probe-cli/v3/internal/engine/version"
"github.com/pborman/getopt/v2"
)
// Options contains the options you can set from the CLI.
type Options struct {
Annotations []string
ExtraOptions []string
HomeDir string
Inputs []string
InputFilePaths []string
NoJSON bool
NoCollector bool
ProbeServicesURL string
Proxy string
Random bool
ReportFile string
SelfCensorSpec string
TorArgs []string
TorBinary string
Tunnel string
Verbose bool
Yes bool
}
const (
softwareName = "miniooni"
softwareVersion = version.Version
)
var (
globalOptions Options
startTime = time.Now()
)
func init() {
getopt.FlagLong(
&globalOptions.Annotations, "annotation", 'A', "Add annotaton", "KEY=VALUE",
)
getopt.FlagLong(
&globalOptions.ExtraOptions, "option", 'O',
"Pass an option to the experiment", "KEY=VALUE",
)
getopt.FlagLong(
&globalOptions.InputFilePaths, "input-file", 'f',
"Path to input file to supply test-dependent input. File must contain one input per line.", "PATH",
)
getopt.FlagLong(
&globalOptions.HomeDir, "home", 0,
"Force specific home directory", "PATH",
)
getopt.FlagLong(
&globalOptions.Inputs, "input", 'i',
"Add test-dependent input to the test input", "INPUT",
)
getopt.FlagLong(
&globalOptions.NoJSON, "no-json", 'N', "Disable writing to disk",
)
getopt.FlagLong(
&globalOptions.NoCollector, "no-collector", 'n', "Don't use a collector",
)
getopt.FlagLong(
&globalOptions.ProbeServicesURL, "probe-services", 0,
"Set the URL of the probe-services instance you want to use", "URL",
)
getopt.FlagLong(
&globalOptions.Proxy, "proxy", 0, "Set the proxy URL", "URL",
)
getopt.FlagLong(
&globalOptions.Random, "random", 0, "Randomize inputs",
)
getopt.FlagLong(
&globalOptions.ReportFile, "reportfile", 'o',
"Set the report file path", "PATH",
)
getopt.FlagLong(
&globalOptions.SelfCensorSpec, "self-censor-spec", 0,
"Enable and configure self censorship", "JSON",
)
getopt.FlagLong(
&globalOptions.TorArgs, "tor-args", 0,
"Extra args for tor binary (may be specified multiple times)",
)
getopt.FlagLong(
&globalOptions.TorBinary, "tor-binary", 0,
"Specify path to a specific tor binary",
)
getopt.FlagLong(
&globalOptions.Tunnel, "tunnel", 0,
"Name of the tunnel to use (one of `tor`, `psiphon`)",
)
getopt.FlagLong(
&globalOptions.Verbose, "verbose", 'v', "Increase verbosity",
)
getopt.FlagLong(
&globalOptions.Yes, "yes", 0, "I accept the risk of running OONI",
)
}
func fatalWithString(msg string) {
panic(msg)
}
func fatalIfFalse(cond bool, msg string) {
if !cond {
panic(msg)
}
}
// Main is the main function of miniooni. This function parses the command line
// options and uses a global state. Use MainWithConfiguration if you want to avoid
// using any global state and relying on command line options.
//
// This function will panic in case of a fatal error. It is up to you that
// integrate this function to either handle the panic of ignore it.
func Main() {
getopt.Parse()
fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name")
MainWithConfiguration(getopt.Arg(0), globalOptions)
}
func split(s string) (string, string, error) {
v := strings.SplitN(s, "=", 2)
if len(v) != 2 {
return "", "", errors.New("invalid key-value pair")
}
return v[0], v[1], nil
}
func fatalOnError(err error, msg string) {
if err != nil {
log.WithError(err).Warn(msg)
panic(msg)
}
}
func warnOnError(err error, msg string) {
if err != nil {
log.WithError(err).Warn(msg)
}
}
func mustMakeMap(input []string) (output map[string]string) {
output = make(map[string]string)
for _, opt := range input {
key, value, err := split(opt)
fatalOnError(err, "cannot split key-value pair")
output[key] = value
}
return
}
func mustParseURL(URL string) *url.URL {
rv, err := url.Parse(URL)
fatalOnError(err, "cannot parse URL")
return rv
}
type logHandler struct {
io.Writer
}
func (h *logHandler) HandleLog(e *log.Entry) (err error) {
s := fmt.Sprintf("[%14.6f] <%s> %s", time.Since(startTime).Seconds(), e.Level, e.Message)
if len(e.Fields) > 0 {
s += fmt.Sprintf(": %+v", e.Fields)
}
s += "\n"
_, err = h.Writer.Write([]byte(s))
return
}
// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da
func gethomedir(optionsHome string) string {
if optionsHome != "" {
return optionsHome
}
if runtime.GOOS == "windows" {
home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
if home == "" {
home = os.Getenv("USERPROFILE")
}
return home
}
if runtime.GOOS == "linux" {
home := os.Getenv("XDG_CONFIG_HOME")
if home != "" {
return home
}
// fallthrough
}
return os.Getenv("HOME")
}
const riskOfRunningOONI = `
Do you consent to OONI Probe data collection?
OONI Probe collects evidence of internet censorship and measures
network performance:
- OONI Probe will likely test objectionable sites and services;
- Anyone monitoring your internet activity (such as a government
or Internet provider) may be able to tell that you are using OONI Probe;
- The network data you collect will be published automatically
unless you use miniooni's -n command line flag.
To learn more, see https://ooni.org/about/risks/.
If you're onboard, re-run the same command and add the --yes flag, to
indicate that you understand the risks. This will create an empty file
named 'consent' in $HOME/.miniooni, meaning that we know you opted in
and we will not ask you this question again.
`
func canOpen(filepath string) bool {
stat, err := os.Stat(filepath)
return err == nil && stat.Mode().IsRegular()
}
func maybeWriteConsentFile(yes bool, filepath string) (err error) {
if yes {
err = ioutil.WriteFile(filepath, []byte("\n"), 0644)
}
return
}
// MainWithConfiguration is the miniooni main with a specific configuration
// represented by the experiment name and the current options.
//
// This function will panic in case of a fatal error. It is up to you that
// integrate this function to either handle the panic of ignore it.
func MainWithConfiguration(experimentName string, currentOptions Options) {
ctx := context.Background()
extraOptions := mustMakeMap(currentOptions.ExtraOptions)
annotations := mustMakeMap(currentOptions.Annotations)
err := selfcensor.MaybeEnable(currentOptions.SelfCensorSpec)
fatalOnError(err, "cannot parse --self-censor-spec argument")
logger := &log.Logger{Level: log.InfoLevel, Handler: &logHandler{Writer: os.Stderr}}
if currentOptions.Verbose {
logger.Level = log.DebugLevel
}
if currentOptions.ReportFile == "" {
currentOptions.ReportFile = "report.jsonl"
}
log.Log = logger
homeDir := gethomedir(currentOptions.HomeDir)
fatalIfFalse(homeDir != "", "home directory is empty")
miniooniDir := path.Join(homeDir, ".miniooni")
assetsDir := path.Join(miniooniDir, "assets")
err = os.MkdirAll(assetsDir, 0700)
fatalOnError(err, "cannot create assets directory")
log.Debugf("miniooni state directory: %s", miniooniDir)
consentFile := path.Join(miniooniDir, "informed")
fatalOnError(maybeWriteConsentFile(currentOptions.Yes, consentFile),
"cannot write informed consent file")
fatalIfFalse(canOpen(consentFile), riskOfRunningOONI)
log.Info("miniooni home directory: $HOME/.miniooni")
var proxyURL *url.URL
if currentOptions.Proxy != "" {
proxyURL = mustParseURL(currentOptions.Proxy)
}
kvstore2dir := filepath.Join(miniooniDir, "kvstore2")
kvstore, err := engine.NewFileSystemKVStore(kvstore2dir)
fatalOnError(err, "cannot create kvstore2 directory")
config := engine.SessionConfig{
AssetsDir: assetsDir,
KVStore: kvstore,
Logger: logger,
ProxyURL: proxyURL,
SoftwareName: softwareName,
SoftwareVersion: softwareVersion,
TorArgs: currentOptions.TorArgs,
TorBinary: currentOptions.TorBinary,
}
if currentOptions.ProbeServicesURL != "" {
config.AvailableProbeServices = []model.Service{{
Address: currentOptions.ProbeServicesURL,
Type: "https",
}}
}
sess, err := engine.NewSession(config)
fatalOnError(err, "cannot create measurement session")
defer func() {
sess.Close()
log.Infof("whole session: recv %s, sent %s",
humanizex.SI(sess.KibiBytesReceived()*1024, "byte"),
humanizex.SI(sess.KibiBytesSent()*1024, "byte"),
)
}()
log.Debugf("miniooni temporary directory: %s", sess.TempDir())
err = sess.MaybeStartTunnel(context.Background(), currentOptions.Tunnel)
fatalOnError(err, "cannot start session tunnel")
log.Info("Looking up OONI backends; please be patient...")
err = sess.MaybeLookupBackends()
fatalOnError(err, "cannot lookup OONI backends")
log.Info("Looking up your location; please be patient...")
err = sess.MaybeLookupLocation()
fatalOnError(err, "cannot lookup your location")
log.Debugf("- IP: %s", sess.ProbeIP())
log.Infof("- country: %s", sess.ProbeCC())
log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString())
log.Infof("- resolver's IP: %s", sess.ResolverIP())
log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(),
sess.ResolverASNString())
builder, err := sess.NewExperimentBuilder(experimentName)
fatalOnError(err, "cannot create experiment builder")
inputLoader := engine.NewInputLoader(engine.InputLoaderConfig{
StaticInputs: currentOptions.Inputs,
SourceFiles: currentOptions.InputFilePaths,
InputPolicy: builder.InputPolicy(),
Session: sess,
URLLimit: 17,
})
inputs, err := inputLoader.Load(context.Background())
fatalOnError(err, "cannot load inputs")
if currentOptions.Random {
rnd := rand.New(rand.NewSource(time.Now().UnixNano()))
rnd.Shuffle(len(inputs), func(i, j int) {
inputs[i], inputs[j] = inputs[j], inputs[i]
})
}
err = builder.SetOptionsGuessType(extraOptions)
fatalOnError(err, "cannot parse extraOptions")
experiment := builder.NewExperiment()
defer func() {
log.Infof("experiment: recv %s, sent %s",
humanizex.SI(experiment.KibiBytesReceived()*1024, "byte"),
humanizex.SI(experiment.KibiBytesSent()*1024, "byte"),
)
}()
submitter, err := engine.NewSubmitter(ctx, engine.SubmitterConfig{
Enabled: currentOptions.NoCollector == false,
Session: sess,
Logger: log.Log,
})
fatalOnError(err, "cannot create submitter")
saver, err := engine.NewSaver(engine.SaverConfig{
Enabled: currentOptions.NoJSON == false,
Experiment: experiment,
FilePath: currentOptions.ReportFile,
Logger: log.Log,
})
fatalOnError(err, "cannot create saver")
inputProcessor := engine.InputProcessor{
Annotations: annotations,
Experiment: &experimentWrapper{
child: engine.NewInputProcessorExperimentWrapper(experiment),
total: len(inputs),
},
Inputs: inputs,
Options: currentOptions.ExtraOptions,
Saver: engine.NewInputProcessorSaverWrapper(saver),
Submitter: submitterWrapper{
child: engine.NewInputProcessorSubmitterWrapper(submitter),
},
}
err = inputProcessor.Run(ctx)
fatalOnError(err, "inputProcessor.Run failed")
}
type experimentWrapper struct {
child engine.InputProcessorExperimentWrapper
total int
}
func (ew *experimentWrapper) MeasureWithContext(
ctx context.Context, idx int, input string) (*model.Measurement, error) {
if input != "" {
log.Infof("[%d/%d] running with input: %s", idx+1, ew.total, input)
}
measurement, err := ew.child.MeasureWithContext(ctx, idx, input)
warnOnError(err, "measurement failed")
// policy: we do not stop the loop if the measurement fails
return measurement, nil
}
type submitterWrapper struct {
child engine.InputProcessorSubmitterWrapper
}
func (sw submitterWrapper) Submit(ctx context.Context, idx int, m *model.Measurement) error {
err := sw.child.Submit(ctx, idx, m)
warnOnError(err, "submitting measurement failed")
// policy: we do not stop the loop if measurement submission fails
return nil
}
internal/engine/libminiooni/libminiooni_integration_test.go
-16
View File
@@ -1,16 +0,0 @@
package libminiooni_test
import (
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/libminiooni"
)
func TestSimple(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
libminiooni.MainWithConfiguration("example", libminiooni.Options{
Yes: true,
})
}