refactor(miniooni): divide et impera (#912)

This diff splits miniooni's implementation in smaller and more
easily tractable blocks ahead of future refactoring.

I'm trying to make `miniooni oonirun -i URL` as possible as
`miniooni -i URL oonirun`, because users typically expect this
kind of flexibity from modern Unix commands.

Part of https://github.com/ooni/probe/issues/2184
This commit is contained in:
Simone Basso 2022-08-31 10:20:04 +02:00 committed by GitHub
parent 196ac55493
commit 7daa686c68
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
9 changed files with 351 additions and 247 deletions

View File

@ -0,0 +1,53 @@
package main
//
// Acquiring user's consent
//
import (
"os"
"path"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// acquireUserConsent ensures the user is okay with using miniooni. This function
// panics if we do not have acquired the user consent.
func acquireUserConsent(miniooniDir string, currentOptions Options) {
consentFile := path.Join(miniooniDir, "informed")
err := maybeWriteConsentFile(currentOptions.Yes, consentFile)
runtimex.PanicOnError(err, "cannot write informed consent file")
runtimex.PanicIfFalse(regularFileExists(consentFile), riskOfRunningOONI)
}
// maybeWriteConsentFile writes the consent file iff the yes argument is true
func maybeWriteConsentFile(yes bool, filepath string) (err error) {
if yes {
err = os.WriteFile(filepath, []byte("\n"), 0644)
}
return
}
// riskOfRunningOONI is miniooni's informed consent text.
const riskOfRunningOONI = `
Do you consent to OONI Probe data collection?
OONI Probe collects evidence of internet censorship and measures
network performance:
- OONI Probe will likely test objectionable sites and services;
- Anyone monitoring your internet activity (such as a government
or Internet provider) may be able to tell that you are using OONI Probe;
- The network data you collect will be published automatically
unless you use miniooni's -n command line flag.
To learn more, see https://ooni.org/about/risks/.
If you're onboard, re-run the same command and add the --yes flag, to
indicate that you understand the risks. This will create an empty file
named 'consent' in $HOME/.miniooni, meaning that we know you opted in
and we will not ask you this question again.
`

View File

@ -1,32 +1,19 @@
// Command miniooni is a simple binary for research and QA purposes
// with a CLI interface similar to MK and OONI Probe v2.x.
package main package main
//
// Core implementation
//
// TODO(bassosimone): we should eventually merge this file and main.go. We still
// have this file becaused we used to have ./internal/libminiooni.
//
import ( import (
"context" "context"
"errors"
"fmt" "fmt"
"io"
"net/url"
"os" "os"
"path" "path"
"path/filepath"
"runtime"
"strings"
"time" "time"
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine" "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/humanize" "github.com/ooni/probe-cli/v3/internal/humanize"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/legacy/assetsdir" "github.com/ooni/probe-cli/v3/internal/legacy/assetsdir"
"github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/oonirun"
"github.com/ooni/probe-cli/v3/internal/runtimex" "github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version" "github.com/ooni/probe-cli/v3/internal/version"
"github.com/pborman/getopt/v2" "github.com/pborman/getopt/v2"
@ -55,15 +42,7 @@ type Options struct {
Yes bool Yes bool
} }
const ( var globalOptions Options
softwareName = "miniooni"
softwareVersion = version.Version
)
var (
globalOptions Options
startTime = time.Now()
)
func init() { func init() {
getopt.FlagLong( getopt.FlagLong(
@ -137,13 +116,13 @@ func init() {
) )
} }
// Main is the main function of miniooni. This function parses the command line // main is the main function of miniooni. This function parses the command line
// options and uses a global state. Use MainWithConfiguration if you want to avoid // options and uses a global state. Use MainWithConfiguration if you want to avoid
// using any global state and relying on command line options. // using any global state and relying on command line options.
// //
// This function will panic in case of a fatal error. It is up to you that // This function will panic in case of a fatal error. It is up to you that
// integrate this function to either handle the panic of ignore it. // integrate this function to either handle the panic of ignore it.
func Main() { func main() {
getopt.Parse() getopt.Parse()
if globalOptions.Version { if globalOptions.Version {
fmt.Printf("%s\n", version.Version) fmt.Printf("%s\n", version.Version)
@ -154,111 +133,6 @@ func Main() {
MainWithConfiguration(getopt.Arg(0), globalOptions) MainWithConfiguration(getopt.Arg(0), globalOptions)
} }
func split(s string) (string, string, error) {
v := strings.SplitN(s, "=", 2)
if len(v) != 2 {
return "", "", errors.New("invalid key-value pair")
}
return v[0], v[1], nil
}
func mustMakeMapString(input []string) (output map[string]string) {
output = make(map[string]string)
for _, opt := range input {
key, value, err := split(opt)
runtimex.PanicOnError(err, "cannot split key-value pair")
output[key] = value
}
return
}
func mustMakeMapAny(input []string) (output map[string]any) {
output = make(map[string]any)
for _, opt := range input {
key, value, err := split(opt)
runtimex.PanicOnError(err, "cannot split key-value pair")
output[key] = value
}
return
}
func mustParseURL(URL string) *url.URL {
rv, err := url.Parse(URL)
runtimex.PanicOnError(err, "cannot parse URL")
return rv
}
type logHandler struct {
io.Writer
}
func (h *logHandler) HandleLog(e *log.Entry) (err error) {
s := fmt.Sprintf("[%14.6f] <%s> %s", time.Since(startTime).Seconds(), e.Level, e.Message)
if len(e.Fields) > 0 {
s += fmt.Sprintf(": %+v", e.Fields)
}
s += "\n"
_, err = h.Writer.Write([]byte(s))
return
}
// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da
func gethomedir(optionsHome string) string {
if optionsHome != "" {
return optionsHome
}
if runtime.GOOS == "windows" {
home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
if home == "" {
home = os.Getenv("USERPROFILE")
}
return home
}
if runtime.GOOS == "linux" {
home := os.Getenv("XDG_CONFIG_HOME")
if home != "" {
return home
}
// fallthrough
}
return os.Getenv("HOME")
}
const riskOfRunningOONI = `
Do you consent to OONI Probe data collection?
OONI Probe collects evidence of internet censorship and measures
network performance:
- OONI Probe will likely test objectionable sites and services;
- Anyone monitoring your internet activity (such as a government
or Internet provider) may be able to tell that you are using OONI Probe;
- The network data you collect will be published automatically
unless you use miniooni's -n command line flag.
To learn more, see https://ooni.org/about/risks/.
If you're onboard, re-run the same command and add the --yes flag, to
indicate that you understand the risks. This will create an empty file
named 'consent' in $HOME/.miniooni, meaning that we know you opted in
and we will not ask you this question again.
`
func canOpen(filepath string) bool {
stat, err := os.Stat(filepath)
return err == nil && stat.Mode().IsRegular()
}
func maybeWriteConsentFile(yes bool, filepath string) (err error) {
if yes {
err = os.WriteFile(filepath, []byte("\n"), 0644)
}
return
}
// tunnelAndProxy is the text printed when the user specifies // tunnelAndProxy is the text printed when the user specifies
// both the --tunnel and the --proxy options // both the --tunnel and the --proxy options
const tunnelAndProxy = `USAGE ERROR: The --tunnel option and the --proxy const tunnelAndProxy = `USAGE ERROR: The --tunnel option and the --proxy
@ -302,8 +176,8 @@ func MainWithConfiguration(experimentName string, currentOptions Options) {
// mainSingleIteration runs a single iteration. There may be multiple iterations // mainSingleIteration runs a single iteration. There may be multiple iterations
// when the user specifies the --repeat-every command line flag. // when the user specifies the --repeat-every command line flag.
func mainSingleIteration(logger model.Logger, experimentName string, currentOptions Options) { func mainSingleIteration(logger model.Logger, experimentName string, currentOptions Options) {
extraOptions := mustMakeMapAny(currentOptions.ExtraOptions) extraOptions := mustMakeMapStringAny(currentOptions.ExtraOptions)
annotations := mustMakeMapString(currentOptions.Annotations) annotations := mustMakeMapStringString(currentOptions.Annotations)
ctx := context.Background() ctx := context.Background()
@ -326,45 +200,11 @@ func mainSingleIteration(logger model.Logger, experimentName string, currentOpti
_, _ = assetsdir.Cleanup(assetsDir) _, _ = assetsdir.Cleanup(assetsDir)
log.Debugf("miniooni state directory: %s", miniooniDir) log.Debugf("miniooni state directory: %s", miniooniDir)
consentFile := path.Join(miniooniDir, "informed")
runtimex.PanicOnError(maybeWriteConsentFile(currentOptions.Yes, consentFile),
"cannot write informed consent file")
runtimex.PanicIfFalse(canOpen(consentFile), riskOfRunningOONI)
log.Info("miniooni home directory: $HOME/.miniooni") log.Info("miniooni home directory: $HOME/.miniooni")
var proxyURL *url.URL acquireUserConsent(miniooniDir, currentOptions)
if currentOptions.Proxy != "" {
proxyURL = mustParseURL(currentOptions.Proxy)
}
kvstore2dir := filepath.Join(miniooniDir, "kvstore2") sess := newSessionOrPanic(ctx, currentOptions, miniooniDir, logger)
kvstore, err := kvstore.NewFS(kvstore2dir)
runtimex.PanicOnError(err, "cannot create kvstore2 directory")
tunnelDir := filepath.Join(miniooniDir, "tunnel")
err = os.MkdirAll(tunnelDir, 0700)
runtimex.PanicOnError(err, "cannot create tunnelDir")
config := engine.SessionConfig{
KVStore: kvstore,
Logger: logger,
ProxyURL: proxyURL,
SoftwareName: softwareName,
SoftwareVersion: softwareVersion,
TorArgs: currentOptions.TorArgs,
TorBinary: currentOptions.TorBinary,
TunnelDir: tunnelDir,
}
if currentOptions.ProbeServicesURL != "" {
config.AvailableProbeServices = []model.OOAPIService{{
Address: currentOptions.ProbeServicesURL,
Type: "https",
}}
}
sess, err := engine.NewSession(ctx, config)
runtimex.PanicOnError(err, "cannot create measurement session")
defer func() { defer func() {
sess.Close() sess.Close()
log.Infof("whole session: recv %s, sent %s", log.Infof("whole session: recv %s, sent %s",
@ -372,20 +212,8 @@ func mainSingleIteration(logger model.Logger, experimentName string, currentOpti
humanize.SI(sess.KibiBytesSent()*1024, "byte"), humanize.SI(sess.KibiBytesSent()*1024, "byte"),
) )
}() }()
log.Debugf("miniooni temporary directory: %s", sess.TempDir()) lookupBackendsOrPanic(ctx, sess)
lookupLocationOrPanic(ctx, sess)
log.Info("Looking up OONI backends; please be patient...")
err = sess.MaybeLookupBackends()
runtimex.PanicOnError(err, "cannot lookup OONI backends")
log.Info("Looking up your location; please be patient...")
err = sess.MaybeLookupLocation()
runtimex.PanicOnError(err, "cannot lookup your location")
log.Debugf("- IP: %s", sess.ProbeIP())
log.Infof("- country: %s", sess.ProbeCC())
log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString())
log.Infof("- resolver's IP: %s", sess.ResolverIP())
log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(),
sess.ResolverASNString())
// We handle the oonirun experiment name specially. The user must specify // We handle the oonirun experiment name specially. The user must specify
// `miniooni -i {OONIRunURL} oonirun` to run a OONI Run URL (v1 or v2). // `miniooni -i {OONIRunURL} oonirun` to run a OONI Run URL (v1 or v2).
@ -395,57 +223,5 @@ func mainSingleIteration(logger model.Logger, experimentName string, currentOpti
} }
// Otherwise just run OONI experiments as we normally do. // Otherwise just run OONI experiments as we normally do.
desc := &oonirun.Experiment{ runx(ctx, sess, experimentName, annotations, extraOptions, currentOptions)
Annotations: annotations,
ExtraOptions: extraOptions,
Inputs: currentOptions.Inputs,
InputFilePaths: currentOptions.InputFilePaths,
MaxRuntime: currentOptions.MaxRuntime,
Name: experimentName,
NoCollector: currentOptions.NoCollector,
NoJSON: currentOptions.NoJSON,
Random: currentOptions.Random,
ReportFile: currentOptions.ReportFile,
Session: sess,
}
err = desc.Run(ctx)
runtimex.PanicOnError(err, "cannot run experiment")
}
// ooniRunMain runs the experiments described by the given OONI Run URLs. This
// function works with both v1 and v2 OONI Run URLs.
func ooniRunMain(ctx context.Context,
sess *engine.Session, currentOptions Options, annotations map[string]string) {
runtimex.PanicIfTrue(
len(currentOptions.Inputs) <= 0,
"in oonirun mode you need to specify at least one URL using `-i URL`",
)
runtimex.PanicIfTrue(
len(currentOptions.InputFilePaths) > 0,
"in oonirun mode you cannot specify any `-f FILE` file",
)
logger := sess.Logger()
cfg := &oonirun.LinkConfig{
AcceptChanges: currentOptions.Yes,
Annotations: annotations,
KVStore: sess.KeyValueStore(),
MaxRuntime: currentOptions.MaxRuntime,
NoCollector: currentOptions.NoCollector,
NoJSON: currentOptions.NoJSON,
Random: currentOptions.Random,
ReportFile: currentOptions.ReportFile,
Session: sess,
}
for _, URL := range currentOptions.Inputs {
r := oonirun.NewLinkRunner(cfg, URL)
if err := r.Run(ctx); err != nil {
if errors.Is(err, oonirun.ErrNeedToAcceptChanges) {
logger.Warnf("oonirun: to accept these changes, rerun adding `-y` to the command line")
logger.Warnf("oonirun: we'll show this error every time the upstream link changes")
panic("oonirun: need to accept changes using `-y`")
}
logger.Warnf("oonirun: running link failed: %s", err.Error())
continue
}
}
} }

View File

@ -0,0 +1,35 @@
package main
//
// Logging functionality
//
import (
"fmt"
"io"
"time"
"github.com/apex/log"
)
// logStartTime is the time when we started logging
var logStartTime = time.Now()
// logHandler implements the log handler required by github.com/apex/log
type logHandler struct {
// Writer is the underlying writer
io.Writer
}
var _ log.Handler = &logHandler{}
// HandleLog implements log.Handler
func (h *logHandler) HandleLog(e *log.Entry) (err error) {
s := fmt.Sprintf("[%14.6f] <%s> %s", time.Since(logStartTime).Seconds(), e.Level, e.Message)
if len(e.Fields) > 0 {
s += fmt.Sprintf(": %+v", e.Fields)
}
s += "\n"
_, err = h.Writer.Write([]byte(s))
return
}

View File

@ -1,11 +0,0 @@
// Command miniooni is a simple binary for research and QA purposes
// with a CLI interface similar to MK and OONI Probe v2.x.
package main
//
// Main function
//
func main() {
Main()
}

View File

@ -0,0 +1,52 @@
package main
//
// OONI Run
//
import (
"context"
"errors"
"github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/oonirun"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// ooniRunMain runs the experiments described by the given OONI Run URLs. This
// function works with both v1 and v2 OONI Run URLs.
func ooniRunMain(ctx context.Context,
sess *engine.Session, currentOptions Options, annotations map[string]string) {
runtimex.PanicIfTrue(
len(currentOptions.Inputs) <= 0,
"in oonirun mode you need to specify at least one URL using `-i URL`",
)
runtimex.PanicIfTrue(
len(currentOptions.InputFilePaths) > 0,
"in oonirun mode you cannot specify any `-f FILE` file",
)
logger := sess.Logger()
cfg := &oonirun.LinkConfig{
AcceptChanges: currentOptions.Yes,
Annotations: annotations,
KVStore: sess.KeyValueStore(),
MaxRuntime: currentOptions.MaxRuntime,
NoCollector: currentOptions.NoCollector,
NoJSON: currentOptions.NoJSON,
Random: currentOptions.Random,
ReportFile: currentOptions.ReportFile,
Session: sess,
}
for _, URL := range currentOptions.Inputs {
r := oonirun.NewLinkRunner(cfg, URL)
if err := r.Run(ctx); err != nil {
if errors.Is(err, oonirun.ErrNeedToAcceptChanges) {
logger.Warnf("oonirun: to accept these changes, rerun adding `-y` to the command line")
logger.Warnf("oonirun: we'll show this error every time the upstream link changes")
panic("oonirun: need to accept changes using `-y`")
}
logger.Warnf("oonirun: running link failed: %s", err.Error())
continue
}
}
}

View File

@ -0,0 +1,32 @@
package main
//
// Run eXperiment by name
//
import (
"context"
"github.com/ooni/probe-cli/v3/internal/oonirun"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// runx runs the given experiment by name
func runx(ctx context.Context, sess oonirun.Session, experimentName string,
annotations map[string]string, extraOptions map[string]any, currentOptions Options) {
desc := &oonirun.Experiment{
Annotations: annotations,
ExtraOptions: extraOptions,
Inputs: currentOptions.Inputs,
InputFilePaths: currentOptions.InputFilePaths,
MaxRuntime: currentOptions.MaxRuntime,
Name: experimentName,
NoCollector: currentOptions.NoCollector,
NoJSON: currentOptions.NoJSON,
Random: currentOptions.Random,
ReportFile: currentOptions.ReportFile,
Session: sess,
}
err := desc.Run(ctx)
runtimex.PanicOnError(err, "cannot run experiment")
}

View File

@ -0,0 +1,79 @@
package main
import (
"context"
"net/url"
"os"
"path/filepath"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/version"
)
const (
softwareName = "miniooni"
softwareVersion = version.Version
)
// newSessionOrPanic creates and starts a new session or panics on failure
func newSessionOrPanic(ctx context.Context, currentOptions Options,
miniooniDir string, logger model.Logger) *engine.Session {
var proxyURL *url.URL
if currentOptions.Proxy != "" {
proxyURL = mustParseURL(currentOptions.Proxy)
}
kvstore2dir := filepath.Join(miniooniDir, "kvstore2")
kvstore, err := kvstore.NewFS(kvstore2dir)
runtimex.PanicOnError(err, "cannot create kvstore2 directory")
tunnelDir := filepath.Join(miniooniDir, "tunnel")
err = os.MkdirAll(tunnelDir, 0700)
runtimex.PanicOnError(err, "cannot create tunnelDir")
config := engine.SessionConfig{
KVStore: kvstore,
Logger: logger,
ProxyURL: proxyURL,
SoftwareName: softwareName,
SoftwareVersion: softwareVersion,
TorArgs: currentOptions.TorArgs,
TorBinary: currentOptions.TorBinary,
TunnelDir: tunnelDir,
}
if currentOptions.ProbeServicesURL != "" {
config.AvailableProbeServices = []model.OOAPIService{{
Address: currentOptions.ProbeServicesURL,
Type: "https",
}}
}
sess, err := engine.NewSession(ctx, config)
runtimex.PanicOnError(err, "cannot create measurement session")
log.Debugf("miniooni temporary directory: %s", sess.TempDir())
return sess
}
func lookupBackendsOrPanic(ctx context.Context, sess *engine.Session) {
log.Info("Looking up OONI backends; please be patient...")
err := sess.MaybeLookupBackendsContext(ctx)
runtimex.PanicOnError(err, "cannot lookup OONI backends")
}
func lookupLocationOrPanic(ctx context.Context, sess *engine.Session) {
log.Info("Looking up your location; please be patient...")
err := sess.MaybeLookupLocationContext(ctx)
runtimex.PanicOnError(err, "cannot lookup your location")
log.Debugf("- IP: %s", sess.ProbeIP()) // make sure it does not appear in default logs
log.Infof("- country: %s", sess.ProbeCC())
log.Infof("- network: %s (%s)", sess.ProbeNetworkName(), sess.ProbeASNString())
log.Infof("- resolver's IP: %s", sess.ResolverIP())
log.Infof("- resolver's network: %s (%s)", sess.ResolverNetworkName(),
sess.ResolverASNString())
}

View File

@ -0,0 +1,88 @@
package main
//
// Utility functions
//
import (
"errors"
"net/url"
"os"
"runtime"
"strings"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
// regularFileExists returns true if the given filepath exists and is a regular file
func regularFileExists(filepath string) bool {
stat, err := os.Stat(filepath)
return err == nil && stat.Mode().IsRegular()
}
// splitPair takes in input a string in the form KEY=VALUE and splits it. This
// function returns an error if it cannot find the = character to split the string.
func splitPair(s string) (string, string, error) {
v := strings.SplitN(s, "=", 2)
if len(v) != 2 {
return "", "", errors.New("invalid key-value pair")
}
return v[0], v[1], nil
}
// mustMakeMapStringString makes a map from string to string using as input a list
// of key-value pairs used to initialize the map, or panics on error
func mustMakeMapStringString(input []string) (output map[string]string) {
output = make(map[string]string)
for _, opt := range input {
key, value, err := splitPair(opt)
runtimex.PanicOnError(err, "cannot split key-value pair")
output[key] = value
}
return
}
// mustMakeMapStringAny makes a map from string to any using as input a list
// of key-value pairs used to initialize the map, or panics on error
func mustMakeMapStringAny(input []string) (output map[string]any) {
output = make(map[string]any)
for _, opt := range input {
key, value, err := splitPair(opt)
runtimex.PanicOnError(err, "cannot split key-value pair")
output[key] = value
}
return
}
// mustParseURL parses the given URL or panics
func mustParseURL(URL string) *url.URL {
rv, err := url.Parse(URL)
runtimex.PanicOnError(err, "cannot parse URL")
return rv
}
// gethomedir returns the home directory. If optionsHome is set, then we
// return that string as the home directory. Otherwise, we use typical
// platform-specific environment variables to determine the home. In case
// of failure to determine the home dir, we return an empty string.
func gethomedir(optionsHome string) string {
// See https://gist.github.com/miguelmota/f30a04a6d64bd52d7ab59ea8d95e54da
if optionsHome != "" {
return optionsHome
}
if runtime.GOOS == "windows" {
home := os.Getenv("HOMEDRIVE") + os.Getenv("HOMEPATH")
if home == "" {
home = os.Getenv("USERPROFILE")
}
return home
}
if runtime.GOOS == "linux" {
home := os.Getenv("XDG_CONFIG_HOME")
if home != "" {
return home
}
// fallthrough
}
return os.Getenv("HOME")
}