c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

fix: import path should be github.com/ooni/probe-cli/v3 (#200)

Browse Source 
See https://github.com/ooni/probe/issues/1335#issuecomment-771499511
This commit is contained in:
Simone Basso
2021-02-02 10:32:46 +01:00
committed by GitHub
parent faa9308b1e
commit b1ce300c8d
68 changed files with 86 additions and 85 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/autorun/autorun.go
-49
View File
@@ -1,49 +0,0 @@
// Package autorun contains code to manage automatic runs
package autorun
import "sync"
const (
// StatusScheduled indicates that OONI is scheduled to run
// periodically in the background.
StatusScheduled = "scheduled"
// StatusStopped indicates that OONI is not scheduled to
// run periodically in the background.
StatusStopped = "stopped"
// StatusRunning indicates that OONI is currently
// running in the background.
StatusRunning = "running"
)
// Manager manages automatic runs
type Manager interface {
LogShow() error
LogStream() error
Start() error
Status() (string, error)
Stop() error
}
var (
registry map[string]Manager
mtx sync.Mutex
)
func register(platform string, manager Manager) {
defer mtx.Unlock()
mtx.Lock()
if registry == nil {
registry = make(map[string]Manager)
}
registry[platform] = manager
}
// Get gets the specified autorun manager. This function
// returns nil if no autorun manager exists.
func Get(platform string) Manager {
defer mtx.Unlock()
mtx.Lock()
return registry[platform]
}
internal/autorun/autorun_darwin.go
-178
View File
@@ -1,178 +0,0 @@
package autorun
import (
"bytes"
"errors"
"fmt"
"io/ioutil"
"os"
"os/exec"
"strconv"
"strings"
"text/template"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/utils"
"github.com/ooni/probe-engine/cmd/jafar/shellx"
"golang.org/x/sys/unix"
)
type managerDarwin struct{}
var (
plistPath = os.ExpandEnv("$HOME/Library/LaunchAgents/org.ooni.cli.plist")
domainTarget = fmt.Sprintf("gui/%d", os.Getuid())
serviceTarget = fmt.Sprintf("%s/org.ooni.cli", domainTarget)
)
var plistTemplate = `
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>Label</key>
<string>org.ooni.cli</string>
<key>KeepAlive</key>
<false/>
<key>RunAtLoad</key>
<true/>
<key>ProgramArguments</key>
<array>
<string>{{ .Executable }}</string>
<string>--log-handler=syslog</string>
<string>run</string>
<string>unattended</string>
</array>
<key>StartInterval</key>
<integer>3600</integer>
</dict>
</plist>
`
func runQuiteQuietly(name string, arg ...string) error {
log.Infof("exec: %s %s", name, strings.Join(arg, " "))
return shellx.RunQuiet(name, arg...)
}
func darwinVersionMajor() (int, error) {
out, err := exec.Command("uname", "-r").Output()
if err != nil {
return 0, err
}
v := bytes.Split(out, []byte("."))
if len(v) != 3 {
return 0, errors.New("cannot split version")
}
major, err := strconv.Atoi(string(v[0]))
if err != nil {
return 0, err
}
return major, nil
}
var errNotImplemented = errors.New(
"autorun: command not implemented in this version of macOS")
func (managerDarwin) LogShow() error {
major, _ := darwinVersionMajor()
if major < 20 /* macOS 11.0 Big Sur */ {
return errNotImplemented
}
return shellx.Run("log", "show", "--info", "--debug",
"--process", "ooniprobe", "--style", "compact")
}
func (managerDarwin) LogStream() error {
return shellx.Run("log", "stream", "--style", "compact", "--level",
"debug", "--process", "ooniprobe")
}
func (managerDarwin) mustNotHavePlist() error {
log.Infof("exec: test -f %s && already_registered()", plistPath)
if utils.FileExists(plistPath) {
// This is not atomic. Do we need atomicity here?
return errors.New("autorun: service already registered")
}
return nil
}
func (managerDarwin) writePlist() error {
executable, err := os.Executable()
if err != nil {
return err
}
var out bytes.Buffer
t := template.Must(template.New("plist").Parse(plistTemplate))
in := struct{ Executable string }{Executable: executable}
if err := t.Execute(&out, in); err != nil {
return err
}
log.Infof("exec: writePlist(%s)", plistPath)
return ioutil.WriteFile(plistPath, out.Bytes(), 0644)
}
func (managerDarwin) start() error {
if err := runQuiteQuietly("launchctl", "enable", serviceTarget); err != nil {
return err
}
return runQuiteQuietly("launchctl", "bootstrap", domainTarget, plistPath)
}
func (m managerDarwin) Start() error {
operations := []func() error{m.mustNotHavePlist, m.writePlist, m.start}
for _, op := range operations {
if err := op(); err != nil {
return err
}
}
return nil
}
func (managerDarwin) stop() error {
var failure *exec.ExitError
err := runQuiteQuietly("launchctl", "bootout", serviceTarget)
if errors.As(err, &failure) && failure.ExitCode() == int(unix.ESRCH) {
err = nil
}
return err
}
func (managerDarwin) removeFile() error {
log.Infof("exec: rm -f %s", plistPath)
err := os.Remove(plistPath)
if errors.Is(err, unix.ENOENT) {
err = nil
}
return err
}
func (m managerDarwin) Stop() error {
operations := []func() error{m.stop, m.removeFile}
for _, op := range operations {
if err := op(); err != nil {
return err
}
}
return nil
}
func (m managerDarwin) Status() (string, error) {
err := runQuiteQuietly("launchctl", "kill", "SIGINFO", serviceTarget)
var failure *exec.ExitError
if errors.As(err, &failure) {
switch failure.ExitCode() {
case int(unix.ESRCH):
return StatusScheduled, nil
case 113: // exit code when there's no plist
return StatusStopped, nil
}
}
if err != nil {
return "", fmt.Errorf("autorun: unexpected error: %w", err)
}
return StatusRunning, nil
}
func init() {
register("darwin", managerDarwin{})
}
internal/bindata/bindata.go
-344
View File
File diff suppressed because one or more lines are too long
internal/cli/app/app.go
-20
View File
@@ -1,20 +0,0 @@
package app
import (
"os"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/version"
)
// Run the app. This is the main app entry point
func Run() {
root.Cmd.Version(version.Version)
_, err := root.Cmd.Parse(os.Args[1:])
if err != nil {
log.WithError(err).Error("failure in main command")
os.Exit(2)
}
return
}
internal/cli/autorun/autorun.go
-95
View File
@@ -1,95 +0,0 @@
package autorun
import (
"errors"
"runtime"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/autorun"
"github.com/ooni/probe-cli/internal/cli/onboard"
"github.com/ooni/probe-cli/internal/cli/root"
)
var errNotImplemented = errors.New("autorun: not implemented on this platform")
func init() {
cmd := root.Command("autorun", "Run automatic tests in the background")
cmd.Action(func(_ *kingpin.ParseContext) error {
probe, err := root.Init()
if err != nil {
log.Errorf("%s", err)
return err
}
if err := onboard.MaybeOnboarding(probe); err != nil {
log.WithError(err).Error("failed to perform onboarding")
return err
}
return nil
})
start := cmd.Command("start", "Start running automatic tests in the background")
start.Action(func(_ *kingpin.ParseContext) error {
svc := autorun.Get(runtime.GOOS)
if svc == nil {
return errNotImplemented
}
if err := svc.Start(); err != nil {
return err
}
log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
return nil
})
stop := cmd.Command("stop", "Stop running automatic tests in the background")
stop.Action(func(_ *kingpin.ParseContext) error {
svc := autorun.Get(runtime.GOOS)
if svc == nil {
return errNotImplemented
}
return svc.Stop()
})
logCmd := cmd.Command("log", "Access background runs logs")
stream := logCmd.Command("stream", "Stream background runs logs")
stream.Action(func(_ *kingpin.ParseContext) error {
svc := autorun.Get(runtime.GOOS)
if svc == nil {
return errNotImplemented
}
return svc.LogStream()
})
show := logCmd.Command("show", "Show background runs logs")
show.Action(func(_ *kingpin.ParseContext) error {
svc := autorun.Get(runtime.GOOS)
if svc == nil {
return errNotImplemented
}
return svc.LogShow()
})
status := cmd.Command("status", "Shows autorun instance status")
status.Action(func(_ *kingpin.ParseContext) error {
svc := autorun.Get(runtime.GOOS)
if svc == nil {
return errNotImplemented
}
out, err := svc.Status()
if err != nil {
return err
}
log.Infof("status: %s", out)
switch out {
case autorun.StatusRunning:
log.Info("hint: use 'ooniprobe autorun stop' to stop")
log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
case autorun.StatusScheduled:
log.Info("hint: use 'ooniprobe autorun stop' to stop")
log.Info("hint: use 'ooniprobe autorun log show' to see previous logs")
case autorun.StatusStopped:
log.Info("hint: use 'ooniprobe autorun start' to start")
}
return nil
})
}
internal/cli/geoip/geoip.go
-57
View File
@@ -1,57 +0,0 @@
package geoip
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/output"
)
func init() {
cmd := root.Command("geoip", "Perform a geoip lookup")
cmd.Action(func(_ *kingpin.ParseContext) error {
return dogeoip(defaultconfig)
})
}
type dogeoipconfig struct {
Logger log.Interface
NewProbeCLI func() (ooni.ProbeCLI, error)
SectionTitle func(string)
}
var defaultconfig = dogeoipconfig{
Logger: log.Log,
NewProbeCLI: root.NewProbeCLI,
SectionTitle: output.SectionTitle,
}
func dogeoip(config dogeoipconfig) error {
config.SectionTitle("GeoIP lookup")
probeCLI, err := config.NewProbeCLI()
if err != nil {
return err
}
engine, err := probeCLI.NewProbeEngine()
if err != nil {
return err
}
defer engine.Close()
err = engine.MaybeLookupLocation()
if err != nil {
return err
}
config.Logger.WithFields(log.Fields{
"type": "table",
"asn": engine.ProbeASNString(),
"network_name": engine.ProbeNetworkName(),
"country_code": engine.ProbeCC(),
"ip": engine.ProbeIP(),
}).Info("Looked up your location")
return nil
}
internal/cli/geoip/geoip_test.go
-134
View File
@@ -1,134 +0,0 @@
package geoip
import (
"errors"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/oonitest"
)
func TestNewProbeCLIFailed(t *testing.T) {
fo := &oonitest.FakeOutput{}
expected := errors.New("mocked error")
err := dogeoip(dogeoipconfig{
SectionTitle: fo.SectionTitle,
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return nil, expected
},
})
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if len(fo.FakeSectionTitle) != 1 {
t.Fatal("invalid section title list size")
}
if fo.FakeSectionTitle[0] != "GeoIP lookup" {
t.Fatal("unexpected string")
}
}
func TestNewProbeEngineFailed(t *testing.T) {
fo := &oonitest.FakeOutput{}
expected := errors.New("mocked error")
cli := &oonitest.FakeProbeCLI{
FakeProbeEngineErr: expected,
}
err := dogeoip(dogeoipconfig{
SectionTitle: fo.SectionTitle,
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return cli, nil
},
})
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if len(fo.FakeSectionTitle) != 1 {
t.Fatal("invalid section title list size")
}
if fo.FakeSectionTitle[0] != "GeoIP lookup" {
t.Fatal("unexpected string")
}
}
func TestMaybeLookupLocationFailed(t *testing.T) {
fo := &oonitest.FakeOutput{}
expected := errors.New("mocked error")
engine := &oonitest.FakeProbeEngine{
FakeMaybeLookupLocation: expected,
}
cli := &oonitest.FakeProbeCLI{
FakeProbeEnginePtr: engine,
}
err := dogeoip(dogeoipconfig{
SectionTitle: fo.SectionTitle,
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return cli, nil
},
})
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if len(fo.FakeSectionTitle) != 1 {
t.Fatal("invalid section title list size")
}
if fo.FakeSectionTitle[0] != "GeoIP lookup" {
t.Fatal("unexpected string")
}
}
func TestMaybeLookupLocationSuccess(t *testing.T) {
fo := &oonitest.FakeOutput{}
engine := &oonitest.FakeProbeEngine{
FakeProbeASNString: "AS30722",
FakeProbeCC: "IT",
FakeProbeNetworkName: "Vodafone Italia S.p.A.",
FakeProbeIP: "130.25.90.216",
}
cli := &oonitest.FakeProbeCLI{
FakeProbeEnginePtr: engine,
}
handler := &oonitest.FakeLoggerHandler{}
err := dogeoip(dogeoipconfig{
SectionTitle: fo.SectionTitle,
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return cli, nil
},
Logger: &log.Logger{
Handler: handler,
Level: log.DebugLevel,
},
})
if err != nil {
t.Fatal(err)
}
if len(fo.FakeSectionTitle) != 1 {
t.Fatal("invalid section title list size")
}
if fo.FakeSectionTitle[0] != "GeoIP lookup" {
t.Fatal("unexpected string")
}
if len(handler.FakeEntries) != 1 {
t.Fatal("invalid number of written entries")
}
entry := handler.FakeEntries[0]
if entry.Level != log.InfoLevel {
t.Fatal("invalid log level")
}
if entry.Message != "Looked up your location" {
t.Fatal("invalid .Message")
}
if entry.Fields["asn"].(string) != "AS30722" {
t.Fatal("invalid asn")
}
if entry.Fields["country_code"].(string) != "IT" {
t.Fatal("invalid asn")
}
if entry.Fields["network_name"].(string) != "Vodafone Italia S.p.A." {
t.Fatal("invalid asn")
}
if entry.Fields["ip"].(string) != "130.25.90.216" {
t.Fatal("invalid asn")
}
}
internal/cli/info/info.go
-36
View File
@@ -1,36 +0,0 @@
package info
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/ooni"
)
func init() {
cmd := root.Command("info", "Display information about OONI Probe")
cmd.Action(func(_ *kingpin.ParseContext) error {
return doinfo(defaultconfig)
})
}
type doinfoconfig struct {
Logger log.Interface
NewProbeCLI func() (ooni.ProbeCLI, error)
}
var defaultconfig = doinfoconfig{
Logger: log.Log,
NewProbeCLI: root.NewProbeCLI,
}
func doinfo(config doinfoconfig) error {
probeCLI, err := config.NewProbeCLI()
if err != nil {
config.Logger.Errorf("%s", err)
return err
}
config.Logger.WithFields(log.Fields{"path": probeCLI.Home()}).Info("Home")
config.Logger.WithFields(log.Fields{"path": probeCLI.TempDir()}).Info("TempDir")
return nil
}
internal/cli/info/info_test.go
-80
View File
@@ -1,80 +0,0 @@
package info
import (
"errors"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/oonitest"
)
func TestNewProbeCLIFailed(t *testing.T) {
expected := errors.New("mocked error")
handler := &oonitest.FakeLoggerHandler{}
err := doinfo(doinfoconfig{
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return nil, expected
},
Logger: &log.Logger{
Handler: handler,
Level: log.DebugLevel,
},
})
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if len(handler.FakeEntries) != 1 {
t.Fatal("invalid number of log entries")
}
entry := handler.FakeEntries[0]
if entry.Level != log.ErrorLevel {
t.Fatal("invalid log level")
}
if entry.Message != "mocked error" {
t.Fatal("invalid .Message")
}
}
func TestSuccess(t *testing.T) {
handler := &oonitest.FakeLoggerHandler{}
cli := &oonitest.FakeProbeCLI{
FakeHome: "fakehome",
FakeTempDir: "faketempdir",
}
err := doinfo(doinfoconfig{
NewProbeCLI: func() (ooni.ProbeCLI, error) {
return cli, nil
},
Logger: &log.Logger{
Handler: handler,
Level: log.DebugLevel,
},
})
if err != nil {
t.Fatal(err)
}
if len(handler.FakeEntries) != 2 {
t.Fatal("invalid number of log entries")
}
entry := handler.FakeEntries[0]
if entry.Level != log.InfoLevel {
t.Fatal("invalid log level")
}
if entry.Message != "Home" {
t.Fatal("invalid .Message")
}
if entry.Fields["path"].(string) != "fakehome" {
t.Fatal("invalid path")
}
entry = handler.FakeEntries[1]
if entry.Level != log.InfoLevel {
t.Fatal("invalid log level")
}
if entry.Message != "TempDir" {
t.Fatal("invalid .Message")
}
if entry.Fields["path"].(string) != "faketempdir" {
t.Fatal("invalid path")
}
}
internal/cli/list/list.go
-128
View File
@@ -1,128 +0,0 @@
package list
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/output"
)
func init() {
cmd := root.Command("list", "List results")
resultID := cmd.Arg("id", "the id of the result to list measurements for").Int64()
cmd.Action(func(_ *kingpin.ParseContext) error {
probeCLI, err := root.Init()
if err != nil {
log.WithError(err).Error("failed to initialize root context")
return err
}
if *resultID > 0 {
measurements, err := database.ListMeasurements(probeCLI.DB(), *resultID)
if err != nil {
log.WithError(err).Error("failed to list measurements")
return err
}
msmtSummary := output.MeasurementSummaryData{
TotalCount: 0,
AnomalyCount: 0,
DataUsageUp: 0.0,
DataUsageDown: 0.0,
TotalRuntime: 0,
ASN: 0,
NetworkName: "",
NetworkCountryCode: "ZZ",
}
isFirst := true
isLast := false
for idx, msmt := range measurements {
if idx > 0 {
isFirst = false
}
if idx == len(measurements)-1 {
isLast = true
}
// We assume that since these are summary level information the first
// item will contain the information necessary.
if isFirst {
msmtSummary.TotalRuntime = msmt.Result.Runtime
msmtSummary.DataUsageUp = msmt.DataUsageUp
msmtSummary.DataUsageDown = msmt.DataUsageDown
msmtSummary.NetworkName = msmt.NetworkName
msmtSummary.NetworkCountryCode = msmt.Network.CountryCode
msmtSummary.ASN = msmt.ASN
msmtSummary.StartTime = msmt.Measurement.StartTime
}
if msmt.IsAnomaly.Bool == true {
msmtSummary.AnomalyCount++
}
msmtSummary.TotalCount++
output.MeasurementItem(msmt, isFirst, isLast)
}
output.MeasurementSummary(msmtSummary)
} else {
doneResults, incompleteResults, err := database.ListResults(probeCLI.DB())
if err != nil {
log.WithError(err).Error("failed to list results")
return err
}
if len(incompleteResults) > 0 {
output.SectionTitle("Incomplete results")
}
for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: result.Network.ASN,
MeasurementCount: 0,
MeasurementAnomalyCount: 0,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
}
resultSummary := output.ResultSummaryData{}
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(probeCLI.DB(), result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
testKeys, err := database.GetResultTestKeys(probeCLI.DB(), result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to get testKeys")
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: result.Network.ASN,
TestKeys: testKeys,
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
resultSummary.TotalTests++
netCount[result.Network.ASN]++
resultSummary.TotalDataUsageUp += result.DataUsageUp
resultSummary.TotalDataUsageDown += result.DataUsageDown
}
resultSummary.TotalNetworks = int64(len(netCount))
output.ResultSummary(resultSummary)
}
return nil
})
}
internal/cli/onboard/onboard.go
-189
View File
@@ -1,189 +0,0 @@
package onboard
import (
"fmt"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/fatih/color"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/config"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/output"
"github.com/pkg/errors"
"gopkg.in/AlecAivazis/survey.v1"
)
// Onboarding start the interactive onboarding procedure
func Onboarding(config *config.Config) error {
output.SectionTitle("What is OONI Probe?")
fmt.Println()
output.Paragraph("Your tool for detecting internet censorship!")
fmt.Println()
output.Paragraph("OONI Probe checks whether your provider blocks access to sites and services. Run OONI Probe to collect evidence of internet censorship and to measure your network performance.")
fmt.Println()
err := output.PressEnterToContinue("Press 'Enter' to continue...")
if err != nil {
return err
}
output.SectionTitle("Heads Up")
fmt.Println()
output.Bullet("Anyone monitoring your internet activity (such as your government or ISP) may be able to see that you are running OONI Probe.")
fmt.Println()
output.Bullet("The network data you will collect will automatically be published (unless you opt-out in the settings).")
fmt.Println()
output.Bullet("You may test objectionable sites.")
fmt.Println()
output.Bullet("Read the documentation to learn more.")
fmt.Println()
err = output.PressEnterToContinue("Press 'Enter' to continue...")
if err != nil {
return err
}
output.SectionTitle("Pop Quiz!")
output.Paragraph("")
answer := ""
quiz1 := &survey.Select{
Message: "Anyone monitoring my internet activity may be able to see that I am running OONI Probe.",
Options: []string{"true", "false"},
Default: "true",
}
if err := survey.AskOne(quiz1, &answer, nil); err != nil {
return err
}
if answer != "true" {
output.Paragraph(color.RedString("Actually..."))
output.Paragraph("OONI Probe is not a privacy tool. Therefore, anyone monitoring your internet activity may be able to see which software you are running.")
} else {
output.Paragraph(color.BlueString("Good job!"))
}
answer = ""
quiz2 := &survey.Select{
Message: "The network data I will collect will automatically be published (unless I opt-out in the settings).",
Options: []string{"true", "false"},
Default: "true",
}
if err := survey.AskOne(quiz2, &answer, nil); err != nil {
return err
}
if answer != "true" {
output.Paragraph(color.RedString("Actually..."))
output.Paragraph("The network data you will collect will automatically be published to increase transparency of internet censorship (unless you opt-out in the settings).")
} else {
output.Paragraph(color.BlueString("Well done!"))
}
changeDefaults := false
prompt := &survey.Confirm{
Message: "Do you want to change the default settings?",
Default: false,
}
if err := survey.AskOne(prompt, &changeDefaults, nil); err != nil {
return err
}
settings := struct {
IncludeIP bool
IncludeNetwork bool
UploadResults bool
SendCrashReports bool
}{}
settings.IncludeIP = false
settings.IncludeNetwork = true
settings.UploadResults = true
settings.SendCrashReports = true
if changeDefaults == true {
var qs = []*survey.Question{
{
Name: "IncludeIP",
Prompt: &survey.Confirm{Message: "Should we include your IP?"},
},
{
Name: "IncludeNetwork",
Prompt: &survey.Confirm{
Message: "Can we include your network name?",
Default: true,
},
},
{
Name: "UploadResults",
Prompt: &survey.Confirm{
Message: "Can we upload your results?",
Default: true,
},
},
{
Name: "SendCrashReports",
Prompt: &survey.Confirm{
Message: "Can we send crash reports to OONI?",
Default: true,
},
},
}
if err := survey.Ask(qs, &settings); err != nil {
log.WithError(err).Error("there was an error in parsing your responses")
return err
}
}
config.Lock()
config.InformedConsent = true
config.Advanced.SendCrashReports = settings.SendCrashReports
config.Sharing.UploadResults = settings.UploadResults
config.Unlock()
if err := config.Write(); err != nil {
log.WithError(err).Error("failed to write config file")
return err
}
return nil
}
// MaybeOnboarding will run the onboarding process only if the informed consent
// config option is set to false
func MaybeOnboarding(probe *ooni.Probe) error {
if probe.Config().InformedConsent == false {
if probe.IsBatch() == true {
return errors.New("cannot run onboarding in batch mode")
}
if err := Onboarding(probe.Config()); err != nil {
return errors.Wrap(err, "onboarding")
}
}
return nil
}
func init() {
cmd := root.Command("onboard", "Starts the onboarding process")
yes := cmd.Flag("yes", "Answer yes to all the onboarding questions.").Bool()
cmd.Action(func(_ *kingpin.ParseContext) error {
probe, err := root.Init()
if err != nil {
return err
}
if *yes == true {
probe.Config().Lock()
probe.Config().InformedConsent = true
probe.Config().Unlock()
if err := probe.Config().Write(); err != nil {
log.WithError(err).Error("failed to write config file")
return err
}
return nil
}
if probe.IsBatch() == true {
return errors.New("cannot do onboarding in batch mode")
}
return Onboarding(probe.Config())
})
}
internal/cli/reset/reset.go
-37
View File
@@ -1,37 +0,0 @@
package reset
import (
"os"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
)
func init() {
cmd := root.Command("reset", "Cleanup an old or experimental installation")
force := cmd.Flag("force", "Force deleting the OONI Home").Bool()
cmd.Action(func(_ *kingpin.ParseContext) error {
ctx, err := root.Init()
if err != nil {
log.WithError(err).Error("failed to init root context")
return err
}
// We need to first the DB otherwise the DB will be rewritten on close when
// we delete the home directory.
err = ctx.DB().Close()
if err != nil {
log.WithError(err).Error("failed to close the DB")
return err
}
if *force == true {
os.RemoveAll(ctx.Home())
log.Infof("Deleted %s", ctx.Home())
} else {
log.Infof("Run with --force to delete %s", ctx.Home())
}
return nil
})
}
internal/cli/rm/rm.go
-94
View File
@@ -1,94 +0,0 @@
package rm
import (
"errors"
"fmt"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/database"
survey "gopkg.in/AlecAivazis/survey.v1"
db "upper.io/db.v3"
"upper.io/db.v3/lib/sqlbuilder"
)
func deleteAll(sess sqlbuilder.Database, skipInteractive bool) error {
if skipInteractive == false {
answer := ""
confirm := &survey.Select{
Message: fmt.Sprintf("Are you sure you wish to delete ALL results"),
Options: []string{"true", "false"},
Default: "false",
}
survey.AskOne(confirm, &answer, nil)
if answer == "false" {
return errors.New("canceled by user")
}
}
doneResults, incompleteResults, err := database.ListResults(sess)
if err != nil {
log.WithError(err).Error("failed to list results")
return err
}
cnt := 0
for _, result := range incompleteResults {
err = database.DeleteResult(sess, result.Result.ID)
if err == db.ErrNoMoreRows {
log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
}
cnt++
}
for _, result := range doneResults {
err = database.DeleteResult(sess, result.Result.ID)
if err == db.ErrNoMoreRows {
log.WithError(err).Errorf("failed to delete result #%d", result.Result.ID)
}
cnt++
}
log.Infof("Deleted #%d measurements", cnt)
return nil
}
func init() {
cmd := root.Command("rm", "Delete a result")
yes := cmd.Flag("yes", "Skip interactive prompt").Bool()
all := cmd.Flag("all", "Delete all measurements").Bool()
resultID := cmd.Arg("id", "the id of the result to delete").Int64()
cmd.Action(func(_ *kingpin.ParseContext) error {
ctx, err := root.Init()
if err != nil {
log.Errorf("%s", err)
return err
}
if *all == true {
return deleteAll(ctx.DB(), *yes)
}
if *yes == true {
err = database.DeleteResult(ctx.DB(), *resultID)
if err == db.ErrNoMoreRows {
return errors.New("result not found")
}
return err
}
answer := ""
confirm := &survey.Select{
Message: fmt.Sprintf("Are you sure you wish to delete the result #%d", *resultID),
Options: []string{"true", "false"},
Default: "false",
}
survey.AskOne(confirm, &answer, nil)
if answer == "false" {
return errors.New("canceled by user")
}
err = database.DeleteResult(ctx.DB(), *resultID)
if err == db.ErrNoMoreRows {
return errors.New("result not found")
}
return err
})
}
internal/cli/root/root.go
-94
View File
@@ -1,94 +0,0 @@
package root
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/log/handlers/batch"
"github.com/ooni/probe-cli/internal/log/handlers/cli"
"github.com/ooni/probe-cli/internal/log/handlers/syslog"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/utils"
"github.com/ooni/probe-cli/internal/version"
)
// Cmd is the root command
var Cmd = kingpin.New("ooniprobe", "")
// Command is syntax sugar for defining sub-commands
var Command = Cmd.Command
// Init should be called by all subcommand that care to have a ooni.Context instance
var Init func() (*ooni.Probe, error)
// NewProbeCLI is like Init but returns a ooni.ProbeCLI instead.
func NewProbeCLI() (ooni.ProbeCLI, error) {
probeCLI, err := Init()
if err != nil {
return nil, err
}
return probeCLI, nil
}
func init() {
configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
isVerbose := Cmd.Flag("verbose", "Enable verbose log output.").Short('v').Bool()
isBatch := Cmd.Flag("batch", "Enable batch command line usage.").Bool()
logHandler := Cmd.Flag(
"log-handler", "Set the desired log handler (one of: batch, cli, syslog)",
).String()
softwareName := Cmd.Flag(
"software-name", "Override application name",
).Default("ooniprobe-cli").String()
softwareVersion := Cmd.Flag(
"software-version", "Override the application version",
).Default(version.Version).String()
Cmd.PreAction(func(ctx *kingpin.ParseContext) error {
// TODO(bassosimone): we need to properly deprecate --batch
// in favour of more granular command line flags.
if *isBatch && *logHandler != "" {
log.Fatal("cannot specify --batch and --log-handler together")
}
if *isBatch {
*logHandler = "batch"
}
switch *logHandler {
case "batch":
log.SetHandler(batch.Default)
case "cli", "":
log.SetHandler(cli.Default)
case "syslog":
log.SetHandler(syslog.Default)
default:
log.Fatalf("unknown --log-handler: %s", *logHandler)
}
if *isVerbose {
log.SetLevel(log.DebugLevel)
log.Debugf("ooni version %s", version.Version)
}
Init = func() (*ooni.Probe, error) {
var err error
homePath, err := utils.GetOONIHome()
if err != nil {
return nil, err
}
probe := ooni.NewProbe(*configPath, homePath)
err = probe.Init(*softwareName, *softwareVersion)
if err != nil {
return nil, err
}
if *isBatch {
probe.SetIsBatch(true)
}
return probe, nil
}
return nil
})
}
internal/cli/run/run.go
-97
View File
@@ -1,97 +0,0 @@
package run
import (
"runtime"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/fatih/color"
"github.com/ooni/probe-cli/internal/cli/onboard"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/nettests"
"github.com/ooni/probe-cli/internal/ooni"
)
func init() {
cmd := root.Command("run", "Run a test group or OONI Run link")
noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool()
var probe *ooni.Probe
cmd.Action(func(_ *kingpin.ParseContext) error {
var err error
probe, err = root.Init()
if err != nil {
log.Errorf("%s", err)
return err
}
if err = onboard.MaybeOnboarding(probe); err != nil {
log.WithError(err).Error("failed to perform onboarding")
return err
}
if *noCollector == true {
probe.Config().Sharing.UploadResults = false
}
return nil
})
functionalRun := func(pred func(name string, gr nettests.Group) bool) error {
for name, group := range nettests.All {
if pred(name, group) != true {
continue
}
log.Infof("Running %s tests", color.BlueString(name))
conf := nettests.RunGroupConfig{GroupName: name, Probe: probe}
if err := nettests.RunGroup(conf); err != nil {
log.WithError(err).Errorf("failed to run %s", name)
}
}
return nil
}
genRunWithGroupName := func(targetName string) func(*kingpin.ParseContext) error {
return func(*kingpin.ParseContext) error {
return functionalRun(func(groupName string, gr nettests.Group) bool {
return groupName == targetName
})
}
}
websitesCmd := cmd.Command("websites", "")
inputFile := websitesCmd.Flag("input-file", "File containing input URLs").Strings()
input := websitesCmd.Flag("input", "Test the specified URL").Strings()
websitesCmd.Action(func(_ *kingpin.ParseContext) error {
log.Infof("Running %s tests", color.BlueString("websites"))
return nettests.RunGroup(nettests.RunGroupConfig{
GroupName: "websites",
Probe: probe,
InputFiles: *inputFile,
Inputs: *input,
})
})
easyRuns := []string{"im", "performance", "circumvention", "middlebox"}
for _, name := range easyRuns {
cmd.Command(name, "").Action(genRunWithGroupName(name))
}
unattendedCmd := cmd.Command("unattended", "")
unattendedCmd.Action(func(_ *kingpin.ParseContext) error {
if runtime.GOOS == "darwin" {
// Until we have enabled the check-in API we're called every
// hour on darwin and we need to self throttle.
// TODO(bassosimone): switch to check-in and remove this hack.
const veryFew = 10
probe.Config().Nettests.WebsitesURLLimit = veryFew
}
return functionalRun(func(name string, gr nettests.Group) bool {
return gr.UnattendedOK == true
})
})
allCmd := cmd.Command("all", "").Default()
allCmd.Action(func(_ *kingpin.ParseContext) error {
return functionalRun(func(name string, gr nettests.Group) bool {
return true
})
})
}
internal/cli/show/show.go
-28
View File
@@ -1,28 +0,0 @@
package nettest
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/output"
)
func init() {
cmd := root.Command("show", "Show a specific measurement")
msmtID := cmd.Arg("id", "the id of the measurement to show").Required().Int64()
cmd.Action(func(_ *kingpin.ParseContext) error {
ctx, err := root.Init()
if err != nil {
log.WithError(err).Error("failed to initialize root context")
return err
}
msmt, err := database.GetMeasurementJSON(ctx.DB(), *msmtID)
if err != nil {
log.Errorf("error: %v", err)
return err
}
output.MeasurementJSON(msmt)
return nil
})
}
internal/cli/upload/upload.go
-17
View File
@@ -1,17 +0,0 @@
package upload
import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/cli/root"
)
func init() {
cmd := root.Command("upload", "Upload a specific measurement")
cmd.Action(func(_ *kingpin.ParseContext) error {
log.Info("Uploading")
log.Error("this function is not implemented")
return nil
})
}
internal/cli/version/version.go
-17
View File
@@ -1,17 +0,0 @@
package version
import (
"fmt"
"github.com/alecthomas/kingpin"
"github.com/ooni/probe-cli/internal/cli/root"
"github.com/ooni/probe-cli/internal/version"
)
func init() {
cmd := root.Command("version", "Show version.")
cmd.Action(func(_ *kingpin.ParseContext) error {
fmt.Println(version.Version)
return nil
})
}
internal/config/parser.go
-100
View File
@@ -1,100 +0,0 @@
package config
import (
"encoding/json"
"io/ioutil"
"sync"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/crashreport"
"github.com/ooni/probe-cli/internal/utils"
"github.com/pkg/errors"
)
// ConfigVersion is the current version of the config
const ConfigVersion = 1
// ReadConfig reads the configuration from the path
func ReadConfig(path string) (*Config, error) {
b, err := ioutil.ReadFile(path)
if err != nil {
return nil, err
}
c, err := ParseConfig(b)
if err != nil {
return nil, errors.Wrap(err, "parsing config")
}
c.path = path
return c, err
}
// ParseConfig returns config from JSON bytes.
func ParseConfig(b []byte) (*Config, error) {
var c Config
if err := json.Unmarshal(b, &c); err != nil {
return nil, errors.Wrap(err, "parsing json")
}
home, err := utils.GetOONIHome()
if err != nil {
return nil, err
}
c.path = utils.ConfigPath(home)
if c.Advanced.SendCrashReports == false {
log.Info("Disabling crash reporting.")
crashreport.Disabled = true
}
return &c, nil
}
// Config for the OONI Probe installation
type Config struct {
// Private settings
Comment string `json:"_"`
Version int64 `json:"_version"`
InformedConsent bool `json:"_informed_consent"`
Sharing Sharing `json:"sharing"`
Nettests Nettests `json:"nettests"`
Advanced Advanced `json:"advanced"`
mutex sync.Mutex
path string
}
// Write the config file in json to the path
func (c *Config) Write() error {
c.Lock()
defer c.Unlock()
configJSON, _ := json.MarshalIndent(c, "", " ")
if c.path == "" {
return errors.New("config file path is empty")
}
if err := ioutil.WriteFile(c.path, configJSON, 0644); err != nil {
return errors.Wrap(err, "writing config JSON")
}
return nil
}
// Lock acquires the write mutex
func (c *Config) Lock() {
c.mutex.Lock()
}
// Unlock releases the write mutex
func (c *Config) Unlock() {
c.mutex.Unlock()
}
// MaybeMigrate checks the current config version and the config file on disk
// and if necessary performs and upgrade of the configuration file.
func (c *Config) MaybeMigrate() error {
if c.Version < ConfigVersion {
return c.Write()
}
return nil
}
internal/config/parser_test.go
-97
View File
@@ -1,97 +0,0 @@
package config
import (
"crypto/sha256"
"encoding/hex"
"io"
"io/ioutil"
"os"
"testing"
)
func getShasum(path string) (string, error) {
hasher := sha256.New()
f, err := os.Open(path)
if err != nil {
return "", err
}
defer f.Close()
if _, err := io.Copy(hasher, f); err != nil {
return "", err
}
return hex.EncodeToString(hasher.Sum(nil)), nil
}
func TestParseConfig(t *testing.T) {
config, err := ReadConfig("testdata/valid-config.json")
if err != nil {
t.Error(err)
}
if config.Sharing.UploadResults != true {
t.Fatal("not the expected value for UploadResults")
}
}
func TestUpdateConfig(t *testing.T) {
tmpFile, err := ioutil.TempFile(os.TempDir(), "ooniconfig-")
if err != nil {
t.Error(err)
}
configPath := tmpFile.Name()
defer os.Remove(configPath)
data, err := ioutil.ReadFile("testdata/config-v0.json")
if err != nil {
t.Error(err)
}
err = ioutil.WriteFile(configPath, data, 0644)
if err != nil {
t.Error(err)
}
origShasum, err := getShasum(configPath)
if err != nil {
t.Error(err)
}
config, err := ReadConfig(configPath)
if err != nil {
t.Error(err)
}
origUploadResults := config.Sharing.UploadResults
origInformedConsent := config.InformedConsent
if err != nil {
t.Error(err)
}
config.MaybeMigrate()
migratedShasum, err := getShasum(configPath)
if err != nil {
t.Error(err)
}
if migratedShasum == origShasum {
t.Fatal("the config was not migrated")
}
newConfig, err := ReadConfig(configPath)
if err != nil {
t.Error(err)
}
if newConfig.Sharing.UploadResults != origUploadResults {
t.Error("UploadResults differs")
}
if newConfig.InformedConsent != origInformedConsent {
t.Error("InformedConsent differs")
}
// Check that the config file stays the same if it's already the most up to
// date version
config.MaybeMigrate()
finalShasum, err := getShasum(configPath)
if err != nil {
t.Error(err)
}
if migratedShasum != finalShasum {
t.Fatal("the config was migrated again")
}
}
internal/config/settings.go
-50
View File
@@ -1,50 +0,0 @@
package config
var websiteCategories = []string{
"ALDR",
"ANON",
"COMM",
"COMT",
"CTRL",
"CULTR",
"DATE",
"ECON",
"ENV",
"FILE",
"GAME",
"GMB",
"GOVT",
"GRP",
"HACK",
"HATE",
"HOST",
"HUMR",
"IGO",
"LGBT",
"MILX",
"MMED",
"NEWS",
"POLR",
"PORN",
"PROV",
"PUBH",
"REL",
"SRCH",
"XED",
}
// Sharing settings
type Sharing struct {
UploadResults bool `json:"upload_results"`
}
// Advanced settings
type Advanced struct {
SendCrashReports bool `json:"send_crash_reports"`
}
// Nettests related settings
type Nettests struct {
WebsitesURLLimit int64 `json:"websites_url_limit"`
WebsitesEnabledCategoryCodes []string `json:"websites_enabled_category_codes"`
}
internal/config/testdata/config-v0.json
Vendored
-62
View File
@@ -1,62 +0,0 @@
{
"_version": 0,
"_informed_consent": true,
"_is_beta": true,
"auto_update": true,
"sharing": {
"include_ip": false,
"include_asn": true,
"include_country": true,
"include_gps": true,
"upload_results": true
},
"notifications": {
"enabled": true,
"notify_on_test_completion": true,
"notify_on_news": false
},
"automated_testing": {
"enabled": false,
"enabled_tests": [
"web-connectivity",
"facebook-messenger",
"whatsapp",
"telegram",
"dash",
"ndt",
"http-invalid-request-line",
"http-header-field-manipulation"
],
"monthly_allowance": "300MB"
},
"test_settings": {
"websites": {
"enabled_categories": []
},
"instant_messaging": {
"enabled_tests": [
"facebook-messenger",
"whatsapp",
"telegram"
]
},
"performance": {
"ndt_server": "auto",
"ndt_server_port": "auto",
"dash_server": "auto",
"dash_server_port": "auto"
},
"middlebox": {
"enabled_tests": [
"http-invalid-request-line",
"http-header-field-manipulation"
]
}
},
"advanced": {
"use_domain_fronting": false,
"send_crash_reports": true,
"collector_url": "",
"bouncer_url": "https://bouncer.ooni.io"
}
}
internal/config/testdata/valid-config.json
Vendored
-13
View File
@@ -1,13 +0,0 @@
{
"_version": 1,
"_informed_consent": false,
"sharing": {
"upload_results": true
},
"nettests": {
"websites_url_limit": 0
},
"advanced": {
"send_crash_reports": true
}
}
internal/crashreport/crashreport.go
-62
View File
@@ -1,62 +0,0 @@
package crashreport
import (
"github.com/apex/log"
"github.com/getsentry/raven-go"
)
// Disabled flag is used to globally disable crash reporting and make all the
// crash reporting logic a no-op.
var Disabled = false
var client *raven.Client
// CapturePanic is a wrapper around raven.CapturePanic that becomes a noop if
// `Disabled` is set to true.
func CapturePanic(f func(), tags map[string]string) (interface{}, string) {
if Disabled == true {
return nil, ""
}
return client.CapturePanic(f, tags)
}
// CapturePanicAndWait is a wrapper around raven.CapturePanicAndWait that becomes a noop if
// `Disabled` is set to true.
func CapturePanicAndWait(f func(), tags map[string]string) (interface{}, string) {
if Disabled == true {
return nil, ""
}
return client.CapturePanicAndWait(f, tags)
}
// CaptureError is a wrapper around raven.CaptureError
func CaptureError(err error, tags map[string]string) string {
if Disabled == true {
return ""
}
return client.CaptureError(err, tags)
}
// CaptureErrorAndWait is a wrapper around raven.CaptureErrorAndWait
func CaptureErrorAndWait(err error, tags map[string]string) string {
if Disabled == true {
return ""
}
return client.CaptureErrorAndWait(err, tags)
}
// Wait will block on sending messages to the sentry server
func Wait() {
if Disabled == false {
log.Info("sending exception backtrace")
client.Wait()
}
}
func init() {
var err error
client, err = raven.NewClient("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954", nil)
if err != nil {
log.WithError(err).Error("failed to create a raven client")
}
}
internal/database/actions.go
-355
View File
@@ -1,355 +0,0 @@
package database
import (
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"net/http"
"net/url"
"os"
"path/filepath"
"reflect"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/enginex"
"github.com/ooni/probe-cli/internal/utils"
"github.com/pkg/errors"
db "upper.io/db.v3"
"upper.io/db.v3/lib/sqlbuilder"
)
// ListMeasurements given a result ID
func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementURLNetwork, error) {
measurements := []MeasurementURLNetwork{}
req := sess.Select(
db.Raw("networks.*"),
db.Raw("urls.*"),
db.Raw("measurements.*"),
db.Raw("results.*"),
).From("results").
Join("measurements").On("results.result_id = measurements.result_id").
Join("networks").On("results.network_id = networks.network_id").
LeftJoin("urls").On("urls.url_id = measurements.url_id").
OrderBy("measurements.measurement_start_time").
Where("results.result_id = ?", resultID)
if err := req.All(&measurements); err != nil {
log.Errorf("failed to run query %s: %v", req.String(), err)
return measurements, err
}
return measurements, nil
}
// GetMeasurementJSON returns a map[string]interface{} given a database and a measurementID
func GetMeasurementJSON(sess sqlbuilder.Database, measurementID int64) (map[string]interface{}, error) {
var (
measurement MeasurementURLNetwork
msmtJSON map[string]interface{}
)
req := sess.Select(
db.Raw("urls.*"),
db.Raw("measurements.*"),
).From("measurements").
LeftJoin("urls").On("urls.url_id = measurements.url_id").
Where("measurements.measurement_id= ?", measurementID)
if err := req.One(&measurement); err != nil {
log.Errorf("failed to run query %s: %v", req.String(), err)
return nil, err
}
if measurement.IsUploaded {
// TODO(bassosimone): this should be a function exposed by probe-engine
reportID := measurement.Measurement.ReportID.String
measurementURL := &url.URL{
Scheme: "https",
Host: "api.ooni.io",
Path: "/api/v1/raw_measurement",
}
query := url.Values{}
query.Add("report_id", reportID)
if measurement.URL.URL.Valid == true {
query.Add("input", measurement.URL.URL.String)
}
measurementURL.RawQuery = query.Encode()
log.Debugf("using %s", measurementURL.String())
resp, err := http.Get(measurementURL.String())
if err != nil {
log.Errorf("failed to fetch the measurement %s %s", reportID, measurement.URL.URL.String)
return nil, err
}
defer resp.Body.Close()
if err := json.NewDecoder(resp.Body).Decode(&msmtJSON); err != nil {
log.Error("failed to unmarshal the measurement_json")
return nil, err
}
return msmtJSON, nil
}
// MeasurementFilePath might be NULL because the measurement from a
// 3.0.0-beta install
if measurement.Measurement.MeasurementFilePath.Valid == false {
log.Error("invalid measurement_file_path")
log.Error("backup your OONI_HOME and run `ooniprobe reset`")
return nil, errors.New("cannot access measurement file")
}
measurementFilePath := measurement.Measurement.MeasurementFilePath.String
b, err := ioutil.ReadFile(measurementFilePath)
if err != nil {
return nil, err
}
if err := json.Unmarshal(b, &msmtJSON); err != nil {
log.Error("failed to unmarshal the measurement_json")
log.Error("backup your OONI_HOME and run `ooniprobe reset`")
return nil, err
}
return msmtJSON, nil
}
// GetResultTestKeys returns a list of TestKeys for a given result
func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) {
res := sess.Collection("measurements").Find("result_id", resultID)
defer res.Close()
var (
msmt Measurement
tk PerformanceTestKeys
)
for res.Next(&msmt) {
// We only really care about performance keys.
// Note: since even in case of failure we still initialise an empty struct,
// it could be that these keys come out as initializes with the default
// values.
// XXX we may want to change this behaviour by adding `omitempty` to the
// struct definition.
if msmt.TestName != "ndt" && msmt.TestName != "dash" {
return "{}", nil
}
if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil {
log.WithError(err).Error("failed to parse testKeys")
return "{}", err
}
}
b, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize testKeys")
return "{}", err
}
return string(b), nil
}
// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
var (
totalCount uint64
anmlyCount uint64
err error
)
col := sess.Collection("measurements")
// XXX these two queries can be done with a single query
totalCount, err = col.Find("result_id", resultID).
Count()
if err != nil {
log.WithError(err).Error("failed to get total count")
return totalCount, anmlyCount, err
}
anmlyCount, err = col.Find("result_id", resultID).
And(db.Cond{"is_anomaly": true}).Count()
if err != nil {
log.WithError(err).Error("failed to get anmly count")
return totalCount, anmlyCount, err
}
log.Debugf("counts: %d, %d, %d", resultID, totalCount, anmlyCount)
return totalCount, anmlyCount, err
}
// ListResults return the list of results
func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
doneResults := []ResultNetwork{}
incompleteResults := []ResultNetwork{}
req := sess.Select(
db.Raw("networks.*"),
db.Raw("results.*"),
).From("results").
Join("networks").On("results.network_id = networks.network_id").
OrderBy("results.result_start_time")
if err := req.Where("result_is_done = true").All(&doneResults); err != nil {
return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
}
if err := req.Where("result_is_done = false").All(&incompleteResults); err != nil {
return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list")
}
return doneResults, incompleteResults, nil
}
// DeleteResult will delete a particular result and the relative measurement on
// disk.
func DeleteResult(sess sqlbuilder.Database, resultID int64) error {
var result Result
res := sess.Collection("results").Find("result_id", resultID)
if err := res.One(&result); err != nil {
if err == db.ErrNoMoreRows {
return err
}
log.WithError(err).Error("error in obtaining the result")
return err
}
if err := res.Delete(); err != nil {
log.WithError(err).Error("failed to delete the result directory")
return err
}
os.RemoveAll(result.MeasurementDir)
return nil
}
// CreateMeasurement writes the measurement to the database a returns a pointer
// to the Measurement
func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, measurementDir string, idx int, resultID int64, urlID sql.NullInt64) (*Measurement, error) {
// TODO we should look into generating this file path in a more robust way.
// If there are two identical test_names in the same test group there is
// going to be a clash of test_name
msmtFilePath := filepath.Join(measurementDir, fmt.Sprintf("msmt-%s-%d.json", testName, idx))
msmt := Measurement{
ReportID: reportID,
TestName: testName,
ResultID: resultID,
MeasurementFilePath: sql.NullString{String: msmtFilePath, Valid: true},
URLID: urlID,
IsFailed: false,
IsDone: false,
// XXX Do we want to have this be part of something else?
StartTime: time.Now().UTC(),
TestKeys: "",
}
newID, err := sess.Collection("measurements").Insert(msmt)
if err != nil {
return nil, errors.Wrap(err, "creating measurement")
}
msmt.ID = newID.(int64)
return &msmt, nil
}
// CreateResult writes the Result to the database a returns a pointer
// to the Result
func CreateResult(sess sqlbuilder.Database, homePath string, testGroupName string, networkID int64) (*Result, error) {
startTime := time.Now().UTC()
p, err := utils.MakeResultsDir(homePath, testGroupName, startTime)
if err != nil {
return nil, err
}
result := Result{
TestGroupName: testGroupName,
StartTime: startTime,
NetworkID: networkID,
}
result.MeasurementDir = p
log.Debugf("Creating result %v", result)
newID, err := sess.Collection("results").Insert(result)
if err != nil {
return nil, errors.Wrap(err, "creating result")
}
result.ID = newID.(int64)
return &result, nil
}
// CreateNetwork will create a new network in the network table
func CreateNetwork(sess sqlbuilder.Database, loc enginex.LocationProvider) (*Network, error) {
network := Network{
ASN: loc.ProbeASN(),
CountryCode: loc.ProbeCC(),
NetworkName: loc.ProbeNetworkName(),
// On desktop we consider it to always be wifi
NetworkType: "wifi",
IP: loc.ProbeIP(),
}
newID, err := sess.Collection("networks").Insert(network)
if err != nil {
return nil, err
}
network.ID = newID.(int64)
return &network, nil
}
// CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't
// exists, otherwise it will update the category code of the one already in
// there.
func CreateOrUpdateURL(sess sqlbuilder.Database, urlStr string, categoryCode string, countryCode string) (int64, error) {
var url URL
tx, err := sess.NewTx(nil)
if err != nil {
log.WithError(err).Error("failed to create transaction")
return 0, err
}
res := tx.Collection("urls").Find(
db.Cond{"url": urlStr, "url_country_code": countryCode},
)
err = res.One(&url)
if err == db.ErrNoMoreRows {
url = URL{
URL: sql.NullString{String: urlStr, Valid: true},
CategoryCode: sql.NullString{String: categoryCode, Valid: true},
CountryCode: sql.NullString{String: countryCode, Valid: true},
}
newID, insErr := tx.Collection("urls").Insert(url)
if insErr != nil {
log.Error("Failed to insert into the URLs table")
return 0, insErr
}
url.ID = sql.NullInt64{Int64: newID.(int64), Valid: true}
} else if err != nil {
log.WithError(err).Error("Failed to get single result")
return 0, err
} else {
url.CategoryCode = sql.NullString{String: categoryCode, Valid: true}
res.Update(url)
}
err = tx.Commit()
if err != nil {
log.WithError(err).Error("Failed to write to the URL table")
return 0, err
}
log.Debugf("returning url %d", url.ID.Int64)
return url.ID.Int64, nil
}
// AddTestKeys writes the summary to the measurement
func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
var (
isAnomaly bool
isAnomalyValid bool
)
tkBytes, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
// This is necessary so that we can extract from the the opaque testKeys just
// the IsAnomaly field of bool type.
// Maybe generics are not so bad after-all, heh golang?
isAnomalyValue := reflect.ValueOf(tk).FieldByName("IsAnomaly")
if isAnomalyValue.IsValid() == true && isAnomalyValue.Kind() == reflect.Bool {
isAnomaly = isAnomalyValue.Bool()
isAnomalyValid = true
}
msmt.TestKeys = string(tkBytes)
msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid}
err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
if err != nil {
log.WithError(err).Error("failed to update measurement")
return errors.Wrap(err, "updating measurement")
}
return nil
}
internal/database/actions_test.go
-363
View File
@@ -1,363 +0,0 @@
package database
import (
"database/sql"
"encoding/json"
"fmt"
"io/ioutil"
"os"
"testing"
db "upper.io/db.v3"
)
type locationInfo struct {
asn uint
countryCode string
ip string
networkName string
resolverIP string
}
func (lp *locationInfo) ProbeASN() uint {
return lp.asn
}
func (lp *locationInfo) ProbeASNString() string {
return fmt.Sprintf("AS%d", lp.asn)
}
func (lp *locationInfo) ProbeCC() string {
return lp.countryCode
}
func (lp *locationInfo) ProbeIP() string {
return lp.ip
}
func (lp *locationInfo) ProbeNetworkName() string {
return lp.networkName
}
func (lp *locationInfo) ResolverIP() string {
return lp.resolverIP
}
func TestMeasurementWorkflow(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
location := locationInfo{
asn: 0,
countryCode: "IT",
networkName: "Unknown",
}
network, err := CreateNetwork(sess, &location)
if err != nil {
t.Fatal(err)
}
result, err := CreateResult(sess, tmpdir, "websites", network.ID)
if err != nil {
t.Fatal(err)
}
reportID := sql.NullString{String: "", Valid: false}
testName := "antani"
resultID := result.ID
msmtFilePath := tmpdir
urlID := sql.NullInt64{Int64: 0, Valid: false}
m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
if err != nil {
t.Fatal(err)
}
var m2 Measurement
err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
if err != nil {
t.Fatal(err)
}
if m2.ResultID != m1.ResultID {
t.Error("result_id mismatch")
}
done, incomplete, err := ListResults(sess)
if err != nil {
t.Fatal(err)
}
if len(incomplete) != 1 {
t.Error("there should be 1 incomplete measurement")
}
if len(done) != 0 {
t.Error("there should be 0 done measurements")
}
msmts, err := ListMeasurements(sess, resultID)
if err != nil {
t.Fatal(err)
}
if msmts[0].Network.NetworkType != "wifi" {
t.Error("network_type should be wifi")
}
}
func TestDeleteResult(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
location := locationInfo{
asn: 0,
countryCode: "IT",
networkName: "Unknown",
}
network, err := CreateNetwork(sess, &location)
if err != nil {
t.Fatal(err)
}
result, err := CreateResult(sess, tmpdir, "websites", network.ID)
if err != nil {
t.Fatal(err)
}
reportID := sql.NullString{String: "", Valid: false}
testName := "antani"
resultID := result.ID
msmtFilePath := tmpdir
urlID := sql.NullInt64{Int64: 0, Valid: false}
m1, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
if err != nil {
t.Fatal(err)
}
var m2 Measurement
err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2)
if err != nil {
t.Fatal(err)
}
if m2.ResultID != m1.ResultID {
t.Error("result_id mismatch")
}
err = DeleteResult(sess, resultID)
if err != nil {
t.Fatal(err)
}
totalResults, err := sess.Collection("results").Find().Count()
if err != nil {
t.Fatal(err)
}
totalMeasurements, err := sess.Collection("measurements").Find().Count()
if err != nil {
t.Fatal(err)
}
if totalResults != 0 {
t.Fatal("results should be zero")
}
if totalMeasurements != 0 {
t.Fatal("measurements should be zero")
}
err = DeleteResult(sess, 20)
if err != db.ErrNoMoreRows {
t.Fatal(err)
}
}
func TestNetworkCreate(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
l1 := locationInfo{
asn: 2,
countryCode: "IT",
networkName: "Antaninet",
}
l2 := locationInfo{
asn: 3,
countryCode: "IT",
networkName: "Fufnet",
}
_, err = CreateNetwork(sess, &l1)
if err != nil {
t.Fatal(err)
}
_, err = CreateNetwork(sess, &l2)
if err != nil {
t.Fatal(err)
}
}
func TestURLCreation(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
newID1, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX")
if err != nil {
t.Fatal(err)
}
newID2, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
if err != nil {
t.Fatal(err)
}
newID3, err := CreateOrUpdateURL(sess, "https://facebook.com", "GRP", "XX")
if err != nil {
t.Fatal(err)
}
newID4, err := CreateOrUpdateURL(sess, "https://facebook.com", "GMP", "XX")
if err != nil {
t.Fatal(err)
}
newID5, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX")
if err != nil {
t.Fatal(err)
}
if newID2 != newID1 {
t.Error("inserting the same URL with different category code should produce the same result")
}
if newID3 == newID1 {
t.Error("inserting different URL should produce different ids")
}
if newID4 != newID3 {
t.Error("inserting the same URL with different category code should produce the same result")
}
if newID5 != newID1 {
t.Error("the ID of google should still be the same")
}
}
func TestPerformanceTestKeys(t *testing.T) {
var tk PerformanceTestKeys
ndtS := "{\"download\":100.0,\"upload\":20.0,\"ping\":2.2}"
dashS := "{\"median_bitrate\":102.0}"
if err := json.Unmarshal([]byte(ndtS), &tk); err != nil {
t.Fatal("failed to parse ndtS")
}
if err := json.Unmarshal([]byte(dashS), &tk); err != nil {
t.Fatal("failed to parse dashS")
}
if tk.Bitrate != 102.0 {
t.Fatalf("error Bitrate %f", tk.Bitrate)
}
if tk.Download != 100.0 {
t.Fatalf("error Download %f", tk.Download)
}
}
func TestGetMeasurementJSON(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
location := locationInfo{
asn: 0,
countryCode: "IT",
networkName: "Unknown",
}
network, err := CreateNetwork(sess, &location)
if err != nil {
t.Fatal(err)
}
result, err := CreateResult(sess, tmpdir, "websites", network.ID)
if err != nil {
t.Fatal(err)
}
reportID := sql.NullString{String: "20210111T085144Z_ndt_RU_3216_n1_qMVnP0PTX7ObUSmD", Valid: true}
testName := "antani"
resultID := result.ID
msmtFilePath := tmpdir
urlID := sql.NullInt64{Int64: 0, Valid: false}
msmt, err := CreateMeasurement(sess, reportID, testName, msmtFilePath, 0, resultID, urlID)
if err != nil {
t.Fatal(err)
}
msmt.IsUploaded = true
err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt)
if err != nil {
t.Fatal(err)
}
tk, err := GetMeasurementJSON(sess, msmt.ID)
if err != nil {
t.Fatal(err)
}
if tk["probe_asn"] != "AS3216" {
t.Error("inconsistent measurement downloaded")
}
}
internal/database/database.go
-47
View File
@@ -1,47 +0,0 @@
package database
import (
"database/sql"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/bindata"
migrate "github.com/rubenv/sql-migrate"
"upper.io/db.v3/lib/sqlbuilder"
"upper.io/db.v3/sqlite"
)
// RunMigrations runs the database migrations
func RunMigrations(db *sql.DB) error {
log.Debugf("running migrations")
migrations := &migrate.AssetMigrationSource{
Asset: bindata.Asset,
AssetDir: bindata.AssetDir,
Dir: "data/migrations",
}
n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up)
if err != nil {
return err
}
log.Debugf("performed %d migrations", n)
return nil
}
// Connect to the database
func Connect(path string) (db sqlbuilder.Database, err error) {
settings := sqlite.ConnectionURL{
Database: path,
Options: map[string]string{"_foreign_keys": "1"},
}
sess, err := sqlite.Open(settings)
if err != nil {
log.WithError(err).Error("failed to open the DB")
return nil, err
}
err = RunMigrations(sess.Driver().(*sql.DB))
if err != nil {
log.WithError(err).Error("failed to run DB migration")
return nil, err
}
return sess, err
}
internal/database/database_test.go
-32
View File
@@ -1,32 +0,0 @@
package database
import (
"io/ioutil"
"os"
"testing"
"github.com/apex/log"
)
func TestConnect(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Error(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Error(err)
}
colls, err := sess.Collections()
if err != nil {
t.Error(err)
}
if len(colls) < 1 {
log.Fatal("missing tables")
}
}
internal/database/models.go
-150
View File
@@ -1,150 +0,0 @@
package database
import (
"database/sql"
"time"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
// ResultNetwork is used to represent the structure made from the JOIN
// between the results and networks tables.
type ResultNetwork struct {
Result `db:",inline"`
Network `db:",inline"`
}
// MeasurementURLNetwork is used for the JOIN between Measurement and URL
type MeasurementURLNetwork struct {
Measurement `db:",inline"`
Network `db:",inline"`
Result `db:",inline"`
URL `db:",inline"`
}
// Network represents a network tested by the user
type Network struct {
ID int64 `db:"network_id,omitempty"`
NetworkName string `db:"network_name"`
NetworkType string `db:"network_type"`
IP string `db:"ip"`
ASN uint `db:"asn"`
CountryCode string `db:"network_country_code"`
}
// URL represents URLs from the testing lists
type URL struct {
ID sql.NullInt64 `db:"url_id,omitempty"`
URL sql.NullString `db:"url"`
CategoryCode sql.NullString `db:"category_code"`
CountryCode sql.NullString `db:"url_country_code"`
}
// Measurement model
type Measurement struct {
ID int64 `db:"measurement_id,omitempty"`
TestName string `db:"test_name"`
StartTime time.Time `db:"measurement_start_time"`
Runtime float64 `db:"measurement_runtime"` // Fractional number of seconds
IsDone bool `db:"measurement_is_done"`
IsUploaded bool `db:"measurement_is_uploaded"`
IsFailed bool `db:"measurement_is_failed"`
FailureMsg sql.NullString `db:"measurement_failure_msg,omitempty"`
IsUploadFailed bool `db:"measurement_is_upload_failed"`
UploadFailureMsg sql.NullString `db:"measurement_upload_failure_msg,omitempty"`
IsRerun bool `db:"measurement_is_rerun"`
ReportID sql.NullString `db:"report_id,omitempty"`
URLID sql.NullInt64 `db:"url_id,omitempty"` // Used to reference URL
MeasurementID sql.NullInt64 `db:"collector_measurement_id,omitempty"`
IsAnomaly sql.NullBool `db:"is_anomaly,omitempty"`
// FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462
TestKeys string `db:"test_keys"`
ResultID int64 `db:"result_id"`
ReportFilePath sql.NullString `db:"report_file_path,omitempty"`
MeasurementFilePath sql.NullString `db:"measurement_file_path,omitempty"`
}
// Result model
type Result struct {
ID int64 `db:"result_id,omitempty"`
TestGroupName string `db:"test_group_name"`
StartTime time.Time `db:"result_start_time"`
NetworkID int64 `db:"network_id"` // Used to include a Network
Runtime float64 `db:"result_runtime"` // Runtime is expressed in fractional seconds
IsViewed bool `db:"result_is_viewed"`
IsDone bool `db:"result_is_done"`
DataUsageUp float64 `db:"result_data_usage_up"`
DataUsageDown float64 `db:"result_data_usage_down"`
MeasurementDir string `db:"measurement_dir"`
}
// PerformanceTestKeys is the result summary for a performance test
type PerformanceTestKeys struct {
Upload float64 `json:"upload"`
Download float64 `json:"download"`
Ping float64 `json:"ping"`
Bitrate float64 `json:"median_bitrate"`
}
// Finished marks the result as done and sets the runtime
func (r *Result) Finished(sess sqlbuilder.Database) error {
if r.IsDone == true || r.Runtime != 0 {
return errors.New("Result is already finished")
}
r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds()
r.IsDone = true
err := sess.Collection("results").Find("result_id", r.ID).Update(r)
if err != nil {
return errors.Wrap(err, "updating finished result")
}
return nil
}
// Failed writes the error string to the measurement
func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error {
m.FailureMsg = sql.NullString{String: failure, Valid: true}
m.IsFailed = true
err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// Done marks the measurement as completed
func (m *Measurement) Done(sess sqlbuilder.Database) error {
runtime := time.Now().UTC().Sub(m.StartTime)
m.Runtime = runtime.Seconds()
m.IsDone = true
err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// UploadFailed writes the error string for the upload failure to the measurement
func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) error {
m.UploadFailureMsg = sql.NullString{String: failure, Valid: true}
m.IsUploaded = false
err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// UploadSucceeded writes the error string for the upload failure to the measurement
func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
m.IsUploaded = true
err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
internal/enginex/enginex.go
-22
View File
@@ -1,22 +0,0 @@
// Package enginex contains ooni/probe-engine extensions.
package enginex
import (
"github.com/apex/log"
)
// Logger is the logger used by the engine.
var Logger = log.WithFields(log.Fields{
"type": "engine",
})
// LocationProvider is an interface that returns the current location. The
// github.com/ooni/probe-engine/session.Session implements it.
type LocationProvider interface {
ProbeASN() uint
ProbeASNString() string
ProbeCC() string
ProbeIP() string
ProbeNetworkName() string
ResolverIP() string
}
internal/log/handlers/batch/batch.go
-33
View File
@@ -1,33 +0,0 @@
package batch
import (
j "encoding/json"
"io"
"os"
"sync"
"github.com/apex/log"
)
// Default handler outputting to stderr.
var Default = New(os.Stderr)
// Handler implementation.
type Handler struct {
*j.Encoder
mu sync.Mutex
}
// New handler.
func New(w io.Writer) *Handler {
return &Handler{
Encoder: j.NewEncoder(w),
}
}
// HandleLog implements log.Handler.
func (h *Handler) HandleLog(e *log.Entry) error {
h.mu.Lock()
defer h.mu.Unlock()
return h.Encoder.Encode(e)
}
internal/log/handlers/cli/cli.go
-173
View File
@@ -1,173 +0,0 @@
package cli
import (
"fmt"
"io"
"os"
"strings"
"sync"
"time"
"github.com/apex/log"
"github.com/fatih/color"
colorable "github.com/mattn/go-colorable"
"github.com/ooni/probe-cli/internal/utils"
)
// Default handler outputting to stderr.
var Default = New(os.Stdout)
// start time.
var start = time.Now()
var bold = color.New(color.Bold)
// Colors mapping.
var Colors = [...]*color.Color{
log.DebugLevel: color.New(color.FgWhite),
log.InfoLevel: color.New(color.FgBlue),
log.WarnLevel: color.New(color.FgYellow),
log.ErrorLevel: color.New(color.FgRed),
log.FatalLevel: color.New(color.FgRed),
}
// Strings mapping.
var Strings = [...]string{
log.DebugLevel: "•",
log.InfoLevel: "•",
log.WarnLevel: "•",
log.ErrorLevel: "⨯",
log.FatalLevel: "⨯",
}
// Handler implementation.
type Handler struct {
mu sync.Mutex
Writer io.Writer
Padding int
}
// New handler.
func New(w io.Writer) *Handler {
if f, ok := w.(*os.File); ok {
return &Handler{
Writer: colorable.NewColorable(f),
Padding: 3,
}
}
return &Handler{
Writer: w,
Padding: 3,
}
}
func logSectionTitle(w io.Writer, f log.Fields) error {
colWidth := 24
title := f.Get("title").(string)
fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
fmt.Fprintf(w, "┃ %s ┃\n", utils.RightPad(title, colWidth))
fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
return nil
}
func logTable(w io.Writer, f log.Fields) error {
color := color.New(color.FgBlue)
names := f.Names()
var lines []string
colWidth := 0
for _, name := range names {
if name == "type" {
continue
}
line := fmt.Sprintf("%s: %s", color.Sprint(name), f.Get(name))
lineLength := utils.EscapeAwareRuneCountInString(line)
lines = append(lines, line)
if colWidth < lineLength {
colWidth = lineLength
}
}
fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth+2)+"┓\n")
for _, line := range lines {
fmt.Fprintf(w, "┃ %s ┃\n",
utils.RightPad(line, colWidth),
)
}
fmt.Fprintf(w, "┗"+strings.Repeat("━", colWidth+2)+"┛\n")
return nil
}
// TypedLog is used for handling special "typed" logs to the CLI
func (h *Handler) TypedLog(t string, e *log.Entry) error {
switch t {
case "engine":
fmt.Fprintf(h.Writer, "[engine] %s\n", e.Message)
return nil
case "progress":
perc := e.Fields.Get("percentage").(float64) * 100
eta := e.Fields.Get("eta").(float64)
var etaMessage string
if eta >= 0 {
etaMessage = fmt.Sprintf("(%ss left)", bold.Sprintf("%.2f", eta))
}
s := fmt.Sprintf(" %s %-25s %s",
bold.Sprintf("%.2f%%", perc),
e.Message, etaMessage)
fmt.Fprint(h.Writer, s)
fmt.Fprintln(h.Writer)
return nil
case "table":
return logTable(h.Writer, e.Fields)
case "measurement_item":
return logMeasurementItem(h.Writer, e.Fields)
case "measurement_json":
return logMeasurementJSON(h.Writer, e.Fields)
case "measurement_summary":
return logMeasurementSummary(h.Writer, e.Fields)
case "result_item":
return logResultItem(h.Writer, e.Fields)
case "result_summary":
return logResultSummary(h.Writer, e.Fields)
case "section_title":
return logSectionTitle(h.Writer, e.Fields)
default:
return h.DefaultLog(e)
}
}
// DefaultLog is the default way of printing out logs
func (h *Handler) DefaultLog(e *log.Entry) error {
color := Colors[e.Level]
level := Strings[e.Level]
names := e.Fields.Names()
s := color.Sprintf("%s %-25s", bold.Sprintf("%*s", h.Padding+1, level), e.Message)
for _, name := range names {
if name == "source" {
continue
}
s += fmt.Sprintf(" %s=%v", color.Sprint(name), e.Fields.Get(name))
}
fmt.Fprint(h.Writer, s)
fmt.Fprintln(h.Writer)
return nil
}
// HandleLog implements log.Handler.
func (h *Handler) HandleLog(e *log.Entry) error {
h.mu.Lock()
defer h.mu.Unlock()
t, isTyped := e.Fields["type"].(string)
if isTyped {
return h.TypedLog(t, e)
}
return h.DefaultLog(e)
}
internal/log/handlers/cli/measurements.go
-142
View File
@@ -1,142 +0,0 @@
package cli
import (
"bytes"
"encoding/json"
"fmt"
"io"
"strings"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/utils"
)
func statusIcon(ok bool) string {
if ok {
return "✓"
}
return "❌"
}
func logTestKeys(w io.Writer, testKeys string) error {
colWidth := 24
var out bytes.Buffer
if err := json.Indent(&out, []byte(testKeys), "", " "); err != nil {
return err
}
testKeysLines := strings.Split(string(out.Bytes()), "\n")
if len(testKeysLines) > 1 {
testKeysLines = testKeysLines[1 : len(testKeysLines)-1]
testKeysLines[0] = "{" + testKeysLines[0][1:]
testKeysLines[len(testKeysLines)-1] = testKeysLines[len(testKeysLines)-1] + "}"
}
for _, line := range testKeysLines {
fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
utils.RightPad(line, colWidth*2)))
}
return nil
}
func logMeasurementItem(w io.Writer, f log.Fields) error {
colWidth := 24
rID := f.Get("id").(int64)
testName := f.Get("test_name").(string)
// We currently don't use these fields in the view
//testGroupName := f.Get("test_group_name").(string)
//networkName := f.Get("network_name").(string)
//asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string))
testKeys := f.Get("test_keys").(string)
isAnomaly := f.Get("is_anomaly").(bool)
isFailed := f.Get("is_failed").(bool)
isUploaded := f.Get("is_uploaded").(bool)
url := f.Get("url").(string)
urlCategoryCode := f.Get("url_category_code").(string)
isFirst := f.Get("is_first").(bool)
isLast := f.Get("is_last").(bool)
if isFirst {
fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n")
} else {
fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n")
}
anomalyStr := fmt.Sprintf("ok: %s", statusIcon(!isAnomaly))
uploadStr := fmt.Sprintf("uploaded: %s", statusIcon(isUploaded))
failureStr := fmt.Sprintf("success: %s", statusIcon(!isFailed))
fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
utils.RightPad(
fmt.Sprintf("#%d", rID), colWidth*2)))
if url != "" {
fmt.Fprintf(w, fmt.Sprintf("│ %s │\n",
utils.RightPad(
fmt.Sprintf("%s (%s)", url, urlCategoryCode), colWidth*2)))
}
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
utils.RightPad(testName, colWidth),
utils.RightPad(anomalyStr, colWidth)))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
utils.RightPad(failureStr, colWidth),
utils.RightPad(uploadStr, colWidth)))
if testKeys != "" {
if err := logTestKeys(w, testKeys); err != nil {
return err
}
}
if isLast {
fmt.Fprintf(w, "└┬────────────────────────────────────────────────┬┘\n")
}
return nil
}
func logMeasurementSummary(w io.Writer, f log.Fields) error {
colWidth := 12
totalCount := f.Get("total_count").(int64)
anomalyCount := f.Get("anomaly_count").(int64)
totalRuntime := f.Get("total_runtime").(float64)
dataUp := f.Get("data_usage_up").(float64)
dataDown := f.Get("data_usage_down").(float64)
startTime := f.Get("start_time").(time.Time)
asn := f.Get("asn").(uint)
countryCode := f.Get("network_country_code").(string)
networkName := f.Get("network_name").(string)
fmt.Fprintf(w, " │ %s │\n",
utils.RightPad(startTime.Format(time.RFC822), (colWidth+3)*3),
)
fmt.Fprintf(w, " │ %s │\n",
utils.RightPad(fmt.Sprintf("AS%d, %s (%s)", asn, networkName, countryCode), (colWidth+3)*3),
)
fmt.Fprintf(w, " │ %s %s %s │\n",
utils.RightPad(fmt.Sprintf("%.2fs", totalRuntime), colWidth),
utils.RightPad(fmt.Sprintf("%d/%d anmls", anomalyCount, totalCount), colWidth),
utils.RightPad(fmt.Sprintf("⬆ %s ⬇ %s", formatSize(dataUp), formatSize(dataDown)), colWidth+4))
fmt.Fprintf(w, " └────────────────────────────────────────────────┘\n")
return nil
}
func logMeasurementJSON(w io.Writer, f log.Fields) error {
m := f.Get("measurement_json").(map[string]interface{})
json, err := json.MarshalIndent(m, "", " ")
if err != nil {
return err
}
fmt.Fprintf(w, string(json))
return nil
}
internal/log/handlers/cli/progress/progress.go
-126
View File
@@ -1,126 +0,0 @@
// Package progress provides a simple terminal progress bar.
package progress
import (
"bytes"
"fmt"
"html/template"
"io"
"math"
"strings"
)
// Bar is a progress bar.
type Bar struct {
StartDelimiter string // StartDelimiter for the bar ("|").
EndDelimiter string // EndDelimiter for the bar ("|").
Filled string // Filled section representation ("█").
Empty string // Empty section representation ("░")
Total float64 // Total value.
Width int // Width of the bar.
value float64
tmpl *template.Template
text string
}
// New returns a new bar with the given total.
func New(total float64) *Bar {
b := &Bar{
StartDelimiter: "|",
EndDelimiter: "|",
Filled: "█",
Empty: "░",
Total: total,
Width: 60,
}
b.Template(`{{.Percent | printf "%3.0f"}}% {{.Bar}} {{.Text}}`)
return b
}
// NewInt returns a new bar with the given total.
func NewInt(total int) *Bar {
return New(float64(total))
}
// Text sets the text value.
func (b *Bar) Text(s string) {
b.text = s
}
// Value sets the value.
func (b *Bar) Value(n float64) {
if n > b.Total {
panic("Bar update value cannot be greater than the total")
}
b.value = n
}
// ValueInt sets the value.
func (b *Bar) ValueInt(n int) {
b.Value(float64(n))
}
// Percent returns the percentage
func (b *Bar) percent() float64 {
return (b.value / b.Total) * 100
}
// Bar returns the progress bar string.
func (b *Bar) bar() string {
p := b.value / b.Total
filled := math.Ceil(float64(b.Width) * p)
empty := math.Floor(float64(b.Width) - filled)
s := b.StartDelimiter
s += strings.Repeat(b.Filled, int(filled))
s += strings.Repeat(b.Empty, int(empty))
s += b.EndDelimiter
return s
}
// String returns the progress bar.
func (b *Bar) String() string {
var buf bytes.Buffer
data := struct {
Value float64
Total float64
Percent float64
StartDelimiter string
EndDelimiter string
Bar string
Text string
}{
Value: b.value,
Text: b.text,
StartDelimiter: b.StartDelimiter,
EndDelimiter: b.EndDelimiter,
Percent: b.percent(),
Bar: b.bar(),
}
if err := b.tmpl.Execute(&buf, data); err != nil {
panic(err)
}
return buf.String()
}
// WriteTo writes the progress bar to w.
func (b *Bar) WriteTo(w io.Writer) (int64, error) {
s := fmt.Sprintf("\r %s ", b.String())
_, err := io.WriteString(w, s)
return int64(len(s)), err
}
// Template for rendering. This method will panic if the template fails to parse.
func (b *Bar) Template(s string) {
t, err := template.New("").Parse(s)
if err != nil {
panic(err)
}
b.tmpl = t
}
internal/log/handlers/cli/results.go
-154
View File
@@ -1,154 +0,0 @@
package cli
import (
"encoding/json"
"fmt"
"io"
"strings"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/utils"
)
func formatSpeed(speed float64) string {
if speed < 1000 {
return fmt.Sprintf("%.2f Kbit/s", speed)
} else if speed < 1000*1000 {
return fmt.Sprintf("%.2f Mbit/s", float32(speed)/1000)
} else if speed < 1000*1000*1000 {
return fmt.Sprintf("%.2f Gbit/s", float32(speed)/(1000*1000))
}
// WTF, you crazy?
return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
}
func formatSize(size float64) string {
if size < 1024 {
return fmt.Sprintf("%.1fK", size)
} else if size < 1024*1024 {
return fmt.Sprintf("%.1fM", size/1024.0)
} else if size < 1024*1024*1024 {
return fmt.Sprintf("%.1fG", size/(1024.0*1024.0))
}
// WTF, you crazy?
return fmt.Sprintf("%.1fT", size/(1024*1024*1024))
}
var summarizers = map[string]func(uint64, uint64, string) []string{
"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var tk database.PerformanceTestKeys
if err := json.Unmarshal([]byte(ss), &tk); err != nil {
return nil
}
return []string{
fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
fmt.Sprintf("Ping: %.2fms", tk.Ping),
}
},
"im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("Detected: %v", anomalyCount > 0),
"",
"",
}
},
"circumvention": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
}
func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
return summarizers[name](totalCount, anomalyCount, ss)
}
func logResultItem(w io.Writer, f log.Fields) error {
colWidth := 24
rID := f.Get("id").(int64)
name := f.Get("name").(string)
isDone := f.Get("is_done").(bool)
startTime := f.Get("start_time").(time.Time)
networkName := f.Get("network_name").(string)
asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string))
//runtime := f.Get("runtime").(float64)
//dataUsageUp := f.Get("dataUsageUp").(int64)
//dataUsageDown := f.Get("dataUsageDown").(int64)
index := f.Get("index").(int)
totalCount := f.Get("total_count").(int)
if index == 0 {
fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n")
} else {
fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n")
}
firstRow := utils.RightPad(fmt.Sprintf("#%d - %s", rID, startTime.Format(time.RFC822)), colWidth*2)
fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
summary := makeSummary(name,
f.Get("measurement_count").(uint64),
f.Get("measurement_anomaly_count").(uint64),
f.Get("test_keys").(string))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
utils.RightPad(name, colWidth),
utils.RightPad(summary[0], colWidth)))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
utils.RightPad(networkName, colWidth),
utils.RightPad(summary[1], colWidth)))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
utils.RightPad(asn, colWidth),
utils.RightPad(summary[2], colWidth)))
if index == totalCount-1 {
if isDone == true {
fmt.Fprintf(w, "└┬──────────────┬─────────────┬───────────────────┬┘\n")
} else {
// We want the incomplete section to not have a footer
fmt.Fprintf(w, "└──────────────────────────────────────────────────┘\n")
}
}
return nil
}
func logResultSummary(w io.Writer, f log.Fields) error {
networks := f.Get("total_networks").(int64)
tests := f.Get("total_tests").(int64)
dataUp := f.Get("total_data_usage_up").(float64)
dataDown := f.Get("total_data_usage_down").(float64)
if tests == 0 {
fmt.Fprintf(w, "No results\n")
fmt.Fprintf(w, "Try running:\n")
fmt.Fprintf(w, " ooniprobe run websites\n")
return nil
}
// └┬──────────────┬─────────────┬───────────────┬
fmt.Fprintf(w, " │ %s │ %s │ %s │\n",
utils.RightPad(fmt.Sprintf("%d tests", tests), 12),
utils.RightPad(fmt.Sprintf("%d nets", networks), 11),
utils.RightPad(fmt.Sprintf("⬆ %s ⬇ %s", formatSize(dataUp), formatSize(dataDown)), 17))
fmt.Fprintf(w, " └──────────────┴─────────────┴───────────────────┘\n")
return nil
}
internal/log/handlers/syslog/syslog.c
-39
View File
@@ -1,39 +0,0 @@
#ifndef _WIN32
#include <syslog.h>
#endif
void ooniprobe_openlog(void) {
#ifndef _WIN32
(void)openlog("ooniprobe", LOG_PID, LOG_USER);
#endif
}
void ooniprobe_log_debug(const char *message) {
#ifndef _WIN32
(void)syslog(LOG_DEBUG, "%s", message);
#endif
}
void ooniprobe_log_info(const char *message) {
#ifndef _WIN32
(void)syslog(LOG_INFO, "%s", message);
#endif
}
void ooniprobe_log_warning(const char *message) {
#ifndef _WIN32
(void)syslog(LOG_WARNING, "%s", message);
#endif
}
void ooniprobe_log_err(const char *message) {
#ifndef _WIN32
(void)syslog(LOG_ERR, "%s", message);
#endif
}
void ooniprobe_log_crit(const char *message) {
#ifndef _WIN32
(void)syslog(LOG_CRIT, "%s", message);
#endif
}
internal/log/handlers/syslog/syslog.go
-53
View File
@@ -1,53 +0,0 @@
// Package syslog contains a syslog handler.
//
// We use this handler on macOS systems to log messages
// when ooniprobe is running in the background.
package syslog
import (
"fmt"
"unsafe"
"github.com/apex/log"
)
/*
#include<stdlib.h>
void ooniprobe_openlog(void);
void ooniprobe_log_debug(const char *message);
void ooniprobe_log_info(const char *message);
void ooniprobe_log_warning(const char *message);
void ooniprobe_log_err(const char *message);
void ooniprobe_log_crit(const char *message);
*/
import "C"
// Default is the handler that emits logs with syslog
var Default log.Handler = newhandler()
type handler struct{}
func newhandler() handler {
C.ooniprobe_openlog()
return handler{}
}
func (h handler) HandleLog(e *log.Entry) error {
message := fmt.Sprintf("%s %+v", e.Message, e.Fields)
cstr := C.CString(message)
defer C.free(unsafe.Pointer(cstr))
switch e.Level {
case log.DebugLevel:
C.ooniprobe_log_debug(cstr)
case log.InfoLevel:
C.ooniprobe_log_info(cstr)
case log.WarnLevel:
C.ooniprobe_log_warning(cstr)
case log.ErrorLevel:
C.ooniprobe_log_err(cstr)
default:
C.ooniprobe_log_crit(cstr)
}
return nil
}
internal/nettests/dash.go
-14
View File
@@ -1,14 +0,0 @@
package nettests
// Dash test implementation
type Dash struct {
}
// Run starts the test
func (d Dash) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder("dash")
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/facebook_messenger.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// FacebookMessenger test implementation
type FacebookMessenger struct {
}
// Run starts the test
func (h FacebookMessenger) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"facebook_messenger",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/groups.go
-52
View File
@@ -1,52 +0,0 @@
package nettests
// Group is a group of nettests
type Group struct {
Label string
Nettests []Nettest
UnattendedOK bool
}
// All contains all the nettests that can be run by the user
var All = map[string]Group{
"websites": {
Label: "Websites",
Nettests: []Nettest{
WebConnectivity{},
},
UnattendedOK: true,
},
"performance": {
Label: "Performance",
Nettests: []Nettest{
Dash{},
NDT{},
},
},
"middlebox": {
Label: "Middleboxes",
Nettests: []Nettest{
HTTPInvalidRequestLine{},
HTTPHeaderFieldManipulation{},
},
UnattendedOK: true,
},
"im": {
Label: "Instant Messaging",
Nettests: []Nettest{
FacebookMessenger{},
Telegram{},
WhatsApp{},
},
UnattendedOK: true,
},
"circumvention": {
Label: "Circumvention Tools",
Nettests: []Nettest{
Psiphon{},
RiseupVPN{},
Tor{},
},
UnattendedOK: true,
},
}
internal/nettests/http_header_field_manipulation.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// HTTPHeaderFieldManipulation test implementation
type HTTPHeaderFieldManipulation struct {
}
// Run starts the test
func (h HTTPHeaderFieldManipulation) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"http_header_field_manipulation",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/http_invalid_request_line.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// HTTPInvalidRequestLine test implementation
type HTTPInvalidRequestLine struct {
}
// Run starts the test
func (h HTTPInvalidRequestLine) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"http_invalid_request_line",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/ndt.go
-15
View File
@@ -1,15 +0,0 @@
package nettests
// NDT test implementation. We use v7 of NDT since 2020-03-12.
type NDT struct {
}
// Run starts the test
func (n NDT) Run(ctl *Controller) error {
// Since 2020-03-18 probe-engine exports v7 as "ndt".
builder, err := ctl.Session.NewExperimentBuilder("ndt")
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/nettests.go
-219
View File
@@ -1,219 +0,0 @@
package nettests
import (
"database/sql"
"fmt"
"time"
"github.com/apex/log"
"github.com/fatih/color"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/output"
engine "github.com/ooni/probe-engine"
"github.com/ooni/probe-engine/model"
"github.com/pkg/errors"
)
// Nettest interface. Every Nettest should implement this.
type Nettest interface {
Run(*Controller) error
}
// NewController creates a nettest controller
func NewController(
nt Nettest, probe *ooni.Probe, res *database.Result, sess *engine.Session) *Controller {
return &Controller{
Probe: probe,
nt: nt,
res: res,
Session: sess,
}
}
// Controller is passed to the run method of every Nettest
// each nettest instance has one controller
type Controller struct {
Probe *ooni.Probe
Session *engine.Session
res *database.Result
nt Nettest
ntCount int
ntIndex int
ntStartTime time.Time // used to calculate the eta
msmts map[int64]*database.Measurement
inputIdxMap map[int64]int64 // Used to map mk idx to database id
// InputFiles optionally contains the names of the input
// files to read inputs from (only for nettests that take
// inputs, of course)
InputFiles []string
// Inputs contains inputs to be tested. These are specified
// using the command line using the --input flag.
Inputs []string
// numInputs is the total number of inputs
numInputs int
// curInputIdx is the current input index
curInputIdx int
}
// SetInputIdxMap is used to set the mapping of index into input. This mapping
// is used to reference, for example, a particular URL based on the index inside
// of the input list and the index of it in the database.
func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
c.inputIdxMap = inputIdxMap
return nil
}
// SetNettestIndex is used to set the current nettest index and total nettest
// count to compute a different progress percentage.
func (c *Controller) SetNettestIndex(i, n int) {
c.ntCount = n
c.ntIndex = i
}
// Run runs the selected nettest using the related experiment
// with the specified inputs.
//
// This function will continue to run in most cases but will
// immediately halt if something's wrong with the file system.
func (c *Controller) Run(builder *engine.ExperimentBuilder, inputs []string) error {
// This will configure the controller as handler for the callbacks
// called by ooni/probe-engine/experiment.Experiment.
builder.SetCallbacks(model.ExperimentCallbacks(c))
c.numInputs = len(inputs)
exp := builder.NewExperiment()
defer func() {
c.res.DataUsageDown += exp.KibiBytesReceived()
c.res.DataUsageUp += exp.KibiBytesSent()
}()
c.msmts = make(map[int64]*database.Measurement)
// These values are shared by every measurement
var reportID sql.NullString
resultID := c.res.ID
log.Debug(color.RedString("status.queued"))
log.Debug(color.RedString("status.started"))
if c.Probe.Config().Sharing.UploadResults {
if err := exp.OpenReport(); err != nil {
log.Debugf(
"%s: %s", color.RedString("failure.report_create"), err.Error(),
)
} else {
log.Debugf(color.RedString("status.report_create"))
reportID = sql.NullString{String: exp.ReportID(), Valid: true}
}
}
c.ntStartTime = time.Now()
for idx, input := range inputs {
if c.Probe.IsTerminated() == true {
log.Debug("isTerminated == true, breaking the input loop")
break
}
c.curInputIdx = idx // allow for precise progress
idx64 := int64(idx)
log.Debug(color.RedString("status.measurement_start"))
var urlID sql.NullInt64
if c.inputIdxMap != nil {
urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true}
}
msmt, err := database.CreateMeasurement(
c.Probe.DB(), reportID, exp.Name(), c.res.MeasurementDir, idx, resultID, urlID,
)
if err != nil {
return errors.Wrap(err, "failed to create measurement")
}
c.msmts[idx64] = msmt
if input != "" {
c.OnProgress(0, fmt.Sprintf("processing input: %s", input))
}
measurement, err := exp.Measure(input)
if err != nil {
log.WithError(err).Debug(color.RedString("failure.measurement"))
if err := c.msmts[idx64].Failed(c.Probe.DB(), err.Error()); err != nil {
return errors.Wrap(err, "failed to mark measurement as failed")
}
// Even with a failed measurement, we want to continue. We want to
// record and submit the information we have. Saving the information
// is useful for local inspection. Submitting it is useful to us to
// undertsand what went wrong (censorship? bug? anomaly?).
}
saveToDisk := true
if c.Probe.Config().Sharing.UploadResults {
// Implementation note: SubmitMeasurement will fail here if we did fail
// to open the report but we still want to continue. There will be a
// bit of a spew in the logs, perhaps, but stopping seems less efficient.
if err := exp.SubmitAndUpdateMeasurement(measurement); err != nil {
log.Debug(color.RedString("failure.measurement_submission"))
if err := c.msmts[idx64].UploadFailed(c.Probe.DB(), err.Error()); err != nil {
return errors.Wrap(err, "failed to mark upload as failed")
}
} else if err := c.msmts[idx64].UploadSucceeded(c.Probe.DB()); err != nil {
return errors.Wrap(err, "failed to mark upload as succeeded")
} else {
// Everything went OK, don't save to disk
saveToDisk = false
}
}
// We only save the measurement to disk if we failed to upload the measurement
if saveToDisk == true {
if err := exp.SaveMeasurement(measurement, msmt.MeasurementFilePath.String); err != nil {
return errors.Wrap(err, "failed to save measurement on disk")
}
}
if err := c.msmts[idx64].Done(c.Probe.DB()); err != nil {
return errors.Wrap(err, "failed to mark measurement as done")
}
// We're not sure whether it's enough to log the error or we should
// instead also mark the measurement as failed. Strictly speaking this
// is an inconsistency between the code that generate the measurement
// and the code that process the measurement. We do have some data
// but we're not gonna have a summary. To be reconsidered.
tk, err := exp.GetSummaryKeys(measurement)
if err != nil {
log.WithError(err).Error("failed to obtain testKeys")
continue
}
log.Debugf("Fetching: %d %v", idx, c.msmts[idx64])
if err := database.AddTestKeys(c.Probe.DB(), c.msmts[idx64], tk); err != nil {
return errors.Wrap(err, "failed to add test keys to summary")
}
}
log.Debugf("status.end")
return nil
}
// OnProgress should be called when a new progress event is available.
func (c *Controller) OnProgress(perc float64, msg string) {
log.Debugf("OnProgress: %f - %s", perc, msg)
var eta float64
eta = -1.0
if c.numInputs > 1 {
// make the percentage relative to the current input over all inputs
floor := (float64(c.curInputIdx) / float64(c.numInputs))
step := 1.0 / float64(c.numInputs)
perc = floor + perc*step
if c.curInputIdx > 0 {
eta = (time.Now().Sub(c.ntStartTime).Seconds() / float64(c.curInputIdx)) * float64(c.numInputs-c.curInputIdx)
}
}
if c.ntCount > 0 {
// make the percentage relative to the current nettest over all nettests
perc = float64(c.ntIndex)/float64(c.ntCount) + perc/float64(c.ntCount)
}
key := fmt.Sprintf("%T", c.nt)
output.Progress(key, perc, eta, msg)
}
internal/nettests/nettests_test.go
-52
View File
@@ -1,52 +0,0 @@
package nettests
import (
"io/ioutil"
"path"
"testing"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/ooni/probe-cli/internal/utils/shutil"
)
func newOONIProbe(t *testing.T) *ooni.Probe {
homePath, err := ioutil.TempDir("", "ooniprobetests")
if err != nil {
t.Fatal(err)
}
configPath := path.Join(homePath, "config.json")
testingConfig := path.Join("..", "..", "testdata", "testing-config.json")
shutil.Copy(testingConfig, configPath, false)
probe := ooni.NewProbe(configPath, homePath)
swName := "ooniprobe-cli-tests"
swVersion := "3.0.0-alpha"
err = probe.Init(swName, swVersion)
if err != nil {
t.Fatal(err)
}
return probe
}
func TestCreateContext(t *testing.T) {
newOONIProbe(t)
}
func TestRun(t *testing.T) {
probe := newOONIProbe(t)
sess, err := probe.NewSession()
if err != nil {
t.Fatal(err)
}
network, err := database.CreateNetwork(probe.DB(), sess)
if err != nil {
t.Fatal(err)
}
res, err := database.CreateResult(probe.DB(), probe.Home(), "middlebox", network.ID)
if err != nil {
t.Fatal(err)
}
nt := HTTPInvalidRequestLine{}
ctl := NewController(nt, probe, res, sess)
nt.Run(ctl)
}
internal/nettests/psiphon.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// Psiphon test implementation
type Psiphon struct {
}
// Run starts the test
func (h Psiphon) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"psiphon",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/riseupvpn.go
-17
View File
@@ -1,17 +0,0 @@
package nettests
// RiseupVPN test implementation
type RiseupVPN struct {
}
// Run starts the test
func (h RiseupVPN) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"riseupvpn",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/run.go
-82
View File
@@ -1,82 +0,0 @@
package nettests
import (
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/ooni"
"github.com/pkg/errors"
)
// RunGroupConfig contains the settings for running a nettest group.
type RunGroupConfig struct {
GroupName string
Probe *ooni.Probe
InputFiles []string
Inputs []string
}
// RunGroup runs a group of nettests according to the specified config.
func RunGroup(config RunGroupConfig) error {
if config.Probe.IsTerminated() == true {
log.Debugf("context is terminated, stopping runNettestGroup early")
return nil
}
sess, err := config.Probe.NewSession()
if err != nil {
log.WithError(err).Error("Failed to create a measurement session")
return err
}
defer sess.Close()
err = sess.MaybeLookupLocation()
if err != nil {
log.WithError(err).Error("Failed to lookup the location of the probe")
return err
}
network, err := database.CreateNetwork(config.Probe.DB(), sess)
if err != nil {
log.WithError(err).Error("Failed to create the network row")
return err
}
if err := sess.MaybeLookupBackends(); err != nil {
log.WithError(err).Warn("Failed to discover OONI backends")
return err
}
group, ok := All[config.GroupName]
if !ok {
log.Errorf("No test group named %s", config.GroupName)
return errors.New("invalid test group name")
}
log.Debugf("Running test group %s", group.Label)
result, err := database.CreateResult(
config.Probe.DB(), config.Probe.Home(), config.GroupName, network.ID)
if err != nil {
log.Errorf("DB result error: %s", err)
return err
}
config.Probe.ListenForSignals()
config.Probe.MaybeListenForStdinClosed()
for i, nt := range group.Nettests {
if config.Probe.IsTerminated() == true {
log.Debugf("context is terminated, stopping group.Nettests early")
break
}
log.Debugf("Running test %T", nt)
ctl := NewController(nt, config.Probe, result, sess)
ctl.InputFiles = config.InputFiles
ctl.Inputs = config.Inputs
ctl.SetNettestIndex(i, len(group.Nettests))
if err = nt.Run(ctl); err != nil {
log.WithError(err).Errorf("Failed to run %s", group.Label)
}
}
if err = result.Finished(config.Probe.DB()); err != nil {
return err
}
return nil
}
internal/nettests/telegram.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// Telegram test implementation
type Telegram struct {
}
// Run starts the test
func (h Telegram) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"telegram",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/tor.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// Tor test implementation
type Tor struct {
}
// Run starts the test
func (h Tor) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"tor",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/nettests/web_connectivity.go
-61
View File
@@ -1,61 +0,0 @@
package nettests
import (
"context"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
engine "github.com/ooni/probe-engine"
)
func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {
inputloader := engine.NewInputLoader(engine.InputLoaderConfig{
InputPolicy: engine.InputOrQueryTestLists,
Session: ctl.Session,
SourceFiles: ctl.InputFiles,
StaticInputs: ctl.Inputs,
URLCategories: categories,
URLLimit: limit,
})
testlist, err := inputloader.Load(context.Background())
var urls []string
urlIDMap := make(map[int64]int64)
if err != nil {
return nil, nil, err
}
for idx, url := range testlist {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(
ctl.Probe.DB(), url.URL, url.CategoryCode, url.CountryCode,
)
if err != nil {
log.Error("failed to add to the URL table")
return nil, nil, err
}
log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID)
urlIDMap[int64(idx)] = urlID
urls = append(urls, url.URL)
}
return urls, urlIDMap, nil
}
// WebConnectivity test implementation
type WebConnectivity struct {
}
// Run starts the test
func (n WebConnectivity) Run(ctl *Controller) error {
log.Debugf("Enabled category codes are the following %v", ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
urls, urlIDMap, err := lookupURLs(ctl, ctl.Probe.Config().Nettests.WebsitesURLLimit, ctl.Probe.Config().Nettests.WebsitesEnabledCategoryCodes)
if err != nil {
return err
}
ctl.SetInputIdxMap(urlIDMap)
builder, err := ctl.Session.NewExperimentBuilder(
"web_connectivity",
)
if err != nil {
return err
}
return ctl.Run(builder, urls)
}
internal/nettests/whatsapp.go
-16
View File
@@ -1,16 +0,0 @@
package nettests
// WhatsApp test implementation
type WhatsApp struct {
}
// Run starts the test
func (h WhatsApp) Run(ctl *Controller) error {
builder, err := ctl.Session.NewExperimentBuilder(
"whatsapp",
)
if err != nil {
return err
}
return ctl.Run(builder, []string{""})
}
internal/ooni/ooni.go
-284
View File
@@ -1,284 +0,0 @@
package ooni
import (
"io/ioutil"
"os"
"os/signal"
"sync/atomic"
"syscall"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/bindata"
"github.com/ooni/probe-cli/internal/config"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/enginex"
"github.com/ooni/probe-cli/internal/utils"
engine "github.com/ooni/probe-engine"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
// ProbeCLI is the OONI Probe CLI context.
type ProbeCLI interface {
Config() *config.Config
DB() sqlbuilder.Database
IsBatch() bool
Home() string
TempDir() string
NewProbeEngine() (ProbeEngine, error)
}
// ProbeEngine is an instance of the OONI Probe engine.
type ProbeEngine interface {
Close() error
MaybeLookupLocation() error
ProbeASNString() string
ProbeCC() string
ProbeIP() string
ProbeNetworkName() string
}
// Probe contains the ooniprobe CLI context.
type Probe struct {
config *config.Config
db sqlbuilder.Database
isBatch bool
home string
tempDir string
dbPath string
configPath string
// We need to use a int32 in order to use the atomic.AddInt32/LoadInt32
// operations to ensure consistent reads of the variables. We do not use
// a 64 bit integer here because that may lead to crashes with 32 bit
// OSes as documented in https://golang.org/pkg/sync/atomic/#pkg-note-BUG.
isTerminatedAtomicInt int32
softwareName string
softwareVersion string
}
// SetIsBatch sets the value of isBatch.
func (p *Probe) SetIsBatch(v bool) {
p.isBatch = v
}
// IsBatch returns whether we're running in batch mode.
func (p *Probe) IsBatch() bool {
return p.isBatch
}
// Config returns the configuration
func (p *Probe) Config() *config.Config {
return p.config
}
// DB returns the database we're using
func (p *Probe) DB() sqlbuilder.Database {
return p.db
}
// Home returns the home directory.
func (p *Probe) Home() string {
return p.home
}
// TempDir returns the temporary directory.
func (p *Probe) TempDir() string {
return p.tempDir
}
// IsTerminated checks to see if the isTerminatedAtomicInt is set to a non zero
// value and therefore we have received the signal to shutdown the running test
func (p *Probe) IsTerminated() bool {
i := atomic.LoadInt32(&p.isTerminatedAtomicInt)
return i != 0
}
// Terminate interrupts the running context
func (p *Probe) Terminate() {
atomic.AddInt32(&p.isTerminatedAtomicInt, 1)
}
// ListenForSignals will listen for SIGINT and SIGTERM. When it receives those
// signals it will set isTerminatedAtomicInt to non-zero, which will cleanly
// shutdown the test logic.
//
// TODO refactor this to use a cancellable context.Context instead of a bool
// flag, probably as part of: https://github.com/ooni/probe-cli/issues/45
func (p *Probe) ListenForSignals() {
s := make(chan os.Signal, 1)
signal.Notify(s, os.Interrupt, syscall.SIGINT, syscall.SIGTERM)
go func() {
<-s
log.Info("caught a stop signal, shutting down cleanly")
p.Terminate()
}()
}
// MaybeListenForStdinClosed will treat any error on stdin just
// like SIGTERM if and only if
//
// os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") == "true"
//
// When this feature is enabled, a collateral effect is that we swallow
// whatever is passed to us on the standard input.
//
// See https://github.com/ooni/probe-cli/pull/111 for more info
// regarding the design of this functionality.
//
// TODO refactor this to use a cancellable context.Context instead of a bool
// flag, probably as part of: https://github.com/ooni/probe-cli/issues/45
func (p *Probe) MaybeListenForStdinClosed() {
if os.Getenv("OONI_STDIN_EOF_IMPLIES_SIGTERM") != "true" {
return
}
go func() {
defer p.Terminate()
defer log.Info("stdin closed, shutting down cleanly")
b := make([]byte, 1<<10)
for {
if _, err := os.Stdin.Read(b); err != nil {
return
}
}
}()
}
// Init the OONI manager
func (p *Probe) Init(softwareName, softwareVersion string) error {
var err error
if err = MaybeInitializeHome(p.home); err != nil {
return err
}
if p.configPath != "" {
log.Debugf("Reading config file from %s", p.configPath)
p.config, err = config.ReadConfig(p.configPath)
} else {
log.Debug("Reading default config file")
p.config, err = InitDefaultConfig(p.home)
}
if err != nil {
return err
}
if err = p.config.MaybeMigrate(); err != nil {
return errors.Wrap(err, "migrating config")
}
p.dbPath = utils.DBDir(p.home, "main")
log.Debugf("Connecting to database sqlite3://%s", p.dbPath)
db, err := database.Connect(p.dbPath)
if err != nil {
return err
}
p.db = db
tempDir, err := ioutil.TempDir("", "ooni")
if err != nil {
return errors.Wrap(err, "creating TempDir")
}
p.tempDir = tempDir
p.softwareName = softwareName
p.softwareVersion = softwareVersion
return nil
}
// NewSession creates a new ooni/probe-engine session using the
// current configuration inside the context. The caller must close
// the session when done using it, by calling sess.Close().
func (p *Probe) NewSession() (*engine.Session, error) {
kvstore, err := engine.NewFileSystemKVStore(
utils.EngineDir(p.home),
)
if err != nil {
return nil, errors.Wrap(err, "creating engine's kvstore")
}
return engine.NewSession(engine.SessionConfig{
AssetsDir: utils.AssetsDir(p.home),
KVStore: kvstore,
Logger: enginex.Logger,
SoftwareName: p.softwareName,
SoftwareVersion: p.softwareVersion,
TempDir: p.tempDir,
})
}
// NewProbeEngine creates a new ProbeEngine instance.
func (p *Probe) NewProbeEngine() (ProbeEngine, error) {
sess, err := p.NewSession()
if err != nil {
return nil, err
}
return sess, nil
}
// NewProbe creates a new probe instance.
func NewProbe(configPath string, homePath string) *Probe {
return &Probe{
home: homePath,
config: &config.Config{},
configPath: configPath,
isTerminatedAtomicInt: 0,
}
}
// MaybeInitializeHome does the setup for a new OONI Home
func MaybeInitializeHome(home string) error {
for _, d := range utils.RequiredDirs(home) {
if _, e := os.Stat(d); e != nil {
if err := os.MkdirAll(d, 0700); err != nil {
return err
}
}
}
return nil
}
// InitDefaultConfig reads the config from common locations or creates it if
// missing.
func InitDefaultConfig(home string) (*config.Config, error) {
var (
err error
c *config.Config
configPath = utils.ConfigPath(home)
)
c, err = config.ReadConfig(configPath)
if err != nil {
if os.IsNotExist(err) {
log.Debugf("writing default config to %s", configPath)
var data []byte
data, err = bindata.Asset("data/default-config.json")
if err != nil {
return nil, err
}
if err = ioutil.WriteFile(configPath, data, 0644); err != nil {
return nil, err
}
// If the user did the informed consent procedure in
// probe-legacy, migrate it over.
if utils.DidLegacyInformedConsent() {
c, err := config.ReadConfig(configPath)
if err != nil {
return nil, err
}
c.Lock()
c.InformedConsent = true
c.Unlock()
if err := c.Write(); err != nil {
return nil, err
}
}
return InitDefaultConfig(home)
}
return nil, err
}
return c, nil
}
internal/ooni/ooni_test.go
-29
View File
@@ -1,29 +0,0 @@
package ooni
import (
"io/ioutil"
"os"
"path"
"testing"
)
func TestInit(t *testing.T) {
ooniHome, err := ioutil.TempDir("", "oonihome")
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(ooniHome)
probe := NewProbe("", ooniHome)
swName := "ooniprobe-cli-tests"
swVersion := "3.0.0-alpha"
if err := probe.Init(swName, swVersion); err != nil {
t.Error(err)
t.Fatal("failed to init the context")
}
configPath := path.Join(ooniHome, "config.json")
if _, err := os.Stat(configPath); os.IsNotExist(err) {
t.Fatal("config file was not created")
}
}
internal/oonitest/oonitest.go
-126
View File
@@ -1,126 +0,0 @@
// Package oonitest contains code used for testing.
package oonitest
import (
"sync"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/config"
"github.com/ooni/probe-cli/internal/ooni"
"upper.io/db.v3/lib/sqlbuilder"
)
// FakeOutput allows to fake the output package.
type FakeOutput struct {
FakeSectionTitle []string
mu sync.Mutex
}
// SectionTitle writes the section title.
func (fo *FakeOutput) SectionTitle(s string) {
fo.mu.Lock()
defer fo.mu.Unlock()
fo.FakeSectionTitle = append(fo.FakeSectionTitle, s)
}
// FakeProbeCLI fakes ooni.ProbeCLI
type FakeProbeCLI struct {
FakeConfig *config.Config
FakeDB sqlbuilder.Database
FakeIsBatch bool
FakeHome string
FakeTempDir string
FakeProbeEnginePtr ooni.ProbeEngine
FakeProbeEngineErr error
}
// Config implements ProbeCLI.Config
func (cli *FakeProbeCLI) Config() *config.Config {
return cli.FakeConfig
}
// DB implements ProbeCLI.DB
func (cli *FakeProbeCLI) DB() sqlbuilder.Database {
return cli.FakeDB
}
// IsBatch implements ProbeCLI.IsBatch
func (cli *FakeProbeCLI) IsBatch() bool {
return cli.FakeIsBatch
}
// Home implements ProbeCLI.Home
func (cli *FakeProbeCLI) Home() string {
return cli.FakeHome
}
// TempDir implements ProbeCLI.TempDir
func (cli *FakeProbeCLI) TempDir() string {
return cli.FakeTempDir
}
// NewProbeEngine implements ProbeCLI.NewProbeEngine
func (cli *FakeProbeCLI) NewProbeEngine() (ooni.ProbeEngine, error) {
return cli.FakeProbeEnginePtr, cli.FakeProbeEngineErr
}
var _ ooni.ProbeCLI = &FakeProbeCLI{}
// FakeProbeEngine fakes ooni.ProbeEngine
type FakeProbeEngine struct {
FakeClose error
FakeMaybeLookupLocation error
FakeProbeASNString string
FakeProbeCC string
FakeProbeIP string
FakeProbeNetworkName string
}
// Close implements ProbeEngine.Close
func (eng *FakeProbeEngine) Close() error {
return eng.FakeClose
}
// MaybeLookupLocation implements ProbeEngine.MaybeLookupLocation
func (eng *FakeProbeEngine) MaybeLookupLocation() error {
return eng.FakeMaybeLookupLocation
}
// ProbeASNString implements ProbeEngine.ProbeASNString
func (eng *FakeProbeEngine) ProbeASNString() string {
return eng.FakeProbeASNString
}
// ProbeCC implements ProbeEngine.ProbeCC
func (eng *FakeProbeEngine) ProbeCC() string {
return eng.FakeProbeCC
}
// ProbeIP implements ProbeEngine.ProbeIP
func (eng *FakeProbeEngine) ProbeIP() string {
return eng.FakeProbeIP
}
// ProbeNetworkName implements ProbeEngine.ProbeNetworkName
func (eng *FakeProbeEngine) ProbeNetworkName() string {
return eng.FakeProbeNetworkName
}
var _ ooni.ProbeEngine = &FakeProbeEngine{}
// FakeLoggerHandler fakes apex.log.Handler.
type FakeLoggerHandler struct {
FakeEntries []*log.Entry
FakeErr error
mu sync.Mutex
}
// HandleLog implements Handler.HandleLog.
func (handler *FakeLoggerHandler) HandleLog(entry *log.Entry) error {
handler.mu.Lock()
defer handler.mu.Unlock()
handler.FakeEntries = append(handler.FakeEntries, entry)
return handler.FakeErr
}
var _ log.Handler = &FakeLoggerHandler{}
internal/output/output.go
-170
View File
@@ -1,170 +0,0 @@
package output
import (
"bufio"
"fmt"
"os"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/internal/utils"
)
// MeasurementJSON prints the JSON of a measurement
func MeasurementJSON(j map[string]interface{}) {
log.WithFields(log.Fields{
"type": "measurement_json",
"measurement_json": j,
}).Info("Measurement JSON")
}
// Progress logs a progress type event
func Progress(key string, perc float64, eta float64, msg string) {
log.WithFields(log.Fields{
"type": "progress",
"key": key,
"percentage": perc,
"eta": eta,
}).Info(msg)
}
type MeasurementSummaryData struct {
TotalRuntime float64
TotalCount int64
AnomalyCount int64
DataUsageUp float64
DataUsageDown float64
ASN uint
NetworkName string
NetworkCountryCode string
StartTime time.Time
}
func MeasurementSummary(msmt MeasurementSummaryData) {
log.WithFields(log.Fields{
"type": "measurement_summary",
"total_runtime": msmt.TotalRuntime,
"total_count": msmt.TotalCount,
"anomaly_count": msmt.AnomalyCount,
"data_usage_down": msmt.DataUsageDown,
"data_usage_up": msmt.DataUsageUp,
"asn": msmt.ASN,
"network_country_code": msmt.NetworkCountryCode,
"network_name": msmt.NetworkName,
"start_time": msmt.StartTime,
}).Info("measurement summary")
}
// MeasurementItem logs a progress type event
func MeasurementItem(msmt database.MeasurementURLNetwork, isFirst bool, isLast bool) {
log.WithFields(log.Fields{
"type": "measurement_item",
"is_first": isFirst,
"is_last": isLast,
"id": msmt.Measurement.ID,
"test_name": msmt.TestName,
"test_group_name": msmt.Result.TestGroupName,
"start_time": msmt.Measurement.StartTime,
"test_keys": msmt.TestKeys,
"network_country_code": msmt.Network.CountryCode,
"network_name": msmt.Network.NetworkName,
"asn": msmt.Network.ASN,
"runtime": msmt.Measurement.Runtime,
"url": msmt.URL.URL.String,
"url_category_code": msmt.URL.CategoryCode.String,
"url_country_code": msmt.URL.CountryCode.String,
"is_anomaly": msmt.IsAnomaly.Bool,
"is_uploaded": msmt.IsUploaded,
"is_upload_failed": msmt.IsUploadFailed,
"upload_failure_msg": msmt.UploadFailureMsg.String,
"is_failed": msmt.IsFailed,
"failure_msg": msmt.FailureMsg.String,
"is_done": msmt.Measurement.IsDone,
"report_file_path": msmt.ReportFilePath.String,
"measurement_file_path": msmt.MeasurementFilePath.String,
}).Info("measurement")
}
// ResultItemData is the metadata about a result
type ResultItemData struct {
ID int64
Name string
StartTime time.Time
TestKeys string
MeasurementCount uint64
MeasurementAnomalyCount uint64
Runtime float64
Country string
NetworkName string
ASN uint
Done bool
DataUsageDown float64
DataUsageUp float64
Index int
TotalCount int
}
// ResultItem logs a progress type event
func ResultItem(result ResultItemData) {
log.WithFields(log.Fields{
"type": "result_item",
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"test_keys": result.TestKeys,
"measurement_count": result.MeasurementCount,
"measurement_anomaly_count": result.MeasurementAnomalyCount,
"network_country_code": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,
"runtime": result.Runtime,
"is_done": result.Done,
"data_usage_down": result.DataUsageDown,
"data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
}).Info("result item")
}
type ResultSummaryData struct {
TotalTests int64
TotalDataUsageUp float64
TotalDataUsageDown float64
TotalNetworks int64
}
func ResultSummary(result ResultSummaryData) {
log.WithFields(log.Fields{
"type": "result_summary",
"total_tests": result.TotalTests,
"total_data_usage_up": result.TotalDataUsageUp,
"total_data_usage_down": result.TotalDataUsageDown,
"total_networks": result.TotalNetworks,
}).Info("result summary")
}
// SectionTitle is the title of a section
func SectionTitle(text string) {
log.WithFields(log.Fields{
"type": "section_title",
"title": text,
}).Info(text)
}
func Paragraph(text string) {
const width = 80
fmt.Println(utils.WrapString(text, width))
}
func Bullet(text string) {
const width = 80
fmt.Printf("• %s\n", utils.WrapString(text, width))
}
func PressEnterToContinue(text string) error {
fmt.Print(text)
_, err := bufio.NewReader(os.Stdin).ReadBytes('\n')
return err
}
internal/utils/homedir/homedir.go
-211
View File
@@ -1,211 +0,0 @@
package homedir
// Stolen from: https://github.com/puma/puma-dev/blob/master/homedir/homedir.go
import (
"bytes"
"errors"
"os"
"os/exec"
"path/filepath"
"runtime"
"strconv"
"strings"
"sync"
)
// DisableCache will disable caching of the home directory. Caching is enabled
// by default.
var DisableCache bool
var homedirCache string
var cacheLock sync.RWMutex
// ErrNoHomeDir when no home dir could be found
var ErrNoHomeDir = errors.New("no home directory available")
// Dir returns the home directory for the executing user.
//
// This uses an OS-specific method for discovering the home directory.
// An error is returned if a home directory cannot be detected.
func Dir() (string, error) {
if !DisableCache {
cacheLock.RLock()
cached := homedirCache
cacheLock.RUnlock()
if cached != "" {
return cached, nil
}
}
cacheLock.Lock()
defer cacheLock.Unlock()
var result string
var err error
switch runtime.GOOS {
case "windows":
result, err = dirWindows()
case "darwin":
result, err = dirDarwin()
default:
// Unix-like system, so just assume Unix
result, err = dirUnix()
}
if err != nil {
return "", err
}
homedirCache = result
return result, nil
}
// Expand expands the path to include the home directory if the path
// is prefixed with `~`. If it isn't prefixed with `~`, the path is
// returned as-is.
func Expand(path string) (string, error) {
if len(path) == 0 {
return path, nil
}
if path[0] != '~' {
return path, nil
}
if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
return "", errors.New("cannot expand user-specific home dir")
}
dir, err := Dir()
if err != nil {
return "", err
}
return filepath.Join(dir, path[1:]), nil
}
func MustExpand(path string) string {
str, err := Expand(path)
if err != nil {
panic(err)
}
return str
}
func dirDarwin() (string, error) {
// First prefer the HOME environmental variable
if home := os.Getenv("HOME"); home != "" {
return home, nil
}
var stdout bytes.Buffer
// If that fails, try OS specific commands
cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`)
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
result := strings.TrimSpace(stdout.String())
if result != "" {
return result, nil
}
}
// try the shell
stdout.Reset()
cmd = exec.Command("sh", "-c", "cd && pwd")
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
result := strings.TrimSpace(stdout.String())
if result != "" {
return result, nil
}
}
// try to figure out the user and check the default location
stdout.Reset()
cmd = exec.Command("whoami")
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
user := strings.TrimSpace(stdout.String())
path := "/Users/" + user
stat, err := os.Stat(path)
if err == nil && stat.IsDir() {
return path, nil
}
}
return "", ErrNoHomeDir
}
func dirUnix() (string, error) {
// First prefer the HOME environmental variable
if home := os.Getenv("HOME"); home != "" {
return home, nil
}
var stdout bytes.Buffer
// If that fails, try OS specific commands
cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid()))
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
if passwd := strings.TrimSpace(stdout.String()); passwd != "" {
// username:password:uid:gid:gecos:home:shell
passwdParts := strings.SplitN(passwd, ":", 7)
if len(passwdParts) > 5 {
return passwdParts[5], nil
}
}
}
// If all else fails, try the shell
stdout.Reset()
cmd = exec.Command("sh", "-c", "cd && pwd")
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
result := strings.TrimSpace(stdout.String())
if result == "" {
return "", errors.New("blank output when reading home directory")
}
}
// try to figure out the user and check the default location
stdout.Reset()
cmd = exec.Command("whoami")
cmd.Stdout = &stdout
if err := cmd.Run(); err == nil {
user := strings.TrimSpace(stdout.String())
path := "/home/" + user
stat, err := os.Stat(path)
if err == nil && stat.IsDir() {
return path, nil
}
}
return "", ErrNoHomeDir
}
func dirWindows() (string, error) {
// First prefer the HOME environmental variable
if home := os.Getenv("HOME"); home != "" {
return home, nil
}
drive := os.Getenv("HOMEDRIVE")
path := os.Getenv("HOMEPATH")
home := drive + path
if drive == "" || path == "" {
home = os.Getenv("USERPROFILE")
}
if home == "" {
return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank")
}
return home, nil
}
internal/utils/paths.go
-98
View File
@@ -1,98 +0,0 @@
package utils
import (
"errors"
"fmt"
"os"
"path/filepath"
"time"
"github.com/ooni/probe-cli/internal/utils/homedir"
)
// RequiredDirs returns the required ooni home directories
func RequiredDirs(home string) []string {
requiredDirs := []string{}
requiredSubdirs := []string{"assets", "db", "msmts"}
for _, d := range requiredSubdirs {
requiredDirs = append(requiredDirs, filepath.Join(home, d))
}
return requiredDirs
}
// ConfigPath returns the default path to the config file
func ConfigPath(home string) string {
return filepath.Join(home, "config.json")
}
// AssetsDir returns the assets data dir for the given OONI Home
func AssetsDir(home string) string {
return filepath.Join(home, "assets")
}
// EngineDir returns the directory where ooni/probe-engine should
// store its private data given a specific OONI Home.
func EngineDir(home string) string {
return filepath.Join(home, "engine")
}
// DBDir returns the database dir for the given name
func DBDir(home string, name string) string {
return filepath.Join(home, "db", fmt.Sprintf("%s.sqlite3", name))
}
// FileExists returns true if the specified path exists and is a
// regular file.
func FileExists(path string) bool {
stat, err := os.Stat(path)
return err == nil && stat.Mode().IsRegular()
}
// ResultTimestamp is a windows friendly timestamp
const ResultTimestamp = "2006-01-02T150405.999999999Z0700"
// MakeResultsDir creates and returns a directory for the result
func MakeResultsDir(home string, name string, ts time.Time) (string, error) {
p := filepath.Join(home, "msmts",
fmt.Sprintf("%s-%s", name, ts.Format(ResultTimestamp)))
// If the path already exists, this is a problem. It should not clash, because
// we are using nanosecond precision for the starttime.
if _, e := os.Stat(p); e == nil {
return "", errors.New("results path already exists")
}
err := os.MkdirAll(p, 0700)
if err != nil {
return "", err
}
return p, nil
}
// GetOONIHome returns the path to the OONI Home
func GetOONIHome() (string, error) {
if ooniHome := os.Getenv("OONI_HOME"); ooniHome != "" {
return ooniHome, nil
}
home, err := homedir.Dir()
if err != nil {
return "", err
}
path := filepath.Join(home, ".ooniprobe")
return path, nil
}
// DidLegacyInformedConsent checks if the user did the informed consent procedure in probe-legacy
func DidLegacyInformedConsent() bool {
home, err := homedir.Dir()
if err != nil {
return false
}
path := filepath.Join(filepath.Join(home, ".ooni"), "initialized")
if FileExists(path) {
return true
}
return false
}
internal/utils/shutil/shutil.go
-319
View File
@@ -1,319 +0,0 @@
package shutil
import (
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
)
type SameFileError struct {
Src string
Dst string
}
func (e SameFileError) Error() string {
return fmt.Sprintf("%s and %s are the same file", e.Src, e.Dst)
}
type SpecialFileError struct {
File string
FileInfo os.FileInfo
}
func (e SpecialFileError) Error() string {
return fmt.Sprintf("`%s` is a named pipe", e.File)
}
type NotADirectoryError struct {
Src string
}
func (e NotADirectoryError) Error() string {
return fmt.Sprintf("`%s` is not a directory", e.Src)
}
type AlreadyExistsError struct {
Dst string
}
func (e AlreadyExistsError) Error() string {
return fmt.Sprintf("`%s` already exists", e.Dst)
}
func samefile(src string, dst string) bool {
srcInfo, _ := os.Stat(src)
dstInfo, _ := os.Stat(dst)
return os.SameFile(srcInfo, dstInfo)
}
func specialfile(fi os.FileInfo) bool {
return (fi.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe
}
func stringInSlice(a string, list []string) bool {
for _, b := range list {
if b == a {
return true
}
}
return false
}
func IsSymlink(fi os.FileInfo) bool {
return (fi.Mode() & os.ModeSymlink) == os.ModeSymlink
}
// Copy data from src to dst
//
// If followSymlinks is not set and src is a symbolic link, a
// new symlink will be created instead of copying the file it points
// to.
func CopyFile(src, dst string, followSymlinks bool) error {
if samefile(src, dst) {
return &SameFileError{src, dst}
}
// Make sure src exists and neither are special files
srcStat, err := os.Lstat(src)
if err != nil {
return err
}
if specialfile(srcStat) {
return &SpecialFileError{src, srcStat}
}
dstStat, err := os.Stat(dst)
if err != nil && !os.IsNotExist(err) {
return err
} else if err == nil {
if specialfile(dstStat) {
return &SpecialFileError{dst, dstStat}
}
}
// If we don't follow symlinks and it's a symlink, just link it and be done
if !followSymlinks && IsSymlink(srcStat) {
return os.Symlink(src, dst)
}
// If we are a symlink, follow it
if IsSymlink(srcStat) {
src, err = os.Readlink(src)
if err != nil {
return err
}
srcStat, err = os.Stat(src)
if err != nil {
return err
}
}
// Do the actual copy
fsrc, err := os.Open(src)
if err != nil {
return err
}
defer fsrc.Close()
fdst, err := os.Create(dst)
if err != nil {
return err
}
defer fdst.Close()
size, err := io.Copy(fdst, fsrc)
if err != nil {
return err
}
if size != srcStat.Size() {
return fmt.Errorf("%s: %d/%d copied", src, size, srcStat.Size())
}
return nil
}
// Copy mode bits from src to dst.
//
// If followSymlinks is false, symlinks aren't followed if and only
// if both `src` and `dst` are symlinks. If `lchmod` isn't available
// and both are symlinks this does nothing. (I don't think lchmod is
// available in Go)
func CopyMode(src, dst string, followSymlinks bool) error {
srcStat, err := os.Lstat(src)
if err != nil {
return err
}
dstStat, err := os.Lstat(dst)
if err != nil {
return err
}
// They are both symlinks and we can't change mode on symlinks.
if !followSymlinks && IsSymlink(srcStat) && IsSymlink(dstStat) {
return nil
}
// Atleast one is not a symlink, get the actual file stats
srcStat, _ = os.Stat(src)
err = os.Chmod(dst, srcStat.Mode())
return err
}
// Copy data and mode bits ("cp src dst"). Return the file's destination.
//
// The destination may be a directory.
//
// If followSymlinks is false, symlinks won't be followed. This
// resembles GNU's "cp -P src dst".
//
// If source and destination are the same file, a SameFileError will be
// rased.
func Copy(src, dst string, followSymlinks bool) (string, error) {
dstInfo, err := os.Stat(dst)
if err == nil && dstInfo.Mode().IsDir() {
dst = filepath.Join(dst, filepath.Base(src))
}
if err != nil && !os.IsNotExist(err) {
return dst, err
}
err = CopyFile(src, dst, followSymlinks)
if err != nil {
return dst, err
}
err = CopyMode(src, dst, followSymlinks)
if err != nil {
return dst, err
}
return dst, nil
}
type CopyTreeOptions struct {
Symlinks bool
IgnoreDanglingSymlinks bool
CopyFunction func(string, string, bool) (string, error)
Ignore func(string, []os.FileInfo) []string
}
// Recursively copy a directory tree.
//
// The destination directory must not already exist.
//
// If the optional Symlinks flag is true, symbolic links in the
// source tree result in symbolic links in the destination tree; if
// it is false, the contents of the files pointed to by symbolic
// links are copied. If the file pointed by the symlink doesn't
// exist, an error will be returned.
//
// You can set the optional IgnoreDanglingSymlinks flag to true if you
// want to silence this error. Notice that this has no effect on
// platforms that don't support os.Symlink.
//
// The optional ignore argument is a callable. If given, it
// is called with the `src` parameter, which is the directory
// being visited by CopyTree(), and `names` which is the list of
// `src` contents, as returned by ioutil.ReadDir():
//
// callable(src, entries) -> ignoredNames
//
// Since CopyTree() is called recursively, the callable will be
// called once for each directory that is copied. It returns a
// list of names relative to the `src` directory that should
// not be copied.
//
// The optional copyFunction argument is a callable that will be used
// to copy each file. It will be called with the source path and the
// destination path as arguments. By default, Copy() is used, but any
// function that supports the same signature (like Copy2() when it
// exists) can be used.
func CopyTree(src, dst string, options *CopyTreeOptions) error {
if options == nil {
options = &CopyTreeOptions{Symlinks: false,
Ignore: nil,
CopyFunction: Copy,
IgnoreDanglingSymlinks: false}
}
srcFileInfo, err := os.Stat(src)
if err != nil {
return err
}
if !srcFileInfo.IsDir() {
return &NotADirectoryError{src}
}
_, err = os.Open(dst)
if !os.IsNotExist(err) {
return &AlreadyExistsError{dst}
}
entries, err := ioutil.ReadDir(src)
if err != nil {
return err
}
err = os.MkdirAll(dst, srcFileInfo.Mode())
if err != nil {
return err
}
ignoredNames := []string{}
if options.Ignore != nil {
ignoredNames = options.Ignore(src, entries)
}
for _, entry := range entries {
if stringInSlice(entry.Name(), ignoredNames) {
continue
}
srcPath := filepath.Join(src, entry.Name())
dstPath := filepath.Join(dst, entry.Name())
entryFileInfo, err := os.Lstat(srcPath)
if err != nil {
return err
}
// Deal with symlinks
if IsSymlink(entryFileInfo) {
linkTo, err := os.Readlink(srcPath)
if err != nil {
return err
}
if options.Symlinks {
os.Symlink(linkTo, dstPath)
//CopyStat(srcPath, dstPath, false)
} else {
// ignore dangling symlink if flag is on
_, err = os.Stat(linkTo)
if os.IsNotExist(err) && options.IgnoreDanglingSymlinks {
continue
}
_, err = options.CopyFunction(srcPath, dstPath, false)
if err != nil {
return err
}
}
} else if entryFileInfo.IsDir() {
err = CopyTree(srcPath, dstPath, options)
if err != nil {
return err
}
} else {
_, err = options.CopyFunction(srcPath, dstPath, false)
if err != nil {
return err
}
}
}
return nil
}
internal/utils/util_test.go
-18
View File
@@ -1,18 +0,0 @@
package utils
import (
"testing"
"github.com/fatih/color"
)
func TestEscapeAwareRuneCountInString(t *testing.T) {
var bold = color.New(color.Bold)
var myColor = color.New(color.FgBlue)
s := myColor.Sprintf("•ABC%s%s", bold.Sprintf("DEF"), "\x1B[00;38;5;244m\x1B[m\x1B[00;38;5;33mGHI\x1B[0m")
count := EscapeAwareRuneCountInString(s)
if count != 10 {
t.Errorf("Count was incorrect, got: %d, want: %d.", count, 10)
}
}
internal/utils/utils.go
-117
View File
@@ -1,117 +0,0 @@
package utils
import (
"bytes"
"fmt"
"os"
"regexp"
"strings"
"unicode"
"unicode/utf8"
"github.com/fatih/color"
)
// Log outputs a log message.
func Log(msg string, v ...interface{}) {
fmt.Printf(" %s\n", color.CyanString(msg, v...))
}
// Fatal error
func Fatal(err error) {
fmt.Fprintf(os.Stderr, "\n %s %s\n\n", color.RedString("Error:"), err)
os.Exit(1)
}
// Finds the ansi escape sequences (like colors)
// Taken from: https://github.com/chalk/ansi-regex/blob/d9d806ecb45d899cf43408906a4440060c5c50e5/index.js
var ansiEscapes = regexp.MustCompile(`[\x1B\x9B][[\]()#;?]*` +
`(?:(?:(?:[a-zA-Z\d]*(?:;[a-zA-Z\\d]*)*)?\x07)` +
`|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PRZcf-ntqry=><~]))`)
// EscapeAwareRuneCountInString counts the number of runes in a
// string taking into account escape sequences.
func EscapeAwareRuneCountInString(s string) int {
n := utf8.RuneCountInString(s)
for _, sm := range ansiEscapes.FindAllString(s, -1) {
n -= utf8.RuneCountInString(sm)
}
return n
}
// RightPadd adds right padding in from of a string
func RightPad(str string, length int) string {
c := length - EscapeAwareRuneCountInString(str)
if c < 0 {
c = 0
}
return str + strings.Repeat(" ", c)
}
// WrapString wraps the given string within lim width in characters.
//
// Wrapping is currently naive and only happens at white-space. A future
// version of the library will implement smarter wrapping. This means that
// pathological cases can dramatically reach past the limit, such as a very
// long word.
// This is taken from: https://github.com/mitchellh/go-wordwrap/tree/f253961a26562056904822f2a52d4692347db1bd
func WrapString(s string, lim uint) string {
// Initialize a buffer with a slightly larger size to account for breaks
init := make([]byte, 0, len(s))
buf := bytes.NewBuffer(init)
var current uint
var wordBuf, spaceBuf bytes.Buffer
for _, char := range s {
if char == '\n' {
if wordBuf.Len() == 0 {
if current+uint(spaceBuf.Len()) > lim {
current = 0
} else {
current += uint(spaceBuf.Len())
spaceBuf.WriteTo(buf)
}
spaceBuf.Reset()
} else {
current += uint(spaceBuf.Len() + wordBuf.Len())
spaceBuf.WriteTo(buf)
spaceBuf.Reset()
wordBuf.WriteTo(buf)
wordBuf.Reset()
}
buf.WriteRune(char)
current = 0
} else if unicode.IsSpace(char) {
if spaceBuf.Len() == 0 || wordBuf.Len() > 0 {
current += uint(spaceBuf.Len() + wordBuf.Len())
spaceBuf.WriteTo(buf)
spaceBuf.Reset()
wordBuf.WriteTo(buf)
wordBuf.Reset()
}
spaceBuf.WriteRune(char)
} else {
wordBuf.WriteRune(char)
if current+uint(spaceBuf.Len()+wordBuf.Len()) > lim && uint(wordBuf.Len()) < lim {
buf.WriteRune('\n')
current = 0
spaceBuf.Reset()
}
}
}
if wordBuf.Len() == 0 {
if current+uint(spaceBuf.Len()) <= lim {
spaceBuf.WriteTo(buf)
}
} else {
spaceBuf.WriteTo(buf)
wordBuf.WriteTo(buf)
}
return buf.String()
}
internal/version/version.go
-7
View File
@@ -1,7 +0,0 @@
// Package version contains version information
package version
const (
// Version is the software version
Version = "3.5.0-alpha"
)