feat: implement darwin launch agent (#192)

* feat: sketch out periodic command

* feat: sketch out periodic command for macOS

* feat: implement darwin's launch agent

* refactor: better way to run on darwin

Make sure we have code that builds on all platforms.

* fix(run): max 10 URLs with darwin in unattended mode

* feat: add support for seeing/streaming logs

* feat: implement the status command and add usage hints

* feat(periodic): run onboarding if needed

* fix: no too confusing function names

* fix: s/periodic/autorun/

Discussed earlier this morning with @hellais.

* fix: we cannot show logs before Big Sur

Bug reported by @hellais.

cmd/ooniprobe/main.go

@@ -3,6 +3,7 @@ package main
 import (
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/internal/cli/app"
+	_ "github.com/ooni/probe-cli/internal/cli/autorun"
 	_ "github.com/ooni/probe-cli/internal/cli/geoip"
 	_ "github.com/ooni/probe-cli/internal/cli/info"
 	_ "github.com/ooni/probe-cli/internal/cli/list"

go.mod

@@ -17,6 +17,7 @@ require (
 	github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
 	github.com/sirupsen/logrus v1.7.0 // indirect
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
+	golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc
 	golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
 	gopkg.in/AlecAivazis/survey.v1 v1.8.8
 	gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect

go.sum

@@ -183,6 +183,7 @@ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXi
 github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
+github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaUGG7oYTSPP8MxqL4YI3kZKwcP4=
 github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
 github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=

internal/autorun/autorun.go

@@ -0,0 +1,49 @@
+// Package autorun contains code to manage automatic runs
+package autorun
+
+import "sync"
+
+const (
+	// StatusScheduled indicates that OONI is scheduled to run
+	// periodically in the background.
+	StatusScheduled = "scheduled"
+
+	// StatusStopped indicates that OONI is not scheduled to
+	// run periodically in the background.
+	StatusStopped = "stopped"
+
+	// StatusRunning indicates that OONI is currently
+	// running in the background.
+	StatusRunning = "running"
+)
+
+// Manager manages automatic runs
+type Manager interface {
+	LogShow() error
+	LogStream() error
+	Start() error
+	Status() (string, error)
+	Stop() error
+}
+
+var (
+	registry map[string]Manager
+	mtx      sync.Mutex
+)
+
+func register(platform string, manager Manager) {
+	defer mtx.Unlock()
+	mtx.Lock()
+	if registry == nil {
+		registry = make(map[string]Manager)
+	}
+	registry[platform] = manager
+}
+
+// Get gets the specified autorun manager. This function
+// returns nil if no autorun manager exists.
+func Get(platform string) Manager {
+	defer mtx.Unlock()
+	mtx.Lock()
+	return registry[platform]
+}

internal/autorun/autorun_darwin.go

@@ -0,0 +1,178 @@
+package autorun
+
+import (
+	"bytes"
+	"errors"
+	"fmt"
+	"io/ioutil"
+	"os"
+	"os/exec"
+	"strconv"
+	"strings"
+	"text/template"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/utils"
+	"github.com/ooni/probe-engine/cmd/jafar/shellx"
+	"golang.org/x/sys/unix"
+)
+
+type managerDarwin struct{}
+
+var (
+	plistPath     = os.ExpandEnv("$HOME/Library/LaunchAgents/org.ooni.cli.plist")
+	domainTarget  = fmt.Sprintf("gui/%d", os.Getuid())
+	serviceTarget = fmt.Sprintf("%s/org.ooni.cli", domainTarget)
+)
+
+var plistTemplate = `
+<?xml version="1.0" encoding="UTF-8"?>
+<!DOCTYPE plist PUBLIC "-//Apple Computer//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
+<plist version="1.0">
+<dict>
+    <key>Label</key>
+    <string>org.ooni.cli</string>
+    <key>KeepAlive</key>
+    <false/>
+    <key>RunAtLoad</key>
+    <true/>
+    <key>ProgramArguments</key>
+    <array>
+        <string>{{ .Executable }}</string>
+        <string>--log-handler=syslog</string>
+        <string>run</string>
+        <string>unattended</string>
+    </array>
+    <key>StartInterval</key>
+    <integer>3600</integer>
+</dict>
+</plist>
+`
+
+func runQuiteQuietly(name string, arg ...string) error {
+	log.Infof("exec: %s %s", name, strings.Join(arg, " "))
+	return shellx.RunQuiet(name, arg...)
+}
+
+func darwinVersionMajor() (int, error) {
+	out, err := exec.Command("uname", "-r").Output()
+	if err != nil {
+		return 0, err
+	}
+	v := bytes.Split(out, []byte("."))
+	if len(v) != 3 {
+		return 0, errors.New("cannot split version")
+	}
+	major, err := strconv.Atoi(string(v[0]))
+	if err != nil {
+		return 0, err
+	}
+	return major, nil
+}
+
+var errNotImplemented = errors.New(
+	"autorun: command not implemented in this version of macOS")
+
+func (managerDarwin) LogShow() error {
+	major, _ := darwinVersionMajor()
+	if major < 20 /* macOS 11.0 Big Sur */ {
+		return errNotImplemented
+	}
+	return shellx.Run("log", "show", "--info", "--debug",
+		"--process", "ooniprobe", "--style", "compact")
+}
+
+func (managerDarwin) LogStream() error {
+	return shellx.Run("log", "stream", "--style", "compact", "--level",
+		"debug", "--process", "ooniprobe")
+}
+
+func (managerDarwin) mustNotHavePlist() error {
+	log.Infof("exec: test -f %s && already_registered()", plistPath)
+	if utils.FileExists(plistPath) {
+		// This is not atomic. Do we need atomicity here?
+		return errors.New("autorun: service already registered")
+	}
+	return nil
+}
+
+func (managerDarwin) writePlist() error {
+	executable, err := os.Executable()
+	if err != nil {
+		return err
+	}
+	var out bytes.Buffer
+	t := template.Must(template.New("plist").Parse(plistTemplate))
+	in := struct{ Executable string }{Executable: executable}
+	if err := t.Execute(&out, in); err != nil {
+		return err
+	}
+	log.Infof("exec: writePlist(%s)", plistPath)
+	return ioutil.WriteFile(plistPath, out.Bytes(), 0644)
+}
+
+func (managerDarwin) start() error {
+	if err := runQuiteQuietly("launchctl", "enable", serviceTarget); err != nil {
+		return err
+	}
+	return runQuiteQuietly("launchctl", "bootstrap", domainTarget, plistPath)
+}
+
+func (m managerDarwin) Start() error {
+	operations := []func() error{m.mustNotHavePlist, m.writePlist, m.start}
+	for _, op := range operations {
+		if err := op(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (managerDarwin) stop() error {
+	var failure *exec.ExitError
+	err := runQuiteQuietly("launchctl", "bootout", serviceTarget)
+	if errors.As(err, &failure) && failure.ExitCode() == int(unix.ESRCH) {
+		err = nil
+	}
+	return err
+}
+
+func (managerDarwin) removeFile() error {
+	log.Infof("exec: rm -f %s", plistPath)
+	err := os.Remove(plistPath)
+	if errors.Is(err, unix.ENOENT) {
+		err = nil
+	}
+	return err
+}
+
+func (m managerDarwin) Stop() error {
+	operations := []func() error{m.stop, m.removeFile}
+	for _, op := range operations {
+		if err := op(); err != nil {
+			return err
+		}
+	}
+	return nil
+}
+
+func (m managerDarwin) Status() (string, error) {
+	err := runQuiteQuietly("launchctl", "kill", "SIGINFO", serviceTarget)
+	var failure *exec.ExitError
+	if errors.As(err, &failure) {
+		switch failure.ExitCode() {
+		case int(unix.ESRCH):
+			return StatusScheduled, nil
+		case 113: // exit code when there's no plist
+			return StatusStopped, nil
+		}
+	}
+	if err != nil {
+		return "", fmt.Errorf("autorun: unexpected error: %w", err)
+	}
+	return StatusRunning, nil
+}
+
+func init() {
+	register("darwin", managerDarwin{})
+}

internal/cli/autorun/autorun.go

@@ -0,0 +1,95 @@
+package autorun
+
+import (
+	"errors"
+	"runtime"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/internal/autorun"
+	"github.com/ooni/probe-cli/internal/cli/onboard"
+	"github.com/ooni/probe-cli/internal/cli/root"
+)
+
+var errNotImplemented = errors.New("autorun: not implemented on this platform")
+
+func init() {
+	cmd := root.Command("autorun", "Run automatic tests in the background")
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		probe, err := root.Init()
+		if err != nil {
+			log.Errorf("%s", err)
+			return err
+		}
+		if err := onboard.MaybeOnboarding(probe); err != nil {
+			log.WithError(err).Error("failed to perform onboarding")
+			return err
+		}
+		return nil
+	})
+
+	start := cmd.Command("start", "Start running automatic tests in the background")
+	start.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		if err := svc.Start(); err != nil {
+			return err
+		}
+		log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
+		return nil
+	})
+
+	stop := cmd.Command("stop", "Stop running automatic tests in the background")
+	stop.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.Stop()
+	})
+
+	logCmd := cmd.Command("log", "Access background runs logs")
+	stream := logCmd.Command("stream", "Stream background runs logs")
+	stream.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.LogStream()
+	})
+
+	show := logCmd.Command("show", "Show background runs logs")
+	show.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		return svc.LogShow()
+	})
+
+	status := cmd.Command("status", "Shows autorun instance status")
+	status.Action(func(_ *kingpin.ParseContext) error {
+		svc := autorun.Get(runtime.GOOS)
+		if svc == nil {
+			return errNotImplemented
+		}
+		out, err := svc.Status()
+		if err != nil {
+			return err
+		}
+		log.Infof("status: %s", out)
+		switch out {
+		case autorun.StatusRunning:
+			log.Info("hint: use 'ooniprobe autorun stop' to stop")
+			log.Info("hint: use 'ooniprobe autorun log stream' to follow logs")
+		case autorun.StatusScheduled:
+			log.Info("hint: use 'ooniprobe autorun stop' to stop")
+			log.Info("hint: use 'ooniprobe autorun log show' to see previous logs")
+		case autorun.StatusStopped:
+			log.Info("hint: use 'ooniprobe autorun start' to start")
+		}
+		return nil
+	})
+}

internal/cli/run/run.go

@@ -1,6 +1,8 @@
 package run
 
 import (
+	"runtime"
+
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/fatih/color"
@@ -74,6 +76,13 @@ func init() {
 
 	unattendedCmd := cmd.Command("unattended", "")
 	unattendedCmd.Action(func(_ *kingpin.ParseContext) error {
+		if runtime.GOOS == "darwin" {
+			// Until we have enabled the check-in API we're called every
+			// hour on darwin and we need to self throttle.
+			// TODO(bassosimone): switch to check-in and remove this hack.
+			const veryFew = 10
+			probe.Config().Nettests.WebsitesURLLimit = veryFew
+		}
 		return functionalRun(func(name string, gr nettests.Group) bool {
 			return gr.UnattendedOK == true
 		})