refactor: move more commands to internal/cmd (#207)

* refactor: move more commands to internal/cmd

Part of https://github.com/ooni/probe/issues/1335.

We would like all commands to be at the same level of engine
rather than inside engine (now that we can do it).

* fix: update .gitignore

* refactor: also move jafar outside engine

* We should be good now?

.gitignore

@@ -2,9 +2,15 @@
 /*.jsonl
 /*.tar.gz
 /*.zip
+/apitool
+/apitool.exe
 /coverage.cov
 /miniooni
 /miniooni.exe
+/oohelper
+/oohelper.exe
+/oohelperd
+/oohelperd.exe
 /ooniprobe
 /ooniprobe.exe
 /ooniprobe_checksums.txt

cmd/ooniprobe/internal/autorun/autorun_darwin.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
 	"golang.org/x/sys/unix"
 )
 

internal/cmd/README.md

@@ -0,0 +1,4 @@
+# Directory github.com/ooni/probe-cli/internal/cmd
+
+This directory contains the source code for the CLI tools we build
+but we don't want to expose to the outside world.

internal/cmd/apitool/main.go

@@ -17,8 +17,8 @@ import (
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 	"github.com/ooni/probe-cli/v3/internal/engine/version"

internal/cmd/jafar/.gitignore

@@ -0,0 +1 @@
+/badproxy.pem

internal/cmd/jafar/flagx/stringarray_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx"
 )
 
 func TestStringArray(t *testing.T) {

internal/cmd/jafar/httpproxy/httpproxy_test.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
 )
 
 func TestPass(t *testing.T) {

internal/cmd/jafar/iptables/iptables_integration_test.go

@@ -14,9 +14,9 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
 )
 
 func init() {

internal/cmd/jafar/iptables/iptables_linux.go

@@ -3,8 +3,8 @@
 package iptables
 
 import (
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
 )
 
 type linuxShell struct{}

internal/cmd/jafar/main.go

@@ -18,15 +18,15 @@ import (
 	"github.com/apex/log"
 	"github.com/apex/log/handlers/cli"
 	"github.com/miekg/dns"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/badproxy"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/httpproxy"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/tlsproxy"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/badproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/flagx"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/httpproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/tlsproxy"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
 )
 
 var (

internal/cmd/jafar/main_test.go

@@ -6,8 +6,8 @@ import (
 	"runtime"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/iptables"
+	"github.com/ooni/probe-cli/v3/internal/engine/shellx"
 )
 
 func ensureWeStartOverWithIPTables() {

internal/cmd/jafar/resolver/resolver_test.go

@@ -5,7 +5,7 @@ import (
 	"testing"
 
 	"github.com/miekg/dns"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
 )
 
 func TestPass(t *testing.T) {

internal/cmd/jafar/tlsproxy/tlsproxy_test.go

@@ -7,7 +7,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
+	"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
 )
 
 func TestPass(t *testing.T) {

internal/cmd/oohelper/internal/client.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 	"github.com/ooni/probe-cli/v3/internal/engine/version"

internal/cmd/oohelper/internal/client_test.go

@@ -8,7 +8,7 @@ import (
 	"reflect"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
 )
 
 func TestMakeTCPEndpoints(t *testing.T) {

internal/cmd/oohelper/oohelper.go

@@ -10,7 +10,7 @@ import (
 	"net/http"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelper/internal"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 )

internal/cmd/oohelperd/internal/http_test.go

@@ -8,7 +8,7 @@ import (
 	"sync"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
 )
 
 func TestHTTPDoWithInvalidURL(t *testing.T) {

internal/cmd/oohelperd/internal/internal_test.go

@@ -10,7 +10,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
 )
 

internal/cmd/oohelperd/oohelperd.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
+	"github.com/ooni/probe-cli/v3/internal/cmd/oohelperd/internal"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 )
 

internal/engine/.gitignore

@@ -2,9 +2,7 @@
 /.vscode
 /apitool
 /asn.mmdb
-/badproxy.pem
 /ca-bundle.pem
-/cmd/jafar/badproxy.pem
 /country.mmdb
 /example.org
 /jafar

internal/engine/cmd/README.md

@@ -1,3 +0,0 @@
-# Directory github.com/ooni/probe-engine/cmd
-
-This directory contains the source code for the CLI tools we build.

internal/engine/experiment/hhfm/hhfm.go

@@ -16,7 +16,7 @@ import (
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"

internal/engine/experiment/tor/tor.go

@@ -13,7 +13,7 @@ import (
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/engine/legacy/netxlogger"
 	"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonidatamodel"

internal/engine/experiment/urlgetter/runner.go

@@ -10,7 +10,7 @@ import (
 	"net/http/cookiejar"
 	"net/url"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"

internal/engine/experiment/urlgetter/runner_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 func TestRunnerWithInvalidURLScheme(t *testing.T) {

internal/engine/experiment/webconnectivity/control.go

@@ -4,7 +4,7 @@ import (
 	"context"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
 )

internal/engine/experiment/webconnectivity/webconnectivity.go

@@ -12,7 +12,7 @@ import (
 	"time"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
 )

internal/engine/geolocate/avast.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 type avastResponse struct {

internal/engine/geolocate/avast_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 func TestIPLookupWorksUsingAvast(t *testing.T) {

internal/engine/geolocate/ipconfig.go

@@ -5,8 +5,8 @@ import (
 	"net/http"
 	"strings"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 func ipConfigIPLookup(

internal/engine/geolocate/ipconfig_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 func TestIPLookupWorksUsingIPConfig(t *testing.T) {

internal/engine/geolocate/ipinfo.go

@@ -4,8 +4,8 @@ import (
 	"context"
 	"net/http"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 type ipInfoResponse struct {

internal/engine/geolocate/ipinfo_test.go

@@ -7,7 +7,7 @@ import (
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 func TestIPLookupWorksUsingIPInfo(t *testing.T) {

internal/engine/geolocate/stun_test.go

@@ -9,7 +9,7 @@ import (
 	"time"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/pion/stun"
 )
 

internal/engine/geolocate/ubuntu.go

@@ -5,7 +5,7 @@ import (
 	"encoding/xml"
 	"net/http"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 type ubuntuResponse struct {

internal/engine/geolocate/ubuntu_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 func TestUbuntuParseError(t *testing.T) {

internal/engine/httpx/fetch_test.go

@@ -9,7 +9,7 @@ import (
 	"testing"
 
 	"github.com/apex/log"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 func TestFetchResourceIntegration(t *testing.T) {

internal/engine/httpx/jsonapi_test.go

@@ -11,7 +11,7 @@ import (
 
 	"github.com/apex/log"
 	"github.com/google/go-cmp/cmp"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
 )
 

internal/engine/inputloader_integration_test.go

@@ -9,7 +9,7 @@ import (
 	"github.com/apex/log"
 	"github.com/google/go-cmp/cmp"
 	engine "github.com/ooni/probe-cli/v3/internal/engine"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 

internal/engine/internal/mockable/mockable.go

@@ -6,7 +6,7 @@ import (
 	"net/http"
 	"net/url"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/psiphonx"
 	"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/torx"

internal/engine/netx/resolver/dnsoverhttps.go

@@ -8,7 +8,7 @@ import (
 	"net/http"
 	"time"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 )
 
 // DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over

internal/engine/netx/resolver/dnsoverhttps_test.go

@@ -9,7 +9,7 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
 	"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
 )
 

internal/engine/probeservices/checkreportid.go

@@ -4,7 +4,7 @@ import (
 	"context"
 	"net/url"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 type checkReportIDResponse struct {

internal/engine/probeservices/checkreportid_test.go

@@ -8,8 +8,8 @@ import (
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 )
 

internal/engine/probeservices/measurementmeta.go

@@ -5,7 +5,7 @@ import (
 	"net/url"
 	"time"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 )
 
 // MeasurementMetaConfig contains configuration for GetMeasurementMeta.

internal/engine/probeservices/measurementmeta_test.go

@@ -9,8 +9,8 @@ import (
 
 	"github.com/apex/log"
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 )
 

internal/engine/probeservices/probeservices.go

@@ -29,7 +29,7 @@ import (
 	"net/url"
 
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 

internal/engine/probeservices/statefile_test.go

@@ -7,7 +7,7 @@ import (
 	"time"
 
 	"github.com/google/go-cmp/cmp"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
 )
 

internal/engine/resources/resources.go

@@ -13,7 +13,7 @@ import (
 	"os"
 	"path/filepath"
 
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
+	"github.com/ooni/probe-cli/v3/internal/engine/httpx"
 	"github.com/ooni/probe-cli/v3/internal/engine/model"
 )
 

internal/engine/session.go

@@ -13,7 +13,7 @@ import (
 
 	"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
 	"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
-	"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
+	"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/platform"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/sessionresolver"
 	"github.com/ooni/probe-cli/v3/internal/engine/internal/tunnel"

pkg/.gitignore

@@ -0,0 +1 @@
+/testdata