cleanup: mark more integration tests as !short mode (#755)

The objective is to make PR checks run much faster.

See https://github.com/ooni/probe/issues/2113 for context.

Regarding netxlite's tests:

Checking for every commit on master or on a release branch is
good enough and makes pull requests faster than one minute since
netxlite for windows is now 1m slower than coverage.

We're losing some coverage but coverage from integration tests
is not so good anyway, so I'm not super sad about this loss.

.github/workflows/netxlite.yml

@@ -1,10 +1,10 @@
 # netxlite runs unit and integration tests on our fundamental net library
 name: netxlite
 on:
-  pull_request:
   push:
     branches:
       - "master"
+      - "release/**"
 jobs:
   test_netxlite_package:
     runs-on: "${{ matrix.os }}"

cmd/ooniprobe/internal/nettests/nettests_test.go

@@ -45,6 +45,9 @@ func TestCreateContext(t *testing.T) {
 }
 
 func TestRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	probe := newOONIProbe(t)
 	sess, err := probe.NewSession(context.Background(), model.RunTypeManual)
 	if err != nil {

internal/cmd/apitool/main_test.go

@@ -11,16 +11,25 @@ func init() {
 }
 
 func TestCheck(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	*mode = "check"
 	main()
 }
 
 func TestRaw(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	*mode = "raw"
 	main()
 }
 
 func TestMeta(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	*mode = "meta"
 	main()
 }

internal/cmd/jafar/badproxy/badproxy_test.go

@@ -49,7 +49,7 @@ func TestListenError(t *testing.T) {
 	}
 }
 
-func TestStarTLS(t *testing.T) {
+func TestStartTLS(t *testing.T) {
 	expected := errors.New("mocked error")
 
 	t.Run("when we cannot create a new authority", func(t *testing.T) {

internal/cmd/jafar/main_test.go

@@ -16,6 +16,9 @@ func ensureWeStartOverWithIPTables() {
 }
 
 func TestNoCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	if runtime.GOOS != "linux" {
 		t.Skip("skip test on non Linux systems")
 	}
@@ -30,6 +33,9 @@ func TestNoCommand(t *testing.T) {
 }
 
 func TestWithCommand(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	if runtime.GOOS != "linux" {
 		t.Skip("skip test on non Linux systems")
 	}
@@ -54,6 +60,7 @@ func TestMustx(t *testing.T) {
 			t.Fatal("should not happen")
 		}
 	})
+
 	t.Run("with non-exit-code error", func(t *testing.T) {
 		var (
 			called   int
@@ -70,6 +77,7 @@ func TestMustx(t *testing.T) {
 			t.Fatal("unexpected exitcode value")
 		}
 	})
+
 	t.Run("with exit-code error", func(t *testing.T) {
 		var (
 			called   int

internal/cmd/jafar/resolver/resolver_test.go

@@ -27,6 +27,9 @@ func TestRedirect(t *testing.T) {
 }
 
 func TestIgnore(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	server := newresolver(t, nil, nil, []string{"ooni.nu"})
 	iotimeout := "i/o timeout"
 	checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout)

internal/cmd/oohelper/internal/client_test.go

@@ -98,6 +98,9 @@ func TestOOClientDoWithInvalidTargetURL(t *testing.T) {
 }
 
 func TestOOClientDoWithResolverFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	ctx := context.Background()
 	config := internal.OOConfig{
 		TargetURL: "http://www.example.com",

internal/cmd/oohelper/oohelper_test.go

@@ -3,6 +3,9 @@ package main
 import "testing"
 
 func TestSmoke(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	*target = "http://www.example.com"
 	main()
 }

internal/cmd/oonireport/oonireport_test.go

@@ -23,6 +23,9 @@ func TestReadLines(t *testing.T) {
 }
 
 func TestNewSessionAndSubmitter(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	ctx := context.Background()
 	sess := newSession(ctx)
 	if sess == nil {
@@ -57,6 +60,9 @@ func TestMainMissingFile(t *testing.T) {
 }
 
 func TestMainEmptyFile(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	defer func() {
 		var s interface{}
 		if s = recover(); s != nil {
@@ -67,6 +73,9 @@ func TestMainEmptyFile(t *testing.T) {
 }
 
 func TestSubmitAllFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	ctx := context.Background()
 	sess := newSession(ctx)
 	subm := newSubmitter(sess, ctx)

internal/engine/experiment/hirl/hirl_test.go

@@ -26,6 +26,9 @@ func TestNewExperimentMeasurer(t *testing.T) {
 }
 
 func TestSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := hirl.NewExperimentMeasurer(hirl.Config{})
 	ctx := context.Background()
 	sess := &mockable.Session{

internal/engine/experiment/ndt7/download_test.go

@@ -117,6 +117,9 @@ func (r *invalidJSONReader) Read(p []byte) (int, error) {
 }
 
 func TestDownloadOnJSONLoop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	mgr := newDownloadManager(
 		&mockableConnMock{
 			NextReaderMsgType: websocket.TextMessage,

internal/engine/experiment/ndt7/ndt7_test.go

@@ -187,6 +187,9 @@ func TestFailUpload(t *testing.T) {
 }
 
 func TestDownloadJSONUnmarshalFail(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := NewExperimentMeasurer(Config{noUpload: true}).(*Measurer)
 	var seenError bool
 	expected := errors.New("expected error")

internal/engine/experiment/ndt7/upload_test.go

@@ -73,6 +73,9 @@ func TestUploadWritePreparedMessageSubsequentFailure(t *testing.T) {
 }
 
 func TestUploadLoop(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	mgr := newUploadManager(
 		&mockableConnMock{},
 		defaultCallbackPerformance,

internal/engine/experiment/quicping/quicping_test.go

@@ -104,6 +104,9 @@ func TestInvalidHost(t *testing.T) {
 }
 
 func TestURLInput(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := NewExperimentMeasurer(Config{
 		Repetitions: 1,
 	})
@@ -123,6 +126,9 @@ func TestURLInput(t *testing.T) {
 }
 
 func TestSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := NewExperimentMeasurer(Config{})
 	measurement := new(model.Measurement)
 	measurement.Input = model.MeasurementTarget("google.com")
@@ -201,6 +207,9 @@ func TestListenFails(t *testing.T) {
 }
 
 func TestWriteFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	expected := errors.New("expected")
 	measurer := NewExperimentMeasurer(Config{
 		networkLib:  &FailStdLib{err: nil, readErr: nil, writeErr: expected},
@@ -229,6 +238,9 @@ func TestWriteFails(t *testing.T) {
 }
 
 func TestReadFails(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	expected := errors.New("expected")
 	measurer := NewExperimentMeasurer(Config{
 		networkLib:  &FailStdLib{err: nil, readErr: expected, writeErr: nil},
@@ -254,6 +266,9 @@ func TestReadFails(t *testing.T) {
 }
 
 func TestNoResponse(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := NewExperimentMeasurer(Config{
 		Repetitions: 1,
 	})

internal/engine/experiment/whatsapp/whatsapp_test.go

@@ -28,6 +28,9 @@ func TestNewExperimentMeasurer(t *testing.T) {
 }
 
 func TestSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	measurer := whatsapp.NewExperimentMeasurer(whatsapp.Config{})
 	ctx := context.Background()
 	sess := &mockable.Session{MockableLogger: log.Log}
@@ -555,6 +558,9 @@ func TestTestKeysOnlyWebHTTPFailureTooManyURLs(t *testing.T) {
 }
 
 func TestWeConfigureWebChecksCorrectly(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	called := &atomicx.Int64{}
 	emptyConfig := urlgetter.Config{}
 	configWithFailOnHTTPError := urlgetter.Config{FailOnHTTPError: true}

internal/engine/probeservices/bouncer_test.go

@@ -6,6 +6,9 @@ import (
 )
 
 func TestGetTestHelpers(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	testhelpers, err := newclient().GetTestHelpers(context.Background())
 	if err != nil {
 		t.Fatal(err)

internal/engine/probeservices/checkreportid_test.go

@@ -14,6 +14,9 @@ import (
 )
 
 func TestCheckReportIDWorkingAsIntended(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	client := probeservices.Client{
 		APIClientTemplate: httpx.APIClientTemplate{
 			BaseURL:    "https://ams-pg.ooni.org/",

internal/engine/probeservices/tor_test.go

@@ -10,6 +10,9 @@ import (
 )
 
 func TestFetchTorTargets(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	clnt := newclient()
 	if err := clnt.MaybeRegister(context.Background(), testorchestra.MetadataFixture()); err != nil {
 		t.Fatal(err)

internal/netxlite/filtering/dns_test.go

@@ -16,6 +16,9 @@ import (
 )
 
 func TestDNSProxy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	newProxyWithCache := func(action DNSAction, cache map[string][]string) (DNSListener, <-chan interface{}, error) {
 		p := &DNSProxy{
 			Cache: cache,

internal/netxlite/filtering/http_test.go

@@ -16,6 +16,9 @@ import (
 )
 
 func TestHTTPProxy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	newproxy := func(action HTTPAction) (net.Listener, error) {
 		p := &HTTPProxy{
 			OnIncomingHost: func(host string) HTTPAction {

internal/netxlite/filtering/tls_test.go

@@ -14,6 +14,9 @@ import (
 )
 
 func TestTLSProxy(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	newproxy := func(action TLSAction) (net.Listener, <-chan interface{}, error) {
 		p := &TLSProxy{
 			OnIncomingSNI: func(sni string) TLSAction {

internal/netxlite/filtering/tproxy_test.go

@@ -389,6 +389,9 @@ func TestTProxyOnIncomingHost(t *testing.T) {
 }
 
 func TestTProxyDial(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	t.Run("with drop SYN", func(t *testing.T) {
 		config := &TProxyConfig{
 			Endpoints: map[string]TProxyPolicy{

internal/ooapi/example_test.go

@@ -1,37 +0,0 @@
-package ooapi_test
-
-import (
-	"context"
-	"fmt"
-	"log"
-
-	"github.com/ooni/probe-cli/v3/internal/kvstore"
-	"github.com/ooni/probe-cli/v3/internal/model"
-	"github.com/ooni/probe-cli/v3/internal/ooapi"
-	"github.com/ooni/probe-cli/v3/internal/ooapi/apimodel"
-)
-
-func ExampleClient() {
-	clnt := &ooapi.Client{
-		KVStore: &kvstore.Memory{},
-	}
-	ctx := context.Background()
-	resp, err := clnt.CheckIn(ctx, &apimodel.CheckInRequest{
-		Charging:        false,
-		OnWiFi:          false,
-		Platform:        "linux",
-		ProbeASN:        "AS30722",
-		ProbeCC:         "IT",
-		RunType:         model.RunTypeTimed,
-		SoftwareName:    "miniooni",
-		SoftwareVersion: "0.1.0-dev",
-		WebConnectivity: apimodel.CheckInRequestWebConnectivity{
-			CategoryCodes: []string{"NEWS"},
-		},
-	})
-	fmt.Printf("%+v\n", err)
-	// Output: <nil>
-	if resp == nil {
-		log.Fatal("expected non-nil response")
-	}
-}

pkg/oonimkall/session_integration_test.go

@@ -5,7 +5,6 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
-	"io/ioutil"
 	"os"
 	"runtime"
 	"strings"
@@ -33,9 +32,6 @@ func NewSessionForTesting() (*oonimkall.Session, error) {
 }
 
 func TestNewSessionWithInvalidStateDir(t *testing.T) {
-	if testing.Short() {
-		t.Skip("skip test in short mode")
-	}
 	sess, err := oonimkall.NewSession(&oonimkall.SessionConfig{
 		StateDir: "",
 	})
@@ -47,29 +43,6 @@ func TestNewSessionWithInvalidStateDir(t *testing.T) {
 	}
 }
 
-func TestMaybeUpdateResourcesWithCancelledContext(t *testing.T) {
-	// Note that MaybeUpdateResources is now a deprecated stub that
-	// does nothing. We will remove it when we bump major.
-	dir, err := ioutil.TempDir("", "xx")
-	if err != nil {
-		t.Fatal(err)
-	}
-	defer os.RemoveAll(dir)
-	sess, err := NewSessionForTestingWithAssetsDir(dir)
-	if err != nil {
-		t.Fatal(err)
-	}
-	ctx := sess.NewContext()
-	ctx.Cancel() // cause immediate failure
-	err = sess.MaybeUpdateResources(ctx)
-	// Explanation: we embed resources. We should change the API
-	// and remove the context. Until we do that, let us just assert
-	// that we have embedding and the context does not matter.
-	if err != nil {
-		t.Fatal(err)
-	}
-}
-
 func ReduceErrorForGeolocate(err error) error {
 	if err == nil {
 		return errors.New("we expected an error here")
@@ -250,6 +223,9 @@ func TestSubmitCancelContextAfterFirstSubmission(t *testing.T) {
 }
 
 func TestCheckInSuccess(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)
@@ -298,6 +274,9 @@ func TestCheckInSuccess(t *testing.T) {
 }
 
 func TestCheckInLookupLocationFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)
@@ -325,6 +304,9 @@ func TestCheckInLookupLocationFailure(t *testing.T) {
 }
 
 func TestCheckInNewProbeServicesFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)
@@ -354,6 +336,9 @@ func TestCheckInNewProbeServicesFailure(t *testing.T) {
 }
 
 func TestCheckInCheckInFailure(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)
@@ -383,6 +368,9 @@ func TestCheckInCheckInFailure(t *testing.T) {
 }
 
 func TestCheckInNoParams(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)
@@ -441,6 +429,9 @@ func TestFetchURLListSuccess(t *testing.T) {
 }
 
 func TestFetchURLListWithCC(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	sess, err := NewSessionForTesting()
 	if err != nil {
 		t.Fatal(err)

pkg/oonimkall/sessioncontext_test.go

@@ -26,6 +26,9 @@ func TestClampTimeout(t *testing.T) {
 }
 
 func TestNewContextWithZeroTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	here := &atomicx.Int64{}
 	ctx, cancel := newContext(0)
 	defer cancel()
@@ -41,6 +44,9 @@ func TestNewContextWithZeroTimeout(t *testing.T) {
 }
 
 func TestNewContextWithNegativeTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	here := &atomicx.Int64{}
 	ctx, cancel := newContext(-1)
 	defer cancel()
@@ -56,6 +62,9 @@ func TestNewContextWithNegativeTimeout(t *testing.T) {
 }
 
 func TestNewContextWithHugeTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	here := &atomicx.Int64{}
 	ctx, cancel := newContext(maxTimeout + 1)
 	defer cancel()
@@ -71,6 +80,9 @@ func TestNewContextWithHugeTimeout(t *testing.T) {
 }
 
 func TestNewContextWithReasonableTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	here := &atomicx.Int64{}
 	ctx, cancel := newContext(1)
 	defer cancel()
@@ -86,6 +98,9 @@ func TestNewContextWithReasonableTimeout(t *testing.T) {
 }
 
 func TestNewContextWithArtificiallyLowMaxTimeout(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	here := &atomicx.Int64{}
 	const maxTimeout = 2
 	ctx, cancel := newContextEx(maxTimeout+1, maxTimeout)

pkg/oonimkall/task_test.go

@@ -14,6 +14,9 @@ type eventlike struct {
 }
 
 func TestStartTaskGood(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	task, err := StartTask(`{
 		"log_level": "DEBUG",
 		"name": "Example",
@@ -63,6 +66,9 @@ func TestStartTaskInvalidJSON(t *testing.T) {
 }
 
 func TestStartTaskCountBytesForExample(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	task, err := StartTask(`{
 		"name": "Example",
 		"options": {
@@ -142,6 +148,9 @@ func TestPrivacyAndScrubbing(t *testing.T) {
 }
 
 func TestNonblockWithFewEvents(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 	// This test tests whether we won't block for a small
 	// number of events emitted by the task
 	task, err := StartTask(`{

pkg/oonimkall/tasklogger_test.go

@@ -1,15 +1,15 @@
 package oonimkall
 
+//
+// This file contains tests for the taskLogger type.
+//
+
 import (
 	"testing"
 
 	"github.com/ooni/probe-cli/v3/internal/model"
 )
 
-//
-// This file contains tests for the taskLogger type.
-//
-
 func TestTaskLogger(t *testing.T) {
 	// debugMessage is the debug message we expect to see.
 	debugMessage := "debug message"

pkg/oonimkall/taskrunner_test.go

@@ -30,6 +30,9 @@ func TestMeasurementSubmissionFailure(t *testing.T) {
 }
 
 func TestTaskRunnerRun(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skip test in short mode")
+	}
 
 	// newRunnerForTesting is a factory for creating a new
 	// runner that wraps newRunner and also sets a specific