feat: port-filtering experiment (#891)

Part of https://github.com/ooni/probe/issues/2005

.gitignore

@@ -14,8 +14,10 @@
 /miniooni.exe
 /oohelper
 /oohelperd
+/ooporthelper
 /oohelperd.exe
 /oohelper.exe
+/ooporthelper.exe
 /ooniprobe
 /ooniprobe_checksums.txt
 /ooniprobe_checksums.txt.asc

internal/cmd/ooporthelper/README.md

@@ -0,0 +1,4 @@
+# ooporthelper
+
+This directory contains the source code of the Port-
+Filtering test helper written in go

internal/cmd/ooporthelper/main.go

@@ -0,0 +1,79 @@
+// command ooporthelper implements the Port Filtering test helper
+package main
+
+import (
+	"context"
+	"flag"
+	"net"
+	"sync"
+	"time"
+
+	"github.com/apex/log"
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/portfiltering"
+	"github.com/ooni/probe-cli/v3/internal/runtimex"
+)
+
+var (
+	srvCtx      context.Context
+	srvCancel   context.CancelFunc
+	srvWg       = new(sync.WaitGroup)
+	srvTestChan = make(chan string, len(TestPorts)) // buffered channel for testing
+	srvTest     bool
+)
+
+func init() {
+	srvCtx, srvCancel = context.WithCancel(context.Background())
+}
+
+func shutdown(ctx context.Context, l net.Listener) {
+	<-ctx.Done()
+	l.Close()
+}
+
+// TODO(DecFox): Add the ability of an echo service to generate some traffic
+func handleConnetion(ctx context.Context, conn net.Conn) {
+	defer conn.Close()
+	ctx, cancel := context.WithTimeout(ctx, 10*time.Second)
+	defer cancel()
+	<-ctx.Done()
+}
+
+func listenTCP(ctx context.Context, port string) {
+	defer srvWg.Done()
+	address := net.JoinHostPort("127.0.0.1", port)
+	listener, err := net.Listen("tcp", address)
+	runtimex.PanicOnError(err, "net.Listen failed")
+	go shutdown(ctx, listener)
+	srvTestChan <- port // send to channel to imply server will start listening on port
+	for {
+		conn, err := listener.Accept()
+		if err != nil {
+			log.Infof("listener unable to accept connections on port%s", port)
+			return
+		}
+		go handleConnetion(ctx, conn)
+	}
+}
+
+func main() {
+	logmap := map[bool]log.Level{
+		true:  log.DebugLevel,
+		false: log.InfoLevel,
+	}
+	debug := flag.Bool("debug", false, "Toggle debug mode")
+	flag.Parse()
+	log.SetLevel(logmap[*debug])
+	defer srvCancel()
+	ports := portfiltering.Ports
+	if srvTest {
+		ports = TestPorts
+	}
+	for _, port := range ports {
+		srvWg.Add(1)
+		ctx, cancel := context.WithCancel(srvCtx)
+		defer cancel()
+		go listenTCP(ctx, port)
+	}
+	<-srvCtx.Done()
+	srvWg.Wait() // wait for listeners on all ports to close
+}

internal/cmd/ooporthelper/main_test.go

@@ -0,0 +1,31 @@
+package main
+
+import (
+	"context"
+	"net"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/netxlite"
+)
+
+func TestMainWorkingAsIntended(t *testing.T) {
+	srvTest = true // toggle to imply that we are running in test mode
+	go main()
+	dialer := netxlite.NewDialerWithoutResolver(model.DiscardLogger)
+	for _, port := range TestPorts {
+		<-srvTestChan
+		addr := net.JoinHostPort("127.0.0.1", port)
+		ctx := context.Background()
+		conn, err := dialer.DialContext(ctx, "tcp", addr)
+		if err != nil {
+			t.Fatal(err)
+		}
+		if conn == nil {
+			t.Fatal("expected non-nil conn")
+		}
+		conn.Close()
+	}
+	srvCancel()  // shutdown server
+	srvWg.Wait() // wait for listeners on all ports to close
+}

internal/cmd/ooporthelper/ports.go

@@ -0,0 +1,12 @@
+package main
+
+//
+// List of ports we want to use for running integration tests
+//
+
+// Ports for testing the testhelper
+// Note: we must only use unprivileged ports here to ensure tests run successfully
+var TestPorts = []string{
+	"8080", // tcp
+	"5050", // tcp
+}

internal/engine/experiment/portfiltering/config.go

@@ -0,0 +1,20 @@
+package portfiltering
+
+//
+// Config for the port-filtering experiment
+//
+
+import "time"
+
+// Config contains the experiment configuration.
+type Config struct {
+	// Delay is the delay between each repetition (in milliseconds).
+	Delay int64 `ooni:"number of milliseconds to wait before testing each port"`
+}
+
+func (c *Config) delay() time.Duration {
+	if c.Delay > 0 {
+		return time.Duration(c.Delay) * time.Millisecond
+	}
+	return 100 * time.Millisecond
+}

internal/engine/experiment/portfiltering/config_test.go

@@ -0,0 +1,13 @@
+package portfiltering
+
+import (
+	"testing"
+	"time"
+)
+
+func TestConfig_delay(t *testing.T) {
+	c := Config{}
+	if c.delay() != 100*time.Millisecond {
+		t.Fatal("invalid default delay")
+	}
+}

internal/engine/experiment/portfiltering/doc.go

@@ -0,0 +1,4 @@
+// Package portfiltering implements the portfiltering experiment
+//
+// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-038-port-filtering.md.
+package portfiltering

internal/engine/experiment/portfiltering/measurer.go

@@ -0,0 +1,67 @@
+package portfiltering
+
+//
+// Measurer for the port-filtering experiment
+//
+
+import (
+	"context"
+	"errors"
+	"net/url"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+const (
+	testName    = "portfiltering"
+	testVersion = "0.1.0"
+)
+
+// Measurer performs the measurement.
+type Measurer struct {
+	config Config
+}
+
+// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
+func (m *Measurer) ExperimentName() string {
+	return testName
+}
+
+// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
+func (m *Measurer) ExperimentVersion() string {
+	return testVersion
+}
+
+var (
+	// errInvalidTestHelper indicates that the given test helper is not an URL
+	errInvalidTestHelper = errors.New("testhelper is not an URL")
+)
+
+// Run implements ExperimentMeasurer.Run.
+func (m *Measurer) Run(
+	ctx context.Context,
+	sess model.ExperimentSession,
+	measurement *model.Measurement,
+	callbacks model.ExperimentCallbacks,
+) error {
+	// TODO(DecFox): Replace the localhost deployment with an OONI testhelper
+	// Ensure that we only do this once we have a deployed testhelper
+	testhelper := "http://127.0.0.1"
+	parsed, err := url.Parse(testhelper)
+	if err != nil {
+		return errInvalidTestHelper
+	}
+	tk := new(TestKeys)
+	measurement.TestKeys = tk
+	out := make(chan *model.ArchivalTCPConnectResult)
+	go m.tcpConnectLoop(ctx, measurement.MeasurementStartTimeSaved, sess.Logger(), parsed.Host, out)
+	for len(tk.TCPConnect) < len(Ports) {
+		tk.TCPConnect = append(tk.TCPConnect, <-out)
+	}
+	return nil // return nil so we always submit the measurement
+}
+
+// NewExperimentMeasurer creates a new ExperimentMeasurer.
+func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
+	return &Measurer{config: config}
+}

internal/engine/experiment/portfiltering/measurer_test.go

@@ -0,0 +1,48 @@
+package portfiltering
+
+import (
+	"context"
+	"testing"
+
+	"github.com/ooni/probe-cli/v3/internal/model"
+	"github.com/ooni/probe-cli/v3/internal/model/mocks"
+)
+
+func TestMeasurerExperimentNameVersion(t *testing.T) {
+	measurer := NewExperimentMeasurer(Config{})
+	if measurer.ExperimentName() != "portfiltering" {
+		t.Fatal("unexpected ExperimentName")
+	}
+	if measurer.ExperimentVersion() != "0.1.0" {
+		t.Fatal("unexpected ExperimentVersion")
+	}
+}
+
+// TODO(DecFox): Skip this test with -short in a future iteration.
+func TestMeasurer_run(t *testing.T) {
+	m := NewExperimentMeasurer(Config{})
+	meas := &model.Measurement{}
+	sess := &mocks.Session{
+		MockLogger: func() model.Logger {
+			return model.DiscardLogger
+		},
+	}
+	callbacks := model.NewPrinterCallbacks(model.DiscardLogger)
+	ctx := context.Background()
+	err := m.Run(ctx, sess, meas, callbacks)
+	if err != nil {
+		t.Fatal(err)
+	}
+	tk := meas.TestKeys.(*TestKeys)
+	if len(tk.TCPConnect) != len(Ports) {
+		t.Fatal("unexpected number of ports")
+	}
+	ask, err := m.GetSummaryKeys(meas)
+	if err != nil {
+		t.Fatal("cannot obtain summary")
+	}
+	summary := ask.(SummaryKeys)
+	if summary.IsAnomaly {
+		t.Fatal("expected no anomaly")
+	}
+}

internal/engine/experiment/portfiltering/ports.go

@@ -0,0 +1,73 @@
+package portfiltering
+
+//
+// List of ports we want to measure
+//
+
+// List generated from nmap-services: https://github.com/nmap/nmap/blob/master/nmap-services
+// Note: Using privileged ports like :80 requires elevated permissions
+var Ports = []string{
+	"80",    // tcp - World Wide Web HTTP
+	"631",   // udp - Internet Printing Protocol
+	"161",   // udp - Simple Net Mgmt Proto
+	"137",   // udp - NETBIOS Name Service
+	"123",   // udp - Network Time Protocol
+	"138",   // udp - NETBIOS Datagram Service
+	"1434",  // udp - Microsoft-SQL-Monitor
+	"135",   // udp, tcp - epmap | Microsoft RPC services | DCE endpoint resolution
+	"67",    // udp - DHCP/Bootstrap Protocol Server
+	"23",    // tcp
+	"53",    // udp, tcp - Domain Name Server
+	"443",   // tcp - secure http (SSL)
+	"21",    // tcp - File Transfer [Control]
+	"22",    // tcp - Secure Shell Login
+	"500",   // udp
+	"68",    // udp - DHCP/Bootstrap Protocol Client
+	"520",   // udp - router routed -- RIP
+	"1900",  // udp - Universal PnP
+	"25",    // tcp - Simple Mail Transfer
+	"4500",  // udp - IKE Nat Traversal negotiation (RFC3947)
+	"514",   // udp - BSD syslogd(8)
+	"49152", // udp
+	"162",   // udp - snmp-trap
+	"69",    // udp - Trivial File Transfer
+	"5353",  // udp - Mac OS X Bonjour/Zeroconf port
+	"49154", // udp
+	"3389",  // tcp - Microsoft Remote Display Protocol (aka ms-term-serv, microsoft-rdp) | MS WBT Server
+	"110",   // tcp - PostOffice V.3 | Post Office Protocol - Version 3
+	"1701",  // udp
+	"998",   // udp
+	"996",   // udp
+	"997",   // udp
+	"999",   // udp - Applix ac
+	"3283",  // udp - Apple Remote Desktop Net Assistant reporting feature
+	"49153", // udp
+	"445",   // tcp - SMB directly over IP
+	"1812",  // udp - RADIUS authentication protocol (RFC 2138)
+	"136",   // udp - PROFILE Naming System
+	"139",   // tcp, udp - NETBIOS Session Service
+	"143",   // tcp - Interim Mail Access Protocol v2 | Internet Message Access Protocol
+	"2222",  // udp - Microsoft Office OS X antipiracy network monitor
+	"3306",  // tcp
+	"2049",  // udp - networked file system
+	"32768", // udp - OpenMosix Autodiscovery Daemon
+	"5060",  // udp - Session Initiation Protocol (SIP)
+	"8080",  // tcp - http-alt | Common HTTP proxy/second web server port | HTTP Alternate (see port 80)
+	"1433",  // udp - Microsoft-SQL-Server
+	"3456",  // udp - also VAT default data
+	"1723",  // tcp - Point-to-point tunnelling protocol
+	"111",   // tcp, udp - sunrpc | portmapper, rpcbind | SUN Remote Procedure Call
+	"995",   // tcp - POP3 protocol over TLS/SSL | pop3 protocol over TLS/SSL (was spop3) | POP3 over TLS protocol
+	"993",   // tcp - imap4 protocol over TLS/SSL | IMAP over TLS protocol
+	"20031", // udp - BakBone NetVault primary communications port
+	"1026",  // udp - Commonly used to send MS Messenger spam
+	"7",     // udp
+	"5900",  // tcp - rfb | Virtual Network Computer display 0 | Remote Framebuffer
+	"1646",  // udp - radius accounting
+	"1645",  // udp - radius authentication
+	"593",   // udp # HTTP RPC Ep Map
+	"1025",  // tcp, udp - blackjack | IIS, NFS, or listener RFS remote_file_sharing | network blackjack
+	"518",   // udp - (talkd)
+	"2048",  // udp
+	"626",   // udp - Mac OS X Server serial number (licensing) daemon
+}

internal/engine/experiment/portfiltering/summary.go

@@ -0,0 +1,20 @@
+package portfiltering
+
+//
+// Summary
+//
+
+import "github.com/ooni/probe-cli/v3/internal/model"
+
+// SummaryKeys contains summary keys for this experiment.
+//
+// Note that this structure is part of the ABI contract with ooniprobe
+// therefore we should be careful when changing it.
+type SummaryKeys struct {
+	IsAnomaly bool `json:"-"`
+}
+
+// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
+func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
+	return SummaryKeys{IsAnomaly: false}, nil
+}

internal/engine/experiment/portfiltering/tcpconnect.go

@@ -0,0 +1,48 @@
+package portfiltering
+
+//
+// TCPConnect for portfiltering
+//
+
+import (
+	"context"
+	"math/rand"
+	"net"
+	"time"
+
+	"github.com/ooni/probe-cli/v3/internal/measurexlite"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+// tcpPingLoop sends the TCP Connect requests to all ports and emits the results onto the out channel
+func (m *Measurer) tcpConnectLoop(ctx context.Context, zeroTime time.Time,
+	logger model.Logger, address string, out chan<- *model.ArchivalTCPConnectResult) {
+	ticker := time.NewTicker(m.config.delay())
+	defer ticker.Stop()
+	rand.Shuffle(len(Ports), func(i, j int) {
+		Ports[i], Ports[j] = Ports[j], Ports[i]
+	})
+	for i, port := range Ports {
+		addr := net.JoinHostPort(address, port)
+		go m.tcpConnectAsync(ctx, int64(i), zeroTime, logger, addr, out)
+		<-ticker.C
+	}
+}
+
+// tcpPingAsync performs a TCP Connect and emits the result onto the out channel.
+func (m *Measurer) tcpConnectAsync(ctx context.Context, index int64,
+	zeroTime time.Time, logger model.Logger, address string, out chan<- *model.ArchivalTCPConnectResult) {
+	out <- m.tcpConnect(ctx, index, zeroTime, logger, address)
+}
+
+// tcpConnect performs a TCP connect and returns the result to the caller.
+func (m *Measurer) tcpConnect(ctx context.Context, index int64,
+	zeroTime time.Time, logger model.Logger, address string) *model.ArchivalTCPConnectResult {
+	trace := measurexlite.NewTrace(index, zeroTime)
+	ol := measurexlite.NewOperationLogger(logger, "TCPConnect #%d %s", index, address)
+	dialer := trace.NewDialerWithoutResolver(logger)
+	conn, err := dialer.DialContext(ctx, "tcp", address)
+	ol.Stop(err)
+	measurexlite.MaybeClose(conn)
+	return trace.FirstTCPConnectOrNil()
+}

internal/engine/experiment/portfiltering/testkeys.go

@@ -0,0 +1,8 @@
+package portfiltering
+
+import "github.com/ooni/probe-cli/v3/internal/model"
+
+// TestKeys contains the experiment results.
+type TestKeys struct {
+	TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"`
+}

internal/registry/portfiltering.go

@@ -0,0 +1,23 @@
+package registry
+
+//
+// Registers the 'portfiltering' experiment
+//
+
+import (
+	"github.com/ooni/probe-cli/v3/internal/engine/experiment/portfiltering"
+	"github.com/ooni/probe-cli/v3/internal/model"
+)
+
+func init() {
+	AllExperiments["portfiltering"] = &Factory{
+		build: func(config any) model.ExperimentMeasurer {
+			return portfiltering.NewExperimentMeasurer(
+				config.(portfiltering.Config),
+			)
+		},
+		config:        portfiltering.Config{},
+		interruptible: false,
+		inputPolicy:   model.InputNone,
+	}
+}