From d6a362d96f1afb869d8c016245ad14a051e9d3e7 Mon Sep 17 00:00:00 2001 From: DecFox <33030671+DecFox@users.noreply.github.com> Date: Wed, 14 Sep 2022 23:24:43 +0530 Subject: [PATCH] feat: port-filtering experiment (#891) Part of https://github.com/ooni/probe/issues/2005 --- .gitignore | 2 + internal/cmd/ooporthelper/README.md | 4 + internal/cmd/ooporthelper/main.go | 79 +++++++++++++++++++ internal/cmd/ooporthelper/main_test.go | 31 ++++++++ internal/cmd/ooporthelper/ports.go | 12 +++ .../engine/experiment/portfiltering/config.go | 20 +++++ .../experiment/portfiltering/config_test.go | 13 +++ .../engine/experiment/portfiltering/doc.go | 4 + .../experiment/portfiltering/measurer.go | 67 ++++++++++++++++ .../experiment/portfiltering/measurer_test.go | 48 +++++++++++ .../engine/experiment/portfiltering/ports.go | 73 +++++++++++++++++ .../experiment/portfiltering/summary.go | 20 +++++ .../experiment/portfiltering/tcpconnect.go | 48 +++++++++++ .../experiment/portfiltering/testkeys.go | 8 ++ internal/registry/portfiltering.go | 23 ++++++ 15 files changed, 452 insertions(+) create mode 100644 internal/cmd/ooporthelper/README.md create mode 100644 internal/cmd/ooporthelper/main.go create mode 100644 internal/cmd/ooporthelper/main_test.go create mode 100644 internal/cmd/ooporthelper/ports.go create mode 100644 internal/engine/experiment/portfiltering/config.go create mode 100644 internal/engine/experiment/portfiltering/config_test.go create mode 100644 internal/engine/experiment/portfiltering/doc.go create mode 100644 internal/engine/experiment/portfiltering/measurer.go create mode 100644 internal/engine/experiment/portfiltering/measurer_test.go create mode 100644 internal/engine/experiment/portfiltering/ports.go create mode 100644 internal/engine/experiment/portfiltering/summary.go create mode 100644 internal/engine/experiment/portfiltering/tcpconnect.go create mode 100644 internal/engine/experiment/portfiltering/testkeys.go create mode 100644 internal/registry/portfiltering.go diff --git a/.gitignore b/.gitignore index ab7fce3..ba35acb 100644 --- a/.gitignore +++ b/.gitignore @@ -14,8 +14,10 @@ /miniooni.exe /oohelper /oohelperd +/ooporthelper /oohelperd.exe /oohelper.exe +/ooporthelper.exe /ooniprobe /ooniprobe_checksums.txt /ooniprobe_checksums.txt.asc diff --git a/internal/cmd/ooporthelper/README.md b/internal/cmd/ooporthelper/README.md new file mode 100644 index 0000000..abdf39a --- /dev/null +++ b/internal/cmd/ooporthelper/README.md @@ -0,0 +1,4 @@ +# ooporthelper + +This directory contains the source code of the Port- +Filtering test helper written in go \ No newline at end of file diff --git a/internal/cmd/ooporthelper/main.go b/internal/cmd/ooporthelper/main.go new file mode 100644 index 0000000..ddb21f5 --- /dev/null +++ b/internal/cmd/ooporthelper/main.go @@ -0,0 +1,79 @@ +// command ooporthelper implements the Port Filtering test helper +package main + +import ( + "context" + "flag" + "net" + "sync" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/portfiltering" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +var ( + srvCtx context.Context + srvCancel context.CancelFunc + srvWg = new(sync.WaitGroup) + srvTestChan = make(chan string, len(TestPorts)) // buffered channel for testing + srvTest bool +) + +func init() { + srvCtx, srvCancel = context.WithCancel(context.Background()) +} + +func shutdown(ctx context.Context, l net.Listener) { + <-ctx.Done() + l.Close() +} + +// TODO(DecFox): Add the ability of an echo service to generate some traffic +func handleConnetion(ctx context.Context, conn net.Conn) { + defer conn.Close() + ctx, cancel := context.WithTimeout(ctx, 10*time.Second) + defer cancel() + <-ctx.Done() +} + +func listenTCP(ctx context.Context, port string) { + defer srvWg.Done() + address := net.JoinHostPort("127.0.0.1", port) + listener, err := net.Listen("tcp", address) + runtimex.PanicOnError(err, "net.Listen failed") + go shutdown(ctx, listener) + srvTestChan <- port // send to channel to imply server will start listening on port + for { + conn, err := listener.Accept() + if err != nil { + log.Infof("listener unable to accept connections on port%s", port) + return + } + go handleConnetion(ctx, conn) + } +} + +func main() { + logmap := map[bool]log.Level{ + true: log.DebugLevel, + false: log.InfoLevel, + } + debug := flag.Bool("debug", false, "Toggle debug mode") + flag.Parse() + log.SetLevel(logmap[*debug]) + defer srvCancel() + ports := portfiltering.Ports + if srvTest { + ports = TestPorts + } + for _, port := range ports { + srvWg.Add(1) + ctx, cancel := context.WithCancel(srvCtx) + defer cancel() + go listenTCP(ctx, port) + } + <-srvCtx.Done() + srvWg.Wait() // wait for listeners on all ports to close +} diff --git a/internal/cmd/ooporthelper/main_test.go b/internal/cmd/ooporthelper/main_test.go new file mode 100644 index 0000000..fe93c66 --- /dev/null +++ b/internal/cmd/ooporthelper/main_test.go @@ -0,0 +1,31 @@ +package main + +import ( + "context" + "net" + "testing" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +func TestMainWorkingAsIntended(t *testing.T) { + srvTest = true // toggle to imply that we are running in test mode + go main() + dialer := netxlite.NewDialerWithoutResolver(model.DiscardLogger) + for _, port := range TestPorts { + <-srvTestChan + addr := net.JoinHostPort("127.0.0.1", port) + ctx := context.Background() + conn, err := dialer.DialContext(ctx, "tcp", addr) + if err != nil { + t.Fatal(err) + } + if conn == nil { + t.Fatal("expected non-nil conn") + } + conn.Close() + } + srvCancel() // shutdown server + srvWg.Wait() // wait for listeners on all ports to close +} diff --git a/internal/cmd/ooporthelper/ports.go b/internal/cmd/ooporthelper/ports.go new file mode 100644 index 0000000..eb1e2f9 --- /dev/null +++ b/internal/cmd/ooporthelper/ports.go @@ -0,0 +1,12 @@ +package main + +// +// List of ports we want to use for running integration tests +// + +// Ports for testing the testhelper +// Note: we must only use unprivileged ports here to ensure tests run successfully +var TestPorts = []string{ + "8080", // tcp + "5050", // tcp +} diff --git a/internal/engine/experiment/portfiltering/config.go b/internal/engine/experiment/portfiltering/config.go new file mode 100644 index 0000000..d1f2e3f --- /dev/null +++ b/internal/engine/experiment/portfiltering/config.go @@ -0,0 +1,20 @@ +package portfiltering + +// +// Config for the port-filtering experiment +// + +import "time" + +// Config contains the experiment configuration. +type Config struct { + // Delay is the delay between each repetition (in milliseconds). + Delay int64 `ooni:"number of milliseconds to wait before testing each port"` +} + +func (c *Config) delay() time.Duration { + if c.Delay > 0 { + return time.Duration(c.Delay) * time.Millisecond + } + return 100 * time.Millisecond +} diff --git a/internal/engine/experiment/portfiltering/config_test.go b/internal/engine/experiment/portfiltering/config_test.go new file mode 100644 index 0000000..9ea4b03 --- /dev/null +++ b/internal/engine/experiment/portfiltering/config_test.go @@ -0,0 +1,13 @@ +package portfiltering + +import ( + "testing" + "time" +) + +func TestConfig_delay(t *testing.T) { + c := Config{} + if c.delay() != 100*time.Millisecond { + t.Fatal("invalid default delay") + } +} diff --git a/internal/engine/experiment/portfiltering/doc.go b/internal/engine/experiment/portfiltering/doc.go new file mode 100644 index 0000000..117ae48 --- /dev/null +++ b/internal/engine/experiment/portfiltering/doc.go @@ -0,0 +1,4 @@ +// Package portfiltering implements the portfiltering experiment +// +// Spec: https://github.com/ooni/spec/blob/master/nettests/ts-038-port-filtering.md. +package portfiltering diff --git a/internal/engine/experiment/portfiltering/measurer.go b/internal/engine/experiment/portfiltering/measurer.go new file mode 100644 index 0000000..76c9a2d --- /dev/null +++ b/internal/engine/experiment/portfiltering/measurer.go @@ -0,0 +1,67 @@ +package portfiltering + +// +// Measurer for the port-filtering experiment +// + +import ( + "context" + "errors" + "net/url" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +const ( + testName = "portfiltering" + testVersion = "0.1.0" +) + +// Measurer performs the measurement. +type Measurer struct { + config Config +} + +// ExperimentName implements ExperimentMeasurer.ExperiExperimentName. +func (m *Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion. +func (m *Measurer) ExperimentVersion() string { + return testVersion +} + +var ( + // errInvalidTestHelper indicates that the given test helper is not an URL + errInvalidTestHelper = errors.New("testhelper is not an URL") +) + +// Run implements ExperimentMeasurer.Run. +func (m *Measurer) Run( + ctx context.Context, + sess model.ExperimentSession, + measurement *model.Measurement, + callbacks model.ExperimentCallbacks, +) error { + // TODO(DecFox): Replace the localhost deployment with an OONI testhelper + // Ensure that we only do this once we have a deployed testhelper + testhelper := "http://127.0.0.1" + parsed, err := url.Parse(testhelper) + if err != nil { + return errInvalidTestHelper + } + tk := new(TestKeys) + measurement.TestKeys = tk + out := make(chan *model.ArchivalTCPConnectResult) + go m.tcpConnectLoop(ctx, measurement.MeasurementStartTimeSaved, sess.Logger(), parsed.Host, out) + for len(tk.TCPConnect) < len(Ports) { + tk.TCPConnect = append(tk.TCPConnect, <-out) + } + return nil // return nil so we always submit the measurement +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return &Measurer{config: config} +} diff --git a/internal/engine/experiment/portfiltering/measurer_test.go b/internal/engine/experiment/portfiltering/measurer_test.go new file mode 100644 index 0000000..e3a3876 --- /dev/null +++ b/internal/engine/experiment/portfiltering/measurer_test.go @@ -0,0 +1,48 @@ +package portfiltering + +import ( + "context" + "testing" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" +) + +func TestMeasurerExperimentNameVersion(t *testing.T) { + measurer := NewExperimentMeasurer(Config{}) + if measurer.ExperimentName() != "portfiltering" { + t.Fatal("unexpected ExperimentName") + } + if measurer.ExperimentVersion() != "0.1.0" { + t.Fatal("unexpected ExperimentVersion") + } +} + +// TODO(DecFox): Skip this test with -short in a future iteration. +func TestMeasurer_run(t *testing.T) { + m := NewExperimentMeasurer(Config{}) + meas := &model.Measurement{} + sess := &mocks.Session{ + MockLogger: func() model.Logger { + return model.DiscardLogger + }, + } + callbacks := model.NewPrinterCallbacks(model.DiscardLogger) + ctx := context.Background() + err := m.Run(ctx, sess, meas, callbacks) + if err != nil { + t.Fatal(err) + } + tk := meas.TestKeys.(*TestKeys) + if len(tk.TCPConnect) != len(Ports) { + t.Fatal("unexpected number of ports") + } + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } +} diff --git a/internal/engine/experiment/portfiltering/ports.go b/internal/engine/experiment/portfiltering/ports.go new file mode 100644 index 0000000..7f0234b --- /dev/null +++ b/internal/engine/experiment/portfiltering/ports.go @@ -0,0 +1,73 @@ +package portfiltering + +// +// List of ports we want to measure +// + +// List generated from nmap-services: https://github.com/nmap/nmap/blob/master/nmap-services +// Note: Using privileged ports like :80 requires elevated permissions +var Ports = []string{ + "80", // tcp - World Wide Web HTTP + "631", // udp - Internet Printing Protocol + "161", // udp - Simple Net Mgmt Proto + "137", // udp - NETBIOS Name Service + "123", // udp - Network Time Protocol + "138", // udp - NETBIOS Datagram Service + "1434", // udp - Microsoft-SQL-Monitor + "135", // udp, tcp - epmap | Microsoft RPC services | DCE endpoint resolution + "67", // udp - DHCP/Bootstrap Protocol Server + "23", // tcp + "53", // udp, tcp - Domain Name Server + "443", // tcp - secure http (SSL) + "21", // tcp - File Transfer [Control] + "22", // tcp - Secure Shell Login + "500", // udp + "68", // udp - DHCP/Bootstrap Protocol Client + "520", // udp - router routed -- RIP + "1900", // udp - Universal PnP + "25", // tcp - Simple Mail Transfer + "4500", // udp - IKE Nat Traversal negotiation (RFC3947) + "514", // udp - BSD syslogd(8) + "49152", // udp + "162", // udp - snmp-trap + "69", // udp - Trivial File Transfer + "5353", // udp - Mac OS X Bonjour/Zeroconf port + "49154", // udp + "3389", // tcp - Microsoft Remote Display Protocol (aka ms-term-serv, microsoft-rdp) | MS WBT Server + "110", // tcp - PostOffice V.3 | Post Office Protocol - Version 3 + "1701", // udp + "998", // udp + "996", // udp + "997", // udp + "999", // udp - Applix ac + "3283", // udp - Apple Remote Desktop Net Assistant reporting feature + "49153", // udp + "445", // tcp - SMB directly over IP + "1812", // udp - RADIUS authentication protocol (RFC 2138) + "136", // udp - PROFILE Naming System + "139", // tcp, udp - NETBIOS Session Service + "143", // tcp - Interim Mail Access Protocol v2 | Internet Message Access Protocol + "2222", // udp - Microsoft Office OS X antipiracy network monitor + "3306", // tcp + "2049", // udp - networked file system + "32768", // udp - OpenMosix Autodiscovery Daemon + "5060", // udp - Session Initiation Protocol (SIP) + "8080", // tcp - http-alt | Common HTTP proxy/second web server port | HTTP Alternate (see port 80) + "1433", // udp - Microsoft-SQL-Server + "3456", // udp - also VAT default data + "1723", // tcp - Point-to-point tunnelling protocol + "111", // tcp, udp - sunrpc | portmapper, rpcbind | SUN Remote Procedure Call + "995", // tcp - POP3 protocol over TLS/SSL | pop3 protocol over TLS/SSL (was spop3) | POP3 over TLS protocol + "993", // tcp - imap4 protocol over TLS/SSL | IMAP over TLS protocol + "20031", // udp - BakBone NetVault primary communications port + "1026", // udp - Commonly used to send MS Messenger spam + "7", // udp + "5900", // tcp - rfb | Virtual Network Computer display 0 | Remote Framebuffer + "1646", // udp - radius accounting + "1645", // udp - radius authentication + "593", // udp # HTTP RPC Ep Map + "1025", // tcp, udp - blackjack | IIS, NFS, or listener RFS remote_file_sharing | network blackjack + "518", // udp - (talkd) + "2048", // udp + "626", // udp - Mac OS X Server serial number (licensing) daemon +} diff --git a/internal/engine/experiment/portfiltering/summary.go b/internal/engine/experiment/portfiltering/summary.go new file mode 100644 index 0000000..f4bb007 --- /dev/null +++ b/internal/engine/experiment/portfiltering/summary.go @@ -0,0 +1,20 @@ +package portfiltering + +// +// Summary +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + return SummaryKeys{IsAnomaly: false}, nil +} diff --git a/internal/engine/experiment/portfiltering/tcpconnect.go b/internal/engine/experiment/portfiltering/tcpconnect.go new file mode 100644 index 0000000..fab2d93 --- /dev/null +++ b/internal/engine/experiment/portfiltering/tcpconnect.go @@ -0,0 +1,48 @@ +package portfiltering + +// +// TCPConnect for portfiltering +// + +import ( + "context" + "math/rand" + "net" + "time" + + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" +) + +// tcpPingLoop sends the TCP Connect requests to all ports and emits the results onto the out channel +func (m *Measurer) tcpConnectLoop(ctx context.Context, zeroTime time.Time, + logger model.Logger, address string, out chan<- *model.ArchivalTCPConnectResult) { + ticker := time.NewTicker(m.config.delay()) + defer ticker.Stop() + rand.Shuffle(len(Ports), func(i, j int) { + Ports[i], Ports[j] = Ports[j], Ports[i] + }) + for i, port := range Ports { + addr := net.JoinHostPort(address, port) + go m.tcpConnectAsync(ctx, int64(i), zeroTime, logger, addr, out) + <-ticker.C + } +} + +// tcpPingAsync performs a TCP Connect and emits the result onto the out channel. +func (m *Measurer) tcpConnectAsync(ctx context.Context, index int64, + zeroTime time.Time, logger model.Logger, address string, out chan<- *model.ArchivalTCPConnectResult) { + out <- m.tcpConnect(ctx, index, zeroTime, logger, address) +} + +// tcpConnect performs a TCP connect and returns the result to the caller. +func (m *Measurer) tcpConnect(ctx context.Context, index int64, + zeroTime time.Time, logger model.Logger, address string) *model.ArchivalTCPConnectResult { + trace := measurexlite.NewTrace(index, zeroTime) + ol := measurexlite.NewOperationLogger(logger, "TCPConnect #%d %s", index, address) + dialer := trace.NewDialerWithoutResolver(logger) + conn, err := dialer.DialContext(ctx, "tcp", address) + ol.Stop(err) + measurexlite.MaybeClose(conn) + return trace.FirstTCPConnectOrNil() +} diff --git a/internal/engine/experiment/portfiltering/testkeys.go b/internal/engine/experiment/portfiltering/testkeys.go new file mode 100644 index 0000000..68dd640 --- /dev/null +++ b/internal/engine/experiment/portfiltering/testkeys.go @@ -0,0 +1,8 @@ +package portfiltering + +import "github.com/ooni/probe-cli/v3/internal/model" + +// TestKeys contains the experiment results. +type TestKeys struct { + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` +} diff --git a/internal/registry/portfiltering.go b/internal/registry/portfiltering.go new file mode 100644 index 0000000..093ccba --- /dev/null +++ b/internal/registry/portfiltering.go @@ -0,0 +1,23 @@ +package registry + +// +// Registers the 'portfiltering' experiment +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/portfiltering" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + AllExperiments["portfiltering"] = &Factory{ + build: func(config any) model.ExperimentMeasurer { + return portfiltering.NewExperimentMeasurer( + config.(portfiltering.Config), + ) + }, + config: portfiltering.Config{}, + interruptible: false, + inputPolicy: model.InputNone, + } +}