From eca06e83d085854ae8d9d92c51d529053fb0b2d6 Mon Sep 17 00:00:00 2001 From: ooninoob Date: Sat, 19 Nov 2022 17:18:46 +0100 Subject: [PATCH] Basic SMTP/IMAP probe: look for bad TLS and StartTLS downgrade attacks --- internal/engine/experiment/imap/imap.go | 325 ++++++++++++++++++ internal/engine/experiment/imap/imap_test.go | 192 +++++++++++ internal/engine/experiment/smtp/smtp.go | 333 +++++++++++++++++++ internal/engine/experiment/smtp/smtp_test.go | 186 +++++++++++ internal/registry/imap.go | 22 ++ internal/registry/smtp.go | 22 ++ internal/tcprunner/tcprunner.go | 192 +++++++++++ 7 files changed, 1272 insertions(+) create mode 100644 internal/engine/experiment/imap/imap.go create mode 100644 internal/engine/experiment/imap/imap_test.go create mode 100644 internal/engine/experiment/smtp/smtp.go create mode 100644 internal/engine/experiment/smtp/smtp_test.go create mode 100644 internal/registry/imap.go create mode 100644 internal/registry/smtp.go create mode 100644 internal/tcprunner/tcprunner.go diff --git a/internal/engine/experiment/imap/imap.go b/internal/engine/experiment/imap/imap.go new file mode 100644 index 0000000..bc4c03e --- /dev/null +++ b/internal/engine/experiment/imap/imap.go @@ -0,0 +1,325 @@ +package imap + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "github.com/pkg/errors" + "net/url" + "strings" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tcprunner" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +var ( + // errNoInputProvided indicates you didn't provide any input + errNoInputProvided = errors.New("not input provided") + + // errInputIsNotAnURL indicates that input is not an URL + errInputIsNotAnURL = errors.New("input is not an URL") + + // errInvalidScheme indicates that the scheme is invalid + errInvalidScheme = errors.New("scheme must be imap(s)") +) + +const ( + testName = "imap" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +type runtimeConfig struct { + host string + port string + forcedTLS bool + noopCount uint8 +} + +func config(input model.MeasurementTarget) (*runtimeConfig, error) { + if input == "" { + // TODO: static input data (eg. gmail/riseup..) + return nil, errNoInputProvided + } + + parsed, err := url.Parse(string(input)) + if err != nil { + return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) + } + if parsed.Scheme != "imap" && parsed.Scheme != "imaps" { + return nil, errInvalidScheme + } + + port := "" + + if parsed.Port() == "" { + // Default ports for StartTLS and forced TLS respectively + if parsed.Scheme == "imap" { + port = "143" + } else { + port = "993" + } + } else { + // Valid port is checked by URL parsing + port = parsed.Port() + } + + validConfig := runtimeConfig{ + host: parsed.Hostname(), + forcedTLS: parsed.Scheme == "imaps", + port: port, + noopCount: 10, + } + + return &validConfig, nil +} + +// TestKeys contains the experiment results for an entire domain host +type TestKeys struct { + Host string `json:"hostname"` + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // Individual IP/port results + Runs []*IndividualTestKeys `json:"runs"` + // Used for global failure (DNS resolution) + Failure string `json:"failure"` +} + +func newTestKeys(host string) *TestKeys { + tk := new(TestKeys) + tk.Host = host + return tk +} + +// Hostname TCPRunnerModel +func (tk *TestKeys) Hostname(host string) { + tk.Host = host +} + +// DNSResults TCPRunnerModel +func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) { + // TODO: not sure if we are passed the overall trace results and should overwrite key, or just append + tk.Queries = append(tk.Queries, res...) +} + +// Failed TCPRunnerModel +func (tk *TestKeys) Failed(msg string) { + tk.Failure = msg +} + +// NewRun TCPRunnerModel +func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel { + itk := newIndividualTestKeys(addr, port) + tk.Runs = append(tk.Runs, itk) + return itk +} + +// IndividualTestKeys contains the experiment results for a single IP/port combo +type IndividualTestKeys struct { + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + Failure string `json:"failure"` + FailureStep string `json:"failed_step"` + IP string `json:"ip"` + Port string `json:"port"` + noopCounter uint8 +} + +func newIndividualTestKeys(addr string, port string) *IndividualTestKeys { + itk := new(IndividualTestKeys) + itk.IP = addr + itk.Port = port + return itk +} + +// IPPort TCPSessionModel +func (itk *IndividualTestKeys) IPPort(ip string, port string) { + itk.IP = ip + itk.Port = port +} + +// ConnectResults TCPSessionModel +func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) { + itk.TCPConnect = append(itk.TCPConnect, res...) +} + +// HandshakeResult TCPSessionModel +func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) { + itk.TLSHandshake = res +} + +// FailedStep TCPSessionModel +func (itk *IndividualTestKeys) FailedStep(failure string, step string) { + itk.Failure = failure + itk.FailureStep = step +} + +// Measurer performs the measurement. +type Measurer struct { + // Config contains the experiment settings. If empty we + // will be using default settings. + Config Config + + // Getter is an optional getter to be used for testing. + Getter urlgetter.MultiGetter +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +// Run implements ExperimentMeasurer.Run +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + sess := args.Session + measurement := args.Measurement + log := sess.Logger() + trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) + + config, err := config(measurement.Input) + if err != nil { + // Invalid input data, we don't even generate report + return err + } + + tk := new(TestKeys) + measurement.TestKeys = tk + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + tlsconfig := tls.Config{ + InsecureSkipVerify: false, + ServerName: config.host, + } + + runner := &tcprunner.TCPRunner{ + Tk: tk, + Trace: trace, + Logger: log, + Ctx: ctx, + Tlsconfig: &tlsconfig, + } + + // First resolve DNS + addrs, success := runner.Resolve(config.host) + if !success { + return nil + } + + for _, addr := range addrs { + tcpSession, success := runner.Conn(addr, config.port) + if !success { + continue + } + defer tcpSession.Close() + + if config.forcedTLS { + log.Infof("Running direct TLS mode to %s:%s", addr, config.port) + + if !tcpSession.Handshake() { + continue + } + + // Try NoOps + if !testIMAP(tcpSession, config.noopCount) { + continue + } + } else { + log.Infof("Running StartTLS mode to %s:%s", addr, config.port) + + // Upgrade via StartTLS and try NoOps + if !tcpSession.StartTLS("A1 STARTTLS\n", "TLS") { + continue + } + + if !testIMAP(tcpSession, config.noopCount) { + continue + } + } + } + + return nil +} + +func testIMAP(s *tcprunner.TCPSession, noop uint8) bool { + // Auto-choose plaintext/TCP session + // TODO: move to Debugf + s.Runner.Logger.Infof("Retrieving existing connection") + conn := s.CurrentConn() + s.Runner.Logger.Infof("Starting IMAP query") + + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + s.FailedStep(*tracex.NewFailure(err), "imap_wait_capability") + return false + } + + if !strings.Contains(command, "CAPABILITY") { + s.FailedStep(fmt.Sprintf("Received unexpected IMAP response: %s", command), "imap_wrong_capability") + return false + } + + s.Runner.Logger.Infof("Finished starting IMAP") + + if noop > 0 { + // Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field + concreteITK := s.Itk.(*IndividualTestKeys) + s.Runner.Logger.Infof("Trying to generate more no-op traffic") + concreteITK.noopCounter = 0 + for concreteITK.noopCounter < noop { + concreteITK.noopCounter++ + s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter) + _, err = conn.Write([]byte("A1 NOOP\n")) + if err != nil { + s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("imap_noop_%d", concreteITK.noopCounter)) + break + } + } + + if concreteITK.noopCounter == noop { + s.Runner.Logger.Infof("Successfully generated no-op traffic") + return true + } + s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter) + return false + } + + return true +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{Config: config} +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + //DNSBlocking bool `json:"facebook_dns_blocking"` + //TCPBlocking bool `json:"facebook_tcp_blocking"` + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{IsAnomaly: false} + _, ok := measurement.TestKeys.(*TestKeys) + if !ok { + return sk, errors.New("invalid test keys type") + } + return sk, nil +} diff --git a/internal/engine/experiment/imap/imap_test.go b/internal/engine/experiment/imap/imap_test.go new file mode 100644 index 0000000..c02c09f --- /dev/null +++ b/internal/engine/experiment/imap/imap_test.go @@ -0,0 +1,192 @@ +package imap + +import ( + "bufio" + "context" + "crypto/tls" + //"encoding/json" + "errors" + "fmt" + "net" + "strings" + "testing" + + "github.com/ooni/probe-cli/v3/internal/engine/mockable" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func plaintextListener() net.Listener { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) + } + } + return l +} + +func tlsListener(l net.Listener) net.Listener { + return tls.NewListener(l, &tls.Config{}) +} + +func listenerAddr(l net.Listener) string { + return l.Addr().String() +} + +func ValidIMAPServer(conn net.Conn) { + starttls := false + for { + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return + } + + if strings.Contains(command, "NOOP") { + conn.Write([]byte("A1 OK NOOP completed.\n")) + } else if command == "STARTTLS" { + starttls = true + conn.Write([]byte("A1 OK Begin TLS negotiation now.\n")) + // TODO: conn.Close does not actually close connection? or does client not detect it? + //conn.Close() + return + } else if starttls { + conn.Write([]byte("GARBAGE TO BREAK STARTTLS")) + } + conn.Write([]byte("\n")) + } +} + +func TCPServer(l net.Listener) { + for { + conn, err := l.Accept() + if err != nil { + continue + } + defer conn.Close() + conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n")) + ValidIMAPServer(conn) + } +} + +func TestMeasurer_run(t *testing.T) { + // runHelper is an helper function to run this set of tests. + runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { + m := NewExperimentMeasurer(Config{}) + if m.ExperimentName() != "imap" { + t.Fatal("invalid experiment name") + } + if m.ExperimentVersion() != "0.0.1" { + t.Fatal("invalid experiment version") + } + ctx := context.Background() + meas := &model.Measurement{ + Input: model.MeasurementTarget(input), + } + sess := &mockable.Session{ + MockableLogger: model.DiscardLogger, + } + + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: meas, + Session: sess, + } + + err := m.Run(ctx, args) + return meas, m, err + } + + t.Run("with empty input", func(t *testing.T) { + _, _, err := runHelper("") + if !errors.Is(err, errNoInputProvided) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid URL", func(t *testing.T) { + _, _, err := runHelper("\t") + if !errors.Is(err, errInputIsNotAnURL) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid scheme", func(t *testing.T) { + _, _, err := runHelper("https://8.8.8.8:443/") + if !errors.Is(err, errInvalidScheme) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with broken TLS", func(t *testing.T) { + p := plaintextListener() + defer p.Close() + + l := tlsListener(p) + defer l.Close() + addr := listenerAddr(l) + go TCPServer(l) + + meas, m, err := runHelper("imaps://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + + for _, run := range tk.Runs { + if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { + t.Fatal("expected unrecognized_name in TLS handshake") + } + + if run.noopCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) + + t.Run("with broken starttls", func(t *testing.T) { + l := plaintextListener() + defer l.Close() + addr := listenerAddr(l) + + go TCPServer(l) + + meas, m, err := runHelper("imap://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + //bs, _ := json.Marshal(tk) + //fmt.Println(string(bs)) + + for _, run := range tk.Runs { + if *run.TLSHandshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" { + t.Fatalf("s%ss", *run.TLSHandshake.Failure) + t.Fatal("expected broken handshake") + } + + if run.noopCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) +} diff --git a/internal/engine/experiment/smtp/smtp.go b/internal/engine/experiment/smtp/smtp.go new file mode 100644 index 0000000..959558b --- /dev/null +++ b/internal/engine/experiment/smtp/smtp.go @@ -0,0 +1,333 @@ +package smtp + +import ( + "context" + "crypto/tls" + "fmt" + "github.com/pkg/errors" + "net/smtp" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tcprunner" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +var ( + // errNoInputProvided indicates you didn't provide any input + errNoInputProvided = errors.New("not input provided") + + // errInputIsNotAnURL indicates that input is not an URL + errInputIsNotAnURL = errors.New("input is not an URL") + + // errInvalidScheme indicates that the scheme is invalid + errInvalidScheme = errors.New("scheme must be smtp(s)") +) + +const ( + testName = "smtp" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +type runtimeConfig struct { + host string + port string + forcedTLS bool + noopCount uint8 +} + +func config(input model.MeasurementTarget) (*runtimeConfig, error) { + if input == "" { + // TODO: static input data (eg. gmail/riseup..) + return nil, errNoInputProvided + } + + parsed, err := url.Parse(string(input)) + if err != nil { + return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) + } + if parsed.Scheme != "smtp" && parsed.Scheme != "smtps" { + return nil, errInvalidScheme + } + + port := "" + + if parsed.Port() == "" { + // Default ports for StartTLS and forced TLS respectively + if parsed.Scheme == "smtp" { + port = "587" + } else { + port = "465" + } + } else { + // Valid port is checked by URL parsing + port = parsed.Port() + } + + validConfig := runtimeConfig{ + host: parsed.Hostname(), + forcedTLS: parsed.Scheme == "smtps", + port: port, + noopCount: 10, + } + + return &validConfig, nil +} + +// TestKeys contains the experiment results for an entire domain host +type TestKeys struct { + Host string `json:"hostname"` + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // Individual IP/port results + Runs []*IndividualTestKeys `json:"runs"` + // Used for global failure (DNS resolution) + Failure string `json:"failure"` +} + +func newTestKeys(host string) *TestKeys { + tk := new(TestKeys) + tk.Host = host + return tk +} + +// Hostname TCPRunnerModel +func (tk *TestKeys) Hostname(host string) { + tk.Host = host +} + +// DNSResults TCPRunnerModel +func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) { + // TODO: not sure if we are passed the overall trace results and should overwrite key, or just append + tk.Queries = append(tk.Queries, res...) +} + +// Failed TCPRunnerModel +func (tk *TestKeys) Failed(msg string) { + tk.Failure = msg +} + +// NewRun TCPRunnerModel +func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel { + itk := newIndividualTestKeys(addr, port) + tk.Runs = append(tk.Runs, itk) + return itk +} + +// IndividualTestKeys contains the experiment results for a single IP/port combo +type IndividualTestKeys struct { + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + Failure string `json:"failure"` + FailureStep string `json:"failed_step"` + IP string `json:"ip"` + Port string `json:"port"` + noopCounter uint8 +} + +func newIndividualTestKeys(addr string, port string) *IndividualTestKeys { + itk := new(IndividualTestKeys) + itk.IP = addr + itk.Port = port + return itk +} + +// IPPort TCPSessionModel +func (itk *IndividualTestKeys) IPPort(ip string, port string) { + itk.IP = ip + itk.Port = port +} + +// ConnectResults TCPSessionModel +func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) { + itk.TCPConnect = append(itk.TCPConnect, res...) +} + +// HandshakeResult TCPSessionModel +func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) { + itk.TLSHandshake = res +} + +// FailedStep TCPSessionModel +func (itk *IndividualTestKeys) FailedStep(failure string, step string) { + itk.Failure = failure + itk.FailureStep = step +} + +// Measurer performs the measurement. +type Measurer struct { + // Config contains the experiment settings. If empty we + // will be using default settings. + Config Config + + // Getter is an optional getter to be used for testing. + Getter urlgetter.MultiGetter +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +// Run implements ExperimentMeasurer.Run +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + sess := args.Session + measurement := args.Measurement + log := sess.Logger() + trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) + + config, err := config(measurement.Input) + if err != nil { + // Invalid input data, we don't even generate report + return err + } + + tk := new(TestKeys) + measurement.TestKeys = tk + + ctx, cancel := context.WithTimeout(ctx, 30*time.Second) + defer cancel() + + tlsconfig := tls.Config{ + InsecureSkipVerify: false, + ServerName: config.host, + } + + runner := &tcprunner.TCPRunner{ + Tk: tk, + Trace: trace, + Logger: log, + Ctx: ctx, + Tlsconfig: &tlsconfig, + } + + // First resolve DNS + addrs, success := runner.Resolve(config.host) + if !success { + return nil + } + + for _, addr := range addrs { + tcpSession, success := runner.Conn(addr, config.port) + if !success { + continue + } + defer tcpSession.Close() + + if config.forcedTLS { + log.Infof("Running direct TLS mode to %s:%s", addr, config.port) + + if !tcpSession.Handshake() { + continue + } + + // Try EHLO + NoOps + if !testSMTP(tcpSession, "localhost", config.noopCount) { + continue + } + } else { + log.Infof("Running StartTLS mode to %s:%s", addr, config.port) + + if !testSMTP(tcpSession, "localhost", 0) { + continue + } + + // Upgrade via StartTLS and try EHLO + NoOps + if !tcpSession.StartTLS("STARTTLS\n", "TLS") { + continue + } + + if !testSMTP(tcpSession, "localhost", config.noopCount) { + continue + } + } + } + + return nil +} + +func testSMTP(s *tcprunner.TCPSession, ehlo string, noop uint8) bool { + // Auto-choose plaintext/TCP session + // TODO: move to Debugf + s.Runner.Logger.Infof("Retrieving existing connection") + conn := s.CurrentConn() + s.Runner.Logger.Infof("Initializing SMTP client") + client, err := smtp.NewClient(conn, ehlo) + if err != nil { + s.FailedStep(*tracex.NewFailure(err), "smtp_init") + return false + } + + s.Runner.Logger.Infof("Starting SMTP EHLO") + err = client.Hello(ehlo) + if err != nil { + if s.TLS { + s.FailedStep(*tracex.NewFailure(err), "smtp_tls_ehlo") + } else { + s.FailedStep(*tracex.NewFailure(err), "smtp_plaintext_ehlo") + } + return false + } + + s.Runner.Logger.Infof("Finished SMTP EHLO") + + if noop > 0 { + // Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field + concreteITK := s.Itk.(*IndividualTestKeys) + s.Runner.Logger.Infof("Trying to generate more no-op traffic") + concreteITK.noopCounter = 0 + for concreteITK.noopCounter < noop { + concreteITK.noopCounter++ + s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter) + err = client.Noop() + if err != nil { + s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("smtp_noop_%d", concreteITK.noopCounter)) + break + } + } + + if concreteITK.noopCounter == noop { + s.Runner.Logger.Infof("Successfully generated no-op traffic") + return true + } + s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter) + return false + } + + return true +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{Config: config} +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + //DNSBlocking bool `json:"facebook_dns_blocking"` + //TCPBlocking bool `json:"facebook_tcp_blocking"` + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{IsAnomaly: false} + _, ok := measurement.TestKeys.(*TestKeys) + if !ok { + return sk, errors.New("invalid test keys type") + } + return sk, nil +} diff --git a/internal/engine/experiment/smtp/smtp_test.go b/internal/engine/experiment/smtp/smtp_test.go new file mode 100644 index 0000000..2866f12 --- /dev/null +++ b/internal/engine/experiment/smtp/smtp_test.go @@ -0,0 +1,186 @@ +package smtp + +import ( + "bufio" + "context" + "crypto/tls" + "errors" + "fmt" + "net" + "strings" + "testing" + + "github.com/ooni/probe-cli/v3/internal/engine/mockable" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func plaintextListener() net.Listener { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) + } + } + return l +} + +func tlsListener(l net.Listener) net.Listener { + return tls.NewListener(l, &tls.Config{}) +} + +func listenerAddr(l net.Listener) string { + return l.Addr().String() +} + +func ValidSMTPServer(conn net.Conn) { + for { + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return + } + + if command == "" { + } else if command == "NOOP" { + conn.Write([]byte("250 2.0.0 Ok\n")) + } else if command == "STARTTLS" { + conn.Write([]byte("220 2.0.0 Ready to start TLS\n")) + // TODO: conn.Close does not actually close connection? or does client not detect it? + conn.Close() + return + } else if strings.HasPrefix(command, "EHLO") { + conn.Write([]byte("250 mock.example.com\n")) + } + conn.Write([]byte("\n")) + } +} + +func TCPServer(l net.Listener) { + for { + conn, err := l.Accept() + if err != nil { + continue + } + defer conn.Close() + conn.Write([]byte("220 mock.example.com ESMTP (spam is not appreciated)\n")) + ValidSMTPServer(conn) + } +} + +func TestMeasurer_run(t *testing.T) { + // runHelper is an helper function to run this set of tests. + runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { + m := NewExperimentMeasurer(Config{}) + if m.ExperimentName() != "smtp" { + t.Fatal("invalid experiment name") + } + if m.ExperimentVersion() != "0.0.1" { + t.Fatal("invalid experiment version") + } + ctx := context.Background() + meas := &model.Measurement{ + Input: model.MeasurementTarget(input), + } + sess := &mockable.Session{ + MockableLogger: model.DiscardLogger, + } + + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) + return meas, m, err + } + + t.Run("with empty input", func(t *testing.T) { + _, _, err := runHelper("") + if !errors.Is(err, errNoInputProvided) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid URL", func(t *testing.T) { + _, _, err := runHelper("\t") + if !errors.Is(err, errInputIsNotAnURL) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid scheme", func(t *testing.T) { + _, _, err := runHelper("https://8.8.8.8:443/") + if !errors.Is(err, errInvalidScheme) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with broken TLS", func(t *testing.T) { + p := plaintextListener() + defer p.Close() + + l := tlsListener(p) + defer l.Close() + addr := listenerAddr(l) + go TCPServer(l) + + meas, m, err := runHelper("smtps://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + + for _, run := range tk.Runs { + if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { + t.Fatal("expected unrecognized_name in TLS handshake") + } + + if run.noopCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) + + t.Run("with broken starttls", func(t *testing.T) { + l := plaintextListener() + defer l.Close() + addr := listenerAddr(l) + + go TCPServer(l) + + meas, m, err := runHelper("smtp://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + + for _, run := range tk.Runs { + if *run.TLSHandshake.Failure != "generic_timeout_error" { + t.Fatal("expected timeout in TLS handshake") + } + + if run.noopCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) +} diff --git a/internal/registry/imap.go b/internal/registry/imap.go new file mode 100644 index 0000000..c607cfc --- /dev/null +++ b/internal/registry/imap.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the 'imap' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/imap" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + AllExperiments["imap"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return imap.NewExperimentMeasurer( + *config.(*imap.Config), + ) + }, + config: &imap.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/registry/smtp.go b/internal/registry/smtp.go new file mode 100644 index 0000000..e3bc805 --- /dev/null +++ b/internal/registry/smtp.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the 'smtp' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/smtp" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + AllExperiments["smtp"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return smtp.NewExperimentMeasurer( + *config.(*smtp.Config), + ) + }, + config: &smtp.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/tcprunner/tcprunner.go b/internal/tcprunner/tcprunner.go new file mode 100644 index 0000000..2faed1d --- /dev/null +++ b/internal/tcprunner/tcprunner.go @@ -0,0 +1,192 @@ +package tcprunner + +import ( + "bufio" + "context" + "crypto/tls" + "net" + "strings" + + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +// Model describes a type that does a DNS lookup(s), then attempts several TCP sessions +type Model interface { + // Stores the provided hostname + Hostname(string) + // Store DNS query result + DNSResults([]*model.ArchivalDNSLookupResult) + // Indicates one or more steps failed (can be overwritten) + Failed(string) + // Stores a new individual test key (for a TCP session) and returns a pointer to it + NewRun(string, string) TCPSessionModel +} + +// TCPSessionModel describes a type that does a single TCP connection and TLS handshake with a given IP/Port combo +type TCPSessionModel interface { + // Store IP/port address used for this session + IPPort(string, string) + // Store TCP connect result + ConnectResults([]*model.ArchivalTCPConnectResult) + // Store TLS handshake result + HandshakeResult(*model.ArchivalTLSOrQUICHandshakeResult) + // Indicates a failure string, as well as an identifier for the failed step + FailedStep(string, string) +} + +// TCPRunner manages sequential TCP sessions to the same hostname (over different IPs) +type TCPRunner struct { + Tk Model + Trace *measurexlite.Trace + Logger model.Logger + Ctx context.Context + Tlsconfig *tls.Config +} + +// TCPSession Manages a single TCP session and TLS handshake to a given ip:port +type TCPSession struct { + Itk TCPSessionModel + Runner *TCPRunner + Addr string + Port string + TLS bool + RawConn *net.Conn + TLSConn *net.Conn +} + +// FailedStep saves a failure (with an associated failed step identifier) into IndividualTestKeys +func (s *TCPSession) FailedStep(failure string, step string) { + // Save FailedStep inside ITK + s.Itk.FailedStep(failure, step) + // Copy FailedStep to global TK + s.Runner.Tk.Failed(failure) + // Print the warning message + s.Runner.Logger.Warn(failure) +} + +// Close closes the open TCP connections +func (s *TCPSession) Close() { + if s.TLS { + var conn = *s.TLSConn + conn.Close() + } else { + // TODO: should raw connection be closed anyway? + var conn = *s.RawConn + conn.Close() + } +} + +// CurrentConn returns the currently active connection (TLS or plaintext) +func (s *TCPSession) CurrentConn() net.Conn { + if s.TLS { + // TODO: move to Debugf + s.Runner.Logger.Infof("Reusing TLS connection") + return *s.TLSConn + } + s.Runner.Logger.Infof("Reusing plaintext connection") + return *s.RawConn +} + +// Conn initializes a new Run and IndividualTestKeys +func (r *TCPRunner) Conn(addr string, port string) (*TCPSession, bool) { + // Get new individual test keys + itk := r.Tk.NewRun(addr, port) + + s := new(TCPSession) + s.Runner = r + s.Itk = itk + s.Addr = addr + s.Port = port + s.TLS = false + + if !s.Conn(addr, port) { + return nil, false + } + return s, true +} + +// Conn starts a new TCP/IP connection to addr/port +func (s *TCPSession) Conn(addr string, port string) bool { + dialer := s.Runner.Trace.NewDialerWithoutResolver(s.Runner.Logger) + s.Runner.Logger.Infof("Dialing to %s:%s", addr, port) + conn, err := dialer.DialContext(s.Runner.Ctx, "tcp", net.JoinHostPort(addr, port)) + s.Itk.ConnectResults(s.Runner.Trace.TCPConnects()) + if err != nil { + s.FailedStep(*tracex.NewFailure(err), "tcp_connect") + return false + } + s.RawConn = &conn + + return true +} + +// Resolve resolves a hostname to a list of addresses +func (r *TCPRunner) Resolve(host string) ([]string, bool) { + r.Logger.Infof("Resolving DNS for %s", host) + resolver := r.Trace.NewStdlibResolver(r.Logger) + addrs, err := resolver.LookupHost(r.Ctx, host) + r.Tk.DNSResults(r.Trace.DNSLookupsFromRoundTrip()) + if err != nil { + r.Tk.Failed(*tracex.NewFailure(err)) + return []string{}, false + } + r.Logger.Infof("Finished DNS for %s: %v", host, addrs) + + return addrs, true +} + +// Handshake performs a TLS handshake over the currently active connection +func (s *TCPSession) Handshake() bool { + if s.TLS { + // TLS already initialized... + return true + } + s.Runner.Logger.Infof("Starting TLS handshake with %s:%s", s.Addr, s.Port) + thx := s.Runner.Trace.NewTLSHandshakerStdlib(s.Runner.Logger) + tconn, _, err := thx.Handshake(s.Runner.Ctx, *s.RawConn, s.Runner.Tlsconfig) + s.Itk.HandshakeResult(s.Runner.Trace.FirstTLSHandshakeOrNil()) + if err != nil { + s.FailedStep(*tracex.NewFailure(err), "tls_handshake") + return false + } + + s.TLS = true + s.TLSConn = &tconn + s.Runner.Logger.Infof("Handshake succeeded") + return true +} + +// StartTLS performs a StartTLS exchange by sending a message over the plaintext connection, waiting for a specific +// response, then performing a TLS handshake +func (s *TCPSession) StartTLS(message string, waitForResponse string) bool { + if s.TLS { + s.Runner.Logger.Warn("Requested TCPSession to do StartTLS when TLS is already enabled") + return true + } + + if message != "" { + s.Runner.Logger.Infof("Asking for StartTLS upgrade") + s.CurrentConn().Write([]byte(message)) + } + + if waitForResponse != "" { + s.Runner.Logger.Infof("Waiting for server response containing: %s", waitForResponse) + conn := s.CurrentConn() + for { + line, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + s.FailedStep(*tracex.NewFailure(err), "starttls_wait_ok") + return false + } + s.Runner.Logger.Debugf("Received: %s", line) + if strings.Contains(line, waitForResponse) { + s.Runner.Logger.Infof("Server is ready for StartTLS") + break + } + } + } + + return s.Handshake() +}