diff --git a/internal/engine/experiment/imap/.smtp.go.swp b/internal/engine/experiment/imap/.smtp.go.swp new file mode 100644 index 0000000..c4f8bb6 Binary files /dev/null and b/internal/engine/experiment/imap/.smtp.go.swp differ diff --git a/internal/engine/experiment/imap/.smtp_test.go.swp b/internal/engine/experiment/imap/.smtp_test.go.swp new file mode 100644 index 0000000..5a742be Binary files /dev/null and b/internal/engine/experiment/imap/.smtp_test.go.swp differ diff --git a/internal/engine/experiment/imap/imap.go b/internal/engine/experiment/imap/imap.go new file mode 100644 index 0000000..93e9386 --- /dev/null +++ b/internal/engine/experiment/imap/imap.go @@ -0,0 +1,416 @@ +package imap + +import ( + "bufio" + "context" + "crypto/tls" + "fmt" + "github.com/pkg/errors" + "net" + //"net/smtp" + "net/url" + "strings" + "time" + + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +var ( + // errNoInputProvided indicates you didn't provide any input + errNoInputProvided = errors.New("not input provided") + + // errInputIsNotAnURL indicates that input is not an URL + errInputIsNotAnURL = errors.New("input is not an URL") + + // errInvalidScheme indicates that the scheme is invalid + errInvalidScheme = errors.New("scheme must be smtp(s)") +) + +const ( + testName = "imap" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +type RuntimeConfig struct { + host string + port string + forced_tls bool + noop_count uint8 +} + +func config(input model.MeasurementTarget) (*RuntimeConfig, error) { + if input == "" { + // TODO: static input data (eg. gmail/riseup..) + return nil, errNoInputProvided + } + + parsed, err := url.Parse(string(input)) + if err != nil { + return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) + } + if parsed.Scheme != "imap" && parsed.Scheme != "imaps" { + return nil, errInvalidScheme + } + + port := "" + + if parsed.Port() == "" { + // Default ports for StartTLS and forced TLS respectively + if parsed.Scheme == "imap" { + port = "143" + } else { + port = "993" + } + } else { + // Valid port is checked by URL parsing + port = parsed.Port() + } + + valid_config := RuntimeConfig{ + host: parsed.Hostname(), + forced_tls: parsed.Scheme == "imaps", + port: port, + noop_count: 10, + } + + return &valid_config, nil +} + +// TestKeys contains the experiment results + +type TestKeys struct { + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + Runs map[string]*IndividualTestKeys `json:"runs"` + // Used for global failure (DNS resolution) + Failure string `json:"failure"` + // Indicates global failure or individual test failure + Failed bool `json:"failed"` +} + +// IndividualTestKeys contains results for TCP/IP level stuff for each address found +// in the DNS lookup +type IndividualTestKeys struct { + NoOpCounter uint8 + TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` + TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` + // Individual failure aborting the test run for this address/port combo + Failure *string `json:"failure"` +} + +type Measurer struct { + // Config contains the experiment settings. If empty we + // will be using default settings. + Config Config + + // Getter is an optional getter to be used for testing. + Getter urlgetter.MultiGetter +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +// Manages sequential TCP sessions to the same hostname (over different IPs) +// don't use in parallel! +type TCPRunner struct { + trace *measurexlite.Trace + logger model.Logger + ctx context.Context + tk *TestKeys + tlsconfig *tls.Config + host string + port string + // addr is changed everytime TCPRunner.conn(addr) is called + addr string +} + +type TCPSession struct { + addr string + port string + runner *TCPRunner + tk *IndividualTestKeys + tls bool + raw_conn *net.Conn + tls_conn *net.Conn +} + +func (s *TCPSession) Close() { + if s.tls { + var conn = *s.tls_conn + conn.Close() + } else { + var conn = *s.raw_conn + conn.Close() + } +} + +func (s *TCPSession) current_conn() net.Conn { + if s.tls { + return *s.tls_conn + } else { + return *s.raw_conn + } +} + +func (r *TCPRunner) run_key() string { + return net.JoinHostPort(r.addr, r.port) +} + +func (r *TCPRunner) get_run() *IndividualTestKeys { + if r.tk.Runs == nil { + r.tk.Runs = make(map[string]*IndividualTestKeys) + } + key := r.run_key() + val, exists := r.tk.Runs[key] + if exists { + return val + } else { + r.tk.Runs[key] = &IndividualTestKeys{} + return r.tk.Runs[key] + } +} + +func (r *TCPRunner) conn(addr string, port string) (*TCPSession, bool) { + r.addr = addr + run := r.get_run() + + s := new(TCPSession) + if !s.conn(addr, port, r, run) { + return nil, false + } + return s, true +} + +func (r *TCPRunner) dial(addr string, port string) (net.Conn, error) { + dialer := r.trace.NewDialerWithoutResolver(r.logger) + conn, err := dialer.DialContext(r.ctx, "tcp", net.JoinHostPort(addr, port)) + run := r.get_run() + run.TCPConnect = append(run.TCPConnect, r.trace.TCPConnects()...) + return conn, err + +} + +func (s *TCPSession) conn(addr string, port string, runner *TCPRunner, tk *IndividualTestKeys) bool { + // Initialize addr field and corresponding errors in TestKeys + s.addr = addr + s.port = port + s.tls = false + s.runner = runner + s.tk = tk + + conn, err := runner.dial(addr, port) + if err != nil { + s.error(err) + return false + } + s.raw_conn = &conn + + return true +} + +func (s *TCPSession) error(err error) { + s.runner.tk.Failed = true + s.tk.Failure = tracex.NewFailure(err) + //s. = append(s.errors, tracex.NewFailure(err)) +} + +func (r *TCPRunner) resolve(host string) ([]string, bool) { + r.logger.Infof("Resolving DNS for %s", host) + resolver := r.trace.NewStdlibResolver(r.logger) + addrs, err := resolver.LookupHost(r.ctx, host) + r.tk.Queries = append(r.tk.Queries, r.trace.DNSLookupsFromRoundTrip()...) + if err != nil { + r.tk.Failure = *tracex.NewFailure(err) + return []string{}, false + } + r.logger.Infof("Finished DNS for %s: %v", host, addrs) + + return addrs, true +} + +func (s *TCPSession) handshake() bool { + if s.tls { + // TLS already initialized... + return true + } + s.runner.logger.Infof("Starting TLS handshake with %s:%s", s.addr, s.port) + thx := s.runner.trace.NewTLSHandshakerStdlib(s.runner.logger) + tconn, _, err := thx.Handshake(s.runner.ctx, *s.raw_conn, s.runner.tlsconfig) + s.tk.TLSHandshakes = append(s.tk.TLSHandshakes, s.runner.trace.FirstTLSHandshakeOrNil()) + if err != nil { + s.error(err) + return false + } + + s.tls = true + s.tls_conn = &tconn + s.runner.logger.Infof("Handshake succeeded") + return true +} + +func (s *TCPSession) starttls(message string) bool { + if s.tls { + // TLS already initialized... + return true + } + if message != "" { + s.runner.logger.Infof("Asking for StartTLS upgrade") + s.current_conn().Write([]byte(message)) + } + return s.handshake() +} + +func (s *TCPSession) imap(noop uint8) bool { + conn := s.current_conn() + + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + s.error(err) + return false + } + if !strings.Contains(command, "CAPABILITY") { + s.error(errors.New("Unexpected IMAP reply: " + command)) + return false + } + + + if noop > 0 { + s.runner.logger.Infof("Trying to generate no-op traffic") + s.tk.NoOpCounter = 0 + for s.tk.NoOpCounter < noop { + s.tk.NoOpCounter += 1 + s.runner.logger.Infof("NoOp Iteration %d", s.tk.NoOpCounter) + + conn.Write([]byte("A1 NOOP\n")) + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + s.error(err) + break + } + if !strings.Contains(command, "OK NOOP") { + s.error(errors.New("Unexpected IMAP reply: " + command)) + break + } + } + + if s.tk.NoOpCounter == noop { + s.runner.logger.Infof("Successfully generated no-op traffic") + return true + } else { + s.runner.logger.Infof("Failed no-op traffic at iteration %d", s.tk.NoOpCounter) + return false + } + } + + return true +} + +// Run implements ExperimentMeasurer.Run +func (m Measurer) Run( + ctx context.Context, sess model.ExperimentSession, + measurement *model.Measurement, callbacks model.ExperimentCallbacks, +) error { + log := sess.Logger() + trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) + + config, err := config(measurement.Input) + if err != nil { + // Invalid input data, we don't even generate report + return err + } + + tk := new(TestKeys) + measurement.TestKeys = tk + + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + tlsconfig := tls.Config{ + InsecureSkipVerify: false, + ServerName: config.host, + } + + runner := &TCPRunner{ + trace: trace, + logger: log, + ctx: ctx, + tk: tk, + tlsconfig: &tlsconfig, + host: config.host, + port: config.port, + } + + // First resolve DNS + addrs, success := runner.resolve(config.host) + if !success { + return nil + } + + for _, addr := range addrs { + tcp_session, success := runner.conn(addr, config.port) + if !success { + continue + } + defer tcp_session.Close() + + if config.forced_tls { + // Direct TLS connection + if !tcp_session.handshake() { + continue + } + + // Try EHLO + NoOps + if !tcp_session.imap(config.noop_count) { + continue + } + } else { + // StartTLS... + if !tcp_session.starttls("A1 STARTTLS\n") { + continue + } + + if !tcp_session.imap(config.noop_count) { + continue + } + } + } + + return nil +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{Config: config} +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + //DNSBlocking bool `json:"facebook_dns_blocking"` + //TCPBlocking bool `json:"facebook_tcp_blocking"` + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{IsAnomaly: false} + _, ok := measurement.TestKeys.(*TestKeys) + if !ok { + return sk, errors.New("invalid test keys type") + } + return sk, nil +} diff --git a/internal/engine/experiment/imap/imap_test.go b/internal/engine/experiment/imap/imap_test.go new file mode 100644 index 0000000..1fbfc15 --- /dev/null +++ b/internal/engine/experiment/imap/imap_test.go @@ -0,0 +1,186 @@ +package imap + +import ( + "bufio" + "context" + "crypto/tls" + //"encoding/json" + "errors" + "fmt" + "net" + "strings" + "testing" + + "github.com/ooni/probe-cli/v3/internal/engine/mockable" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func plaintextListener() net.Listener { + l, err := net.Listen("tcp", "127.0.0.1:0") + if err != nil { + if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { + panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) + } + } + return l +} + +func tlsListener(l net.Listener) net.Listener { + return tls.NewListener(l, &tls.Config{}) +} + +func listener_addr(l net.Listener) string { + return l.Addr().String() +} + +func ValidIMAPServer(conn net.Conn) { + for { + command, err := bufio.NewReader(conn).ReadString('\n') + if err != nil { + return + } + + if strings.Contains(command, "NOOP") { + conn.Write([]byte("A1 OK NOOP completed.\n")) + } else if command == "STARTTLS" { + conn.Write([]byte("A1 OK Begin TLS negotiation now.\n")) + // TODO: conn.Close does not actually close connection? or does client not detect it? + conn.Close() + return + } + conn.Write([]byte("\n")) + } +} + +func TCPServer(l net.Listener) { + for { + conn, err := l.Accept() + if err != nil { + continue + } + defer conn.Close() + conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n")) + ValidIMAPServer(conn) + } +} + +func TestMeasurer_run(t *testing.T) { + // runHelper is an helper function to run this set of tests. + runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { + m := NewExperimentMeasurer(Config{}) + if m.ExperimentName() != "imap" { + t.Fatal("invalid experiment name") + } + if m.ExperimentVersion() != "0.0.1" { + t.Fatal("invalid experiment version") + } + ctx := context.Background() + meas := &model.Measurement{ + Input: model.MeasurementTarget(input), + } + sess := &mockable.Session{ + MockableLogger: model.DiscardLogger, + } + callbacks := model.NewPrinterCallbacks(model.DiscardLogger) + err := m.Run(ctx, sess, meas, callbacks) + return meas, m, err + } + + t.Run("with empty input", func(t *testing.T) { + _, _, err := runHelper("") + if !errors.Is(err, errNoInputProvided) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid URL", func(t *testing.T) { + _, _, err := runHelper("\t") + if !errors.Is(err, errInputIsNotAnURL) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid scheme", func(t *testing.T) { + _, _, err := runHelper("https://8.8.8.8:443/") + if !errors.Is(err, errInvalidScheme) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with broken TLS", func(t *testing.T) { + p := plaintextListener() + defer p.Close() + + l := tlsListener(p) + defer l.Close() + addr := listener_addr(l) + go TCPServer(l) + + meas, m, err := runHelper("imaps://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + + for _, run := range tk.Runs { + for _, handshake := range run.TLSHandshakes { + if *handshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { + t.Fatal("expected unrecognized_name in TLS handshake") + } + } + + if run.NoOpCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) + + t.Run("with broken starttls", func(t *testing.T) { + l := plaintextListener() + defer l.Close() + addr := listener_addr(l) + + go TCPServer(l) + + meas, m, err := runHelper("imap://" + addr) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + //bs, _ := json.Marshal(tk) + //fmt.Println(string(bs)) + + for _, run := range tk.Runs { + for _, handshake := range run.TLSHandshakes { + if *handshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" { + + t.Fatal("expected broken handshake") + } + } + + if run.NoOpCounter != 0 { + t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) + } + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) +}