diff --git a/internal/engine/experiment/imap/.smtp.go.swp b/internal/engine/experiment/imap/.smtp.go.swp deleted file mode 100644 index c4f8bb6..0000000 Binary files a/internal/engine/experiment/imap/.smtp.go.swp and /dev/null differ diff --git a/internal/engine/experiment/imap/.smtp_test.go.swp b/internal/engine/experiment/imap/.smtp_test.go.swp deleted file mode 100644 index 5a742be..0000000 Binary files a/internal/engine/experiment/imap/.smtp_test.go.swp and /dev/null differ diff --git a/internal/engine/experiment/imap/imap.go b/internal/engine/experiment/imap/imap.go deleted file mode 100644 index 93e9386..0000000 --- a/internal/engine/experiment/imap/imap.go +++ /dev/null @@ -1,416 +0,0 @@ -package imap - -import ( - "bufio" - "context" - "crypto/tls" - "fmt" - "github.com/pkg/errors" - "net" - //"net/smtp" - "net/url" - "strings" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/tracex" -) - -var ( - // errNoInputProvided indicates you didn't provide any input - errNoInputProvided = errors.New("not input provided") - - // errInputIsNotAnURL indicates that input is not an URL - errInputIsNotAnURL = errors.New("input is not an URL") - - // errInvalidScheme indicates that the scheme is invalid - errInvalidScheme = errors.New("scheme must be smtp(s)") -) - -const ( - testName = "imap" - testVersion = "0.0.1" -) - -// Config contains the experiment config. -type Config struct{} - -type RuntimeConfig struct { - host string - port string - forced_tls bool - noop_count uint8 -} - -func config(input model.MeasurementTarget) (*RuntimeConfig, error) { - if input == "" { - // TODO: static input data (eg. gmail/riseup..) - return nil, errNoInputProvided - } - - parsed, err := url.Parse(string(input)) - if err != nil { - return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) - } - if parsed.Scheme != "imap" && parsed.Scheme != "imaps" { - return nil, errInvalidScheme - } - - port := "" - - if parsed.Port() == "" { - // Default ports for StartTLS and forced TLS respectively - if parsed.Scheme == "imap" { - port = "143" - } else { - port = "993" - } - } else { - // Valid port is checked by URL parsing - port = parsed.Port() - } - - valid_config := RuntimeConfig{ - host: parsed.Hostname(), - forced_tls: parsed.Scheme == "imaps", - port: port, - noop_count: 10, - } - - return &valid_config, nil -} - -// TestKeys contains the experiment results - -type TestKeys struct { - Queries []*model.ArchivalDNSLookupResult `json:"queries"` - Runs map[string]*IndividualTestKeys `json:"runs"` - // Used for global failure (DNS resolution) - Failure string `json:"failure"` - // Indicates global failure or individual test failure - Failed bool `json:"failed"` -} - -// IndividualTestKeys contains results for TCP/IP level stuff for each address found -// in the DNS lookup -type IndividualTestKeys struct { - NoOpCounter uint8 - TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` - TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` - // Individual failure aborting the test run for this address/port combo - Failure *string `json:"failure"` -} - -type Measurer struct { - // Config contains the experiment settings. If empty we - // will be using default settings. - Config Config - - // Getter is an optional getter to be used for testing. - Getter urlgetter.MultiGetter -} - -// ExperimentName implements ExperimentMeasurer.ExperimentName -func (m Measurer) ExperimentName() string { - return testName -} - -// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion -func (m Measurer) ExperimentVersion() string { - return testVersion -} - -// Manages sequential TCP sessions to the same hostname (over different IPs) -// don't use in parallel! -type TCPRunner struct { - trace *measurexlite.Trace - logger model.Logger - ctx context.Context - tk *TestKeys - tlsconfig *tls.Config - host string - port string - // addr is changed everytime TCPRunner.conn(addr) is called - addr string -} - -type TCPSession struct { - addr string - port string - runner *TCPRunner - tk *IndividualTestKeys - tls bool - raw_conn *net.Conn - tls_conn *net.Conn -} - -func (s *TCPSession) Close() { - if s.tls { - var conn = *s.tls_conn - conn.Close() - } else { - var conn = *s.raw_conn - conn.Close() - } -} - -func (s *TCPSession) current_conn() net.Conn { - if s.tls { - return *s.tls_conn - } else { - return *s.raw_conn - } -} - -func (r *TCPRunner) run_key() string { - return net.JoinHostPort(r.addr, r.port) -} - -func (r *TCPRunner) get_run() *IndividualTestKeys { - if r.tk.Runs == nil { - r.tk.Runs = make(map[string]*IndividualTestKeys) - } - key := r.run_key() - val, exists := r.tk.Runs[key] - if exists { - return val - } else { - r.tk.Runs[key] = &IndividualTestKeys{} - return r.tk.Runs[key] - } -} - -func (r *TCPRunner) conn(addr string, port string) (*TCPSession, bool) { - r.addr = addr - run := r.get_run() - - s := new(TCPSession) - if !s.conn(addr, port, r, run) { - return nil, false - } - return s, true -} - -func (r *TCPRunner) dial(addr string, port string) (net.Conn, error) { - dialer := r.trace.NewDialerWithoutResolver(r.logger) - conn, err := dialer.DialContext(r.ctx, "tcp", net.JoinHostPort(addr, port)) - run := r.get_run() - run.TCPConnect = append(run.TCPConnect, r.trace.TCPConnects()...) - return conn, err - -} - -func (s *TCPSession) conn(addr string, port string, runner *TCPRunner, tk *IndividualTestKeys) bool { - // Initialize addr field and corresponding errors in TestKeys - s.addr = addr - s.port = port - s.tls = false - s.runner = runner - s.tk = tk - - conn, err := runner.dial(addr, port) - if err != nil { - s.error(err) - return false - } - s.raw_conn = &conn - - return true -} - -func (s *TCPSession) error(err error) { - s.runner.tk.Failed = true - s.tk.Failure = tracex.NewFailure(err) - //s. = append(s.errors, tracex.NewFailure(err)) -} - -func (r *TCPRunner) resolve(host string) ([]string, bool) { - r.logger.Infof("Resolving DNS for %s", host) - resolver := r.trace.NewStdlibResolver(r.logger) - addrs, err := resolver.LookupHost(r.ctx, host) - r.tk.Queries = append(r.tk.Queries, r.trace.DNSLookupsFromRoundTrip()...) - if err != nil { - r.tk.Failure = *tracex.NewFailure(err) - return []string{}, false - } - r.logger.Infof("Finished DNS for %s: %v", host, addrs) - - return addrs, true -} - -func (s *TCPSession) handshake() bool { - if s.tls { - // TLS already initialized... - return true - } - s.runner.logger.Infof("Starting TLS handshake with %s:%s", s.addr, s.port) - thx := s.runner.trace.NewTLSHandshakerStdlib(s.runner.logger) - tconn, _, err := thx.Handshake(s.runner.ctx, *s.raw_conn, s.runner.tlsconfig) - s.tk.TLSHandshakes = append(s.tk.TLSHandshakes, s.runner.trace.FirstTLSHandshakeOrNil()) - if err != nil { - s.error(err) - return false - } - - s.tls = true - s.tls_conn = &tconn - s.runner.logger.Infof("Handshake succeeded") - return true -} - -func (s *TCPSession) starttls(message string) bool { - if s.tls { - // TLS already initialized... - return true - } - if message != "" { - s.runner.logger.Infof("Asking for StartTLS upgrade") - s.current_conn().Write([]byte(message)) - } - return s.handshake() -} - -func (s *TCPSession) imap(noop uint8) bool { - conn := s.current_conn() - - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - s.error(err) - return false - } - if !strings.Contains(command, "CAPABILITY") { - s.error(errors.New("Unexpected IMAP reply: " + command)) - return false - } - - - if noop > 0 { - s.runner.logger.Infof("Trying to generate no-op traffic") - s.tk.NoOpCounter = 0 - for s.tk.NoOpCounter < noop { - s.tk.NoOpCounter += 1 - s.runner.logger.Infof("NoOp Iteration %d", s.tk.NoOpCounter) - - conn.Write([]byte("A1 NOOP\n")) - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - s.error(err) - break - } - if !strings.Contains(command, "OK NOOP") { - s.error(errors.New("Unexpected IMAP reply: " + command)) - break - } - } - - if s.tk.NoOpCounter == noop { - s.runner.logger.Infof("Successfully generated no-op traffic") - return true - } else { - s.runner.logger.Infof("Failed no-op traffic at iteration %d", s.tk.NoOpCounter) - return false - } - } - - return true -} - -// Run implements ExperimentMeasurer.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { - log := sess.Logger() - trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) - - config, err := config(measurement.Input) - if err != nil { - // Invalid input data, we don't even generate report - return err - } - - tk := new(TestKeys) - measurement.TestKeys = tk - - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - tlsconfig := tls.Config{ - InsecureSkipVerify: false, - ServerName: config.host, - } - - runner := &TCPRunner{ - trace: trace, - logger: log, - ctx: ctx, - tk: tk, - tlsconfig: &tlsconfig, - host: config.host, - port: config.port, - } - - // First resolve DNS - addrs, success := runner.resolve(config.host) - if !success { - return nil - } - - for _, addr := range addrs { - tcp_session, success := runner.conn(addr, config.port) - if !success { - continue - } - defer tcp_session.Close() - - if config.forced_tls { - // Direct TLS connection - if !tcp_session.handshake() { - continue - } - - // Try EHLO + NoOps - if !tcp_session.imap(config.noop_count) { - continue - } - } else { - // StartTLS... - if !tcp_session.starttls("A1 STARTTLS\n") { - continue - } - - if !tcp_session.imap(config.noop_count) { - continue - } - } - } - - return nil -} - -// NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { - return Measurer{Config: config} -} - -// SummaryKeys contains summary keys for this experiment. -// -// Note that this structure is part of the ABI contract with ooniprobe -// therefore we should be careful when changing it. -type SummaryKeys struct { - //DNSBlocking bool `json:"facebook_dns_blocking"` - //TCPBlocking bool `json:"facebook_tcp_blocking"` - IsAnomaly bool `json:"-"` -} - -// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. -func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { - sk := SummaryKeys{IsAnomaly: false} - _, ok := measurement.TestKeys.(*TestKeys) - if !ok { - return sk, errors.New("invalid test keys type") - } - return sk, nil -} diff --git a/internal/engine/experiment/imap/imap_test.go b/internal/engine/experiment/imap/imap_test.go deleted file mode 100644 index 1fbfc15..0000000 --- a/internal/engine/experiment/imap/imap_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package imap - -import ( - "bufio" - "context" - "crypto/tls" - //"encoding/json" - "errors" - "fmt" - "net" - "strings" - "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/model" -) - -func plaintextListener() net.Listener { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { - panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) - } - } - return l -} - -func tlsListener(l net.Listener) net.Listener { - return tls.NewListener(l, &tls.Config{}) -} - -func listener_addr(l net.Listener) string { - return l.Addr().String() -} - -func ValidIMAPServer(conn net.Conn) { - for { - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - return - } - - if strings.Contains(command, "NOOP") { - conn.Write([]byte("A1 OK NOOP completed.\n")) - } else if command == "STARTTLS" { - conn.Write([]byte("A1 OK Begin TLS negotiation now.\n")) - // TODO: conn.Close does not actually close connection? or does client not detect it? - conn.Close() - return - } - conn.Write([]byte("\n")) - } -} - -func TCPServer(l net.Listener) { - for { - conn, err := l.Accept() - if err != nil { - continue - } - defer conn.Close() - conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n")) - ValidIMAPServer(conn) - } -} - -func TestMeasurer_run(t *testing.T) { - // runHelper is an helper function to run this set of tests. - runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { - m := NewExperimentMeasurer(Config{}) - if m.ExperimentName() != "imap" { - t.Fatal("invalid experiment name") - } - if m.ExperimentVersion() != "0.0.1" { - t.Fatal("invalid experiment version") - } - ctx := context.Background() - meas := &model.Measurement{ - Input: model.MeasurementTarget(input), - } - sess := &mockable.Session{ - MockableLogger: model.DiscardLogger, - } - callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) - return meas, m, err - } - - t.Run("with empty input", func(t *testing.T) { - _, _, err := runHelper("") - if !errors.Is(err, errNoInputProvided) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid URL", func(t *testing.T) { - _, _, err := runHelper("\t") - if !errors.Is(err, errInputIsNotAnURL) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid scheme", func(t *testing.T) { - _, _, err := runHelper("https://8.8.8.8:443/") - if !errors.Is(err, errInvalidScheme) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with broken TLS", func(t *testing.T) { - p := plaintextListener() - defer p.Close() - - l := tlsListener(p) - defer l.Close() - addr := listener_addr(l) - go TCPServer(l) - - meas, m, err := runHelper("imaps://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { - t.Fatal("expected unrecognized_name in TLS handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) - - t.Run("with broken starttls", func(t *testing.T) { - l := plaintextListener() - defer l.Close() - addr := listener_addr(l) - - go TCPServer(l) - - meas, m, err := runHelper("imap://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - //bs, _ := json.Marshal(tk) - //fmt.Println(string(bs)) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" { - - t.Fatal("expected broken handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) -}