c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Basic SMTP/IMAP probe: look for bad TLS and StartTLS downgrade attacks

Browse Source
This commit is contained in:
ooni noob 2022-11-19 17:18:46 +01:00
parent a0dc65641d
commit eca06e83d0
7 changed files with 1272 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

325
internal/engine/experiment/imap/imap.go Normal file
View File

@ -0,0 +1,325 @@
package imap
import (
"bufio"
"context"
"crypto/tls"
"fmt"
"github.com/pkg/errors"
"net/url"
"strings"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/tcprunner"
"github.com/ooni/probe-cli/v3/internal/tracex"
)
var (
// errNoInputProvided indicates you didn't provide any input
errNoInputProvided = errors.New("not input provided")
// errInputIsNotAnURL indicates that input is not an URL
errInputIsNotAnURL = errors.New("input is not an URL")
// errInvalidScheme indicates that the scheme is invalid
errInvalidScheme = errors.New("scheme must be imap(s)")
)
const (
testName = "imap"
testVersion = "0.0.1"
)
// Config contains the experiment config.
type Config struct{}
type runtimeConfig struct {
host string
port string
forcedTLS bool
noopCount uint8
}
func config(input model.MeasurementTarget) (*runtimeConfig, error) {
if input == "" {
// TODO: static input data (eg. gmail/riseup..)
return nil, errNoInputProvided
}
parsed, err := url.Parse(string(input))
if err != nil {
return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error())
}
if parsed.Scheme != "imap" && parsed.Scheme != "imaps" {
return nil, errInvalidScheme
}
port := ""
if parsed.Port() == "" {
// Default ports for StartTLS and forced TLS respectively
if parsed.Scheme == "imap" {
port = "143"
} else {
port = "993"
}
} else {
// Valid port is checked by URL parsing
port = parsed.Port()
}
validConfig := runtimeConfig{
host: parsed.Hostname(),
forcedTLS: parsed.Scheme == "imaps",
port: port,
noopCount: 10,
}
return &validConfig, nil
}
// TestKeys contains the experiment results for an entire domain host
type TestKeys struct {
Host string `json:"hostname"`
Queries []*model.ArchivalDNSLookupResult `json:"queries"`
// Individual IP/port results
Runs []*IndividualTestKeys `json:"runs"`
// Used for global failure (DNS resolution)
Failure string `json:"failure"`
}
func newTestKeys(host string) *TestKeys {
tk := new(TestKeys)
tk.Host = host
return tk
}
// Hostname TCPRunnerModel
func (tk *TestKeys) Hostname(host string) {
tk.Host = host
}
// DNSResults TCPRunnerModel
func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) {
// TODO: not sure if we are passed the overall trace results and should overwrite key, or just append
tk.Queries = append(tk.Queries, res...)
}
// Failed TCPRunnerModel
func (tk *TestKeys) Failed(msg string) {
tk.Failure = msg
}
// NewRun TCPRunnerModel
func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel {
itk := newIndividualTestKeys(addr, port)
tk.Runs = append(tk.Runs, itk)
return itk
}
// IndividualTestKeys contains the experiment results for a single IP/port combo
type IndividualTestKeys struct {
TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"`
TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
Failure string `json:"failure"`
FailureStep string `json:"failed_step"`
IP string `json:"ip"`
Port string `json:"port"`
noopCounter uint8
}
func newIndividualTestKeys(addr string, port string) *IndividualTestKeys {
itk := new(IndividualTestKeys)
itk.IP = addr
itk.Port = port
return itk
}
// IPPort TCPSessionModel
func (itk *IndividualTestKeys) IPPort(ip string, port string) {
itk.IP = ip
itk.Port = port
}
// ConnectResults TCPSessionModel
func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) {
itk.TCPConnect = append(itk.TCPConnect, res...)
}
// HandshakeResult TCPSessionModel
func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) {
itk.TLSHandshake = res
}
// FailedStep TCPSessionModel
func (itk *IndividualTestKeys) FailedStep(failure string, step string) {
itk.Failure = failure
itk.FailureStep = step
}
// Measurer performs the measurement.
type Measurer struct {
// Config contains the experiment settings. If empty we
// will be using default settings.
Config Config
// Getter is an optional getter to be used for testing.
Getter urlgetter.MultiGetter
}
// ExperimentName implements ExperimentMeasurer.ExperimentName
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// Run implements ExperimentMeasurer.Run
func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
sess := args.Session
measurement := args.Measurement
log := sess.Logger()
trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved)
config, err := config(measurement.Input)
if err != nil {
// Invalid input data, we don't even generate report
return err
}
tk := new(TestKeys)
measurement.TestKeys = tk
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
tlsconfig := tls.Config{
InsecureSkipVerify: false,
ServerName: config.host,
}
runner := &tcprunner.TCPRunner{
Tk: tk,
Trace: trace,
Logger: log,
Ctx: ctx,
Tlsconfig: &tlsconfig,
}
// First resolve DNS
addrs, success := runner.Resolve(config.host)
if !success {
return nil
}
for _, addr := range addrs {
tcpSession, success := runner.Conn(addr, config.port)
if !success {
continue
}
defer tcpSession.Close()
if config.forcedTLS {
log.Infof("Running direct TLS mode to %s:%s", addr, config.port)
if !tcpSession.Handshake() {
continue
}
// Try NoOps
if !testIMAP(tcpSession, config.noopCount) {
continue
}
} else {
log.Infof("Running StartTLS mode to %s:%s", addr, config.port)
// Upgrade via StartTLS and try NoOps
if !tcpSession.StartTLS("A1 STARTTLS\n", "TLS") {
continue
}
if !testIMAP(tcpSession, config.noopCount) {
continue
}
}
}
return nil
}
func testIMAP(s *tcprunner.TCPSession, noop uint8) bool {
// Auto-choose plaintext/TCP session
// TODO: move to Debugf
s.Runner.Logger.Infof("Retrieving existing connection")
conn := s.CurrentConn()
s.Runner.Logger.Infof("Starting IMAP query")
command, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
s.FailedStep(*tracex.NewFailure(err), "imap_wait_capability")
return false
}
if !strings.Contains(command, "CAPABILITY") {
s.FailedStep(fmt.Sprintf("Received unexpected IMAP response: %s", command), "imap_wrong_capability")
return false
}
s.Runner.Logger.Infof("Finished starting IMAP")
if noop > 0 {
// Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field
concreteITK := s.Itk.(*IndividualTestKeys)
s.Runner.Logger.Infof("Trying to generate more no-op traffic")
concreteITK.noopCounter = 0
for concreteITK.noopCounter < noop {
concreteITK.noopCounter++
s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter)
_, err = conn.Write([]byte("A1 NOOP\n"))
if err != nil {
s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("imap_noop_%d", concreteITK.noopCounter))
break
}
}
if concreteITK.noopCounter == noop {
s.Runner.Logger.Infof("Successfully generated no-op traffic")
return true
}
s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter)
return false
}
return true
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
//DNSBlocking bool `json:"facebook_dns_blocking"`
//TCPBlocking bool `json:"facebook_tcp_blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{IsAnomaly: false}
_, ok := measurement.TestKeys.(*TestKeys)
if !ok {
return sk, errors.New("invalid test keys type")
}
return sk, nil
}

192
internal/engine/experiment/imap/imap_test.go Normal file
View File

@ -0,0 +1,192 @@
package imap
import (
"bufio"
"context"
"crypto/tls"
//"encoding/json"
"errors"
"fmt"
"net"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/mockable"
"github.com/ooni/probe-cli/v3/internal/model"
)
func plaintextListener() net.Listener {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
}
}
return l
}
func tlsListener(l net.Listener) net.Listener {
return tls.NewListener(l, &tls.Config{})
}
func listenerAddr(l net.Listener) string {
return l.Addr().String()
}
func ValidIMAPServer(conn net.Conn) {
starttls := false
for {
command, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return
}
if strings.Contains(command, "NOOP") {
conn.Write([]byte("A1 OK NOOP completed.\n"))
} else if command == "STARTTLS" {
starttls = true
conn.Write([]byte("A1 OK Begin TLS negotiation now.\n"))
// TODO: conn.Close does not actually close connection? or does client not detect it?
//conn.Close()
return
} else if starttls {
conn.Write([]byte("GARBAGE TO BREAK STARTTLS"))
}
conn.Write([]byte("\n"))
}
}
func TCPServer(l net.Listener) {
for {
conn, err := l.Accept()
if err != nil {
continue
}
defer conn.Close()
conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n"))
ValidIMAPServer(conn)
}
}
func TestMeasurer_run(t *testing.T) {
// runHelper is an helper function to run this set of tests.
runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) {
m := NewExperimentMeasurer(Config{})
if m.ExperimentName() != "imap" {
t.Fatal("invalid experiment name")
}
if m.ExperimentVersion() != "0.0.1" {
t.Fatal("invalid experiment version")
}
ctx := context.Background()
meas := &model.Measurement{
Input: model.MeasurementTarget(input),
}
sess := &mockable.Session{
MockableLogger: model.DiscardLogger,
}
args := &model.ExperimentArgs{
Callbacks: model.NewPrinterCallbacks(model.DiscardLogger),
Measurement: meas,
Session: sess,
}
err := m.Run(ctx, args)
return meas, m, err
}
t.Run("with empty input", func(t *testing.T) {
_, _, err := runHelper("")
if !errors.Is(err, errNoInputProvided) {
t.Fatal("unexpected error", err)
}
})
t.Run("with invalid URL", func(t *testing.T) {
_, _, err := runHelper("\t")
if !errors.Is(err, errInputIsNotAnURL) {
t.Fatal("unexpected error", err)
}
})
t.Run("with invalid scheme", func(t *testing.T) {
_, _, err := runHelper("https://8.8.8.8:443/")
if !errors.Is(err, errInvalidScheme) {
t.Fatal("unexpected error", err)
}
})
t.Run("with broken TLS", func(t *testing.T) {
p := plaintextListener()
defer p.Close()
l := tlsListener(p)
defer l.Close()
addr := listenerAddr(l)
go TCPServer(l)
meas, m, err := runHelper("imaps://" + addr)
if err != nil {
t.Fatal(err)
}
tk := meas.TestKeys.(*TestKeys)
for _, run := range tk.Runs {
if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" {
t.Fatal("expected unrecognized_name in TLS handshake")
}
if run.noopCounter != 0 {
t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
}
}
ask, err := m.GetSummaryKeys(meas)
if err != nil {
t.Fatal("cannot obtain summary")
}
summary := ask.(SummaryKeys)
if summary.IsAnomaly {
t.Fatal("expected no anomaly")
}
})
t.Run("with broken starttls", func(t *testing.T) {
l := plaintextListener()
defer l.Close()
addr := listenerAddr(l)
go TCPServer(l)
meas, m, err := runHelper("imap://" + addr)
if err != nil {
t.Fatal(err)
}
tk := meas.TestKeys.(*TestKeys)
//bs, _ := json.Marshal(tk)
//fmt.Println(string(bs))
for _, run := range tk.Runs {
if *run.TLSHandshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" {
t.Fatalf("s%ss", *run.TLSHandshake.Failure)
t.Fatal("expected broken handshake")
}
if run.noopCounter != 0 {
t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
}
}
ask, err := m.GetSummaryKeys(meas)
if err != nil {
t.Fatal("cannot obtain summary")
}
summary := ask.(SummaryKeys)
if summary.IsAnomaly {
t.Fatal("expected no anomaly")
}
})
}

333
internal/engine/experiment/smtp/smtp.go Normal file
View File

@ -0,0 +1,333 @@
package smtp
import (
"context"
"crypto/tls"
"fmt"
"github.com/pkg/errors"
"net/smtp"
"net/url"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/tcprunner"
"github.com/ooni/probe-cli/v3/internal/tracex"
)
var (
// errNoInputProvided indicates you didn't provide any input
errNoInputProvided = errors.New("not input provided")
// errInputIsNotAnURL indicates that input is not an URL
errInputIsNotAnURL = errors.New("input is not an URL")
// errInvalidScheme indicates that the scheme is invalid
errInvalidScheme = errors.New("scheme must be smtp(s)")
)
const (
testName = "smtp"
testVersion = "0.0.1"
)
// Config contains the experiment config.
type Config struct{}
type runtimeConfig struct {
host string
port string
forcedTLS bool
noopCount uint8
}
func config(input model.MeasurementTarget) (*runtimeConfig, error) {
if input == "" {
// TODO: static input data (eg. gmail/riseup..)
return nil, errNoInputProvided
}
parsed, err := url.Parse(string(input))
if err != nil {
return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error())
}
if parsed.Scheme != "smtp" && parsed.Scheme != "smtps" {
return nil, errInvalidScheme
}
port := ""
if parsed.Port() == "" {
// Default ports for StartTLS and forced TLS respectively
if parsed.Scheme == "smtp" {
port = "587"
} else {
port = "465"
}
} else {
// Valid port is checked by URL parsing
port = parsed.Port()
}
validConfig := runtimeConfig{
host: parsed.Hostname(),
forcedTLS: parsed.Scheme == "smtps",
port: port,
noopCount: 10,
}
return &validConfig, nil
}
// TestKeys contains the experiment results for an entire domain host
type TestKeys struct {
Host string `json:"hostname"`
Queries []*model.ArchivalDNSLookupResult `json:"queries"`
// Individual IP/port results
Runs []*IndividualTestKeys `json:"runs"`
// Used for global failure (DNS resolution)
Failure string `json:"failure"`
}
func newTestKeys(host string) *TestKeys {
tk := new(TestKeys)
tk.Host = host
return tk
}
// Hostname TCPRunnerModel
func (tk *TestKeys) Hostname(host string) {
tk.Host = host
}
// DNSResults TCPRunnerModel
func (tk *TestKeys) DNSResults(res []*model.ArchivalDNSLookupResult) {
// TODO: not sure if we are passed the overall trace results and should overwrite key, or just append
tk.Queries = append(tk.Queries, res...)
}
// Failed TCPRunnerModel
func (tk *TestKeys) Failed(msg string) {
tk.Failure = msg
}
// NewRun TCPRunnerModel
func (tk *TestKeys) NewRun(addr string, port string) tcprunner.TCPSessionModel {
itk := newIndividualTestKeys(addr, port)
tk.Runs = append(tk.Runs, itk)
return itk
}
// IndividualTestKeys contains the experiment results for a single IP/port combo
type IndividualTestKeys struct {
TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"`
TLSHandshake *model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"`
Failure string `json:"failure"`
FailureStep string `json:"failed_step"`
IP string `json:"ip"`
Port string `json:"port"`
noopCounter uint8
}
func newIndividualTestKeys(addr string, port string) *IndividualTestKeys {
itk := new(IndividualTestKeys)
itk.IP = addr
itk.Port = port
return itk
}
// IPPort TCPSessionModel
func (itk *IndividualTestKeys) IPPort(ip string, port string) {
itk.IP = ip
itk.Port = port
}
// ConnectResults TCPSessionModel
func (itk *IndividualTestKeys) ConnectResults(res []*model.ArchivalTCPConnectResult) {
itk.TCPConnect = append(itk.TCPConnect, res...)
}
// HandshakeResult TCPSessionModel
func (itk *IndividualTestKeys) HandshakeResult(res *model.ArchivalTLSOrQUICHandshakeResult) {
itk.TLSHandshake = res
}
// FailedStep TCPSessionModel
func (itk *IndividualTestKeys) FailedStep(failure string, step string) {
itk.Failure = failure
itk.FailureStep = step
}
// Measurer performs the measurement.
type Measurer struct {
// Config contains the experiment settings. If empty we
// will be using default settings.
Config Config
// Getter is an optional getter to be used for testing.
Getter urlgetter.MultiGetter
}
// ExperimentName implements ExperimentMeasurer.ExperimentName
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// Run implements ExperimentMeasurer.Run
func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error {
sess := args.Session
measurement := args.Measurement
log := sess.Logger()
trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved)
config, err := config(measurement.Input)
if err != nil {
// Invalid input data, we don't even generate report
return err
}
tk := new(TestKeys)
measurement.TestKeys = tk
ctx, cancel := context.WithTimeout(ctx, 30*time.Second)
defer cancel()
tlsconfig := tls.Config{
InsecureSkipVerify: false,
ServerName: config.host,
}
runner := &tcprunner.TCPRunner{
Tk: tk,
Trace: trace,
Logger: log,
Ctx: ctx,
Tlsconfig: &tlsconfig,
}
// First resolve DNS
addrs, success := runner.Resolve(config.host)
if !success {
return nil
}
for _, addr := range addrs {
tcpSession, success := runner.Conn(addr, config.port)
if !success {
continue
}
defer tcpSession.Close()
if config.forcedTLS {
log.Infof("Running direct TLS mode to %s:%s", addr, config.port)
if !tcpSession.Handshake() {
continue
}
// Try EHLO + NoOps
if !testSMTP(tcpSession, "localhost", config.noopCount) {
continue
}
} else {
log.Infof("Running StartTLS mode to %s:%s", addr, config.port)
if !testSMTP(tcpSession, "localhost", 0) {
continue
}
// Upgrade via StartTLS and try EHLO + NoOps
if !tcpSession.StartTLS("STARTTLS\n", "TLS") {
continue
}
if !testSMTP(tcpSession, "localhost", config.noopCount) {
continue
}
}
}
return nil
}
func testSMTP(s *tcprunner.TCPSession, ehlo string, noop uint8) bool {
// Auto-choose plaintext/TCP session
// TODO: move to Debugf
s.Runner.Logger.Infof("Retrieving existing connection")
conn := s.CurrentConn()
s.Runner.Logger.Infof("Initializing SMTP client")
client, err := smtp.NewClient(conn, ehlo)
if err != nil {
s.FailedStep(*tracex.NewFailure(err), "smtp_init")
return false
}
s.Runner.Logger.Infof("Starting SMTP EHLO")
err = client.Hello(ehlo)
if err != nil {
if s.TLS {
s.FailedStep(*tracex.NewFailure(err), "smtp_tls_ehlo")
} else {
s.FailedStep(*tracex.NewFailure(err), "smtp_plaintext_ehlo")
}
return false
}
s.Runner.Logger.Infof("Finished SMTP EHLO")
if noop > 0 {
// Downcast TCPSession's itk into typed IndividualTestKeys to access noopCounter field
concreteITK := s.Itk.(*IndividualTestKeys)
s.Runner.Logger.Infof("Trying to generate more no-op traffic")
concreteITK.noopCounter = 0
for concreteITK.noopCounter < noop {
concreteITK.noopCounter++
s.Runner.Logger.Infof("NoOp Iteration %d", concreteITK.noopCounter)
err = client.Noop()
if err != nil {
s.FailedStep(*tracex.NewFailure(err), fmt.Sprintf("smtp_noop_%d", concreteITK.noopCounter))
break
}
}
if concreteITK.noopCounter == noop {
s.Runner.Logger.Infof("Successfully generated no-op traffic")
return true
}
s.Runner.Logger.Warnf("Failed no-op traffic at iteration %d", concreteITK.noopCounter)
return false
}
return true
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with ooniprobe
// therefore we should be careful when changing it.
type SummaryKeys struct {
//DNSBlocking bool `json:"facebook_dns_blocking"`
//TCPBlocking bool `json:"facebook_tcp_blocking"`
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
sk := SummaryKeys{IsAnomaly: false}
_, ok := measurement.TestKeys.(*TestKeys)
if !ok {
return sk, errors.New("invalid test keys type")
}
return sk, nil
}

186
internal/engine/experiment/smtp/smtp_test.go Normal file
View File

@ -0,0 +1,186 @@
package smtp
import (
"bufio"
"context"
"crypto/tls"
"errors"
"fmt"
"net"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/mockable"
"github.com/ooni/probe-cli/v3/internal/model"
)
func plaintextListener() net.Listener {
l, err := net.Listen("tcp", "127.0.0.1:0")
if err != nil {
if l, err = net.Listen("tcp6", "[::1]:0"); err != nil {
panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err))
}
}
return l
}
func tlsListener(l net.Listener) net.Listener {
return tls.NewListener(l, &tls.Config{})
}
func listenerAddr(l net.Listener) string {
return l.Addr().String()
}
func ValidSMTPServer(conn net.Conn) {
for {
command, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
return
}
if command == "" {
} else if command == "NOOP" {
conn.Write([]byte("250 2.0.0 Ok\n"))
} else if command == "STARTTLS" {
conn.Write([]byte("220 2.0.0 Ready to start TLS\n"))
// TODO: conn.Close does not actually close connection? or does client not detect it?
conn.Close()
return
} else if strings.HasPrefix(command, "EHLO") {
conn.Write([]byte("250 mock.example.com\n"))
}
conn.Write([]byte("\n"))
}
}
func TCPServer(l net.Listener) {
for {
conn, err := l.Accept()
if err != nil {
continue
}
defer conn.Close()
conn.Write([]byte("220 mock.example.com ESMTP (spam is not appreciated)\n"))
ValidSMTPServer(conn)
}
}
func TestMeasurer_run(t *testing.T) {
// runHelper is an helper function to run this set of tests.
runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) {
m := NewExperimentMeasurer(Config{})
if m.ExperimentName() != "smtp" {
t.Fatal("invalid experiment name")
}
if m.ExperimentVersion() != "0.0.1" {
t.Fatal("invalid experiment version")
}
ctx := context.Background()
meas := &model.Measurement{
Input: model.MeasurementTarget(input),
}
sess := &mockable.Session{
MockableLogger: model.DiscardLogger,
}
args := &model.ExperimentArgs{
Callbacks: model.NewPrinterCallbacks(model.DiscardLogger),
Measurement: meas,
Session: sess,
}
err := m.Run(ctx, args)
return meas, m, err
}
t.Run("with empty input", func(t *testing.T) {
_, _, err := runHelper("")
if !errors.Is(err, errNoInputProvided) {
t.Fatal("unexpected error", err)
}
})
t.Run("with invalid URL", func(t *testing.T) {
_, _, err := runHelper("\t")
if !errors.Is(err, errInputIsNotAnURL) {
t.Fatal("unexpected error", err)
}
})
t.Run("with invalid scheme", func(t *testing.T) {
_, _, err := runHelper("https://8.8.8.8:443/")
if !errors.Is(err, errInvalidScheme) {
t.Fatal("unexpected error", err)
}
})
t.Run("with broken TLS", func(t *testing.T) {
p := plaintextListener()
defer p.Close()
l := tlsListener(p)
defer l.Close()
addr := listenerAddr(l)
go TCPServer(l)
meas, m, err := runHelper("smtps://" + addr)
if err != nil {
t.Fatal(err)
}
tk := meas.TestKeys.(*TestKeys)
for _, run := range tk.Runs {
if *run.TLSHandshake.Failure != "unknown_failure: remote error: tls: unrecognized name" {
t.Fatal("expected unrecognized_name in TLS handshake")
}
if run.noopCounter != 0 {
t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
}
}
ask, err := m.GetSummaryKeys(meas)
if err != nil {
t.Fatal("cannot obtain summary")
}
summary := ask.(SummaryKeys)
if summary.IsAnomaly {
t.Fatal("expected no anomaly")
}
})
t.Run("with broken starttls", func(t *testing.T) {
l := plaintextListener()
defer l.Close()
addr := listenerAddr(l)
go TCPServer(l)
meas, m, err := runHelper("smtp://" + addr)
if err != nil {
t.Fatal(err)
}
tk := meas.TestKeys.(*TestKeys)
for _, run := range tk.Runs {
if *run.TLSHandshake.Failure != "generic_timeout_error" {
t.Fatal("expected timeout in TLS handshake")
}
if run.noopCounter != 0 {
t.Fatalf("expected to not have any noops, not %d noops", run.noopCounter)
}
}
ask, err := m.GetSummaryKeys(meas)
if err != nil {
t.Fatal("cannot obtain summary")
}
summary := ask.(SummaryKeys)
if summary.IsAnomaly {
t.Fatal("expected no anomaly")
}
})
}

22
internal/registry/imap.go Normal file
View File

@ -0,0 +1,22 @@
package registry
//
// Registers the 'imap' experiment.
//
import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/imap"
"github.com/ooni/probe-cli/v3/internal/model"
)
func init() {
AllExperiments["imap"] = &Factory{
build: func(config interface{}) model.ExperimentMeasurer {
return imap.NewExperimentMeasurer(
*config.(*imap.Config),
)
},
config: &imap.Config{},
inputPolicy: model.InputOrStaticDefault,
}
}

22
internal/registry/smtp.go Normal file
View File

@ -0,0 +1,22 @@
package registry
//
// Registers the 'smtp' experiment.
//
import (
"github.com/ooni/probe-cli/v3/internal/engine/experiment/smtp"
"github.com/ooni/probe-cli/v3/internal/model"
)
func init() {
AllExperiments["smtp"] = &Factory{
build: func(config interface{}) model.ExperimentMeasurer {
return smtp.NewExperimentMeasurer(
*config.(*smtp.Config),
)
},
config: &smtp.Config{},
inputPolicy: model.InputOrStaticDefault,
}
}

192
internal/tcprunner/tcprunner.go Normal file
View File

@ -0,0 +1,192 @@
package tcprunner
import (
"bufio"
"context"
"crypto/tls"
"net"
"strings"
"github.com/ooni/probe-cli/v3/internal/measurexlite"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/tracex"
)
// Model describes a type that does a DNS lookup(s), then attempts several TCP sessions
type Model interface {
// Stores the provided hostname
Hostname(string)
// Store DNS query result
DNSResults([]*model.ArchivalDNSLookupResult)
// Indicates one or more steps failed (can be overwritten)
Failed(string)
// Stores a new individual test key (for a TCP session) and returns a pointer to it
NewRun(string, string) TCPSessionModel
}
// TCPSessionModel describes a type that does a single TCP connection and TLS handshake with a given IP/Port combo
type TCPSessionModel interface {
// Store IP/port address used for this session
IPPort(string, string)
// Store TCP connect result
ConnectResults([]*model.ArchivalTCPConnectResult)
// Store TLS handshake result
HandshakeResult(*model.ArchivalTLSOrQUICHandshakeResult)
// Indicates a failure string, as well as an identifier for the failed step
FailedStep(string, string)
}
// TCPRunner manages sequential TCP sessions to the same hostname (over different IPs)
type TCPRunner struct {
Tk Model
Trace *measurexlite.Trace
Logger model.Logger
Ctx context.Context
Tlsconfig *tls.Config
}
// TCPSession Manages a single TCP session and TLS handshake to a given ip:port
type TCPSession struct {
Itk TCPSessionModel
Runner *TCPRunner
Addr string
Port string
TLS bool
RawConn *net.Conn
TLSConn *net.Conn
}
// FailedStep saves a failure (with an associated failed step identifier) into IndividualTestKeys
func (s *TCPSession) FailedStep(failure string, step string) {
// Save FailedStep inside ITK
s.Itk.FailedStep(failure, step)
// Copy FailedStep to global TK
s.Runner.Tk.Failed(failure)
// Print the warning message
s.Runner.Logger.Warn(failure)
}
// Close closes the open TCP connections
func (s *TCPSession) Close() {
if s.TLS {
var conn = *s.TLSConn
conn.Close()
} else {
// TODO: should raw connection be closed anyway?
var conn = *s.RawConn
conn.Close()
}
}
// CurrentConn returns the currently active connection (TLS or plaintext)
func (s *TCPSession) CurrentConn() net.Conn {
if s.TLS {
// TODO: move to Debugf
s.Runner.Logger.Infof("Reusing TLS connection")
return *s.TLSConn
}
s.Runner.Logger.Infof("Reusing plaintext connection")
return *s.RawConn
}
// Conn initializes a new Run and IndividualTestKeys
func (r *TCPRunner) Conn(addr string, port string) (*TCPSession, bool) {
// Get new individual test keys
itk := r.Tk.NewRun(addr, port)
s := new(TCPSession)
s.Runner = r
s.Itk = itk
s.Addr = addr
s.Port = port
s.TLS = false
if !s.Conn(addr, port) {
return nil, false
}
return s, true
}
// Conn starts a new TCP/IP connection to addr/port
func (s *TCPSession) Conn(addr string, port string) bool {
dialer := s.Runner.Trace.NewDialerWithoutResolver(s.Runner.Logger)
s.Runner.Logger.Infof("Dialing to %s:%s", addr, port)
conn, err := dialer.DialContext(s.Runner.Ctx, "tcp", net.JoinHostPort(addr, port))
s.Itk.ConnectResults(s.Runner.Trace.TCPConnects())
if err != nil {
s.FailedStep(*tracex.NewFailure(err), "tcp_connect")
return false
}
s.RawConn = &conn
return true
}
// Resolve resolves a hostname to a list of addresses
func (r *TCPRunner) Resolve(host string) ([]string, bool) {
r.Logger.Infof("Resolving DNS for %s", host)
resolver := r.Trace.NewStdlibResolver(r.Logger)
addrs, err := resolver.LookupHost(r.Ctx, host)
r.Tk.DNSResults(r.Trace.DNSLookupsFromRoundTrip())
if err != nil {
r.Tk.Failed(*tracex.NewFailure(err))
return []string{}, false
}
r.Logger.Infof("Finished DNS for %s: %v", host, addrs)
return addrs, true
}
// Handshake performs a TLS handshake over the currently active connection
func (s *TCPSession) Handshake() bool {
if s.TLS {
// TLS already initialized...
return true
}
s.Runner.Logger.Infof("Starting TLS handshake with %s:%s", s.Addr, s.Port)
thx := s.Runner.Trace.NewTLSHandshakerStdlib(s.Runner.Logger)
tconn, _, err := thx.Handshake(s.Runner.Ctx, *s.RawConn, s.Runner.Tlsconfig)
s.Itk.HandshakeResult(s.Runner.Trace.FirstTLSHandshakeOrNil())
if err != nil {
s.FailedStep(*tracex.NewFailure(err), "tls_handshake")
return false
}
s.TLS = true
s.TLSConn = &tconn
s.Runner.Logger.Infof("Handshake succeeded")
return true
}
// StartTLS performs a StartTLS exchange by sending a message over the plaintext connection, waiting for a specific
// response, then performing a TLS handshake
func (s *TCPSession) StartTLS(message string, waitForResponse string) bool {
if s.TLS {
s.Runner.Logger.Warn("Requested TCPSession to do StartTLS when TLS is already enabled")
return true
}
if message != "" {
s.Runner.Logger.Infof("Asking for StartTLS upgrade")
s.CurrentConn().Write([]byte(message))
}
if waitForResponse != "" {
s.Runner.Logger.Infof("Waiting for server response containing: %s", waitForResponse)
conn := s.CurrentConn()
for {
line, err := bufio.NewReader(conn).ReadString('\n')
if err != nil {
s.FailedStep(*tracex.NewFailure(err), "starttls_wait_ok")
return false
}
s.Runner.Logger.Debugf("Received: %s", line)
if strings.Contains(line, waitForResponse) {
s.Runner.Logger.Infof("Server is ready for StartTLS")
break
}
}
}
return s.Handshake()
}