c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: interfaces and data types into the model package (#642)

Browse Source 
## Checklist

- [x] I have read the [contribution guidelines](https://github.com/ooni/probe-cli/blob/master/CONTRIBUTING.md)
- [x] reference issue for this pull request: https://github.com/ooni/probe/issues/1885
- [x] related ooni/spec pull request: N/A

Location of the issue tracker: https://github.com/ooni/probe

## Description

This PR contains a set of changes to move important interfaces and data types into the `./internal/model` package.

The criteria for including an interface or data type in here is roughly that the type should be important and used by several packages. We are especially interested to move more interfaces here to increase modularity.

An additional side effect is that, by reading this package, one should be able to understand more quickly how different parts of the codebase interact with each other.

This is what I want to move in `internal/model`:

- [x] most important interfaces from `internal/netxlite`
- [x] everything that was previously part of `internal/engine/model`
- [x] mocks from `internal/netxlite/mocks` should also be moved in here as a subpackage
This commit is contained in:
Simone Basso
2022-01-03 13:53:23 +01:00
committed by GitHub
parent 69aca619ea
commit 273b70bacc
275 changed files with 1281 additions and 1375 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/measurex/dialer.go
+6 -8
View File
@@ -11,29 +11,27 @@ import (
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// Conn is a network connection.
type Conn = net.Conn
// Dialer dials network connections.
type Dialer = netxlite.Dialer
// WrapDialer creates a new dialer that writes events
// into the given WritableDB. The net.Conns created by
// a wrapped dialer also write into the WritableDB.
func (mx *Measurer) WrapDialer(db WritableDB, dialer netxlite.Dialer) Dialer {
func (mx *Measurer) WrapDialer(db WritableDB, dialer model.Dialer) model.Dialer {
return WrapDialer(mx.Begin, db, dialer)
}
// WrapDialer wraps a dialer.
func WrapDialer(begin time.Time, db WritableDB, dialer netxlite.Dialer) Dialer {
func WrapDialer(begin time.Time, db WritableDB, dialer model.Dialer) model.Dialer {
return &dialerDB{Dialer: dialer, db: db, begin: begin}
}
// NewDialerWithSystemResolver creates a
func (mx *Measurer) NewDialerWithSystemResolver(db WritableDB, logger Logger) Dialer {
func (mx *Measurer) NewDialerWithSystemResolver(db WritableDB, logger model.Logger) model.Dialer {
r := mx.NewResolverSystem(db, logger)
return mx.WrapDialer(db, netxlite.NewDialerWithResolver(logger, r))
}
@@ -41,12 +39,12 @@ func (mx *Measurer) NewDialerWithSystemResolver(db WritableDB, logger Logger) Di
// NewDialerWithoutResolver is a convenience factory for creating
// a dialer that saves measurements into the DB and that is not attached
// to any resolver (hence only works when passed IP addresses).
func (mx *Measurer) NewDialerWithoutResolver(db WritableDB, logger Logger) Dialer {
func (mx *Measurer) NewDialerWithoutResolver(db WritableDB, logger model.Logger) model.Dialer {
return mx.WrapDialer(db, netxlite.NewDialerWithoutResolver(logger))
}
type dialerDB struct {
netxlite.Dialer
model.Dialer
begin time.Time
db WritableDB
}
internal/measurex/dnsx.go
+3 -8
View File
@@ -10,22 +10,17 @@ import (
"context"
"time"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/model"
)
// DNSXRoundTripper is a transport for sending raw DNS queries
// and receiving raw DNS replies. The internal/netxlite/dnsx
// package implements a bunch of these transports.
type DNSTransport = netxlite.DNSTransport
// WrapDNSXRoundTripper creates a new DNSXRoundTripper that
// saves events into the given WritableDB.
func (mx *Measurer) WrapDNSXRoundTripper(db WritableDB, rtx netxlite.DNSTransport) DNSTransport {
func (mx *Measurer) WrapDNSXRoundTripper(db WritableDB, rtx model.DNSTransport) model.DNSTransport {
return &dnsxRoundTripperDB{db: db, DNSTransport: rtx, begin: mx.Begin}
}
type dnsxRoundTripperDB struct {
netxlite.DNSTransport
model.DNSTransport
begin time.Time
db WritableDB
}
internal/measurex/http.go
+10 -12
View File
@@ -27,18 +27,16 @@ import (
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/runtimex"
"golang.org/x/net/publicsuffix"
)
// HTTPTransport is the HTTP transport type we use.
type HTTPTransport = netxlite.HTTPTransport
// WrapHTTPTransport creates a new transport that saves
// HTTP events into the WritableDB.
func (mx *Measurer) WrapHTTPTransport(
db WritableDB, txp HTTPTransport) *HTTPTransportDB {
db WritableDB, txp model.HTTPTransport) *HTTPTransportDB {
return WrapHTTPTransport(mx.Begin, db, txp)
}
@@ -48,7 +46,7 @@ func (mx *Measurer) WrapHTTPTransport(
const httpMaxBodySnapshot = 1 << 11
func WrapHTTPTransport(
begin time.Time, db WritableDB, txp HTTPTransport) *HTTPTransportDB {
begin time.Time, db WritableDB, txp model.HTTPTransport) *HTTPTransportDB {
return &HTTPTransportDB{
HTTPTransport: txp,
Begin: begin,
@@ -60,7 +58,7 @@ func WrapHTTPTransport(
// NewHTTPTransportWithConn creates and wraps an HTTPTransport that
// does not dial and only uses the given conn.
func (mx *Measurer) NewHTTPTransportWithConn(
logger Logger, db WritableDB, conn Conn) *HTTPTransportDB {
logger model.Logger, db WritableDB, conn Conn) *HTTPTransportDB {
return mx.WrapHTTPTransport(db, netxlite.NewHTTPTransport(
logger, netxlite.NewSingleUseDialer(conn), netxlite.NewNullTLSDialer()))
}
@@ -68,7 +66,7 @@ func (mx *Measurer) NewHTTPTransportWithConn(
// NewHTTPTransportWithTLSConn creates and wraps an HTTPTransport that
// does not dial and only uses the given conn.
func (mx *Measurer) NewHTTPTransportWithTLSConn(
logger Logger, db WritableDB, conn netxlite.TLSConn) *HTTPTransportDB {
logger model.Logger, db WritableDB, conn netxlite.TLSConn) *HTTPTransportDB {
return mx.WrapHTTPTransport(db, netxlite.NewHTTPTransport(
logger, netxlite.NewNullDialer(), netxlite.NewSingleUseTLSDialer(conn)))
}
@@ -76,7 +74,7 @@ func (mx *Measurer) NewHTTPTransportWithTLSConn(
// NewHTTPTransportWithQUICSess creates and wraps an HTTPTransport that
// does not dial and only uses the given QUIC session.
func (mx *Measurer) NewHTTPTransportWithQUICSess(
logger Logger, db WritableDB, sess quic.EarlySession) *HTTPTransportDB {
logger model.Logger, db WritableDB, sess quic.EarlySession) *HTTPTransportDB {
return mx.WrapHTTPTransport(db, netxlite.NewHTTP3Transport(
logger, netxlite.NewSingleUseQUICDialer(sess), &tls.Config{}))
}
@@ -88,7 +86,7 @@ func (mx *Measurer) NewHTTPTransportWithQUICSess(
// you can construct it manually. In which case, do not modify
// public fields during usage, since this may cause a data race.
type HTTPTransportDB struct {
netxlite.HTTPTransport
model.HTTPTransport
// Begin is when we started measuring.
Begin time.Time
@@ -206,14 +204,14 @@ type HTTPClient interface {
// NewHTTPClient creates a new HTTPClient instance that
// does not automatically perform redirects.
func NewHTTPClientWithoutRedirects(
db WritableDB, jar http.CookieJar, txp HTTPTransport) HTTPClient {
db WritableDB, jar http.CookieJar, txp model.HTTPTransport) HTTPClient {
return newHTTPClient(db, jar, txp, http.ErrUseLastResponse)
}
// NewHTTPClientWithRedirects creates a new HTTPClient
// instance that automatically perform redirects.
func NewHTTPClientWithRedirects(
db WritableDB, jar http.CookieJar, txp HTTPTransport) HTTPClient {
db WritableDB, jar http.CookieJar, txp model.HTTPTransport) HTTPClient {
return newHTTPClient(db, jar, txp, nil)
}
@@ -243,7 +241,7 @@ type HTTPRedirectEvent struct {
var ErrHTTPTooManyRedirects = errors.New("stopped after 10 redirects")
func newHTTPClient(db WritableDB, cookiejar http.CookieJar,
txp HTTPTransport, defaultErr error) HTTPClient {
txp model.HTTPTransport, defaultErr error) HTTPClient {
return netxlite.WrapHTTPClient(&http.Client{
Transport: txp,
Jar: cookiejar,
internal/measurex/logger.go
+3 -15
View File
@@ -11,24 +11,12 @@ import (
"sync"
"time"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/model"
)
// Logger is the logger type we use. This type is compatible
// with the logger type of github.com/apex/log.
type Logger interface {
netxlite.Logger
Info(msg string)
Infof(format string, v ...interface{})
Warn(msg string)
Warnf(format string, v ...interface{})
}
// NewOperationLogger creates a new logger that logs
// about an in-progress operation.
func NewOperationLogger(logger Logger, format string, v ...interface{}) *OperationLogger {
func NewOperationLogger(logger model.Logger, format string, v ...interface{}) *OperationLogger {
ol := &OperationLogger{
sighup: make(chan interface{}),
logger: logger,
@@ -43,7 +31,7 @@ func NewOperationLogger(logger Logger, format string, v ...interface{}) *Operati
// OperationLogger logs about an in-progress operation
type OperationLogger struct {
logger Logger
logger model.Logger
message string
once *sync.Once
sighup chan interface{}
internal/measurex/measurer.go
+6 -5
View File
@@ -21,6 +21,7 @@ import (
"github.com/apex/log"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
@@ -34,7 +35,7 @@ type Measurer struct {
HTTPClient HTTPClient
// Logger is the MANDATORY logger to use.
Logger Logger
Logger model.Logger
// MeasureURLHelper is the OPTIONAL test helper to use when
// we're measuring using the MeasureURL function. If this field
@@ -45,7 +46,7 @@ type Measurer struct {
Resolvers []*ResolverInfo
// TLSHandshaker is the MANDATORY TLS handshaker.
TLSHandshaker netxlite.TLSHandshaker
TLSHandshaker model.TLSHandshaker
}
// NewMeasurerWithDefaultSettings creates a new Measurer
@@ -85,7 +86,7 @@ func (mx *Measurer) LookupHostSystem(ctx context.Context, domain string) *DNSMea
// lookupHostForeign performs a LookupHost using a "foreign" resolver.
func (mx *Measurer) lookupHostForeign(
ctx context.Context, domain string, r Resolver) *DNSMeasurement {
ctx context.Context, domain string, r model.Resolver) *DNSMeasurement {
const timeout = 4 * time.Second
ol := NewOperationLogger(mx.Logger, "LookupHost %s with %s", domain, r.Network())
ctx, cancel := context.WithTimeout(ctx, timeout)
@@ -158,7 +159,7 @@ func (mx *Measurer) LookupHTTPSSvcUDP(
// lookupHTTPSSvcUDPForeign is like LookupHTTPSSvcUDP
// except that it uses a "foreign" resolver.
func (mx *Measurer) lookupHTTPSSvcUDPForeign(
ctx context.Context, domain string, r Resolver) *DNSMeasurement {
ctx context.Context, domain string, r model.Resolver) *DNSMeasurement {
const timeout = 4 * time.Second
ol := NewOperationLogger(mx.Logger, "LookupHTTPSvc %s with %s", domain, r.Address())
ctx, cancel := context.WithTimeout(ctx, timeout)
@@ -591,7 +592,7 @@ type ResolverInfo struct {
// ForeignResolver is only used when Network's
// value equals the ResolverForeign constant.
ForeignResolver Resolver
ForeignResolver model.Resolver
}
// LookupURLHostParallel performs an LookupHost-like operation for each
internal/measurex/quic.go
+7 -16
View File
@@ -13,26 +13,17 @@ import (
"time"
"github.com/lucas-clemente/quic-go"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/netxlite/quicx"
)
// QUICConn is the kind of conn used by QUIC.
type QUICConn = quicx.UDPLikeConn
// QUICDialer creates QUICSesssions.
type QUICDialer = netxlite.QUICDialer
// QUICListener creates listening connections for QUIC.
type QUICListener = netxlite.QUICListener
type quicListenerDB struct {
netxlite.QUICListener
model.QUICListener
begin time.Time
db WritableDB
}
func (ql *quicListenerDB) Listen(addr *net.UDPAddr) (QUICConn, error) {
func (ql *quicListenerDB) Listen(addr *net.UDPAddr) (model.UDPLikeConn, error) {
pconn, err := ql.QUICListener.Listen(addr)
if err != nil {
return nil, err
@@ -45,7 +36,7 @@ func (ql *quicListenerDB) Listen(addr *net.UDPAddr) (QUICConn, error) {
}
type udpLikeConnDB struct {
quicx.UDPLikeConn
model.UDPLikeConn
begin time.Time
db WritableDB
}
@@ -103,15 +94,15 @@ func (c *udpLikeConnDB) Close() error {
// address containing a domain name will fail. This QUICDialer will
// save any event into the WritableDB. Any QUICConn created by it will
// likewise save any event into the WritableDB.
func (mx *Measurer) NewQUICDialerWithoutResolver(db WritableDB, logger Logger) QUICDialer {
func (mx *Measurer) NewQUICDialerWithoutResolver(db WritableDB, logger model.Logger) model.QUICDialer {
return &quicDialerDB{db: db, logger: logger, begin: mx.Begin}
}
type quicDialerDB struct {
netxlite.QUICDialer
model.QUICDialer
begin time.Time
db WritableDB
logger Logger
logger model.Logger
}
func (qh *quicDialerDB) DialContext(ctx context.Context, network, address string,
internal/measurex/resolver.go
+8 -15
View File
@@ -11,30 +11,23 @@ import (
"strings"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
"github.com/ooni/probe-cli/v3/internal/netxlite/dnsx"
)
// HTTPSSvc is the result returned by HTTPSSvc queries.
type HTTPSSvc = dnsx.HTTPSSvc
// Resolver is the resolver type we use. This resolver will
// store resolve events into the DB.
type Resolver = netxlite.Resolver
// WrapResolver creates a new Resolver that saves events into the WritableDB.
func (mx *Measurer) WrapResolver(db WritableDB, r netxlite.Resolver) Resolver {
func (mx *Measurer) WrapResolver(db WritableDB, r model.Resolver) model.Resolver {
return WrapResolver(mx.Begin, db, r)
}
// WrapResolver wraps a resolver.
func WrapResolver(begin time.Time, db WritableDB, r netxlite.Resolver) Resolver {
func WrapResolver(begin time.Time, db WritableDB, r model.Resolver) model.Resolver {
return &resolverDB{Resolver: r, db: db, begin: begin}
}
// NewResolverSystem creates a system resolver and then wraps
// it using the WrapResolver function/
func (mx *Measurer) NewResolverSystem(db WritableDB, logger Logger) Resolver {
func (mx *Measurer) NewResolverSystem(db WritableDB, logger model.Logger) model.Resolver {
return mx.WrapResolver(db, netxlite.NewResolverStdlib(logger))
}
@@ -48,7 +41,7 @@ func (mx *Measurer) NewResolverSystem(db WritableDB, logger Logger) Resolver {
// - logger is the logger;
//
// - address is the resolver address (e.g., "1.1.1.1:53").
func (mx *Measurer) NewResolverUDP(db WritableDB, logger Logger, address string) Resolver {
func (mx *Measurer) NewResolverUDP(db WritableDB, logger model.Logger, address string) model.Resolver {
return mx.WrapResolver(db, netxlite.WrapResolver(
logger, netxlite.NewSerialResolver(
mx.WrapDNSXRoundTripper(db, netxlite.NewDNSOverUDP(
@@ -59,7 +52,7 @@ func (mx *Measurer) NewResolverUDP(db WritableDB, logger Logger, address string)
}
type resolverDB struct {
netxlite.Resolver
model.Resolver
begin time.Time
db WritableDB
}
@@ -160,7 +153,7 @@ func (r *resolverDB) computeOddityLookupHost(addrs []string, err error) Oddity {
return ""
}
func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*HTTPSSvc, error) {
func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*model.HTTPSSvc, error) {
started := time.Since(r.begin).Seconds()
https, err := r.Resolver.LookupHTTPS(ctx, domain)
finished := time.Since(r.begin).Seconds()
@@ -183,7 +176,7 @@ func (r *resolverDB) LookupHTTPS(ctx context.Context, domain string) (*HTTPSSvc,
return https, err
}
func (r *resolverDB) computeOddityHTTPSSvc(https *HTTPSSvc, err error) Oddity {
func (r *resolverDB) computeOddityHTTPSSvc(https *model.HTTPSSvc, err error) Oddity {
if err != nil {
return r.computeOddityLookupHost(nil, err)
}
internal/measurex/tls.go
+4 -6
View File
@@ -14,26 +14,24 @@ import (
"net"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
// TLSHandshaker performs TLS handshakes.
type TLSHandshaker = netxlite.TLSHandshaker
// WrapTLSHandshaker wraps a netxlite.TLSHandshaker to return a new
// instance of TLSHandshaker that saves events into the DB.
func (mx *Measurer) WrapTLSHandshaker(db WritableDB, thx netxlite.TLSHandshaker) TLSHandshaker {
func (mx *Measurer) WrapTLSHandshaker(db WritableDB, thx model.TLSHandshaker) model.TLSHandshaker {
return &tlsHandshakerDB{TLSHandshaker: thx, db: db, begin: mx.Begin}
}
// NewTLSHandshakerStdlib creates a new TLS handshaker that
// saves results into the DB and uses the stdlib for TLS.
func (mx *Measurer) NewTLSHandshakerStdlib(db WritableDB, logger Logger) TLSHandshaker {
func (mx *Measurer) NewTLSHandshakerStdlib(db WritableDB, logger model.Logger) model.TLSHandshaker {
return mx.WrapTLSHandshaker(db, netxlite.NewTLSHandshakerStdlib(logger))
}
type tlsHandshakerDB struct {
netxlite.TLSHandshaker
model.TLSHandshaker
begin time.Time
db WritableDB
}
internal/measurex/tracing.go
+5 -4
View File
@@ -5,6 +5,7 @@ import (
"net/url"
"time"
"github.com/ooni/probe-cli/v3/internal/model"
"github.com/ooni/probe-cli/v3/internal/netxlite"
)
@@ -25,8 +26,8 @@ import (
// - resolver is the underlying resolver to use
//
// - handshake is the TLS handshaker to use
func NewTracingHTTPTransport(logger Logger, begin time.Time, db WritableDB,
resolver Resolver, dialer Dialer, handshaker TLSHandshaker) *HTTPTransportDB {
func NewTracingHTTPTransport(logger model.Logger, begin time.Time, db WritableDB,
resolver model.Resolver, dialer model.Dialer, handshaker model.TLSHandshaker) *HTTPTransportDB {
resolver = WrapResolver(begin, db, resolver)
dialer = netxlite.WrapDialer(logger, resolver, WrapDialer(begin, db, dialer))
tlsDialer := netxlite.NewTLSDialer(dialer, handshaker)
@@ -47,7 +48,7 @@ func NewTracingHTTPTransport(logger Logger, begin time.Time, db WritableDB,
// eventually become the measurement
//
func NewTracingHTTPTransportWithDefaultSettings(
begin time.Time, logger Logger, db WritableDB) *HTTPTransportDB {
begin time.Time, logger model.Logger, db WritableDB) *HTTPTransportDB {
return NewTracingHTTPTransport(logger, begin, db,
netxlite.NewResolverStdlib(logger),
netxlite.NewDialerWithoutResolver(logger),
@@ -55,7 +56,7 @@ func NewTracingHTTPTransportWithDefaultSettings(
}
func (mx *Measurer) NewTracingHTTPTransportWithDefaultSettings(
logger Logger, db WritableDB) *HTTPTransportDB {
logger model.Logger, db WritableDB) *HTTPTransportDB {
return NewTracingHTTPTransport(
mx.Logger, mx.Begin, db, mx.NewResolverSystem(db, mx.Logger),
mx.NewDialerWithoutResolver(db, mx.Logger),