c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse Source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/experiment/urlgetter/.gitignore
+1
View File
@@ -0,0 +1 @@
/urlgetter-tunnel
internal/engine/experiment/urlgetter/configurer.go
+102
View File
@@ -0,0 +1,102 @@
package urlgetter
import (
"crypto/tls"
"errors"
"net"
"net/url"
"regexp"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
// The Configurer job is to construct a Configuration that can
// later be used by the measurer to perform measurements.
type Configurer struct {
Config Config
Logger model.Logger
ProxyURL *url.URL
Saver *trace.Saver
}
// The Configuration is the configuration for running a measurement.
type Configuration struct {
HTTPConfig netx.Config
DNSClient netx.DNSClient
}
// CloseIdleConnections will close idle connections, if needed.
func (c Configuration) CloseIdleConnections() {
c.DNSClient.CloseIdleConnections()
}
// NewConfiguration builds a new measurement configuration.
func (c Configurer) NewConfiguration() (Configuration, error) {
// set up defaults
configuration := Configuration{
HTTPConfig: netx.Config{
BogonIsError: c.Config.RejectDNSBogons,
CacheResolutions: true,
CertPool: c.Config.CertPool,
ContextByteCounting: true,
DialSaver: c.Saver,
HTTP3Enabled: c.Config.HTTP3Enabled,
HTTPSaver: c.Saver,
Logger: c.Logger,
ReadWriteSaver: c.Saver,
ResolveSaver: c.Saver,
TLSSaver: c.Saver,
},
}
// fill DNS cache
if c.Config.DNSCache != "" {
entry := strings.Split(c.Config.DNSCache, " ")
if len(entry) < 2 {
return configuration, errors.New("invalid DNSCache string")
}
domainregex := regexp.MustCompile(`^([a-z0-9]+(-[a-z0-9]+)*\.)+[a-z]{2,}$`)
if !domainregex.MatchString(entry[0]) {
return configuration, errors.New("invalid domain in DNSCache")
}
var addresses []string
for i := 1; i < len(entry); i++ {
if net.ParseIP(entry[i]) == nil {
return configuration, errors.New("invalid IP in DNSCache")
}
addresses = append(addresses, entry[i])
}
configuration.HTTPConfig.DNSCache = map[string][]string{
entry[0]: addresses,
}
}
dnsclient, err := netx.NewDNSClientWithOverrides(
configuration.HTTPConfig, c.Config.ResolverURL,
c.Config.DNSHTTPHost, c.Config.DNSTLSServerName,
c.Config.DNSTLSVersion,
)
if err != nil {
return configuration, err
}
configuration.DNSClient = dnsclient
configuration.HTTPConfig.BaseResolver = dnsclient.Resolver
// configure TLS
configuration.HTTPConfig.TLSConfig = &tls.Config{
NextProtos: []string{"h2", "http/1.1"},
}
if c.Config.TLSServerName != "" {
configuration.HTTPConfig.TLSConfig.ServerName = c.Config.TLSServerName
}
err = netx.ConfigureTLSVersion(
configuration.HTTPConfig.TLSConfig, c.Config.TLSVersion,
)
if err != nil {
return configuration, err
}
configuration.HTTPConfig.NoTLSVerify = c.Config.NoTLSVerify
// configure proxy
configuration.HTTPConfig.ProxyURL = c.ProxyURL
return configuration, nil
}
internal/engine/experiment/urlgetter/configurer_test.go
+734
View File
@@ -0,0 +1,734 @@
package urlgetter_test
import (
"crypto/tls"
"errors"
"net/url"
"strings"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
func TestConfigurerNewConfigurationVanilla(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
defer configuration.CloseIdleConnections()
if configuration.HTTPConfig.BogonIsError != false {
t.Fatal("not the BogonIsError we expected")
}
if configuration.HTTPConfig.CacheResolutions != true {
t.Fatal("not the CacheResolutions we expected")
}
if configuration.HTTPConfig.ContextByteCounting != true {
t.Fatal("not the ContextByteCounting we expected")
}
if configuration.HTTPConfig.DialSaver != saver {
t.Fatal("not the DialSaver we expected")
}
if configuration.HTTPConfig.HTTPSaver != saver {
t.Fatal("not the HTTPSaver we expected")
}
if configuration.HTTPConfig.Logger != log.Log {
t.Fatal("not the Logger we expected")
}
if configuration.HTTPConfig.ReadWriteSaver != saver {
t.Fatal("not the ReadWriteSaver we expected")
}
if configuration.HTTPConfig.ResolveSaver != saver {
t.Fatal("not the ResolveSaver we expected")
}
if configuration.HTTPConfig.TLSSaver != saver {
t.Fatal("not the TLSSaver we expected")
}
if configuration.HTTPConfig.BaseResolver == nil {
t.Fatal("not the BaseResolver we expected")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.NoTLSVerify == true {
t.Fatal("not the NoTLSVerify we expected")
}
if configuration.HTTPConfig.ProxyURL != nil {
t.Fatal("not the ProxyURL we expected")
}
}
func TestConfigurerNewConfigurationResolverDNSOverHTTPSPowerdns(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "doh://google",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
defer configuration.CloseIdleConnections()
if configuration.HTTPConfig.BogonIsError != false {
t.Fatal("not the BogonIsError we expected")
}
if configuration.HTTPConfig.CacheResolutions != true {
t.Fatal("not the CacheResolutions we expected")
}
if configuration.HTTPConfig.ContextByteCounting != true {
t.Fatal("not the ContextByteCounting we expected")
}
if configuration.HTTPConfig.DialSaver != saver {
t.Fatal("not the DialSaver we expected")
}
if configuration.HTTPConfig.HTTPSaver != saver {
t.Fatal("not the HTTPSaver we expected")
}
if configuration.HTTPConfig.Logger != log.Log {
t.Fatal("not the Logger we expected")
}
if configuration.HTTPConfig.ReadWriteSaver != saver {
t.Fatal("not the ReadWriteSaver we expected")
}
if configuration.HTTPConfig.ResolveSaver != saver {
t.Fatal("not the ResolveSaver we expected")
}
if configuration.HTTPConfig.TLSSaver != saver {
t.Fatal("not the TLSSaver we expected")
}
if configuration.HTTPConfig.BaseResolver == nil {
t.Fatal("not the BaseResolver we expected")
}
sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
if !ok {
t.Fatal("not the resolver we expected")
}
stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
if !ok {
t.Fatal("not the DNS transport we expected")
}
dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
if !ok {
t.Fatal("not the DNS transport we expected")
}
if dohtxp.URL != "https://dns.google/dns-query" {
t.Fatal("not the DoH URL we expected")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.NoTLSVerify == true {
t.Fatal("not the NoTLSVerify we expected")
}
if configuration.HTTPConfig.ProxyURL != nil {
t.Fatal("not the ProxyURL we expected")
}
}
func TestConfigurerNewConfigurationResolverDNSOverHTTPSGoogle(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "doh://google",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
defer configuration.CloseIdleConnections()
if configuration.HTTPConfig.BogonIsError != false {
t.Fatal("not the BogonIsError we expected")
}
if configuration.HTTPConfig.CacheResolutions != true {
t.Fatal("not the CacheResolutions we expected")
}
if configuration.HTTPConfig.ContextByteCounting != true {
t.Fatal("not the ContextByteCounting we expected")
}
if configuration.HTTPConfig.DialSaver != saver {
t.Fatal("not the DialSaver we expected")
}
if configuration.HTTPConfig.HTTPSaver != saver {
t.Fatal("not the HTTPSaver we expected")
}
if configuration.HTTPConfig.Logger != log.Log {
t.Fatal("not the Logger we expected")
}
if configuration.HTTPConfig.ReadWriteSaver != saver {
t.Fatal("not the ReadWriteSaver we expected")
}
if configuration.HTTPConfig.ResolveSaver != saver {
t.Fatal("not the ResolveSaver we expected")
}
if configuration.HTTPConfig.TLSSaver != saver {
t.Fatal("not the TLSSaver we expected")
}
if configuration.HTTPConfig.BaseResolver == nil {
t.Fatal("not the BaseResolver we expected")
}
sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
if !ok {
t.Fatal("not the resolver we expected")
}
stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
if !ok {
t.Fatal("not the DNS transport we expected")
}
dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
if !ok {
t.Fatal("not the DNS transport we expected")
}
if dohtxp.URL != "https://dns.google/dns-query" {
t.Fatal("not the DoH URL we expected")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.NoTLSVerify == true {
t.Fatal("not the NoTLSVerify we expected")
}
if configuration.HTTPConfig.ProxyURL != nil {
t.Fatal("not the ProxyURL we expected")
}
}
func TestConfigurerNewConfigurationResolverDNSOverHTTPSCloudflare(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "doh://cloudflare",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
defer configuration.CloseIdleConnections()
if configuration.HTTPConfig.BogonIsError != false {
t.Fatal("not the BogonIsError we expected")
}
if configuration.HTTPConfig.CacheResolutions != true {
t.Fatal("not the CacheResolutions we expected")
}
if configuration.HTTPConfig.ContextByteCounting != true {
t.Fatal("not the ContextByteCounting we expected")
}
if configuration.HTTPConfig.DialSaver != saver {
t.Fatal("not the DialSaver we expected")
}
if configuration.HTTPConfig.HTTPSaver != saver {
t.Fatal("not the HTTPSaver we expected")
}
if configuration.HTTPConfig.Logger != log.Log {
t.Fatal("not the Logger we expected")
}
if configuration.HTTPConfig.ReadWriteSaver != saver {
t.Fatal("not the ReadWriteSaver we expected")
}
if configuration.HTTPConfig.ResolveSaver != saver {
t.Fatal("not the ResolveSaver we expected")
}
if configuration.HTTPConfig.TLSSaver != saver {
t.Fatal("not the TLSSaver we expected")
}
if configuration.HTTPConfig.BaseResolver == nil {
t.Fatal("not the BaseResolver we expected")
}
sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
if !ok {
t.Fatal("not the resolver we expected")
}
stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
if !ok {
t.Fatal("not the DNS transport we expected")
}
dohtxp, ok := stxp.RoundTripper.(resolver.DNSOverHTTPS)
if !ok {
t.Fatal("not the DNS transport we expected")
}
if dohtxp.URL != "https://cloudflare-dns.com/dns-query" {
t.Fatal("not the DoH URL we expected")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.NoTLSVerify == true {
t.Fatal("not the NoTLSVerify we expected")
}
if configuration.HTTPConfig.ProxyURL != nil {
t.Fatal("not the ProxyURL we expected")
}
}
func TestConfigurerNewConfigurationResolverUDP(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "udp://8.8.8.8:53",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
defer configuration.CloseIdleConnections()
if configuration.HTTPConfig.BogonIsError != false {
t.Fatal("not the BogonIsError we expected")
}
if configuration.HTTPConfig.CacheResolutions != true {
t.Fatal("not the CacheResolutions we expected")
}
if configuration.HTTPConfig.ContextByteCounting != true {
t.Fatal("not the ContextByteCounting we expected")
}
if configuration.HTTPConfig.DialSaver != saver {
t.Fatal("not the DialSaver we expected")
}
if configuration.HTTPConfig.HTTPSaver != saver {
t.Fatal("not the HTTPSaver we expected")
}
if configuration.HTTPConfig.Logger != log.Log {
t.Fatal("not the Logger we expected")
}
if configuration.HTTPConfig.ReadWriteSaver != saver {
t.Fatal("not the ReadWriteSaver we expected")
}
if configuration.HTTPConfig.ResolveSaver != saver {
t.Fatal("not the ResolveSaver we expected")
}
if configuration.HTTPConfig.TLSSaver != saver {
t.Fatal("not the TLSSaver we expected")
}
if configuration.HTTPConfig.BaseResolver == nil {
t.Fatal("not the BaseResolver we expected")
}
sr, ok := configuration.HTTPConfig.BaseResolver.(resolver.SerialResolver)
if !ok {
t.Fatal("not the resolver we expected")
}
stxp, ok := sr.Txp.(resolver.SaverDNSTransport)
if !ok {
t.Fatal("not the DNS transport we expected")
}
udptxp, ok := stxp.RoundTripper.(resolver.DNSOverUDP)
if !ok {
t.Fatal("not the DNS transport we expected")
}
if udptxp.Address() != "8.8.8.8:53" {
t.Fatal("not the DoH URL we expected")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("not the TLSConfig we expected")
}
if configuration.HTTPConfig.NoTLSVerify == true {
t.Fatal("not the NoTLSVerify we expected")
}
if configuration.HTTPConfig.ProxyURL != nil {
t.Fatal("not the ProxyURL we expected")
}
}
func TestConfigurerNewConfigurationDNSCacheInvalidString(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
DNSCache: "a",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if err == nil || !strings.HasSuffix(err.Error(), "invalid DNSCache string") {
t.Fatal("not the error we expected")
}
}
func TestConfigurerNewConfigurationDNSCacheNotDomain(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
DNSCache: "b b",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if err == nil || !strings.HasSuffix(err.Error(), "invalid domain in DNSCache") {
t.Fatal("not the error we expected")
}
}
func TestConfigurerNewConfigurationDNSCacheNotIP(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
DNSCache: "x.org b",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if err == nil || !strings.HasSuffix(err.Error(), "invalid IP in DNSCache") {
t.Fatal("not the error we expected")
}
}
func TestConfigurerNewConfigurationDNSCacheGood(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
DNSCache: "dns.google.com 8.8.8.8 8.8.4.4",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.DNSCache) != 1 {
t.Fatal("invalid number of entries in DNSCache")
}
if len(configuration.HTTPConfig.DNSCache["dns.google.com"]) != 2 {
t.Fatal("invalid number of IPs saved in DNSCache")
}
if configuration.HTTPConfig.DNSCache["dns.google.com"][0] != "8.8.8.8" {
t.Fatal("invalid IPs saved in DNSCache")
}
if configuration.HTTPConfig.DNSCache["dns.google.com"][1] != "8.8.4.4" {
t.Fatal("invalid IPs saved in DNSCache")
}
}
func TestConfigurerNewConfigurationResolverInvalidURL(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "\t",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
}
func TestConfigurerNewConfigurationResolverInvalidURLScheme(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: "antani://8.8.8.8:53",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if err == nil || !strings.HasSuffix(err.Error(), "unsupported resolver scheme") {
t.Fatal("not the error we expected")
}
}
func TestConfigurerNewConfigurationTLSServerName(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSServerName: "www.x.org",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if configuration.HTTPConfig.TLSConfig.ServerName != "www.x.org" {
t.Fatal("invalid ServerName")
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
}
func TestConfigurerNewConfigurationNoTLSVerify(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
NoTLSVerify: true,
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if configuration.HTTPConfig.NoTLSVerify != true {
t.Fatal("not the NoTLSVerify we expected")
}
}
func TestConfigurerNewConfigurationTLSv1(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "TLSv1",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS10 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS10 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSv1dot0(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "TLSv1.0",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS10 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS10 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSv1dot1(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "TLSv1.1",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS11 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS11 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSv1dot2(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "TLSv1.2",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS12 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS12 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSv1dot3(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "TLSv1.3",
},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != tls.VersionTLS13 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != tls.VersionTLS13 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSvDefault(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{},
Logger: log.Log,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if len(configuration.HTTPConfig.TLSConfig.NextProtos) != 2 {
t.Fatal("invalid len(NextProtos)")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[0] != "h2" {
t.Fatal("invalid NextProtos[0]")
}
if configuration.HTTPConfig.TLSConfig.NextProtos[1] != "http/1.1" {
t.Fatal("invalid NextProtos[1]")
}
if configuration.HTTPConfig.TLSConfig.MinVersion != 0 {
t.Fatal("invalid MinVersion")
}
if configuration.HTTPConfig.TLSConfig.MaxVersion != 0 {
t.Fatal("invalid MaxVersion")
}
}
func TestConfigurerNewConfigurationTLSvInvalid(t *testing.T) {
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Config: urlgetter.Config{
TLSVersion: "SSLv3",
},
Logger: log.Log,
Saver: saver,
}
_, err := configurer.NewConfiguration()
if !errors.Is(err, netx.ErrInvalidTLSVersion) {
t.Fatalf("not the error we expected: %+v", err)
}
}
func TestConfigurerNewConfigurationProxyURL(t *testing.T) {
URL, _ := url.Parse("socks5://127.0.0.1:9050")
saver := new(trace.Saver)
configurer := urlgetter.Configurer{
Logger: log.Log,
Saver: saver,
ProxyURL: URL,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
t.Fatal(err)
}
if configuration.HTTPConfig.ProxyURL != URL {
t.Fatal("invalid ProxyURL")
}
}
internal/engine/experiment/urlgetter/getter.go
+132
View File
@@ -0,0 +1,132 @@
package urlgetter
import (
"context"
"net/url"
"path/filepath"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/internal/tunnel"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
)
// The Getter gets the specified target in the context of the
// given session and with the specified config.
//
// Other OONI experiment should use the Getter to factor code when
// the Getter implements the operations they wanna perform.
type Getter struct {
// Begin is the time when the experiment begun. If you do not
// set this field, every target is measured independently.
Begin time.Time
// Config contains settings for this run. If not set, then
// we will use the default config.
Config Config
// Session is the session for this run. This field must
// be set otherwise the code will panic.
Session model.ExperimentSession
// Target is the thing to measure in this run. This field must
// be set otherwise the code won't know what to do.
Target string
}
// Get performs the action described by g using the given context
// and returning the test keys and eventually an error
func (g Getter) Get(ctx context.Context) (TestKeys, error) {
if g.Config.Timeout > 0 {
var cancel context.CancelFunc
ctx, cancel = context.WithTimeout(ctx, g.Config.Timeout)
defer cancel()
}
if g.Begin.IsZero() {
g.Begin = time.Now()
}
saver := new(trace.Saver)
tk, err := g.get(ctx, saver)
// Make sure we have an operation in cases where we fail before
// hitting our httptransport that does error wrapping.
err = errorx.SafeErrWrapperBuilder{
Error: err,
Operation: errorx.TopLevelOperation,
}.MaybeBuild()
tk.FailedOperation = archival.NewFailedOperation(err)
tk.Failure = archival.NewFailure(err)
events := saver.Read()
tk.Queries = append(
tk.Queries, archival.NewDNSQueriesList(
g.Begin, events, g.Session.ASNDatabasePath())...,
)
tk.NetworkEvents = append(
tk.NetworkEvents, archival.NewNetworkEventsList(g.Begin, events)...,
)
tk.Requests = append(
tk.Requests, archival.NewRequestList(g.Begin, events)...,
)
if len(tk.Requests) > 0 {
// OONI's convention is that the last request appears first
tk.HTTPResponseStatus = tk.Requests[0].Response.Code
tk.HTTPResponseBody = tk.Requests[0].Response.Body.Value
tk.HTTPResponseLocations = tk.Requests[0].Response.Locations
}
tk.TCPConnect = append(
tk.TCPConnect, archival.NewTCPConnectList(g.Begin, events)...,
)
tk.TLSHandshakes = append(
tk.TLSHandshakes, archival.NewTLSHandshakesList(g.Begin, events)...,
)
return tk, err
}
func (g Getter) get(ctx context.Context, saver *trace.Saver) (TestKeys, error) {
tk := TestKeys{
Agent: "redirect",
Tunnel: g.Config.Tunnel,
}
if g.Config.DNSCache != "" {
tk.DNSCache = []string{g.Config.DNSCache}
}
if g.Config.NoFollowRedirects {
tk.Agent = "agent"
}
// start tunnel
var proxyURL *url.URL
if g.Config.Tunnel != "" {
tun, err := tunnel.Start(ctx, tunnel.Config{
Name: g.Config.Tunnel,
Session: g.Session,
WorkDir: filepath.Join(g.Session.TempDir(), "urlgetter-tunnel"),
})
if err != nil {
return tk, err
}
tk.BootstrapTime = tun.BootstrapTime().Seconds()
proxyURL = tun.SOCKS5ProxyURL()
tk.SOCKSProxy = proxyURL.String()
defer tun.Stop()
}
// create configuration
configurer := Configurer{
Config: g.Config,
Logger: g.Session.Logger(),
ProxyURL: proxyURL,
Saver: saver,
}
configuration, err := configurer.NewConfiguration()
if err != nil {
return tk, err
}
defer configuration.CloseIdleConnections()
// run the measurement
runner := Runner{
Config: g.Config,
HTTPConfig: configuration.HTTPConfig,
Target: g.Target,
}
return tk, runner.Run(ctx)
}
internal/engine/experiment/urlgetter/getter_test.go
+777
View File
@@ -0,0 +1,777 @@
package urlgetter_test
import (
"context"
"errors"
"net/http"
"strings"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
func TestGetterWithVeryShortTimeout(t *testing.T) {
g := urlgetter.Getter{
Config: urlgetter.Config{
Timeout: 1,
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(context.Background())
if !errors.Is(err, context.DeadlineExceeded) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "generic_timeout_error" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextVanilla(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextAndMethod(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{Method: "POST"},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "POST" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextNoFollowRedirects(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true,
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || !strings.HasSuffix(*tk.Failure, "interrupted") {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 3 {
t.Fatal("not the NetworkEvents we expected")
}
if tk.NetworkEvents[0].Operation != "http_transaction_start" {
t.Fatal("not the NetworkEvents[0].Operation we expected")
}
if tk.NetworkEvents[1].Operation != "http_request_metadata" {
t.Fatal("not the NetworkEvents[1].Operation we expected")
}
if tk.NetworkEvents[2].Operation != "http_transaction_done" {
t.Fatal("not the NetworkEvents[2].Operation we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextCannotStartTunnel(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // fail immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
Tunnel: "psiphon",
},
Session: &mockable.Session{MockableLogger: log.Log},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "interrupted" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 0 {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "psiphon" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterWithCancelledContextUnknownResolverURL(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel() // faily immediately
g := urlgetter.Getter{
Config: urlgetter.Config{
ResolverURL: "antani://8.8.8.8:53",
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err == nil || err.Error() != "unknown_failure: unsupported resolver scheme" {
t.Fatal("not the error we expected")
}
if tk.Agent != "redirect" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation == nil || *tk.FailedOperation != errorx.TopLevelOperation {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure == nil || *tk.Failure != "unknown_failure: unsupported resolver scheme" {
t.Fatal("not the Failure we expected")
}
if len(tk.NetworkEvents) != 0 {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 0 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 0 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterIntegrationHTTPS(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
},
Session: &mockable.Session{},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpWroteHeaders bool
httpWroteRequest bool
httpFirstResponseByte bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case errorx.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_wrote_headers":
httpWroteHeaders = true
case "http_wrote_request":
httpWroteRequest = true
case "http_first_response_byte":
httpFirstResponseByte = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && httpTransactionStart
ok = ok && httpRequestMetadata
ok = ok && resolveStart
ok = ok && resolveDone
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && httpWroteHeaders
ok = ok && httpWroteRequest
ok = ok && httpFirstResponseByte
ok = ok && httpResponseMetadata
ok = ok && httpResponseBodySnapshot
ok = ok && httpTransactionDone
if !ok {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 2 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 200 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if len(tk.HTTPResponseBody) <= 0 {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterIntegrationRedirect(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{NoFollowRedirects: true},
Session: &mockable.Session{},
Target: "http://web.whatsapp.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.HTTPResponseStatus != 302 {
t.Fatal("unexpected status code")
}
if len(tk.HTTPResponseLocations) != 1 {
t.Fatal("missing redirect URL")
}
if tk.HTTPResponseLocations[0] != "https://web.whatsapp.com/" {
t.Fatal("invalid redirect URL")
}
}
func TestGetterIntegrationTLSHandshake(t *testing.T) {
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
},
Session: &mockable.Session{},
Target: "tlshandshake://www.google.com:443",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime != 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpWroteHeaders bool
httpWroteRequest bool
httpFirstResponseByte bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case errorx.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_wrote_headers":
httpWroteHeaders = true
case "http_wrote_request":
httpWroteRequest = true
case "http_first_response_byte":
httpFirstResponseByte = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && !httpTransactionStart
ok = ok && !httpRequestMetadata
ok = ok && resolveStart
ok = ok && resolveDone
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && !httpWroteHeaders
ok = ok && !httpWroteRequest
ok = ok && !httpFirstResponseByte
ok = ok && !httpResponseMetadata
ok = ok && !httpResponseBodySnapshot
ok = ok && !httpTransactionDone
if !ok {
t.Fatal("not the NetworkEvents we expected")
}
if len(tk.Queries) != 2 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 0 {
t.Fatal("not the Requests we expected")
}
if tk.SOCKSProxy != "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 0 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if tk.HTTPResponseBody != "" {
t.Fatal("not the HTTPResponseBody we expected")
}
}
func TestGetterIntegrationHTTPSWithTunnel(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
ctx := context.Background()
g := urlgetter.Getter{
Config: urlgetter.Config{
NoFollowRedirects: true, // reduce number of events
Tunnel: "psiphon",
},
Session: &mockable.Session{
MockableHTTPClient: http.DefaultClient,
MockableLogger: log.Log,
},
Target: "https://www.google.com",
}
tk, err := g.Get(ctx)
if err != nil {
t.Fatal(err)
}
if tk.Agent != "agent" {
t.Fatal("not the Agent we expected")
}
if tk.BootstrapTime <= 0 {
t.Fatal("not the BootstrapTime we expected")
}
if tk.FailedOperation != nil {
t.Fatal("not the FailedOperation we expected")
}
if tk.Failure != nil {
t.Fatal("not the Failure we expected")
}
var (
httpTransactionStart bool
httpRequestMetadata bool
resolveStart bool
resolveDone bool
connect bool
tlsHandshakeStart bool
tlsHandshakeDone bool
httpWroteHeaders bool
httpWroteRequest bool
httpFirstResponseByte bool
httpResponseMetadata bool
httpResponseBodySnapshot bool
httpTransactionDone bool
)
for _, ev := range tk.NetworkEvents {
switch ev.Operation {
case "http_transaction_start":
httpTransactionStart = true
case "http_request_metadata":
httpRequestMetadata = true
case "resolve_start":
resolveStart = true
case "resolve_done":
resolveDone = true
case errorx.ConnectOperation:
connect = true
case "tls_handshake_start":
tlsHandshakeStart = true
case "tls_handshake_done":
tlsHandshakeDone = true
case "http_wrote_headers":
httpWroteHeaders = true
case "http_wrote_request":
httpWroteRequest = true
case "http_first_response_byte":
httpFirstResponseByte = true
case "http_response_metadata":
httpResponseMetadata = true
case "http_response_body_snapshot":
httpResponseBodySnapshot = true
case "http_transaction_done":
httpTransactionDone = true
}
}
ok := true
ok = ok && httpTransactionStart
ok = ok && httpRequestMetadata
ok = ok && resolveStart == false
ok = ok && resolveDone == false
ok = ok && connect
ok = ok && tlsHandshakeStart
ok = ok && tlsHandshakeDone
ok = ok && httpWroteHeaders
ok = ok && httpWroteRequest
ok = ok && httpFirstResponseByte
ok = ok && httpResponseMetadata
ok = ok && httpResponseBodySnapshot
ok = ok && httpTransactionDone
if !ok {
t.Fatalf("not the NetworkEvents we expected: %+v", tk.NetworkEvents)
}
if len(tk.Queries) != 0 {
t.Fatal("not the Queries we expected")
}
if len(tk.TCPConnect) != 1 {
t.Fatal("not the TCPConnect we expected")
}
if len(tk.Requests) != 1 {
t.Fatal("not the Requests we expected")
}
if tk.Requests[0].Request.Method != "GET" {
t.Fatal("not the Method we expected")
}
if tk.Requests[0].Request.URL != "https://www.google.com" {
t.Fatal("not the URL we expected")
}
if tk.SOCKSProxy == "" {
t.Fatal("not the SOCKSProxy we expected")
}
if len(tk.TLSHandshakes) != 1 {
t.Fatal("not the TLSHandshakes we expected")
}
if tk.Tunnel != "psiphon" {
t.Fatal("not the Tunnel we expected")
}
if tk.HTTPResponseStatus != 200 {
t.Fatal("not the HTTPResponseStatus we expected")
}
if len(tk.HTTPResponseBody) <= 0 {
t.Fatal("not the HTTPResponseBody we expected")
}
}
internal/engine/experiment/urlgetter/multi.go
+143
View File
@@ -0,0 +1,143 @@
package urlgetter
import (
"context"
"fmt"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
// MultiInput is the input for Multi.Run().
type MultiInput struct {
// Config contains the configuration for this target.
Config Config
// Target contains the target URL to measure.
Target string
}
// MultiOutput is the output returned by Multi.Run()
type MultiOutput struct {
// Input is the input for which we measured.
Input MultiInput
// Err contains the measurement error.
Err error
// TestKeys contains the measured test keys.
TestKeys TestKeys
}
// MultiGetter allows to override the behaviour of Multi for testing purposes.
type MultiGetter func(ctx context.Context, g Getter) (TestKeys, error)
// DefaultMultiGetter is the default MultiGetter
func DefaultMultiGetter(ctx context.Context, g Getter) (TestKeys, error) {
return g.Get(ctx)
}
// Multi allows to run several urlgetters in paraller.
type Multi struct {
// Begin is the time when the experiment begun. If you do not
// set this field, every target is measured independently.
Begin time.Time
// Getter is the Getter func to be used. If this is nil we use
// the default getter, which is what you typically want.
Getter MultiGetter
// Parallelism is the optional parallelism to be used. If this is
// zero, or negative, we use a reasonable default.
Parallelism int
// Session is the session to be used. If this is nil, the Run
// method will panic with a nil pointer error.
Session model.ExperimentSession
}
// Run performs several urlgetters in parallel. This function returns a channel
// where each result is posted. This function will always perform all the requested
// measurements: if the ctx is canceled or its deadline expires, then you will see
// a bunch of failed measurements. Since all measurements are always performed,
// you know you're done when you've read len(inputs) results in output.
func (m Multi) Run(ctx context.Context, inputs []MultiInput) <-chan MultiOutput {
parallelism := m.Parallelism
if parallelism <= 0 {
const defaultParallelism = 3
parallelism = defaultParallelism
}
inputch := make(chan MultiInput)
outputch := make(chan MultiOutput)
go m.source(inputs, inputch)
for i := 0; i < parallelism; i++ {
go m.do(ctx, inputch, outputch)
}
return outputch
}
// Collect prints on the output channel the result of running urlgetter
// on every provided input. It closes the output channel when done.
func (m Multi) Collect(ctx context.Context, inputs []MultiInput,
prefix string, callbacks model.ExperimentCallbacks) <-chan MultiOutput {
return m.CollectOverall(ctx, inputs, 0, len(inputs), prefix, callbacks)
}
// CollectOverall prints on the output channel the result of running urlgetter
// on every provided input. You can use this method if you perform multiple collection
// tasks within one experiment as it allows to calculate the overall progress correctly
func (m Multi) CollectOverall(ctx context.Context, inputChunk []MultiInput, overallStartIndex int, overallCount int,
prefix string, callbacks model.ExperimentCallbacks) <-chan MultiOutput {
outputch := make(chan MultiOutput)
go m.collect(len(inputChunk), overallStartIndex, overallCount, prefix, callbacks, m.Run(ctx, inputChunk), outputch)
return outputch
}
// collect drains inputch, prints progress, and emits to outputch. When done, this
// function will close outputch to notify the calller.
func (m Multi) collect(expect int, overallStartIndex int, overallCount int, prefix string, callbacks model.ExperimentCallbacks,
inputch <-chan MultiOutput, outputch chan<- MultiOutput) {
count := overallStartIndex
var index int
defer close(outputch)
for index < expect {
entry := <-inputch
index++
count++
percentage := float64(count) / float64(overallCount)
callbacks.OnProgress(percentage, fmt.Sprintf(
"%s: measure %s: %+v", prefix, entry.Input.Target, entry.Err,
))
outputch <- entry
}
}
// source posts all the inputs in the inputch. When done, this
// method will close the input channel to notify the reader.
func (m Multi) source(inputs []MultiInput, inputch chan<- MultiInput) {
defer close(inputch)
for _, input := range inputs {
inputch <- input
}
}
// do performs urlgetter on all the inputs read from the in channel and
// writes the results on the out channel. If the context is canceled, or
// its deadline expires, this function will continue performing all the
// required measurements, which will all fail.
func (m Multi) do(ctx context.Context, in <-chan MultiInput, out chan<- MultiOutput) {
for input := range in {
g := Getter{
Begin: m.Begin,
Config: input.Config,
Session: m.Session,
Target: input.Target,
}
fn := m.Getter
if fn == nil {
fn = DefaultMultiGetter
}
tk, err := fn(ctx, g)
out <- MultiOutput{Input: input, Err: err, TestKeys: tk}
}
}
internal/engine/experiment/urlgetter/multi_test.go
+256
View File
@@ -0,0 +1,256 @@
package urlgetter_test
import (
"context"
"crypto/x509"
"errors"
"fmt"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
func TestMultiIntegration(t *testing.T) {
multi := urlgetter.Multi{Session: &mockable.Session{}}
inputs := []urlgetter.MultiInput{{
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.google.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.facebook.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.kernel.org",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.instagram.com",
}}
outputs := multi.Collect(context.Background(), inputs, "integration-test",
model.NewPrinterCallbacks(log.Log))
var count int
for result := range outputs {
count++
switch result.Input.Target {
case "https://www.google.com":
case "https://www.facebook.com":
case "https://www.kernel.org":
case "https://www.instagram.com":
default:
t.Fatal("unexpected Input.Target")
}
if result.Input.Config.Method != "HEAD" {
t.Fatal("unexpected Input.Config.Method")
}
if result.Err != nil {
t.Fatal(result.Err)
}
if result.TestKeys.Agent != "agent" {
t.Fatal("invalid TestKeys.Agent")
}
if len(result.TestKeys.Queries) != 2 {
t.Fatal("invalid number of Queries")
}
if len(result.TestKeys.Requests) != 1 {
t.Fatal("invalid number of Requests")
}
if len(result.TestKeys.TCPConnect) != 1 {
t.Fatal("invalid number of TCPConnects")
}
if len(result.TestKeys.TLSHandshakes) != 1 {
t.Fatal("invalid number of TLSHandshakes")
}
}
if count != 4 {
t.Fatal("invalid number of outputs")
}
}
func TestMultiIntegrationWithBaseTime(t *testing.T) {
// We set a beginning of time that's significantly in the past and then
// fail the test if we see any T smaller than 3600 seconds.
begin := time.Date(2009, time.November, 10, 23, 0, 0, 0, time.UTC)
multi := urlgetter.Multi{
Begin: begin,
Session: &mockable.Session{},
}
inputs := []urlgetter.MultiInput{{
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.google.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.instagram.com",
}}
outputs := multi.Collect(context.Background(), inputs, "integration-test",
model.NewPrinterCallbacks(log.Log))
var count int
for result := range outputs {
for _, entry := range result.TestKeys.NetworkEvents {
if entry.T < 3600 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.Queries {
if entry.T < 3600 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.TCPConnect {
if entry.T < 3600 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.TLSHandshakes {
if entry.T < 3600 {
t.Fatal("base time not correctly set")
}
count++
}
}
if count <= 0 {
t.Fatal("unexpected number of entries processed")
}
}
func TestMultiIntegrationWithoutBaseTime(t *testing.T) {
// We use the default beginning of time and then fail the test
// if we see any T smaller than 60 seconds.
multi := urlgetter.Multi{Session: &mockable.Session{}}
inputs := []urlgetter.MultiInput{{
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.google.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.instagram.com",
}}
outputs := multi.Collect(context.Background(), inputs, "integration-test",
model.NewPrinterCallbacks(log.Log))
var count int
for result := range outputs {
for _, entry := range result.TestKeys.NetworkEvents {
if entry.T > 60 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.Queries {
if entry.T > 60 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.TCPConnect {
if entry.T > 60 {
t.Fatal("base time not correctly set")
}
count++
}
for _, entry := range result.TestKeys.TLSHandshakes {
if entry.T > 60 {
t.Fatal("base time not correctly set")
}
count++
}
}
if count <= 0 {
t.Fatal("unexpected number of entries processed")
}
}
func TestMultiContextCanceled(t *testing.T) {
multi := urlgetter.Multi{Session: &mockable.Session{}}
inputs := []urlgetter.MultiInput{{
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.google.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.facebook.com",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.kernel.org",
}, {
Config: urlgetter.Config{Method: "HEAD", NoFollowRedirects: true},
Target: "https://www.instagram.com",
}}
ctx, cancel := context.WithCancel(context.Background())
cancel()
outputs := multi.Collect(ctx, inputs, "integration-test",
model.NewPrinterCallbacks(log.Log))
var count int
for result := range outputs {
count++
switch result.Input.Target {
case "https://www.google.com":
case "https://www.facebook.com":
case "https://www.kernel.org":
case "https://www.instagram.com":
default:
t.Fatal("unexpected Input.Target")
}
if result.Input.Config.Method != "HEAD" {
t.Fatal("unexpected Input.Config.Method")
}
if !errors.Is(result.Err, context.Canceled) {
t.Fatal("unexpected error")
}
if result.TestKeys.Agent != "agent" {
t.Fatal("invalid TestKeys.Agent")
}
if len(result.TestKeys.Queries) != 0 {
t.Fatal("invalid number of Queries")
}
if len(result.TestKeys.Requests) != 1 {
t.Fatal("invalid number of Requests")
}
if len(result.TestKeys.TCPConnect) != 0 {
t.Fatal("invalid number of TCPConnects")
}
if len(result.TestKeys.TLSHandshakes) != 0 {
t.Fatal("invalid number of TLSHandshakes")
}
}
if count != 4 {
t.Fatal("invalid number of outputs")
}
}
func TestMultiWithSpecificCertPool(t *testing.T) {
server := httptest.NewTLSServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
fmt.Fprintln(w, "Hello, client")
}))
defer server.Close()
cert := server.Certificate()
certpool := x509.NewCertPool()
certpool.AddCert(cert)
multi := urlgetter.Multi{Session: &mockable.Session{}}
inputs := []urlgetter.MultiInput{{
Config: urlgetter.Config{
CertPool: certpool,
Method: "GET",
NoFollowRedirects: true,
},
Target: server.URL,
}}
ctx, cancel := context.WithCancel(context.Background())
defer cancel()
outputs := multi.Collect(ctx, inputs, "integration-test",
model.NewPrinterCallbacks(log.Log))
var count int
for result := range outputs {
count++
if result.Err != nil {
t.Fatal(result.Err)
}
}
if count != 1 {
t.Fatal("unexpected count value")
}
}
internal/engine/experiment/urlgetter/runner.go
+129
View File
@@ -0,0 +1,129 @@
package urlgetter
import (
"context"
"errors"
"fmt"
"io"
"io/ioutil"
"net/http"
"net/http/cookiejar"
"net/url"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
)
const httpRequestFailed = "http_request_failed"
// ErrHTTPRequestFailed indicates that the HTTP request failed.
var ErrHTTPRequestFailed = &errorx.ErrWrapper{
Failure: httpRequestFailed,
Operation: errorx.TopLevelOperation,
WrappedErr: errors.New(httpRequestFailed),
}
// The Runner job is to run a single measurement
type Runner struct {
Config Config
HTTPConfig netx.Config
Target string
}
// Run runs a measurement and returns the measurement result
func (r Runner) Run(ctx context.Context) error {
targetURL, err := url.Parse(r.Target)
if err != nil {
return fmt.Errorf("urlgetter: invalid target URL: %w", err)
}
switch targetURL.Scheme {
case "http", "https":
return r.httpGet(ctx, r.Target)
case "dnslookup":
return r.dnsLookup(ctx, targetURL.Hostname())
case "tlshandshake":
return r.tlsHandshake(ctx, targetURL.Host)
case "tcpconnect":
return r.tcpConnect(ctx, targetURL.Host)
default:
return errors.New("unknown targetURL scheme")
}
}
// MaybeUserAgent returns ua if ua is not empty. Otherwise it
// returns httpheader.RandomUserAgent().
func MaybeUserAgent(ua string) string {
if ua == "" {
ua = httpheader.UserAgent()
}
return ua
}
func (r Runner) httpGet(ctx context.Context, url string) error {
// Implementation note: empty Method implies using the GET method
req, err := http.NewRequest(r.Config.Method, url, nil)
runtimex.PanicOnError(err, "http.NewRequest failed")
req = req.WithContext(ctx)
req.Header.Set("Accept", httpheader.Accept())
req.Header.Set("Accept-Language", httpheader.AcceptLanguage())
req.Header.Set("User-Agent", MaybeUserAgent(r.Config.UserAgent))
if r.Config.HTTPHost != "" {
req.Host = r.Config.HTTPHost
}
// Implementation note: the following cookiejar accepts all cookies
// from all domains. As such, would not be safe for usage where cookies
// matter, but it's totally fine for performing measurements.
jar, err := cookiejar.New(nil)
runtimex.PanicOnError(err, "cookiejar.New failed")
httpClient := &http.Client{
Jar: jar,
Transport: netx.NewHTTPTransport(r.HTTPConfig),
}
if r.Config.NoFollowRedirects {
httpClient.CheckRedirect = func(*http.Request, []*http.Request) error {
return http.ErrUseLastResponse
}
}
defer httpClient.CloseIdleConnections()
resp, err := httpClient.Do(req)
if err != nil {
return err
}
defer resp.Body.Close()
if _, err = io.Copy(ioutil.Discard, resp.Body); err != nil {
return err
}
// Implementation note: we shall check for this error once we have read the
// whole body. Even though we discard the body, we want to know whether we
// see any error when reading the body before inspecting the HTTP status code.
if resp.StatusCode >= 400 && r.Config.FailOnHTTPError {
return ErrHTTPRequestFailed
}
return nil
}
func (r Runner) dnsLookup(ctx context.Context, hostname string) error {
resolver := netx.NewResolver(r.HTTPConfig)
_, err := resolver.LookupHost(ctx, hostname)
return err
}
func (r Runner) tlsHandshake(ctx context.Context, address string) error {
tlsDialer := netx.NewTLSDialer(r.HTTPConfig)
conn, err := tlsDialer.DialTLSContext(ctx, "tcp", address)
if conn != nil {
conn.Close()
}
return err
}
func (r Runner) tcpConnect(ctx context.Context, address string) error {
dialer := netx.NewDialer(r.HTTPConfig)
conn, err := dialer.DialContext(ctx, "tcp", address)
if conn != nil {
conn.Close()
}
return err
}
internal/engine/experiment/urlgetter/runner_test.go
+287
View File
@@ -0,0 +1,287 @@
package urlgetter_test
import (
"context"
"errors"
"io"
"net/http"
"net/http/httptest"
"strings"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
)
func TestRunnerWithInvalidURLScheme(t *testing.T) {
r := urlgetter.Runner{Target: "antani://www.google.com"}
err := r.Run(context.Background())
if err == nil || err.Error() != "unknown targetURL scheme" {
t.Fatal("not the error we expected")
}
}
func TestRunnerHTTPWithContextCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
r := urlgetter.Runner{Target: "https://www.google.com"}
err := r.Run(ctx)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
}
func TestRunnerDNSLookupWithContextCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
err := r.Run(ctx)
if err == nil || err.Error() != "interrupted" {
t.Fatal("not the error we expected")
}
}
func TestRunnerTLSHandshakeWithContextCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
err := r.Run(ctx)
if err == nil || err.Error() != "interrupted" {
t.Fatal("not the error we expected")
}
}
func TestRunnerTCPConnectWithContextCanceled(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
err := r.Run(ctx)
if err == nil || err.Error() != "interrupted" {
t.Fatal("not the error we expected")
}
}
func TestRunnerWithInvalidURL(t *testing.T) {
r := urlgetter.Runner{Target: "\t"}
err := r.Run(context.Background())
if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") {
t.Fatal("not the error we expected")
}
}
func TestRunnerWithEmptyHostname(t *testing.T) {
r := urlgetter.Runner{Target: "http:///foo.txt"}
err := r.Run(context.Background())
if err == nil || !strings.HasSuffix(err.Error(), "no Host in request URL") {
t.Fatal("not the error we expected")
}
}
func TestRunnerTLSHandshakeSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
r := urlgetter.Runner{Target: "tlshandshake://www.google.com:443"}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestRunnerTCPConnectSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
r := urlgetter.Runner{Target: "tcpconnect://www.google.com:443"}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestRunnerDNSLookupSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
r := urlgetter.Runner{Target: "dnslookup://www.google.com"}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestRunnerHTTPSSuccess(t *testing.T) {
if testing.Short() {
t.Skip("skip test in short mode")
}
r := urlgetter.Runner{Target: "https://www.google.com"}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestRunnerHTTPSetHostHeader(t *testing.T) {
var host string
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
host = r.Host
w.WriteHeader(200)
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
HTTPHost: "x.org",
},
Target: server.URL,
}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if host != "x.org" {
t.Fatal("not the host we expected")
}
}
func TestRunnerHTTPNoRedirect(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Location", "http:///") // cause failure if we redirect
w.WriteHeader(302)
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
NoFollowRedirects: true,
},
Target: server.URL,
}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
}
func TestRunnerHTTPCannotReadBody(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hijacker, ok := w.(http.Hijacker)
if !ok {
panic("hijacking not supported by this server")
}
conn, _, _ := hijacker.Hijack()
conn.Write([]byte("HTTP/1.1 200 Ok\r\n"))
conn.Write([]byte("Content-Length: 1024\r\n"))
conn.Write([]byte("\r\n"))
conn.Write([]byte("123456789"))
conn.Close()
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
NoFollowRedirects: true,
},
Target: server.URL,
}
err := r.Run(context.Background())
if !errors.Is(err, io.EOF) {
t.Fatal("not the error we expected")
}
}
func TestRunnerHTTPWeHandle400Correctly(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(400)
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
FailOnHTTPError: true,
NoFollowRedirects: true,
},
Target: server.URL,
}
err := r.Run(context.Background())
if !errors.Is(err, urlgetter.ErrHTTPRequestFailed) {
t.Fatal("not the error we expected")
}
}
func TestRunnerHTTPCannotReadBodyWinsOver400(t *testing.T) {
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
hijacker, ok := w.(http.Hijacker)
if !ok {
panic("hijacking not supported by this server")
}
conn, _, _ := hijacker.Hijack()
conn.Write([]byte("HTTP/1.1 400 Bad Request\r\n"))
conn.Write([]byte("Content-Length: 1024\r\n"))
conn.Write([]byte("\r\n"))
conn.Write([]byte("123456789"))
conn.Close()
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
FailOnHTTPError: true,
NoFollowRedirects: true,
},
Target: server.URL,
}
err := r.Run(context.Background())
if !errors.Is(err, io.EOF) {
t.Fatal("not the error we expected")
}
}
func TestRunnerWeCanForceUserAgent(t *testing.T) {
expected := "antani/1.23.4-dev"
found := atomicx.NewInt64()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("User-Agent") == expected {
found.Add(1)
}
w.WriteHeader(200)
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
FailOnHTTPError: true,
NoFollowRedirects: true,
UserAgent: expected,
},
Target: server.URL,
}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if found.Load() != 1 {
t.Fatal("we didn't override the user agent")
}
}
func TestRunnerDefaultUserAgent(t *testing.T) {
expected := httpheader.UserAgent()
found := atomicx.NewInt64()
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
if r.Header.Get("User-Agent") == expected {
found.Add(1)
}
w.WriteHeader(200)
}))
defer server.Close()
r := urlgetter.Runner{
Config: urlgetter.Config{
FailOnHTTPError: true,
NoFollowRedirects: true,
},
Target: server.URL,
}
err := r.Run(context.Background())
if err != nil {
t.Fatal(err)
}
if found.Load() != 1 {
t.Fatal("we didn't override the user agent")
}
}
internal/engine/experiment/urlgetter/urlgetter.go
+133
View File
@@ -0,0 +1,133 @@
// Package urlgetter implements a nettest that fetches a URL.
//
// See https://github.com/ooni/spec/blob/master/nettests/ts-027-urlgetter.md.
package urlgetter
import (
"context"
"crypto/x509"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
)
const (
testName = "urlgetter"
testVersion = "0.1.0"
)
// Config contains the experiment's configuration.
type Config struct {
// not settable from command line
CertPool *x509.CertPool
Timeout time.Duration
// settable from command line
DNSCache string `ooni:"Add 'DOMAIN IP...' to cache"`
DNSHTTPHost string `ooni:"Force using specific HTTP Host header for DNS requests"`
DNSTLSServerName string `ooni:"Force TLS to using a specific SNI for encrypted DNS requests"`
DNSTLSVersion string `ooni:"Force specific TLS version used for DoT/DoH (e.g. 'TLSv1.3')"`
FailOnHTTPError bool `ooni:"Fail HTTP request if status code is 400 or above"`
HTTP3Enabled bool `ooni:"use http3 instead of http/1.1 or http2"`
HTTPHost string `ooni:"Force using specific HTTP Host header"`
Method string `ooni:"Force HTTP method different than GET"`
NoFollowRedirects bool `ooni:"Disable following redirects"`
NoTLSVerify bool `ooni:"Disable TLS verification"`
RejectDNSBogons bool `ooni:"Fail DNS lookup if response contains bogons"`
ResolverURL string `ooni:"URL describing the resolver to use"`
TLSServerName string `ooni:"Force TLS to using a specific SNI in Client Hello"`
TLSVersion string `ooni:"Force specific TLS version (e.g. 'TLSv1.3')"`
Tunnel string `ooni:"Run experiment over a tunnel, e.g. psiphon"`
UserAgent string `ooni:"Use the specified User-Agent"`
}
// TestKeys contains the experiment's result.
type TestKeys struct {
// The following fields are part of the typical JSON emitted by OONI.
Agent string `json:"agent"`
BootstrapTime float64 `json:"bootstrap_time,omitempty"`
DNSCache []string `json:"dns_cache,omitempty"`
FailedOperation *string `json:"failed_operation"`
Failure *string `json:"failure"`
NetworkEvents []archival.NetworkEvent `json:"network_events"`
Queries []archival.DNSQueryEntry `json:"queries"`
Requests []archival.RequestEntry `json:"requests"`
SOCKSProxy string `json:"socksproxy,omitempty"`
TCPConnect []archival.TCPConnectEntry `json:"tcp_connect"`
TLSHandshakes []archival.TLSHandshake `json:"tls_handshakes"`
Tunnel string `json:"tunnel,omitempty"`
// The following fields are not serialised but are useful to simplify
// analysing the measurements in telegram, whatsapp, etc.
HTTPResponseStatus int64 `json:"-"`
HTTPResponseBody string `json:"-"`
HTTPResponseLocations []string `json:"-"`
}
// RegisterExtensions registers the extensions used by the urlgetter
// experiment into the provided measurement.
func RegisterExtensions(m *model.Measurement) {
archival.ExtHTTP.AddTo(m)
archival.ExtDNS.AddTo(m)
archival.ExtNetevents.AddTo(m)
archival.ExtTCPConnect.AddTo(m)
archival.ExtTLSHandshake.AddTo(m)
archival.ExtTunnel.AddTo(m)
}
// Measurer performs the measurement.
type Measurer struct {
Config
}
// ExperimentName implements model.ExperimentSession.ExperimentName
func (m Measurer) ExperimentName() string {
return testName
}
// ExperimentVersion implements model.ExperimentSession.ExperimentVersion
func (m Measurer) ExperimentVersion() string {
return testVersion
}
// Run implements model.ExperimentSession.Run
func (m Measurer) Run(
ctx context.Context, sess model.ExperimentSession,
measurement *model.Measurement, callbacks model.ExperimentCallbacks,
) error {
// When using the urlgetter experiment directly, there is a nonconfigurable
// default timeout that applies. When urlgetter is used as a library, it's
// instead the responsibility of the user of urlgetter to set timeouts. Note
// that this code is indeed only called when using urlgetter directly.
if m.Config.Timeout <= 0 {
m.Config.Timeout = 45 * time.Second
}
RegisterExtensions(measurement)
g := Getter{
Config: m.Config,
Session: sess,
Target: string(measurement.Input),
}
tk, err := g.Get(ctx)
measurement.TestKeys = &tk
return err
}
// NewExperimentMeasurer creates a new ExperimentMeasurer.
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
return Measurer{Config: config}
}
// SummaryKeys contains summary keys for this experiment.
//
// Note that this structure is part of the ABI contract with probe-cli
// therefore we should be careful when changing it.
type SummaryKeys struct {
IsAnomaly bool `json:"-"`
}
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
return SummaryKeys{IsAnomaly: false}, nil
}
internal/engine/experiment/urlgetter/urlgetter_test.go
+90
View File
@@ -0,0 +1,90 @@
package urlgetter_test
import (
"context"
"errors"
"testing"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
func TestMeasurer(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
m := urlgetter.NewExperimentMeasurer(urlgetter.Config{})
if m.ExperimentName() != "urlgetter" {
t.Fatal("invalid experiment name")
}
if m.ExperimentVersion() != "0.1.0" {
t.Fatal("invalid experiment version")
}
measurement := new(model.Measurement)
measurement.Input = "https://www.google.com"
err := m.Run(
ctx, &mockable.Session{},
measurement, model.NewPrinterCallbacks(log.Log),
)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if len(measurement.Extensions) != 6 {
t.Fatal("not the expected number of extensions")
}
tk := measurement.TestKeys.(*urlgetter.TestKeys)
if len(tk.DNSCache) != 0 {
t.Fatal("not the DNSCache value we expected")
}
sk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
if _, ok := sk.(urlgetter.SummaryKeys); !ok {
t.Fatal("invalid type for summary keys")
}
}
func TestMeasurerDNSCache(t *testing.T) {
ctx, cancel := context.WithCancel(context.Background())
cancel()
m := urlgetter.NewExperimentMeasurer(urlgetter.Config{
DNSCache: "dns.google 8.8.8.8 8.8.4.4",
})
if m.ExperimentName() != "urlgetter" {
t.Fatal("invalid experiment name")
}
if m.ExperimentVersion() != "0.1.0" {
t.Fatal("invalid experiment version")
}
measurement := new(model.Measurement)
measurement.Input = "https://www.google.com"
err := m.Run(
ctx, &mockable.Session{},
measurement, model.NewPrinterCallbacks(log.Log),
)
if !errors.Is(err, context.Canceled) {
t.Fatal("not the error we expected")
}
if len(measurement.Extensions) != 6 {
t.Fatal("not the expected number of extensions")
}
tk := measurement.TestKeys.(*urlgetter.TestKeys)
if len(tk.DNSCache) != 1 || tk.DNSCache[0] != "dns.google 8.8.8.8 8.8.4.4" {
t.Fatal("invalid tk.DNSCache")
}
}
func TestSummaryKeysGeneric(t *testing.T) {
measurement := &model.Measurement{TestKeys: &urlgetter.TestKeys{}}
m := &urlgetter.Measurer{}
osk, err := m.GetSummaryKeys(measurement)
if err != nil {
t.Fatal(err)
}
sk := osk.(urlgetter.SummaryKeys)
if sk.IsAnomaly {
t.Fatal("invalid isAnomaly")
}
}