refactor: move more commands to internal/cmd (#207)
* refactor: move more commands to internal/cmd Part of https://github.com/ooni/probe/issues/1335. We would like all commands to be at the same level of engine rather than inside engine (now that we can do it). * fix: update .gitignore * refactor: also move jafar outside engine * We should be good now?
This commit is contained in:
Simone Basso
GitHub
@@ -2,9 +2,7 @@
|
||||
/.vscode
|
||||
/apitool
|
||||
/asn.mmdb
|
||||
/badproxy.pem
|
||||
/ca-bundle.pem
|
||||
/cmd/jafar/badproxy.pem
|
||||
/country.mmdb
|
||||
/example.org
|
||||
/jafar
|
||||
|
||||
@@ -1,3 +0,0 @@
|
||||
# Directory github.com/ooni/probe-engine/cmd
|
||||
|
||||
This directory contains the source code for the CLI tools we build.
|
||||
@@ -1,8 +0,0 @@
|
||||
# apitool
|
||||
|
||||
This directory contains a tool to fetch measurements. This tool is
|
||||
intended to sporadically fetch measurements, not for batch downloading.
|
||||
|
||||
Please, see https://ooni.org/data for information pertaining how to
|
||||
access OONI data in bulk. Please see https://explorer.ooni.org if your
|
||||
intent is to navigate and explore OONI data.
|
||||
@@ -1,115 +0,0 @@
|
||||
// Command apitool is a simple tool to fetch individual OONI measurements.
|
||||
//
|
||||
// This tool IS NOT intended for batch downloading.
|
||||
//
|
||||
// Please, see https://ooni.org/data for information pertaining how to
|
||||
// access OONI data in bulk. Please see https://explorer.ooni.org if your
|
||||
// intent is to navigate and explore OONI data
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"os"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/version"
|
||||
)
|
||||
|
||||
func newclient() probeservices.Client {
|
||||
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
|
||||
ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version)
|
||||
return probeservices.Client{
|
||||
Client: httpx.Client{
|
||||
BaseURL: "https://ams-pg.ooni.org/",
|
||||
HTTPClient: &http.Client{Transport: txp},
|
||||
Logger: log.Log,
|
||||
UserAgent: ua,
|
||||
},
|
||||
LoginCalls: atomicx.NewInt64(),
|
||||
RegisterCalls: atomicx.NewInt64(),
|
||||
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
|
||||
}
|
||||
}
|
||||
|
||||
var osExit = os.Exit
|
||||
|
||||
func fatalOnError(err error, message string) {
|
||||
if err != nil {
|
||||
log.WithError(err).Error(message)
|
||||
osExit(1) // overridable from tests
|
||||
}
|
||||
}
|
||||
|
||||
var (
|
||||
debug = flag.Bool("v", false, "Enable verbose mode")
|
||||
input = flag.String("input", "", "Input of the measurement")
|
||||
mode = flag.String("mode", "", "One of: check, meta, raw")
|
||||
reportid = flag.String("report-id", "", "Report ID of the measurement")
|
||||
)
|
||||
|
||||
var logmap = map[bool]log.Level{
|
||||
true: log.DebugLevel,
|
||||
false: log.InfoLevel,
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
log.SetLevel(logmap[*debug])
|
||||
client := newclient()
|
||||
switch *mode {
|
||||
case "check":
|
||||
check(client)
|
||||
case "meta":
|
||||
meta(client)
|
||||
case "raw":
|
||||
raw(client)
|
||||
default:
|
||||
fatalOnError(fmt.Errorf("invalid -mode flag value: %s", *mode), "usage error")
|
||||
}
|
||||
}
|
||||
|
||||
func check(c probeservices.Client) {
|
||||
found, err := c.CheckReportID(context.Background(), *reportid)
|
||||
fatalOnError(err, "c.CheckReportID failed")
|
||||
fmt.Printf("%+v\n", found)
|
||||
}
|
||||
|
||||
func meta(c probeservices.Client) {
|
||||
pprint(mmeta(c, false))
|
||||
}
|
||||
|
||||
func raw(c probeservices.Client) {
|
||||
m := mmeta(c, true)
|
||||
rm := []byte(m.RawMeasurement)
|
||||
var opaque interface{}
|
||||
err := json.Unmarshal(rm, &opaque)
|
||||
fatalOnError(err, "json.Unmarshal failed")
|
||||
pprint(opaque)
|
||||
}
|
||||
|
||||
func pprint(opaque interface{}) {
|
||||
data, err := json.MarshalIndent(opaque, "", " ")
|
||||
fatalOnError(err, "json.MarshalIndent failed")
|
||||
fmt.Printf("%s\n", data)
|
||||
}
|
||||
|
||||
func mmeta(c probeservices.Client, full bool) *probeservices.MeasurementMeta {
|
||||
config := probeservices.MeasurementMetaConfig{
|
||||
ReportID: *reportid,
|
||||
Full: full,
|
||||
Input: *input,
|
||||
}
|
||||
ctx := context.Background()
|
||||
m, err := c.GetMeasurementMeta(ctx, config)
|
||||
fatalOnError(err, "client.GetMeasurementMeta failed")
|
||||
return m
|
||||
}
|
||||
@@ -1,39 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func init() {
|
||||
*reportid = `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
|
||||
*input = `https://www.example.org`
|
||||
}
|
||||
|
||||
func TestCheck(t *testing.T) {
|
||||
*mode = "check"
|
||||
main()
|
||||
}
|
||||
|
||||
func TestRaw(t *testing.T) {
|
||||
*mode = "raw"
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMeta(t *testing.T) {
|
||||
*mode = "meta"
|
||||
main()
|
||||
}
|
||||
|
||||
func TestInvalidMode(t *testing.T) {
|
||||
defer func() {
|
||||
if recover() == nil {
|
||||
t.Fatal("the code did not panic")
|
||||
}
|
||||
}()
|
||||
osExit = func(code int) {
|
||||
panic(fmt.Errorf("%d", code))
|
||||
}
|
||||
*mode = "antani"
|
||||
main()
|
||||
}
|
||||
@@ -1,258 +0,0 @@
|
||||
# Jafar
|
||||
|
||||
> We stepped up the game of simulating censorship upgrading from the
|
||||
> evil genius to the evil grand vizier.
|
||||
|
||||
Jafar is a censorship simulation tool used for testing OONI. It builds on
|
||||
any system but it really on works on Linux.
|
||||
|
||||
## Building
|
||||
|
||||
We use Go >= 1.14. Jafar also needs the C library headers,
|
||||
iptables installed, and root permissions.
|
||||
|
||||
With Linux Alpine edge, you can compile Jafar with:
|
||||
|
||||
```bash
|
||||
apk add go git musl-dev iptables
|
||||
go build -v .
|
||||
```
|
||||
|
||||
Otherwise, using Docker:
|
||||
|
||||
```bash
|
||||
docker build -t jafar-runner .
|
||||
docker run -it --privileged -v`pwd`:/jafar -w/jafar jafar-runner
|
||||
go build -v .
|
||||
```
|
||||
|
||||
## Usage
|
||||
|
||||
You need to run Jafar as root. You can get a complete list
|
||||
of all flags using `./jafar -help`. Jafar is composed of modules. Each
|
||||
module is controllable via flags. We describe modules below.
|
||||
|
||||
### main
|
||||
|
||||
The main module starts all the other modules. If you don't provide the
|
||||
`-main-command <command>` flag, the code will run until interrupted. If
|
||||
instead you use the `-main-command` flag, you can specify a command to
|
||||
run inside the censored environment. In such case, the main module
|
||||
will exit when the specified command terminates. Note that the main
|
||||
module will propagate the child exit code, if the child fails.
|
||||
|
||||
The command can also include arguments. Make sure you quote the arguments
|
||||
such that your shell passes the whole string to the specified option, as
|
||||
in `-main-command 'ls -lha'`. This will execute the `ls -lha` command line
|
||||
inside the censored Jafar context. You can also combine that with quoting
|
||||
and variables interpolation, e.g., `-main-command "echo '$USER is the
|
||||
walrus'"`. The `$USER` variable will be expanded by your shell. Assuming
|
||||
your user name is `paul`, then Jafar will lex the main command as `echo
|
||||
"paul is the walrus"` and will execute it.
|
||||
|
||||
Use the `-main-user <username>` flag to select the user to use for
|
||||
running child commands. By default, we use the `nobody` user for this
|
||||
purpose. We implement this feature using `sudo`, therefore you need
|
||||
to make sure that `sudo` is installed.
|
||||
|
||||
### iptables
|
||||
|
||||
The iptables module is only available on Linux. It exports these flags:
|
||||
|
||||
```bash
|
||||
-iptables-drop-ip value
|
||||
Drop traffic to the specified IP address
|
||||
-iptables-drop-keyword-hex value
|
||||
Drop traffic containing the specified hex keyword
|
||||
-iptables-drop-keyword value
|
||||
Drop traffic containing the specified keyword
|
||||
-iptables-hijack-dns-to string
|
||||
Hijack all DNS UDP traffic to the specified endpoint
|
||||
-iptables-hijack-https-to string
|
||||
Hijack all HTTPS traffic to the specified endpoint
|
||||
-iptables-hijack-http-to string
|
||||
Hijack all HTTP traffic to the specified endpoint
|
||||
-iptables-reset-ip value
|
||||
Reset TCP/IP traffic to the specified IP address
|
||||
-iptables-reset-keyword-hex value
|
||||
Reset TCP/IP traffic containing the specified hex keyword
|
||||
-iptables-reset-keyword value
|
||||
Reset TCP/IP traffic containing the specified keyword
|
||||
```
|
||||
|
||||
The difference between `drop` and `reset` is that in the former case
|
||||
a packet is dropped, in the latter case a RST is sent.
|
||||
|
||||
The difference between `ip` and `keyword` flags is that the former
|
||||
match an outgoing IP, the latter uses DPI.
|
||||
|
||||
The `drop` and `reset` rules allow you to simulate, respectively, when
|
||||
operations timeout and when a connection cannot be established (with
|
||||
`reset` and `ip`) or is reset after a keyword is seen (with `keyword`).
|
||||
|
||||
Hijacking DNS traffic is useful, for example, to redirect all DNS UDP
|
||||
traffic from the box to the `dns-proxy` module.
|
||||
|
||||
Hijacking HTTP and HTTPS traffic actually hijacks based on ports rather
|
||||
than on DPI. As a known bug, when hijacking HTTP or HTTPS traffic, we
|
||||
do not hijack traffic owned by root. This is because Jafar runs as root
|
||||
and therefore its traffic must not match the hijack rule.
|
||||
|
||||
When matching keywords, the simplest option is to use ASCII strings as
|
||||
in `-iptables-drop-keyword ooni`. However, you can also specify a sequence
|
||||
of hex bytes, as in `-iptables-drop-keyword-hex |6f 6f 6e 69|`.
|
||||
|
||||
Note that with `-iptables-drop-keyword`, DNS queries containing such
|
||||
keyword will fail returning `EPERM`. For a more realistic approach to
|
||||
dropping specific DNS packets, combine DNS traffic hijacking with
|
||||
`-dns-proxy-ignore`, to "drop" packets at the DNS proxy.
|
||||
|
||||
### dns-proxy (aka resolver)
|
||||
|
||||
The DNS proxy or resolver allows to manipulate DNS. Unless you use DNS
|
||||
hijacking, you will need to configure your application explicitly to use
|
||||
the proxy with application specific command line flags.
|
||||
|
||||
```bash
|
||||
-dns-proxy-address string
|
||||
Address where the DNS proxy should listen (default "127.0.0.1:53")
|
||||
-dns-proxy-block value
|
||||
Register keyword triggering NXDOMAIN censorship
|
||||
-dns-proxy-hijack value
|
||||
Register keyword triggering redirection to 127.0.0.1
|
||||
-dns-proxy-ignore value
|
||||
Register keyword causing the proxy to ignore the query
|
||||
```
|
||||
|
||||
The `-dns-proxy-address` flag controls the endpoint where the proxy is
|
||||
listening.
|
||||
|
||||
The `-dns-proxy-block` tells the resolver that every incoming request whose
|
||||
query contains the specifed string shall receive an `NXDOMAIN` reply.
|
||||
|
||||
The `-dns-proxy-hijack` is similar but instead lies and returns to the
|
||||
client that the requested domain is at `127.0.0.1`. This is an opportunity
|
||||
to redirect traffic to the HTTP and TLS proxies.
|
||||
|
||||
The `-dns-proxy-ignore` is similar but instead just ignores the query.
|
||||
|
||||
### http-proxy
|
||||
|
||||
The HTTP proxy is an HTTP proxy that may refuse to forward some
|
||||
specific requests. It's controlled by these flags:
|
||||
|
||||
```bash
|
||||
-http-proxy-address string
|
||||
Address where the HTTP proxy should listen (default "127.0.0.1:80")
|
||||
-http-proxy-block value
|
||||
Register keyword triggering HTTP 451 censorship
|
||||
```
|
||||
|
||||
The `-http-proxy-address` flag has the same semantics it has for the DNS
|
||||
proxy.
|
||||
|
||||
The `-http-proxy-block` flag tells the proxy that it should return a `451`
|
||||
response for every request whose `Host` contains the specified string.
|
||||
|
||||
### tls-proxy
|
||||
|
||||
TLS proxy is a proxy that routes traffic to specific servers depending
|
||||
on their SNI value. It is controlled by the following flags:
|
||||
|
||||
```bash
|
||||
-tls-proxy-address string
|
||||
Address where the HTTP proxy should listen (default "127.0.0.1:443")
|
||||
-tls-proxy-block value
|
||||
Register keyword triggering TLS censorship
|
||||
```
|
||||
|
||||
The `-tls-proxy-address` flags has the same semantics it has for the DNS
|
||||
proxy.
|
||||
|
||||
The `-tls-proxy-block` specifies which string or strings should cause the
|
||||
proxy to return an internal-erorr alert when the incoming ClientHello's SNI
|
||||
contains one of the strings provided with this option.
|
||||
|
||||
### bad-proxy
|
||||
|
||||
```bash
|
||||
-bad-proxy-address string
|
||||
Address where to listen for TCP connections (default "127.0.0.1:7117")
|
||||
-bad-proxy-address-tls string
|
||||
Address where to listen for TLS connections (default "127.0.0.1:4114")
|
||||
-bad-proxy-tls-output-ca string
|
||||
File where to write the CA used by the bad proxy (default "badproxy.pem")
|
||||
```
|
||||
|
||||
The bad proxy is a proxy that reads some bytes from any incoming connection
|
||||
and then closes the connection without replying anything. This simulates a
|
||||
proxy that is not working properly, hence the name of the module.
|
||||
|
||||
When connecting using TLS, the above behaviour happens after the handshake.
|
||||
|
||||
We write the CA on the file specified using `-bad-proxy-tls-output-ca` such that
|
||||
tools like curl(1) can use such CA to avoid TLS handshake errors. The code will
|
||||
generate on the fly a certificate for the provided SNI. Not providing any SNI in
|
||||
the client Hello message will cause the TLS handshake to fail.
|
||||
|
||||
### uncensored
|
||||
|
||||
```bash
|
||||
-uncensored-resolver-url string
|
||||
URL of an hopefully uncensored resolver (default "dot://1.1.1.1:853")
|
||||
```
|
||||
|
||||
The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS
|
||||
censorship, they may be affected as well. To avoid this issue, we use a different
|
||||
resolver for them, which by default is `dot://1.1.1.1:853`. You can change such
|
||||
default by using the `-uncensored-resolver-url` command line flag. The input
|
||||
URL is `<transport>://<domain>[:<port>][/<path>]`. Here are some examples:
|
||||
|
||||
* `system:///` uses the system resolver (i.e. `getaddrinfo`)
|
||||
* `udp://8.8.8.8:53` uses DNS over UDP
|
||||
* `tcp://8.8.8.8:53` used DNS over TCP
|
||||
* `dot://8.8.8.8:853` uses DNS over TLS
|
||||
* `https://dns.google/dns-query` uses DNS over HTTPS
|
||||
|
||||
So, for example, if you are using Jafar to censor `1.1.1.1:853`, then you
|
||||
most likely want to use `-uncensored-resolver-url`.
|
||||
|
||||
## Examples
|
||||
|
||||
Block `play.google.com` with RST injection, force DNS traffic to use the our
|
||||
DNS proxy, and force it to censor `play.google.com` with `NXDOMAIN`.
|
||||
|
||||
```bash
|
||||
# ./jafar -iptables-reset-keyword play.google.com \
|
||||
-iptables-hijack-dns-to 127.0.0.1:5353 \
|
||||
-dns-proxy-address 127.0.0.1:5353 \
|
||||
-dns-proxy-block play.google.com
|
||||
```
|
||||
|
||||
Force all traffic through the HTTP and TLS proxy and use them to censor
|
||||
`play.google.com` using HTTP 451 and responding with TLS alerts:
|
||||
|
||||
```bash
|
||||
# ./jafar -iptables-hijack-dns-to 127.0.0.1:5353 \
|
||||
-dns-proxy-address 127.0.0.1:5353 \
|
||||
-dns-proxy-hijack play.google.com \
|
||||
-http-proxy-block play.google.com \
|
||||
-tls-proxy-block play.google.com
|
||||
```
|
||||
|
||||
Run `ping` in a censored environment:
|
||||
|
||||
```bash
|
||||
# ./jafar -iptables-drop-ip 8.8.8.8 -main-command 'ping -c3 8.8.8.8'
|
||||
```
|
||||
|
||||
Run `curl` in a censored environment where it cannot connect to
|
||||
`play.google.com` using `https`:
|
||||
|
||||
```bash
|
||||
# ./jafar -iptables-hijack-https-to 127.0.0.1:443 \
|
||||
-tls-proxy-block play.google.com \
|
||||
-main-command 'curl -Lv http://play.google.com'
|
||||
```
|
||||
|
||||
For more usage examples, see `../../testjafar.bash`.
|
||||
@@ -1,113 +0,0 @@
|
||||
// Package badproxy implements misbehaving proxies. We have a single
|
||||
// CensoringProxy that exports two misbehaving endpoints. Each endpoint
|
||||
// implements a different proxy-censorsing technique. The first one
|
||||
// reads some bytes from the connection then closes the connection. The
|
||||
// other instead replies with a self signed x509 certificate.
|
||||
package badproxy
|
||||
|
||||
import (
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"strings"
|
||||
"time"
|
||||
|
||||
"github.com/google/martian/v3/mitm"
|
||||
)
|
||||
|
||||
// CensoringProxy is a proxy that does not behave correctly.
|
||||
type CensoringProxy struct {
|
||||
mitmNewAuthority func(
|
||||
name string, organization string,
|
||||
validity time.Duration,
|
||||
) (*x509.Certificate, *rsa.PrivateKey, error)
|
||||
|
||||
mitmNewConfig func(
|
||||
ca *x509.Certificate, privateKey interface{},
|
||||
) (*mitm.Config, error)
|
||||
|
||||
tlsListen func(
|
||||
network string, laddr string, config *tls.Config,
|
||||
) (net.Listener, error)
|
||||
}
|
||||
|
||||
// NewCensoringProxy creates a new instance of a misbehaving proxy.
|
||||
func NewCensoringProxy() *CensoringProxy {
|
||||
return &CensoringProxy{
|
||||
mitmNewAuthority: mitm.NewAuthority,
|
||||
mitmNewConfig: mitm.NewConfig,
|
||||
tlsListen: tls.Listen,
|
||||
}
|
||||
}
|
||||
|
||||
func (p *CensoringProxy) serve(conn net.Conn) {
|
||||
deadline := time.Now().Add(250 * time.Millisecond)
|
||||
conn.SetDeadline(deadline)
|
||||
// To simulate the case where the proxy isn't willing to forward our
|
||||
// traffic, we close the connection (1) right after the handshake for
|
||||
// TLS connections and (2) reasonably after we've received the HTTP
|
||||
// request for cleartext connections. This may break in several cases
|
||||
// but is good enough approximation of these bad proxies for now.
|
||||
if tlsconn, ok := conn.(*tls.Conn); ok {
|
||||
tlsconn.Handshake()
|
||||
} else {
|
||||
const maxread = 1 << 17
|
||||
reader := io.LimitReader(conn, maxread)
|
||||
ioutil.ReadAll(reader)
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
func (p *CensoringProxy) run(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil && strings.Contains(
|
||||
err.Error(), "use of closed network connection") {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
// It's difficult to make accept fail, so restructure
|
||||
// the code such that we enter into the happy path
|
||||
go p.serve(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the misbehaving proxy for TCP. This endpoint will read some
|
||||
// bytes from the request and then close the connection. This behaviour is
|
||||
// implemented by a bunch of censoring proxy around the world. Usually such
|
||||
// proxies only close the connection with offending SNIs/Host headers.
|
||||
func (p *CensoringProxy) Start(address string) (net.Listener, error) {
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go p.run(listener)
|
||||
return listener, nil
|
||||
}
|
||||
|
||||
// StartTLS starts the misbehaving proxy for TLS. This endpoint will return
|
||||
// to the client a self signed certificate. Thus, it models the case where a
|
||||
// MITM forces users to accept a rogue certificate. After sending such a
|
||||
// certificate, this proxy will close the TCP connection.
|
||||
func (p *CensoringProxy) StartTLS(address string) (net.Listener, *x509.Certificate, error) {
|
||||
cert, privkey, err := p.mitmNewAuthority(
|
||||
"jafar", "OONI", 24*time.Hour,
|
||||
)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
config, err := p.mitmNewConfig(cert, privkey)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
listener, err := p.tlsListen("tcp", address, config.TLS())
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
go p.run(listener)
|
||||
return listener, cert, nil
|
||||
}
|
||||
@@ -1,157 +0,0 @@
|
||||
package badproxy
|
||||
|
||||
import (
|
||||
"crypto/rsa"
|
||||
"crypto/tls"
|
||||
"crypto/x509"
|
||||
"errors"
|
||||
"net"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/google/martian/v3/mitm"
|
||||
)
|
||||
|
||||
func TestCleartext(t *testing.T) {
|
||||
listener := newproxy(t)
|
||||
checkdial(t, listener.Addr().String(), nil, net.Dial)
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestTLS(t *testing.T) {
|
||||
listener := newproxytls(t)
|
||||
checkdial(t, listener.Addr().String(), nil,
|
||||
func(network, address string) (net.Conn, error) {
|
||||
conn, err := tls.Dial(network, address, &tls.Config{
|
||||
InsecureSkipVerify: true,
|
||||
ServerName: "antani.local",
|
||||
})
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
if err = conn.Handshake(); err != nil {
|
||||
conn.Close()
|
||||
return nil, err
|
||||
}
|
||||
return conn, nil
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestListenError(t *testing.T) {
|
||||
proxy := NewCensoringProxy()
|
||||
listener, err := proxy.Start("8.8.8.8:80")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if listener != nil {
|
||||
t.Fatal("expected nil listener here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestStarTLS(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
|
||||
t.Run("when we cannot create a new authority", func(t *testing.T) {
|
||||
proxy := NewCensoringProxy()
|
||||
proxy.mitmNewAuthority = func(
|
||||
name string, organization string,
|
||||
validity time.Duration,
|
||||
) (*x509.Certificate, *rsa.PrivateKey, error) {
|
||||
return nil, nil, expected
|
||||
}
|
||||
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if cert != nil {
|
||||
t.Fatal("expected nil cert")
|
||||
}
|
||||
if privkey != nil {
|
||||
t.Fatal("expected nil privkey")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("when we cannot create a new config", func(t *testing.T) {
|
||||
proxy := NewCensoringProxy()
|
||||
proxy.mitmNewConfig = func(
|
||||
ca *x509.Certificate, privateKey interface{},
|
||||
) (*mitm.Config, error) {
|
||||
return nil, expected
|
||||
}
|
||||
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if cert != nil {
|
||||
t.Fatal("expected nil cert")
|
||||
}
|
||||
if privkey != nil {
|
||||
t.Fatal("expected nil privkey")
|
||||
}
|
||||
})
|
||||
|
||||
t.Run("when we cannot listen", func(t *testing.T) {
|
||||
proxy := NewCensoringProxy()
|
||||
proxy.tlsListen = func(
|
||||
network string, laddr string, config *tls.Config,
|
||||
) (net.Listener, error) {
|
||||
return nil, expected
|
||||
}
|
||||
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if cert != nil {
|
||||
t.Fatal("expected nil cert")
|
||||
}
|
||||
if privkey != nil {
|
||||
t.Fatal("expected nil privkey")
|
||||
}
|
||||
})
|
||||
}
|
||||
|
||||
func newproxy(t *testing.T) net.Listener {
|
||||
proxy := NewCensoringProxy()
|
||||
listener, err := proxy.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return listener
|
||||
}
|
||||
|
||||
func newproxytls(t *testing.T) net.Listener {
|
||||
proxy := NewCensoringProxy()
|
||||
listener, _, err := proxy.StartTLS("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return listener
|
||||
}
|
||||
|
||||
func killproxy(t *testing.T, listener net.Listener) {
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkdial(
|
||||
t *testing.T, proxyAddr string, expectErr error,
|
||||
dial func(network, address string) (net.Conn, error),
|
||||
) {
|
||||
conn, err := dial("tcp", proxyAddr)
|
||||
if err != expectErr {
|
||||
t.Fatal("not the result we expected")
|
||||
}
|
||||
if conn == nil && expectErr == nil {
|
||||
t.Fatal("expected actionable conn")
|
||||
}
|
||||
if conn != nil && expectErr != nil {
|
||||
t.Fatal("expected nil conn")
|
||||
}
|
||||
if conn != nil {
|
||||
conn.Write([]byte("123454321"))
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
@@ -1,46 +0,0 @@
|
||||
// Package flagx contains extensions for the standard library
|
||||
// flag package. The code is adapted from github.com/m-lab/go and more
|
||||
// specifically from <https://git.io/JJ8UA>. This file is licensed under
|
||||
// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
|
||||
package flagx
|
||||
|
||||
import (
|
||||
"fmt"
|
||||
"strings"
|
||||
)
|
||||
|
||||
// StringArray is a new flag type. It appends the flag parameter to an
|
||||
// `[]string` allowing the parameter to be specified multiple times or using ","
|
||||
// separated items. Unlike other Flag types, the default argument should almost
|
||||
// always be the empty array, because there is no way to remove an element, only
|
||||
// to add one.
|
||||
type StringArray []string
|
||||
|
||||
// Get retrieves the value contained in the flag.
|
||||
func (sa StringArray) Get() interface{} {
|
||||
return sa
|
||||
}
|
||||
|
||||
// Set accepts a string parameter and appends it to the associated StringArray.
|
||||
// Set attempts to split the given string on commas "," and appends each element
|
||||
// to the StringArray.
|
||||
func (sa *StringArray) Set(s string) error {
|
||||
f := strings.Split(s, ",")
|
||||
*sa = append(*sa, f...)
|
||||
return nil
|
||||
}
|
||||
|
||||
// String reports the StringArray as a Go value.
|
||||
func (sa StringArray) String() string {
|
||||
return fmt.Sprintf("%#v", []string(sa))
|
||||
}
|
||||
|
||||
// Contains returns true when the given value equals one of the StringArray values.
|
||||
func (sa StringArray) Contains(value string) bool {
|
||||
for _, v := range sa {
|
||||
if v == value {
|
||||
return true
|
||||
}
|
||||
}
|
||||
return false
|
||||
}
|
||||
@@ -1,73 +0,0 @@
|
||||
package flagx_test
|
||||
|
||||
// The code in this file is adapted from github.com/m-lab/go and more
|
||||
// specifically from <https://git.io/JJ8UA>. This file is licensed under
|
||||
// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
|
||||
|
||||
import (
|
||||
"flag"
|
||||
"testing"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
|
||||
)
|
||||
|
||||
func TestStringArray(t *testing.T) {
|
||||
tests := []struct {
|
||||
name string
|
||||
args []string
|
||||
expt flagx.StringArray
|
||||
repr string
|
||||
contains string
|
||||
wantErr bool
|
||||
}{
|
||||
{
|
||||
name: "okay",
|
||||
args: []string{"a", "b"},
|
||||
expt: flagx.StringArray{"a", "b"},
|
||||
repr: `[]string{"a", "b"}`,
|
||||
contains: "b",
|
||||
},
|
||||
{
|
||||
name: "okay-split-commas",
|
||||
args: []string{"a", "b", "c,d"},
|
||||
expt: flagx.StringArray{"a", "b", "c", "d"},
|
||||
repr: `[]string{"a", "b", "c", "d"}`,
|
||||
contains: "d",
|
||||
},
|
||||
{
|
||||
name: "empty",
|
||||
args: []string{},
|
||||
expt: flagx.StringArray{},
|
||||
repr: `[]string{}`,
|
||||
contains: "a",
|
||||
wantErr: true,
|
||||
},
|
||||
}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
sa := &flagx.StringArray{}
|
||||
for i := range tt.args {
|
||||
if err := sa.Set(tt.args[i]); err != nil {
|
||||
t.Errorf("StringArray.Set() error = %v, want nil", err)
|
||||
}
|
||||
}
|
||||
v := (sa.Get().(flagx.StringArray))
|
||||
if diff := cmp.Diff(v, tt.expt); diff != "" {
|
||||
t.Errorf("StringArray.Get() unexpected differences %v", diff)
|
||||
}
|
||||
if tt.repr != sa.String() {
|
||||
t.Errorf("StringArray.String() want = %q, got %q", tt.repr, sa.String())
|
||||
}
|
||||
if sa.Contains(tt.contains) == tt.wantErr {
|
||||
t.Errorf("StringArray.Contains() want = %q, got %t", tt.repr, sa.Contains(tt.contains))
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
// Successful compilation of this function means that StringArray implements the
|
||||
// flag.Getter interface. The function need not be called.
|
||||
func assertFlagGetterStringArray(b flagx.StringArray) {
|
||||
func(in flag.Getter) {}(&b)
|
||||
}
|
||||
@@ -1,80 +0,0 @@
|
||||
// Package httpproxy contains a censoring HTTP proxy. This proxy will
|
||||
// vet all the traffic and reply with 451 responses for a configurable
|
||||
// set of offending Host headers in incoming requests.
|
||||
package httpproxy
|
||||
|
||||
import (
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httputil"
|
||||
"net/url"
|
||||
"strings"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
const product = "jafar/0.1.0"
|
||||
|
||||
// CensoringProxy is a censoring HTTP proxy
|
||||
type CensoringProxy struct {
|
||||
keywords []string
|
||||
transport http.RoundTripper
|
||||
}
|
||||
|
||||
// NewCensoringProxy creates a new CensoringProxy instance using
|
||||
// the specified list of keywords to censor. keywords is the list
|
||||
// of keywords that trigger censorship if any of them appears in
|
||||
// the Host header of a request. dnsNetwork and dnsAddress are
|
||||
// settings to configure the upstream, non censored DNS.
|
||||
func NewCensoringProxy(
|
||||
keywords []string, uncensored netx.HTTPRoundTripper,
|
||||
) *CensoringProxy {
|
||||
return &CensoringProxy{keywords: keywords, transport: uncensored}
|
||||
}
|
||||
|
||||
var blockpage = []byte(`<html><head>
|
||||
<title>451 Unavailable For Legal Reasons</title>
|
||||
</head><body>
|
||||
<center><h1>451 Unavailable For Legal Reasons</h1></center>
|
||||
<p>This content is not available in your jurisdiction.</p>
|
||||
</body></html>
|
||||
`)
|
||||
|
||||
// ServeHTTP serves HTTP requests
|
||||
func (p *CensoringProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
|
||||
// Implementation note: use Via header to detect in a loose way
|
||||
// requests originated by us and directed to us
|
||||
if r.Header.Get("Via") != "" || r.Host == "" {
|
||||
w.WriteHeader(http.StatusBadRequest)
|
||||
return
|
||||
}
|
||||
for _, pattern := range p.keywords {
|
||||
if strings.Contains(r.Host, pattern) {
|
||||
w.WriteHeader(http.StatusUnavailableForLegalReasons)
|
||||
w.Write(blockpage)
|
||||
return
|
||||
}
|
||||
}
|
||||
r.Header.Add("Via", product) // see above
|
||||
proxy := httputil.NewSingleHostReverseProxy(&url.URL{
|
||||
Host: r.Host,
|
||||
Scheme: "http",
|
||||
})
|
||||
proxy.ModifyResponse = func(resp *http.Response) error {
|
||||
resp.Header.Add("Via", product) // see above
|
||||
return nil
|
||||
}
|
||||
proxy.Transport = p.transport
|
||||
proxy.ServeHTTP(w, r)
|
||||
}
|
||||
|
||||
// Start starts the censoring proxy.
|
||||
func (p *CensoringProxy) Start(address string) (*http.Server, net.Addr, error) {
|
||||
server := &http.Server{Handler: p}
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
return nil, nil, err
|
||||
}
|
||||
go server.Serve(listener)
|
||||
return server, listener.Addr(), nil
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
package httpproxy
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
|
||||
)
|
||||
|
||||
func TestPass(t *testing.T) {
|
||||
server, addr := newproxy(t, "ooni.io")
|
||||
// We're filtering ooni.io, so we expect example.com to pass
|
||||
// through the proxy with 200 and we also expect to see the
|
||||
// Via header in the responses we receive, of course.
|
||||
checkrequest(t, addr.String(), "example.com", 200, true)
|
||||
killproxy(t, server)
|
||||
}
|
||||
|
||||
func TestBlock(t *testing.T) {
|
||||
server, addr := newproxy(t, "ooni.io")
|
||||
// Here we're filtering any domain containing ooni.io, so we
|
||||
// expect the proxy to send 451 without actually proxing, thus
|
||||
// there should not be any Via header in the output.
|
||||
checkrequest(t, addr.String(), "mia-ps.ooni.io", 451, false)
|
||||
killproxy(t, server)
|
||||
}
|
||||
|
||||
func TestLoop(t *testing.T) {
|
||||
server, addr := newproxy(t, "ooni.io")
|
||||
// Here we're forcing the proxy to connect to itself. It does
|
||||
// does that and recognizes itself because of the Via header
|
||||
// being set in the request generated by the connection to itself,
|
||||
// which should cause a 400. The response should have the Via
|
||||
// header set because the 400 is received by the connection that
|
||||
// this code has made to the proxy.
|
||||
checkrequest(t, addr.String(), addr.String(), 400, true)
|
||||
killproxy(t, server)
|
||||
}
|
||||
|
||||
func TestListenError(t *testing.T) {
|
||||
proxy := NewCensoringProxy([]string{""}, uncensored.DefaultClient)
|
||||
server, addr, err := proxy.Start("8.8.8.8:80")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if server != nil {
|
||||
t.Fatal("expected nil server here")
|
||||
}
|
||||
if addr != nil {
|
||||
t.Fatal("expected nil addr here")
|
||||
}
|
||||
}
|
||||
|
||||
func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) {
|
||||
proxy := NewCensoringProxy([]string{blocked}, uncensored.DefaultClient)
|
||||
server, addr, err := proxy.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return server, addr
|
||||
}
|
||||
|
||||
func killproxy(t *testing.T, server *http.Server) {
|
||||
err := server.Shutdown(context.Background())
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkrequest(
|
||||
t *testing.T, proxyAddr, host string,
|
||||
expectStatus int, expectVia bool,
|
||||
) {
|
||||
req, err := http.NewRequest("GET", "http://"+proxyAddr, nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
req.Host = host
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != expectStatus {
|
||||
t.Fatal("unexpected value of status code")
|
||||
}
|
||||
t.Log(resp)
|
||||
values, _ := resp.Header["Via"]
|
||||
var foundProduct bool
|
||||
for _, value := range values {
|
||||
if value == product {
|
||||
foundProduct = true
|
||||
}
|
||||
}
|
||||
if foundProduct && !expectVia {
|
||||
t.Fatal("unexpectedly found Via header")
|
||||
}
|
||||
if !foundProduct && expectVia {
|
||||
t.Fatal("Via header not found")
|
||||
}
|
||||
proxiedData, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if expectStatus == 200 {
|
||||
checkbody(t, proxiedData, host)
|
||||
}
|
||||
}
|
||||
|
||||
func checkbody(t *testing.T, proxiedData []byte, host string) {
|
||||
resp, err := http.Get("http://" + host)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatal("unexpected status code")
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if bytes.Equal(data, proxiedData) == false {
|
||||
t.Fatal("body mismatch")
|
||||
}
|
||||
}
|
||||
@@ -1,98 +0,0 @@
|
||||
// Package iptables contains code for managing firewall rules. This package
|
||||
// really only works reliably on Linux. In all other systems the functionality
|
||||
// in here is just a set of stubs returning errors.
|
||||
package iptables
|
||||
|
||||
import (
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
)
|
||||
|
||||
type shell interface {
|
||||
createChains() error
|
||||
dropIfDestinationEquals(ip string) error
|
||||
rstIfDestinationEqualsAndIsTCP(ip string) error
|
||||
dropIfContainsKeywordHex(keyword string) error
|
||||
dropIfContainsKeyword(keyword string) error
|
||||
rstIfContainsKeywordHexAndIsTCP(keyword string) error
|
||||
rstIfContainsKeywordAndIsTCP(keyword string) error
|
||||
hijackDNS(address string) error
|
||||
hijackHTTPS(address string) error
|
||||
hijackHTTP(address string) error
|
||||
waive() error
|
||||
}
|
||||
|
||||
// CensoringPolicy implements a censoring policy.
|
||||
type CensoringPolicy struct {
|
||||
DropIPs []string // drop IP traffic to these IPs
|
||||
DropKeywordsHex []string // drop IP packets with these hex keywords
|
||||
DropKeywords []string // drop IP packets with these keywords
|
||||
HijackDNSAddress string // where to hijack DNS to
|
||||
HijackHTTPSAddress string // where to hijack HTTPS to
|
||||
HijackHTTPAddress string // where to hijack HTTP to
|
||||
ResetIPs []string // RST TCP/IP traffic to these IPs
|
||||
ResetKeywordsHex []string // RST TCP/IP flows with these hex keywords
|
||||
ResetKeywords []string // RST TCP/IP flows with these keywords
|
||||
sh shell
|
||||
}
|
||||
|
||||
// NewCensoringPolicy returns a new censoring policy.
|
||||
func NewCensoringPolicy() *CensoringPolicy {
|
||||
return &CensoringPolicy{
|
||||
sh: newShell(),
|
||||
}
|
||||
}
|
||||
|
||||
// Apply applies the censorship policy
|
||||
func (c *CensoringPolicy) Apply() (err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
// JUST KNOW WE'VE BEEN HERE
|
||||
}
|
||||
}()
|
||||
err = c.sh.createChains()
|
||||
runtimex.PanicOnError(err, "c.sh.createChains failed")
|
||||
// Implementation note: we want the RST rules to be first such
|
||||
// that we end up enforcing them before the drop rules.
|
||||
for _, keyword := range c.ResetKeywordsHex {
|
||||
err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword)
|
||||
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed")
|
||||
}
|
||||
for _, keyword := range c.ResetKeywords {
|
||||
err = c.sh.rstIfContainsKeywordAndIsTCP(keyword)
|
||||
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed")
|
||||
}
|
||||
for _, ip := range c.ResetIPs {
|
||||
err = c.sh.rstIfDestinationEqualsAndIsTCP(ip)
|
||||
runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed")
|
||||
}
|
||||
for _, keyword := range c.DropKeywordsHex {
|
||||
err = c.sh.dropIfContainsKeywordHex(keyword)
|
||||
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed")
|
||||
}
|
||||
for _, keyword := range c.DropKeywords {
|
||||
err = c.sh.dropIfContainsKeyword(keyword)
|
||||
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed")
|
||||
}
|
||||
for _, ip := range c.DropIPs {
|
||||
err = c.sh.dropIfDestinationEquals(ip)
|
||||
runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed")
|
||||
}
|
||||
if c.HijackDNSAddress != "" {
|
||||
err = c.sh.hijackDNS(c.HijackDNSAddress)
|
||||
runtimex.PanicOnError(err, "c.sh.hijackDNS failed")
|
||||
}
|
||||
if c.HijackHTTPSAddress != "" {
|
||||
err = c.sh.hijackHTTPS(c.HijackHTTPSAddress)
|
||||
runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed")
|
||||
}
|
||||
if c.HijackHTTPAddress != "" {
|
||||
err = c.sh.hijackHTTP(c.HijackHTTPAddress)
|
||||
runtimex.PanicOnError(err, "c.sh.hijackHTTP failed")
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// Waive removes any censorship policy
|
||||
func (c *CensoringPolicy) Waive() error {
|
||||
return c.sh.waive()
|
||||
}
|
||||
@@ -1,345 +0,0 @@
|
||||
package iptables
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"os/exec"
|
||||
"runtime"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
|
||||
)
|
||||
|
||||
func init() {
|
||||
log.SetLevel(log.ErrorLevel)
|
||||
}
|
||||
|
||||
func newCensoringPolicy() *CensoringPolicy {
|
||||
policy := NewCensoringPolicy()
|
||||
policy.Waive() // start over to allow for repeated tests on failure
|
||||
return policy
|
||||
}
|
||||
|
||||
func TestCannotApplyPolicy(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.DropIPs = []string{"antani"}
|
||||
if err := policy.Apply(); err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestCreateChainsError(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
// you should not be able to apply the policy when there is
|
||||
// already a policy, you need to waive it first
|
||||
if err := policy.Apply(); err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropIP(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.DropIPs = []string{"1.1.1.1"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853")
|
||||
if err == nil {
|
||||
t.Fatalf("expected an error here")
|
||||
}
|
||||
if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Fatal("expected nil connection here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropKeyword(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.DropKeywords = []string{"ooni.io"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if !strings.HasSuffix(err.Error(), "context deadline exceeded") {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if resp != nil {
|
||||
t.Fatal("expected nil response here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDropKeywordHex(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
|
||||
defer cancel()
|
||||
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
// the error we see with GitHub Actions is different from the error
|
||||
// we see when testing locally on Fedora
|
||||
if !strings.HasSuffix(err.Error(), "operation not permitted") &&
|
||||
!strings.HasSuffix(err.Error(), "Temporary failure in name resolution") &&
|
||||
!strings.HasSuffix(err.Error(), "no such host") {
|
||||
t.Fatalf("unexpected error occurred: %+v", err)
|
||||
}
|
||||
if resp != nil {
|
||||
t.Fatal("expected nil response here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetIP(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.ResetIPs = []string{"1.1.1.1"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853")
|
||||
if err == nil {
|
||||
t.Fatalf("expected an error here")
|
||||
}
|
||||
if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Fatal("expected nil connection here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetKeyword(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.ResetKeywords = []string{"ooni.io"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := http.Get("http://www.ooni.io")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if resp != nil {
|
||||
t.Fatal("expected nil response here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestResetKeywordHex(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"}
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
resp, err := http.Get("http://www.ooni.io")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if resp != nil {
|
||||
t.Fatal("expected nil response here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHijackDNS(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
resolver := resolver.NewCensoringResolver(
|
||||
[]string{"ooni.io"}, nil, nil,
|
||||
uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")),
|
||||
)
|
||||
server, err := resolver.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer server.Shutdown()
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
policy.HijackDNSAddress = server.PacketConn.LocalAddr().String()
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
addrs, err := net.LookupHost("www.ooni.io")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if strings.Contains(err.Error(), "no such host") == false {
|
||||
t.Fatal("unexpected error occurred")
|
||||
}
|
||||
if addrs != nil {
|
||||
t.Fatal("expected nil addrs here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHijackHTTP(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
// Implementation note: this test is complicated by the fact
|
||||
// that we are running as root and so we're whitelisted.
|
||||
server := httptest.NewServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(451)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
pu, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policy.HijackHTTPAddress = pu.Host
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = shellx.Run("sudo", "-u", "nobody", "--",
|
||||
"curl", "-sf", "http://example.com")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) {
|
||||
t.Fatal("not the error type we expected")
|
||||
}
|
||||
if exitErr.ExitCode() != 22 {
|
||||
t.Fatal("not the exit code we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHijackHTTPS(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("not implemented on this platform")
|
||||
}
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
// Implementation note: this test is complicated by the fact
|
||||
// that we are running as root and so we're whitelisted.
|
||||
server := httptest.NewTLSServer(http.HandlerFunc(
|
||||
func(w http.ResponseWriter, r *http.Request) {
|
||||
w.WriteHeader(451)
|
||||
}),
|
||||
)
|
||||
defer server.Close()
|
||||
policy := newCensoringPolicy()
|
||||
defer policy.Waive()
|
||||
pu, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
policy.HijackHTTPSAddress = pu.Host
|
||||
if err := policy.Apply(); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
err = shellx.Run("sudo", "-u", "nobody", "--",
|
||||
"curl", "-sf", "https://example.com")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
t.Log(err)
|
||||
var exitErr *exec.ExitError
|
||||
if !errors.As(err, &exitErr) {
|
||||
t.Fatal("not the error type we expected")
|
||||
}
|
||||
if exitErr.ExitCode() != 60 {
|
||||
t.Fatal("not the exit code we expected")
|
||||
}
|
||||
}
|
||||
@@ -1,117 +0,0 @@
|
||||
// +build linux
|
||||
|
||||
package iptables
|
||||
|
||||
import (
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
)
|
||||
|
||||
type linuxShell struct{}
|
||||
|
||||
func (s *linuxShell) createChains() (err error) {
|
||||
defer func() {
|
||||
if recover() != nil {
|
||||
// JUST KNOW WE'VE BEEN HERE
|
||||
}
|
||||
}()
|
||||
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_INPUT")
|
||||
runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain")
|
||||
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_OUTPUT")
|
||||
runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain")
|
||||
err = shellx.Run("sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT")
|
||||
runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain")
|
||||
err = shellx.Run("sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT")
|
||||
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT")
|
||||
err = shellx.Run("sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT")
|
||||
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT")
|
||||
err = shellx.Run("sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
|
||||
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT")
|
||||
return nil
|
||||
}
|
||||
|
||||
func (s *linuxShell) dropIfDestinationEquals(ip string) error {
|
||||
return shellx.Run("sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP")
|
||||
}
|
||||
|
||||
func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip,
|
||||
"-j", "REJECT", "--reject-with", "tcp-reset",
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error {
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
|
||||
"--hex-string", keyword, "-j", "DROP",
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) dropIfContainsKeyword(keyword string) error {
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
|
||||
"--string", keyword, "-j", "DROP",
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
|
||||
"kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
|
||||
"kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) hijackDNS(address string) error {
|
||||
// Hijack any DNS query, like the Vodafone station does when using the
|
||||
// secure network feature. Our transparent proxies will use DoT, in order
|
||||
// to bypass this restriction and avoid routing loop.
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp",
|
||||
"--dport", "53", "-j", "DNAT", "--to", address,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) hijackHTTPS(address string) error {
|
||||
// We need to whitelist root otherwise the traffic sent by Jafar
|
||||
// itself will match the rule and loop.
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
|
||||
"--dport", "443", "-m", "owner", "!", "--uid-owner", "0",
|
||||
"-j", "DNAT", "--to", address,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) hijackHTTP(address string) error {
|
||||
// We need to whitelist root otherwise the traffic sent by Jafar
|
||||
// itself will match the rule and loop.
|
||||
return shellx.Run(
|
||||
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
|
||||
"--dport", "80", "-m", "owner", "!", "--uid-owner", "0",
|
||||
"-j", "DNAT", "--to", address,
|
||||
)
|
||||
}
|
||||
|
||||
func (s *linuxShell) waive() error {
|
||||
shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT")
|
||||
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT")
|
||||
return nil
|
||||
}
|
||||
|
||||
func newShell() *linuxShell {
|
||||
return &linuxShell{}
|
||||
}
|
||||
@@ -1,45 +0,0 @@
|
||||
// +build !linux
|
||||
|
||||
package iptables
|
||||
|
||||
import "errors"
|
||||
|
||||
type otherwiseShell struct{}
|
||||
|
||||
func (*otherwiseShell) createChains() error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) dropIfDestinationEquals(ip string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) dropIfContainsKeyword(keyword string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) hijackDNS(address string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) hijackHTTPS(address string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) hijackHTTP(address string) error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
func (*otherwiseShell) waive() error {
|
||||
return errors.New("not implemented")
|
||||
}
|
||||
|
||||
func newShell() *otherwiseShell {
|
||||
return &otherwiseShell{}
|
||||
}
|
||||
@@ -1,286 +0,0 @@
|
||||
// Jafar is a censorship simulation tool used for testing OONI.
|
||||
package main
|
||||
|
||||
import (
|
||||
"encoding/pem"
|
||||
"errors"
|
||||
"flag"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"os"
|
||||
"os/exec"
|
||||
"os/signal"
|
||||
"strings"
|
||||
"syscall"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/apex/log/handlers/cli"
|
||||
"github.com/miekg/dns"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/badproxy"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/httpproxy"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/tlsproxy"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
)
|
||||
|
||||
var (
|
||||
badProxyAddress *string
|
||||
badProxyAddressTLS *string
|
||||
badProxyTLSOutputCA *string
|
||||
|
||||
dnsProxyAddress *string
|
||||
dnsProxyBlock flagx.StringArray
|
||||
dnsProxyHijack flagx.StringArray
|
||||
dnsProxyIgnore flagx.StringArray
|
||||
|
||||
httpProxyAddress *string
|
||||
httpProxyBlock flagx.StringArray
|
||||
|
||||
iptablesDropIP flagx.StringArray
|
||||
iptablesDropKeywordHex flagx.StringArray
|
||||
iptablesDropKeyword flagx.StringArray
|
||||
iptablesHijackDNSTo *string
|
||||
iptablesHijackHTTPSTo *string
|
||||
iptablesHijackHTTPTo *string
|
||||
iptablesResetIP flagx.StringArray
|
||||
iptablesResetKeywordHex flagx.StringArray
|
||||
iptablesResetKeyword flagx.StringArray
|
||||
|
||||
mainCh chan os.Signal
|
||||
mainCommand *string
|
||||
mainUser *string
|
||||
|
||||
tag *string
|
||||
|
||||
tlsProxyAddress *string
|
||||
tlsProxyBlock flagx.StringArray
|
||||
|
||||
uncensoredResolverURL *string
|
||||
)
|
||||
|
||||
func init() {
|
||||
// badProxy
|
||||
badProxyAddress = flag.String(
|
||||
"bad-proxy-address", "127.0.0.1:7117",
|
||||
"Address where to listen for TCP connections",
|
||||
)
|
||||
badProxyAddressTLS = flag.String(
|
||||
"bad-proxy-address-tls", "127.0.0.1:4114",
|
||||
"Address where to listen for TLS connections",
|
||||
)
|
||||
badProxyTLSOutputCA = flag.String(
|
||||
"bad-proxy-tls-output-ca", "badproxy.pem",
|
||||
"File where to write the CA used by the bad proxy",
|
||||
)
|
||||
|
||||
// dnsProxy
|
||||
dnsProxyAddress = flag.String(
|
||||
"dns-proxy-address", "127.0.0.1:53",
|
||||
"Address where the DNS proxy should listen",
|
||||
)
|
||||
flag.Var(
|
||||
&dnsProxyBlock, "dns-proxy-block",
|
||||
"Register keyword triggering NXDOMAIN censorship",
|
||||
)
|
||||
flag.Var(
|
||||
&dnsProxyHijack, "dns-proxy-hijack",
|
||||
"Register keyword triggering redirection to 127.0.0.1",
|
||||
)
|
||||
flag.Var(
|
||||
&dnsProxyIgnore, "dns-proxy-ignore",
|
||||
"Register keyword causing the proxy to ignore the query",
|
||||
)
|
||||
|
||||
// httpProxy
|
||||
httpProxyAddress = flag.String(
|
||||
"http-proxy-address", "127.0.0.1:80",
|
||||
"Address where the HTTP proxy should listen",
|
||||
)
|
||||
flag.Var(
|
||||
&httpProxyBlock, "http-proxy-block",
|
||||
"Register keyword triggering HTTP 451 censorship",
|
||||
)
|
||||
|
||||
// iptables
|
||||
flag.Var(
|
||||
&iptablesDropIP, "iptables-drop-ip",
|
||||
"Drop traffic to the specified IP address",
|
||||
)
|
||||
flag.Var(
|
||||
&iptablesDropKeywordHex, "iptables-drop-keyword-hex",
|
||||
"Drop traffic containing the specified keyword in hex",
|
||||
)
|
||||
flag.Var(
|
||||
&iptablesDropKeyword, "iptables-drop-keyword",
|
||||
"Drop traffic containing the specified keyword",
|
||||
)
|
||||
iptablesHijackDNSTo = flag.String(
|
||||
"iptables-hijack-dns-to", "",
|
||||
"Hijack all DNS UDP traffic to the specified endpoint",
|
||||
)
|
||||
iptablesHijackHTTPSTo = flag.String(
|
||||
"iptables-hijack-https-to", "",
|
||||
"Hijack all HTTPS traffic to the specified endpoint",
|
||||
)
|
||||
iptablesHijackHTTPTo = flag.String(
|
||||
"iptables-hijack-http-to", "",
|
||||
"Hijack all HTTP traffic to the specified endpoint",
|
||||
)
|
||||
flag.Var(
|
||||
&iptablesResetIP, "iptables-reset-ip",
|
||||
"Reset TCP/IP traffic to the specified IP address",
|
||||
)
|
||||
flag.Var(
|
||||
&iptablesResetKeywordHex, "iptables-reset-keyword-hex",
|
||||
"Reset TCP/IP traffic containing the specified keyword in hex",
|
||||
)
|
||||
flag.Var(
|
||||
&iptablesResetKeyword, "iptables-reset-keyword",
|
||||
"Reset TCP/IP traffic containing the specified keyword",
|
||||
)
|
||||
|
||||
// main
|
||||
mainCh = make(chan os.Signal, 1)
|
||||
signal.Notify(
|
||||
mainCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT,
|
||||
)
|
||||
mainCommand = flag.String("main-command", "", "Optional command to execute")
|
||||
mainUser = flag.String("main-user", "nobody", "Run command as user")
|
||||
|
||||
// tag
|
||||
tag = flag.String("tag", "", "Add tag to a specific run")
|
||||
|
||||
// tlsProxy
|
||||
tlsProxyAddress = flag.String(
|
||||
"tls-proxy-address", "127.0.0.1:443",
|
||||
"Address where the HTTP proxy should listen",
|
||||
)
|
||||
flag.Var(
|
||||
&tlsProxyBlock, "tls-proxy-block",
|
||||
"Register keyword triggering TLS censorship",
|
||||
)
|
||||
|
||||
// uncensored
|
||||
uncensoredResolverURL = flag.String(
|
||||
"uncensored-resolver-url", "dot://1.1.1.1:853",
|
||||
"URL of an hopefully uncensored resolver",
|
||||
)
|
||||
}
|
||||
|
||||
func badProxyStart() net.Listener {
|
||||
proxy := badproxy.NewCensoringProxy()
|
||||
listener, err := proxy.Start(*badProxyAddress)
|
||||
runtimex.PanicOnError(err, "proxy.Start failed")
|
||||
return listener
|
||||
}
|
||||
|
||||
func badProxyStartTLS() net.Listener {
|
||||
proxy := badproxy.NewCensoringProxy()
|
||||
listener, cert, err := proxy.StartTLS(*badProxyAddressTLS)
|
||||
runtimex.PanicOnError(err, "proxy.StartTLS failed")
|
||||
err = ioutil.WriteFile(*badProxyTLSOutputCA, pem.EncodeToMemory(&pem.Block{
|
||||
Type: "CERTIFICATE",
|
||||
Bytes: cert.Raw,
|
||||
}), 0644)
|
||||
runtimex.PanicOnError(err, "ioutil.WriteFile failed")
|
||||
return listener
|
||||
}
|
||||
|
||||
func dnsProxyStart(uncensored *uncensored.Client) *dns.Server {
|
||||
proxy := resolver.NewCensoringResolver(
|
||||
dnsProxyBlock, dnsProxyHijack, dnsProxyIgnore, uncensored,
|
||||
)
|
||||
server, err := proxy.Start(*dnsProxyAddress)
|
||||
runtimex.PanicOnError(err, "proxy.Start failed")
|
||||
return server
|
||||
}
|
||||
|
||||
func httpProxyStart(uncensored *uncensored.Client) *http.Server {
|
||||
proxy := httpproxy.NewCensoringProxy(httpProxyBlock, uncensored)
|
||||
server, _, err := proxy.Start(*httpProxyAddress)
|
||||
runtimex.PanicOnError(err, "proxy.Start failed")
|
||||
return server
|
||||
}
|
||||
|
||||
func iptablesStart() *iptables.CensoringPolicy {
|
||||
policy := iptables.NewCensoringPolicy()
|
||||
// For robustness waive the policy so we start afresh
|
||||
policy.Waive()
|
||||
policy.DropIPs = iptablesDropIP
|
||||
policy.DropKeywordsHex = iptablesDropKeywordHex
|
||||
policy.DropKeywords = iptablesDropKeyword
|
||||
policy.HijackDNSAddress = *iptablesHijackDNSTo
|
||||
policy.HijackHTTPSAddress = *iptablesHijackHTTPSTo
|
||||
policy.HijackHTTPAddress = *iptablesHijackHTTPTo
|
||||
policy.ResetIPs = iptablesResetIP
|
||||
policy.ResetKeywordsHex = iptablesResetKeywordHex
|
||||
policy.ResetKeywords = iptablesResetKeyword
|
||||
err := policy.Apply()
|
||||
runtimex.PanicOnError(err, "policy.Apply failed")
|
||||
return policy
|
||||
}
|
||||
|
||||
func tlsProxyStart(uncensored *uncensored.Client) net.Listener {
|
||||
proxy := tlsproxy.NewCensoringProxy(tlsProxyBlock, uncensored)
|
||||
listener, err := proxy.Start(*tlsProxyAddress)
|
||||
runtimex.PanicOnError(err, "proxy.Start failed")
|
||||
return listener
|
||||
}
|
||||
|
||||
func newUncensoredClient() *uncensored.Client {
|
||||
clnt, err := uncensored.NewClient(*uncensoredResolverURL)
|
||||
runtimex.PanicOnError(err, "uncensored.NewClient failed")
|
||||
return clnt
|
||||
}
|
||||
|
||||
func mustx(err error, message string, osExit func(int)) {
|
||||
if err != nil {
|
||||
var (
|
||||
exitcode = 1
|
||||
exiterr *exec.ExitError
|
||||
)
|
||||
if errors.As(err, &exiterr) {
|
||||
exitcode = exiterr.ExitCode()
|
||||
}
|
||||
log.Errorf("%s", message)
|
||||
osExit(exitcode)
|
||||
}
|
||||
}
|
||||
|
||||
func main() {
|
||||
flag.Parse()
|
||||
// TODO(bassosimone): we may want a verbose flag
|
||||
log.SetLevel(log.InfoLevel)
|
||||
log.SetHandler(cli.Default)
|
||||
log.Infof("jafar command line: [%s]", strings.Join(os.Args, ", "))
|
||||
log.Infof("jafar tag: %s", *tag)
|
||||
uncensoredClient := newUncensoredClient()
|
||||
defer uncensoredClient.CloseIdleConnections()
|
||||
badlistener := badProxyStart()
|
||||
defer badlistener.Close()
|
||||
badtlslistener := badProxyStartTLS()
|
||||
defer badtlslistener.Close()
|
||||
dnsproxy := dnsProxyStart(uncensoredClient)
|
||||
defer dnsproxy.Shutdown()
|
||||
httpproxy := httpProxyStart(uncensoredClient)
|
||||
defer httpproxy.Close()
|
||||
tlslistener := tlsProxyStart(uncensoredClient)
|
||||
defer tlslistener.Close()
|
||||
policy := iptablesStart()
|
||||
var err error
|
||||
if *mainCommand != "" {
|
||||
err = shellx.RunCommandline(fmt.Sprintf(
|
||||
"sudo -u '%s' -- %s", *mainUser, *mainCommand,
|
||||
))
|
||||
} else {
|
||||
<-mainCh
|
||||
}
|
||||
policy.Waive()
|
||||
mustx(err, "subcommand failed", os.Exit)
|
||||
}
|
||||
@@ -1,89 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"errors"
|
||||
"os"
|
||||
"runtime"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
|
||||
)
|
||||
|
||||
func ensureWeStartOverWithIPTables() {
|
||||
iptables.NewCensoringPolicy().Waive()
|
||||
}
|
||||
|
||||
func TestNoCommand(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("skip test on non Linux systems")
|
||||
}
|
||||
ensureWeStartOverWithIPTables()
|
||||
*dnsProxyAddress = "127.0.0.1:0"
|
||||
*httpProxyAddress = "127.0.0.1:0"
|
||||
*tlsProxyAddress = "127.0.0.1:0"
|
||||
go func() {
|
||||
mainCh <- os.Interrupt
|
||||
}()
|
||||
main()
|
||||
}
|
||||
|
||||
func TestWithCommand(t *testing.T) {
|
||||
if runtime.GOOS != "linux" {
|
||||
t.Skip("skip test on non Linux systems")
|
||||
}
|
||||
ensureWeStartOverWithIPTables()
|
||||
*dnsProxyAddress = "127.0.0.1:0"
|
||||
*httpProxyAddress = "127.0.0.1:0"
|
||||
*tlsProxyAddress = "127.0.0.1:0"
|
||||
*mainCommand = "whoami"
|
||||
defer func() {
|
||||
*mainCommand = ""
|
||||
}()
|
||||
main()
|
||||
}
|
||||
|
||||
func TestMustx(t *testing.T) {
|
||||
t.Run("with no error", func(t *testing.T) {
|
||||
var called int
|
||||
mustx(nil, "", func(int) {
|
||||
called++
|
||||
})
|
||||
if called != 0 {
|
||||
t.Fatal("should not happen")
|
||||
}
|
||||
})
|
||||
t.Run("with non-exit-code error", func(t *testing.T) {
|
||||
var (
|
||||
called int
|
||||
exitcode int
|
||||
)
|
||||
mustx(errors.New("antani"), "", func(ec int) {
|
||||
called++
|
||||
exitcode = ec
|
||||
})
|
||||
if called != 1 {
|
||||
t.Fatal("not called?!")
|
||||
}
|
||||
if exitcode != 1 {
|
||||
t.Fatal("unexpected exitcode value")
|
||||
}
|
||||
})
|
||||
t.Run("with exit-code error", func(t *testing.T) {
|
||||
var (
|
||||
called int
|
||||
exitcode int
|
||||
)
|
||||
err := shellx.Run("curl", "-sf", "") // cause exitcode == 3
|
||||
mustx(err, "", func(ec int) {
|
||||
called++
|
||||
exitcode = ec
|
||||
})
|
||||
if called != 1 {
|
||||
t.Fatal("not called?!")
|
||||
}
|
||||
if exitcode != 3 {
|
||||
t.Fatal("unexpected exitcode value")
|
||||
}
|
||||
})
|
||||
}
|
||||
@@ -1,130 +0,0 @@
|
||||
// Package resolver contains a censoring DNS resolver. Most queries are
|
||||
// answered without censorship, but selected queries could either be
|
||||
// discarded or replied to with a bogon or NXDOMAIN answer.
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"strings"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
// CensoringResolver is a censoring resolver.
|
||||
type CensoringResolver struct {
|
||||
blocked []string
|
||||
hijacked []string
|
||||
ignored []string
|
||||
lookupHost func(ctx context.Context, host string) ([]string, error)
|
||||
}
|
||||
|
||||
// NewCensoringResolver creates a new CensoringResolver instance using
|
||||
// the specified list of keywords to censor. blocked is the list of
|
||||
// keywords that trigger NXDOMAIN if they appear in a query. hijacked
|
||||
// is similar but redirects to 127.0.0.1, where the transparent HTTP
|
||||
// and TLS proxies will pick them up. dnsNetwork and dnsAddress are the
|
||||
// settings to configure the upstream, non censored DNS.
|
||||
func NewCensoringResolver(
|
||||
blocked, hijacked, ignored []string, uncensored netx.Resolver,
|
||||
) *CensoringResolver {
|
||||
return &CensoringResolver{
|
||||
blocked: blocked,
|
||||
hijacked: hijacked,
|
||||
ignored: ignored,
|
||||
lookupHost: uncensored.LookupHost,
|
||||
}
|
||||
}
|
||||
|
||||
func (r *CensoringResolver) roundtrip(rw dns.ResponseWriter, req *dns.Msg) {
|
||||
name := req.Question[0].Name
|
||||
addrs, err := r.lookupHost(context.Background(), name)
|
||||
var ips []net.IP
|
||||
if err == nil {
|
||||
for _, addr := range addrs {
|
||||
if ip := net.ParseIP(addr); ip != nil {
|
||||
ips = append(ips, ip)
|
||||
}
|
||||
}
|
||||
}
|
||||
r.reply(rw, req, ips)
|
||||
}
|
||||
|
||||
func (r *CensoringResolver) reply(
|
||||
rw dns.ResponseWriter, req *dns.Msg, ips []net.IP,
|
||||
) {
|
||||
m := new(dns.Msg)
|
||||
m.Compress = true
|
||||
m.MsgHdr.RecursionAvailable = true
|
||||
m.SetReply(req)
|
||||
for _, ip := range ips {
|
||||
ipv6 := strings.Contains(ip.String(), ":")
|
||||
if !ipv6 && req.Question[0].Qtype == dns.TypeA {
|
||||
m.Answer = append(m.Answer, &dns.A{
|
||||
Hdr: dns.RR_Header{
|
||||
Name: req.Question[0].Name,
|
||||
Rrtype: dns.TypeA,
|
||||
Class: dns.ClassINET,
|
||||
Ttl: 0,
|
||||
},
|
||||
A: ip,
|
||||
})
|
||||
}
|
||||
}
|
||||
if m.Answer == nil {
|
||||
m.SetRcode(req, dns.RcodeNameError)
|
||||
}
|
||||
rw.WriteMsg(m)
|
||||
}
|
||||
|
||||
func (r *CensoringResolver) failure(rw dns.ResponseWriter, req *dns.Msg) {
|
||||
m := new(dns.Msg)
|
||||
m.Compress = true
|
||||
m.MsgHdr.RecursionAvailable = true
|
||||
m.SetRcode(req, dns.RcodeServerFailure)
|
||||
rw.WriteMsg(m)
|
||||
}
|
||||
|
||||
// ServeDNS serves a DNS request
|
||||
func (r *CensoringResolver) ServeDNS(rw dns.ResponseWriter, req *dns.Msg) {
|
||||
if len(req.Question) < 1 {
|
||||
r.failure(rw, req)
|
||||
return
|
||||
}
|
||||
name := req.Question[0].Name
|
||||
for _, pattern := range r.blocked {
|
||||
if strings.Contains(name, pattern) {
|
||||
r.reply(rw, req, nil)
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, pattern := range r.hijacked {
|
||||
if strings.Contains(name, pattern) {
|
||||
r.reply(rw, req, []net.IP{net.IPv4(127, 0, 0, 1)})
|
||||
return
|
||||
}
|
||||
}
|
||||
for _, pattern := range r.ignored {
|
||||
if strings.Contains(name, pattern) {
|
||||
return
|
||||
}
|
||||
}
|
||||
r.roundtrip(rw, req)
|
||||
}
|
||||
|
||||
// Start starts the DNS resolver
|
||||
func (r *CensoringResolver) Start(address string) (*dns.Server, error) {
|
||||
packetconn, err := net.ListenPacket("udp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
server := &dns.Server{
|
||||
Addr: address,
|
||||
Handler: r,
|
||||
Net: "udp",
|
||||
PacketConn: packetconn,
|
||||
}
|
||||
go server.ActivateAndServe()
|
||||
return server, nil
|
||||
}
|
||||
@@ -1,173 +0,0 @@
|
||||
package resolver
|
||||
|
||||
import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/miekg/dns"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
|
||||
)
|
||||
|
||||
func TestPass(t *testing.T) {
|
||||
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
|
||||
checkrequest(t, server, "example.com", "success", nil)
|
||||
killserver(t, server)
|
||||
}
|
||||
|
||||
func TestBlock(t *testing.T) {
|
||||
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
|
||||
checkrequest(t, server, "mia-ps.ooni.io", "blocked", nil)
|
||||
killserver(t, server)
|
||||
}
|
||||
|
||||
func TestRedirect(t *testing.T) {
|
||||
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
|
||||
checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", nil)
|
||||
killserver(t, server)
|
||||
}
|
||||
|
||||
func TestIgnore(t *testing.T) {
|
||||
server := newresolver(t, nil, nil, []string{"ooni.nu"})
|
||||
iotimeout := "i/o timeout"
|
||||
checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout)
|
||||
killserver(t, server)
|
||||
}
|
||||
|
||||
func TestLookupFailure(t *testing.T) {
|
||||
server := newresolver(t, nil, nil, nil)
|
||||
// we should receive same response as when we're blocked
|
||||
checkrequest(t, server, "example.antani", "blocked", nil)
|
||||
killserver(t, server)
|
||||
}
|
||||
|
||||
func TestFailureNoQuestion(t *testing.T) {
|
||||
resolver := NewCensoringResolver(
|
||||
nil, nil, nil, uncensored.DefaultClient,
|
||||
)
|
||||
resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg))
|
||||
}
|
||||
|
||||
func TestListenFailure(t *testing.T) {
|
||||
resolver := NewCensoringResolver(
|
||||
nil, nil, nil, uncensored.DefaultClient,
|
||||
)
|
||||
server, err := resolver.Start("8.8.8.8:53")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if server != nil {
|
||||
t.Fatal("expected nil server here")
|
||||
}
|
||||
}
|
||||
|
||||
func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server {
|
||||
resolver := NewCensoringResolver(
|
||||
blocked, hijacked, ignored,
|
||||
// using faster dns because dot here causes miekg/dns's
|
||||
// dns.Exchange to timeout and I don't want more complexity
|
||||
uncensored.Must(uncensored.NewClient("system:///")),
|
||||
)
|
||||
server, err := resolver.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return server
|
||||
}
|
||||
|
||||
func killserver(t *testing.T, server *dns.Server) {
|
||||
err := server.Shutdown()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkrequest(
|
||||
t *testing.T, server *dns.Server, host string, expectStatus string,
|
||||
expectErrorSuffix *string,
|
||||
) {
|
||||
address := server.PacketConn.LocalAddr().String()
|
||||
query := newquery(host)
|
||||
reply, err := dns.Exchange(query, address)
|
||||
if err != nil {
|
||||
if expectErrorSuffix != nil &&
|
||||
strings.HasSuffix(err.Error(), *expectErrorSuffix) {
|
||||
return
|
||||
}
|
||||
t.Fatal(err)
|
||||
}
|
||||
switch expectStatus {
|
||||
case "success":
|
||||
checksuccess(t, reply)
|
||||
case "hijacked":
|
||||
checkhijacked(t, reply)
|
||||
case "blocked":
|
||||
checkblocked(t, reply)
|
||||
default:
|
||||
panic("unexpected value")
|
||||
}
|
||||
}
|
||||
|
||||
func checksuccess(t *testing.T, reply *dns.Msg) {
|
||||
if reply.Rcode != dns.RcodeSuccess {
|
||||
t.Fatal("unexpected rcode")
|
||||
}
|
||||
if len(reply.Answer) < 1 {
|
||||
t.Fatal("too few answers")
|
||||
}
|
||||
for _, answer := range reply.Answer {
|
||||
if rr, ok := answer.(*dns.A); ok {
|
||||
if rr.A.String() == "127.0.0.1" {
|
||||
t.Fatal("unexpected hijacked response here")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkhijacked(t *testing.T, reply *dns.Msg) {
|
||||
if reply.Rcode != dns.RcodeSuccess {
|
||||
t.Fatal("unexpected rcode")
|
||||
}
|
||||
if len(reply.Answer) < 1 {
|
||||
t.Fatal("too few answers")
|
||||
}
|
||||
for _, answer := range reply.Answer {
|
||||
if rr, ok := answer.(*dns.A); ok {
|
||||
if rr.A.String() != "127.0.0.1" {
|
||||
t.Fatal("unexpected non-hijacked response here")
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
func checkblocked(t *testing.T, reply *dns.Msg) {
|
||||
if reply.Rcode != dns.RcodeNameError {
|
||||
t.Fatal("unexpected rcode")
|
||||
}
|
||||
if len(reply.Answer) >= 1 {
|
||||
t.Fatal("too many answers")
|
||||
}
|
||||
}
|
||||
|
||||
func newquery(name string) *dns.Msg {
|
||||
query := new(dns.Msg)
|
||||
query.Id = dns.Id()
|
||||
query.RecursionDesired = true
|
||||
query.Question = append(query.Question, dns.Question{
|
||||
Name: dns.Fqdn(name),
|
||||
Qclass: dns.ClassINET,
|
||||
Qtype: dns.TypeA,
|
||||
})
|
||||
return query
|
||||
}
|
||||
|
||||
type fakeResponseWriter struct {
|
||||
dns.ResponseWriter
|
||||
t *testing.T
|
||||
}
|
||||
|
||||
func (rw *fakeResponseWriter) WriteMsg(m *dns.Msg) error {
|
||||
if m.Rcode != dns.RcodeServerFailure {
|
||||
rw.t.Fatal("unexpected rcode")
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -1,194 +0,0 @@
|
||||
// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed
|
||||
// through using the SNI to choose the hostname to connect to. Specific offending
|
||||
// SNIs are censored by returning a TLS alert to the client.
|
||||
package tlsproxy
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
// CensoringProxy is a censoring TLS proxy
|
||||
type CensoringProxy struct {
|
||||
keywords []string
|
||||
dial func(network, address string) (net.Conn, error)
|
||||
}
|
||||
|
||||
// NewCensoringProxy creates a new CensoringProxy instance using
|
||||
// the specified list of keywords to censor. keywords is the list
|
||||
// of keywords that trigger censorship if any of them appears in
|
||||
// the SNII record of a ClientHello. dnsNetwork and dnsAddress are
|
||||
// settings to configure the upstream, non censored DNS.
|
||||
func NewCensoringProxy(
|
||||
keywords []string, uncensored netx.Dialer,
|
||||
) *CensoringProxy {
|
||||
return &CensoringProxy{
|
||||
keywords: keywords,
|
||||
dial: func(network, address string) (net.Conn, error) {
|
||||
return uncensored.DialContext(context.Background(), network, address)
|
||||
},
|
||||
}
|
||||
}
|
||||
|
||||
// handshakeReader is a hack to perform the initial part of the
|
||||
// TLS handshake so to know the SNI and then replay the bytes of
|
||||
// this initial part of the handshake with the server.
|
||||
type handshakeReader struct {
|
||||
net.Conn
|
||||
incoming []byte
|
||||
}
|
||||
|
||||
// Read saves the initial bytes of the handshake such that later
|
||||
// we can replay the handshake with the real TLS server.
|
||||
func (c *handshakeReader) Read(b []byte) (int, error) {
|
||||
count, err := c.Conn.Read(b)
|
||||
if err == nil {
|
||||
c.incoming = append(c.incoming, b[:count]...)
|
||||
}
|
||||
return count, err
|
||||
}
|
||||
|
||||
// Write prevents writing on the real connection
|
||||
func (c *handshakeReader) Write(b []byte) (int, error) {
|
||||
return 0, errors.New("cannot write on this connection")
|
||||
}
|
||||
|
||||
// forward forwards left traffic to right
|
||||
func forward(wg *sync.WaitGroup, left, right net.Conn) {
|
||||
data := make([]byte, 1<<18)
|
||||
for {
|
||||
n, err := left.Read(data)
|
||||
if err != nil {
|
||||
break
|
||||
}
|
||||
if _, err = right.Write(data[:n]); err != nil {
|
||||
break
|
||||
}
|
||||
}
|
||||
wg.Done()
|
||||
}
|
||||
|
||||
// reset closes the connection with a RST segment
|
||||
func reset(conn net.Conn) {
|
||||
if tc, ok := conn.(*net.TCPConn); ok {
|
||||
tc.SetLinger(0)
|
||||
}
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// alertclose sends a TLS alert and then closes the connection
|
||||
func alertclose(conn net.Conn) {
|
||||
alertdata := []byte{
|
||||
21, // alert
|
||||
3, // version[0]
|
||||
3, // version[1]
|
||||
0, // length[0]
|
||||
2, // length[1]
|
||||
2, // fatal
|
||||
80, // internal error
|
||||
}
|
||||
conn.Write(alertdata)
|
||||
conn.Close()
|
||||
}
|
||||
|
||||
// getsni attempts the handshakeReader hack to obtain the SNI by reading
|
||||
// the beginning of the TLS handshake. On success a nonempty SNI string
|
||||
// is returned. Otherwise we cannot distinguish between the absence of a
|
||||
// SNI and any other reading network error that may have occurred.
|
||||
func getsni(conn *handshakeReader) string {
|
||||
var (
|
||||
sni string
|
||||
mutex sync.Mutex // just for safety
|
||||
)
|
||||
tls.Server(conn, &tls.Config{
|
||||
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
|
||||
mutex.Lock()
|
||||
sni = info.ServerName
|
||||
mutex.Unlock()
|
||||
return nil, errors.New("tlsproxy: we can't really continue handshake")
|
||||
},
|
||||
}).Handshake()
|
||||
return sni
|
||||
}
|
||||
|
||||
func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool {
|
||||
local := conn.LocalAddr().String()
|
||||
localAddr, _, localErr := net.SplitHostPort(local)
|
||||
remote := conn.RemoteAddr().String()
|
||||
remoteAddr, _, remoteErr := net.SplitHostPort(remote)
|
||||
return localErr != nil || remoteErr != nil || localAddr == remoteAddr
|
||||
}
|
||||
|
||||
// handle implements the TLS SNI proxy
|
||||
func (p *CensoringProxy) handle(clientconn net.Conn) {
|
||||
hr := &handshakeReader{Conn: clientconn}
|
||||
sni := getsni(hr)
|
||||
if sni == "" {
|
||||
log.Warn("tlsproxy: network failure or SNI not provided")
|
||||
reset(clientconn)
|
||||
return
|
||||
}
|
||||
for _, pattern := range p.keywords {
|
||||
if strings.Contains(sni, pattern) {
|
||||
log.Warnf("tlsproxy: reject SNI by policy: %s", sni)
|
||||
alertclose(clientconn)
|
||||
return
|
||||
}
|
||||
}
|
||||
serverconn, err := p.dial("tcp", net.JoinHostPort(sni, "443"))
|
||||
if err != nil {
|
||||
log.WithError(err).Warn("tlsproxy: p.dial failed")
|
||||
alertclose(clientconn)
|
||||
return
|
||||
}
|
||||
if p.connectingToMyself(serverconn) {
|
||||
log.Warn("tlsproxy: connecting to myself")
|
||||
alertclose(clientconn)
|
||||
return
|
||||
}
|
||||
if _, err := serverconn.Write(hr.incoming); err != nil {
|
||||
log.WithError(err).Warn("tlsproxy: serverconn.Write failed")
|
||||
alertclose(clientconn)
|
||||
return
|
||||
}
|
||||
log.Debugf("tlsproxy: routing for %s", sni)
|
||||
defer clientconn.Close()
|
||||
defer serverconn.Close()
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(2)
|
||||
go forward(&wg, clientconn, serverconn)
|
||||
go forward(&wg, serverconn, clientconn)
|
||||
wg.Wait()
|
||||
}
|
||||
|
||||
func (p *CensoringProxy) run(listener net.Listener) {
|
||||
for {
|
||||
conn, err := listener.Accept()
|
||||
if err != nil && strings.Contains(
|
||||
err.Error(), "use of closed network connection") {
|
||||
return
|
||||
}
|
||||
if err == nil {
|
||||
// It's difficult to make accept fail, so restructure
|
||||
// the code such that we enter into the happy path
|
||||
go p.handle(conn)
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Start starts the censoring proxy.
|
||||
func (p *CensoringProxy) Start(address string) (net.Listener, error) {
|
||||
listener, err := net.Listen("tcp", address)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
go p.run(listener)
|
||||
return listener, nil
|
||||
}
|
||||
@@ -1,183 +0,0 @@
|
||||
package tlsproxy
|
||||
|
||||
import (
|
||||
"crypto/tls"
|
||||
"errors"
|
||||
"net"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
|
||||
)
|
||||
|
||||
func TestPass(t *testing.T) {
|
||||
listener := newproxy(t, "ooni.io")
|
||||
checkdialtls(t, listener.Addr().String(), true, &tls.Config{
|
||||
ServerName: "example.com",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestBlock(t *testing.T) {
|
||||
listener := newproxy(t, "ooni.io")
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "mia-ps.ooni.io",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestNoSNI(t *testing.T) {
|
||||
listener := newproxy(t, "ooni.io")
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestInvalidDomain(t *testing.T) {
|
||||
listener := newproxy(t, "ooni.io")
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "antani.local",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestFailHandshake(t *testing.T) {
|
||||
listener := newproxy(t, "ooni.io")
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "expired.badssl.com",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestFailConnectingToSelf(t *testing.T) {
|
||||
proxy := &CensoringProxy{
|
||||
dial: func(network string, address string) (net.Conn, error) {
|
||||
return &mockedConnWriteError{}, nil
|
||||
},
|
||||
}
|
||||
listener, err := proxy.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if listener == nil {
|
||||
t.Fatal("expected non nil listener here")
|
||||
}
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "www.google.com",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestFailWriteAfterConnect(t *testing.T) {
|
||||
proxy := &CensoringProxy{
|
||||
dial: func(network string, address string) (net.Conn, error) {
|
||||
return &mockedConnWriteError{
|
||||
// must be different or it refuses connecting to self
|
||||
localIP: net.IPv4(127, 0, 0, 1),
|
||||
remoteIP: net.IPv4(127, 0, 0, 2),
|
||||
}, nil
|
||||
},
|
||||
}
|
||||
listener, err := proxy.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if listener == nil {
|
||||
t.Fatal("expected non nil listener here")
|
||||
}
|
||||
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
|
||||
ServerName: "www.google.com",
|
||||
})
|
||||
killproxy(t, listener)
|
||||
}
|
||||
|
||||
func TestListenError(t *testing.T) {
|
||||
proxy := NewCensoringProxy(
|
||||
[]string{""}, uncensored.DefaultClient,
|
||||
)
|
||||
listener, err := proxy.Start("8.8.8.8:80")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if listener != nil {
|
||||
t.Fatal("expected nil listener here")
|
||||
}
|
||||
}
|
||||
|
||||
func newproxy(t *testing.T, blocked string) net.Listener {
|
||||
proxy := NewCensoringProxy(
|
||||
[]string{blocked}, uncensored.DefaultClient,
|
||||
)
|
||||
listener, err := proxy.Start("127.0.0.1:0")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
return listener
|
||||
}
|
||||
|
||||
func killproxy(t *testing.T, listener net.Listener) {
|
||||
err := listener.Close()
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func checkdialtls(
|
||||
t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config,
|
||||
) {
|
||||
conn, err := tls.Dial("tcp", proxyAddr, config)
|
||||
if err != nil && expectSuccess {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if err == nil && !expectSuccess {
|
||||
t.Fatal("expected failure here")
|
||||
}
|
||||
if conn == nil && expectSuccess {
|
||||
t.Fatal("expected actionable conn")
|
||||
}
|
||||
if conn != nil && !expectSuccess {
|
||||
t.Fatal("expected nil conn")
|
||||
}
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
}
|
||||
|
||||
type mockedConnWriteError struct {
|
||||
net.Conn
|
||||
localIP net.IP
|
||||
remoteIP net.IP
|
||||
}
|
||||
|
||||
func (c *mockedConnWriteError) Write(b []byte) (int, error) {
|
||||
return 0, errors.New("cannot write sorry")
|
||||
}
|
||||
|
||||
func (c *mockedConnWriteError) LocalAddr() net.Addr {
|
||||
return &net.TCPAddr{
|
||||
IP: c.localIP,
|
||||
}
|
||||
}
|
||||
|
||||
func (c *mockedConnWriteError) RemoteAddr() net.Addr {
|
||||
return &net.TCPAddr{
|
||||
IP: c.remoteIP,
|
||||
}
|
||||
}
|
||||
|
||||
func TestForwardWriteError(t *testing.T) {
|
||||
var wg sync.WaitGroup
|
||||
wg.Add(1)
|
||||
forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{})
|
||||
}
|
||||
|
||||
type mockedConnReadOkay struct {
|
||||
net.Conn
|
||||
localIP net.IP
|
||||
remoteIP net.IP
|
||||
}
|
||||
|
||||
func (c *mockedConnReadOkay) Read(b []byte) (int, error) {
|
||||
return len(b), nil
|
||||
}
|
||||
@@ -1,86 +0,0 @@
|
||||
// Package uncensored contains code used by Jafar to evade its own
|
||||
// censorship efforts by taking alternate routes.
|
||||
package uncensored
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
// Client is DNS, HTTP, and TCP client.
|
||||
type Client struct {
|
||||
dnsClient *netx.DNSClient
|
||||
httpTransport netx.HTTPRoundTripper
|
||||
dialer netx.Dialer
|
||||
}
|
||||
|
||||
// NewClient creates a new Client.
|
||||
func NewClient(resolverURL string) (*Client, error) {
|
||||
configuration, err := urlgetter.Configurer{
|
||||
Config: urlgetter.Config{
|
||||
ResolverURL: resolverURL,
|
||||
},
|
||||
Logger: log.Log,
|
||||
}.NewConfiguration()
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
return &Client{
|
||||
dnsClient: &configuration.DNSClient,
|
||||
httpTransport: netx.NewHTTPTransport(configuration.HTTPConfig),
|
||||
dialer: netx.NewDialer(configuration.HTTPConfig),
|
||||
}, nil
|
||||
}
|
||||
|
||||
// Must panics if it's not possible to create a Client. Usually you should
|
||||
// use it like `uncensored.Must(uncensored.NewClient(URL))`.
|
||||
func Must(client *Client, err error) *Client {
|
||||
runtimex.PanicOnError(err, "cannot create uncensored client")
|
||||
return client
|
||||
}
|
||||
|
||||
// DefaultClient is the default client for DNS, HTTP, and TCP.
|
||||
var DefaultClient = Must(NewClient(""))
|
||||
|
||||
var _ netx.Resolver = DefaultClient
|
||||
|
||||
// Address implements netx.Resolver.Address
|
||||
func (c *Client) Address() string {
|
||||
return c.dnsClient.Address()
|
||||
}
|
||||
|
||||
// LookupHost implements netx.Resolver.LookupHost
|
||||
func (c *Client) LookupHost(ctx context.Context, domain string) ([]string, error) {
|
||||
return c.dnsClient.LookupHost(ctx, domain)
|
||||
}
|
||||
|
||||
// Network implements netx.Resolver.Network
|
||||
func (c *Client) Network() string {
|
||||
return c.dnsClient.Network()
|
||||
}
|
||||
|
||||
var _ netx.Dialer = DefaultClient
|
||||
|
||||
// DialContext implements netx.Dialer.DialContext
|
||||
func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
|
||||
return c.dialer.DialContext(ctx, network, address)
|
||||
}
|
||||
|
||||
var _ netx.HTTPRoundTripper = DefaultClient
|
||||
|
||||
// CloseIdleConnections implement netx.HTTPRoundTripper.CloseIdleConnections
|
||||
func (c *Client) CloseIdleConnections() {
|
||||
c.dnsClient.CloseIdleConnections()
|
||||
c.httpTransport.CloseIdleConnections()
|
||||
}
|
||||
|
||||
// RoundTrip implement netx.HTTPRoundTripper.RoundTrip
|
||||
func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
return c.httpTransport.RoundTrip(req)
|
||||
}
|
||||
@@ -1,75 +0,0 @@
|
||||
package uncensored
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestGood(t *testing.T) {
|
||||
client, err := NewClient("dot://1.1.1.1:853")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer client.CloseIdleConnections()
|
||||
if client.Address() != "1.1.1.1:853" {
|
||||
t.Fatal("invalid address")
|
||||
}
|
||||
if client.Network() != "dot" {
|
||||
t.Fatal("invalid network")
|
||||
}
|
||||
ctx := context.Background()
|
||||
addrs, err := client.LookupHost(ctx, "dns.google")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
var quad8, two8two4 bool
|
||||
for _, addr := range addrs {
|
||||
quad8 = quad8 || (addr == "8.8.8.8")
|
||||
two8two4 = two8two4 || (addr == "8.8.4.4")
|
||||
}
|
||||
if quad8 != true && two8two4 != true {
|
||||
t.Fatal("invalid response")
|
||||
}
|
||||
conn, err := client.DialContext(ctx, "tcp", "8.8.8.8:853")
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer conn.Close()
|
||||
resp, err := client.RoundTrip(&http.Request{
|
||||
Method: "GET",
|
||||
URL: &url.URL{
|
||||
Scheme: "https",
|
||||
Host: "www.google.com",
|
||||
Path: "/humans.txt",
|
||||
},
|
||||
Header: http.Header{},
|
||||
})
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
t.Fatal("invalid status-code")
|
||||
}
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !bytes.HasPrefix(data, []byte("Google is built by a large team")) {
|
||||
t.Fatal("not the expected body")
|
||||
}
|
||||
}
|
||||
|
||||
func TestNewClientFailure(t *testing.T) {
|
||||
clnt, err := NewClient("antani:///")
|
||||
if err == nil {
|
||||
t.Fatal("expected an error here")
|
||||
}
|
||||
if clnt != nil {
|
||||
t.Fatal("expected nil client here")
|
||||
}
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
# oohelper
|
||||
|
||||
This directory contains the source code of a simple client
|
||||
for the Web Connectivity test helper.
|
||||
@@ -1,138 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"bytes"
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/version"
|
||||
)
|
||||
|
||||
type (
|
||||
// CtrlResponse is the type of response returned by the test helper.
|
||||
CtrlResponse = webconnectivity.ControlResponse
|
||||
|
||||
// ctrlRequest is the type of the request sent to the test helper.
|
||||
ctrlRequest = webconnectivity.ControlRequest
|
||||
)
|
||||
|
||||
// The following errors may be returned by this implementation.
|
||||
var (
|
||||
ErrHTTPStatusCode = errors.New("oohelper: http status code indicates failure")
|
||||
ErrUnsupportedURLScheme = errors.New("oohelper: unsupported URL scheme")
|
||||
ErrUnsupportedExplicitPort = errors.New("oohelper: unsupported explicit port")
|
||||
ErrEmptyURL = errors.New("oohelper: empty server and/or target URL")
|
||||
ErrInvalidURL = errors.New("oohelper: cannot parse URL")
|
||||
ErrCannotCreateRequest = errors.New("oohelper: cannot create HTTP request")
|
||||
ErrCannotParseJSONReply = errors.New("oohelper: cannot parse JSON reply")
|
||||
)
|
||||
|
||||
// OOClient is a client for the OONI Web Connectivity test helper.
|
||||
type OOClient struct {
|
||||
// HTTPClient is the HTTP client to use.
|
||||
HTTPClient *http.Client
|
||||
|
||||
// Resolver is the resolver to user.
|
||||
Resolver netx.Resolver
|
||||
}
|
||||
|
||||
// OOConfig contains configuration for the client.
|
||||
type OOConfig struct {
|
||||
// ServerURL is the URL of the test helper server.
|
||||
ServerURL string
|
||||
|
||||
// TargetURL is the URL that we want to measure.
|
||||
TargetURL string
|
||||
}
|
||||
|
||||
// MakeTCPEndpoints constructs the list of TCP endpoints to send
|
||||
// to the Web Connectivity test helper.
|
||||
func MakeTCPEndpoints(URL *url.URL, addrs []string) ([]string, error) {
|
||||
var (
|
||||
port string
|
||||
out []string
|
||||
)
|
||||
if URL.Host != URL.Hostname() {
|
||||
return nil, ErrUnsupportedExplicitPort
|
||||
}
|
||||
switch URL.Scheme {
|
||||
case "https":
|
||||
port = "443"
|
||||
case "http":
|
||||
port = "80"
|
||||
default:
|
||||
return nil, ErrUnsupportedURLScheme
|
||||
}
|
||||
for _, addr := range addrs {
|
||||
out = append(out, net.JoinHostPort(addr, port))
|
||||
}
|
||||
return out, nil
|
||||
}
|
||||
|
||||
// Do sends a measurement request to the Web Connectivity test
|
||||
// helper and receives the corresponding response.
|
||||
func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, error) {
|
||||
if config.TargetURL == "" || config.ServerURL == "" {
|
||||
return nil, ErrEmptyURL
|
||||
}
|
||||
targetURL, err := url.Parse(config.TargetURL)
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrInvalidURL, err.Error())
|
||||
}
|
||||
addrs, err := oo.Resolver.LookupHost(ctx, targetURL.Hostname())
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
endpoints, err := MakeTCPEndpoints(targetURL, addrs)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
creq := ctrlRequest{
|
||||
HTTPRequest: config.TargetURL,
|
||||
HTTPRequestHeaders: map[string][]string{
|
||||
"Accept": {httpheader.Accept()},
|
||||
"Accept-Language": {httpheader.AcceptLanguage()},
|
||||
"User-Agent": {httpheader.UserAgent()},
|
||||
},
|
||||
TCPConnect: endpoints,
|
||||
}
|
||||
data, err := json.Marshal(creq)
|
||||
runtimex.PanicOnError(err, "oohelper: cannot marshal control request")
|
||||
log.Debugf("out: %s", string(data))
|
||||
req, err := http.NewRequestWithContext(ctx, "POST", config.ServerURL, bytes.NewReader(data))
|
||||
if err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrCannotCreateRequest, err.Error())
|
||||
}
|
||||
req.Header.Add("user-agent", fmt.Sprintf(
|
||||
"oohelper/%s ooniprobe-engine/%s", version.Version, version.Version,
|
||||
))
|
||||
req.Header.Add("content-type", "application/json")
|
||||
resp, err := oo.HTTPClient.Do(req)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != 200 {
|
||||
return nil, ErrHTTPStatusCode
|
||||
}
|
||||
data, err = ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
var cresp CtrlResponse
|
||||
if err := json.Unmarshal(data, &cresp); err != nil {
|
||||
return nil, fmt.Errorf("%w: %s", ErrCannotParseJSONReply, err.Error())
|
||||
}
|
||||
return &cresp, nil
|
||||
}
|
||||
@@ -1,338 +0,0 @@
|
||||
package internal_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"reflect"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
|
||||
)
|
||||
|
||||
func TestMakeTCPEndpoints(t *testing.T) {
|
||||
type args struct {
|
||||
URL *url.URL
|
||||
addrs []string
|
||||
}
|
||||
tests := []struct {
|
||||
name string
|
||||
args args
|
||||
want []string
|
||||
err error
|
||||
}{{
|
||||
name: "with host != hostname",
|
||||
args: args{URL: &url.URL{Host: "127.0.0.1:8080"}},
|
||||
err: internal.ErrUnsupportedExplicitPort,
|
||||
}, {
|
||||
name: "with unsupported URL scheme",
|
||||
args: args{URL: &url.URL{Host: "127.0.0.1", Scheme: "imap"}},
|
||||
err: internal.ErrUnsupportedURLScheme,
|
||||
}, {
|
||||
name: "with http scheme",
|
||||
args: args{
|
||||
URL: &url.URL{Host: "www.kernel.org", Scheme: "http"},
|
||||
addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
|
||||
},
|
||||
want: []string{"1.1.1.1:80", "2.2.2.2:80", "[::1]:80"},
|
||||
}, {
|
||||
name: "with https scheme",
|
||||
args: args{
|
||||
URL: &url.URL{Host: "www.kernel.org", Scheme: "https"},
|
||||
addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
|
||||
},
|
||||
want: []string{"1.1.1.1:443", "2.2.2.2:443", "[::1]:443"},
|
||||
}}
|
||||
for _, tt := range tests {
|
||||
t.Run(tt.name, func(t *testing.T) {
|
||||
got, err := internal.MakeTCPEndpoints(tt.args.URL, tt.args.addrs)
|
||||
if !errors.Is(err, tt.err) {
|
||||
t.Errorf("MakeTCPEndpoints() error = %v, wantErr %v", err, tt.err)
|
||||
return
|
||||
}
|
||||
if !reflect.DeepEqual(got, tt.want) {
|
||||
t.Errorf("MakeTCPEndpoints() = %v, want %v", got, tt.want)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithEmptyTargetURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{}
|
||||
clnt := internal.OOClient{}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrEmptyURL) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithEmptyServerURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{TargetURL: "http://www.example.com"}
|
||||
clnt := internal.OOClient{}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrEmptyURL) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithInvalidTargetURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{TargetURL: "\t", ServerURL: "https://wcth.ooni.io"}
|
||||
clnt := internal.OOClient{}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrInvalidURL) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithResolverFailure(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverThatFails(),
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrNotFound) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithUnsupportedExplicitPort(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com:8080",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrUnsupportedExplicitPort) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithInvalidServerURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "\t",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrCannotCreateRequest) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithRoundTripError(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
HTTPClient: &http.Client{
|
||||
Transport: internal.FakeTransport{Err: expected},
|
||||
},
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithInvalidStatusCode(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
HTTPClient: &http.Client{Transport: internal.FakeTransport{
|
||||
Resp: &http.Response{
|
||||
StatusCode: 400,
|
||||
Body: &internal.FakeBody{},
|
||||
},
|
||||
}},
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrHTTPStatusCode) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithBodyReadError(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
HTTPClient: &http.Client{Transport: internal.FakeTransport{
|
||||
Resp: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: &internal.FakeBody{
|
||||
Err: expected,
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
func TestOOClientDoWithInvalidJSON(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
HTTPClient: &http.Client{Transport: internal.FakeTransport{
|
||||
Resp: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: &internal.FakeBody{
|
||||
Data: []byte("{"),
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, internal.ErrCannotParseJSONReply) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp != nil {
|
||||
t.Fatal("expected nil response")
|
||||
}
|
||||
}
|
||||
|
||||
const goodresponse = `{
|
||||
"tcp_connect": {
|
||||
"172.217.21.68:80": {
|
||||
"status": true,
|
||||
"failure": null
|
||||
}
|
||||
},
|
||||
"http_request": {
|
||||
"body_length": 207878,
|
||||
"failure": null,
|
||||
"title": "Google",
|
||||
"headers": {
|
||||
"Content-Type": "text/html"
|
||||
},
|
||||
"status_code": 200
|
||||
},
|
||||
"dns": {
|
||||
"failure": null,
|
||||
"addrs": [
|
||||
"172.217.17.68"
|
||||
]
|
||||
}
|
||||
}`
|
||||
|
||||
func TestOOClientDoWithParseableJSON(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
config := internal.OOConfig{
|
||||
TargetURL: "http://www.example.com",
|
||||
ServerURL: "https://wcth.ooni.io",
|
||||
}
|
||||
clnt := internal.OOClient{
|
||||
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
|
||||
HTTPClient: &http.Client{Transport: internal.FakeTransport{
|
||||
Resp: &http.Response{
|
||||
StatusCode: 200,
|
||||
Body: &internal.FakeBody{
|
||||
Data: []byte(goodresponse),
|
||||
},
|
||||
},
|
||||
}},
|
||||
}
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
if !errors.Is(err, nil) {
|
||||
t.Fatalf("not the error we expected: %+v", err)
|
||||
}
|
||||
if cresp.DNS.Failure != nil {
|
||||
t.Fatal("unexpected Failure value")
|
||||
}
|
||||
if len(cresp.DNS.Addrs) != 1 {
|
||||
t.Fatal("unexpected number of DNS entries")
|
||||
}
|
||||
if cresp.DNS.Addrs[0] != "172.217.17.68" {
|
||||
t.Fatal("unexpected DNS addrs [0]")
|
||||
}
|
||||
if cresp.HTTPRequest.BodyLength != 207878 {
|
||||
t.Fatal("invalid http body length")
|
||||
}
|
||||
if cresp.HTTPRequest.Failure != nil {
|
||||
t.Fatal("invalid http failure")
|
||||
}
|
||||
if cresp.HTTPRequest.Title != "Google" {
|
||||
t.Fatal("invalid http title")
|
||||
}
|
||||
if len(cresp.HTTPRequest.Headers) != 1 {
|
||||
t.Fatal("invalid http headers length")
|
||||
}
|
||||
if cresp.HTTPRequest.Headers["Content-Type"] != "text/html" {
|
||||
t.Fatal("invalid http content-type header")
|
||||
}
|
||||
if cresp.HTTPRequest.StatusCode != 200 {
|
||||
t.Fatal("invalid http status code")
|
||||
}
|
||||
if len(cresp.TCPConnect) != 1 {
|
||||
t.Fatal("invalid tcp connect length")
|
||||
}
|
||||
entry, ok := cresp.TCPConnect["172.217.21.68:80"]
|
||||
if !ok {
|
||||
t.Fatal("cannot find expected TCP connect entry")
|
||||
}
|
||||
if entry.Status != true {
|
||||
t.Fatal("unexpected TCP connect entry status")
|
||||
}
|
||||
if entry.Failure != nil {
|
||||
t.Fatal("unexpected TCP connect entry failure value")
|
||||
}
|
||||
}
|
||||
@@ -1,102 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
type FakeResolver struct {
|
||||
NumFailures *atomicx.Int64
|
||||
Err error
|
||||
Result []string
|
||||
}
|
||||
|
||||
func NewFakeResolverThatFails() FakeResolver {
|
||||
return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
|
||||
}
|
||||
|
||||
func NewFakeResolverWithResult(r []string) FakeResolver {
|
||||
return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
|
||||
}
|
||||
|
||||
var ErrNotFound = &net.DNSError{
|
||||
Err: "no such host",
|
||||
}
|
||||
|
||||
func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if c.Err != nil {
|
||||
if c.NumFailures != nil {
|
||||
c.NumFailures.Add(1)
|
||||
}
|
||||
return nil, c.Err
|
||||
}
|
||||
return c.Result, nil
|
||||
}
|
||||
|
||||
func (c FakeResolver) Network() string {
|
||||
return "fake"
|
||||
}
|
||||
|
||||
func (c FakeResolver) Address() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
var _ netx.Resolver = FakeResolver{}
|
||||
|
||||
type FakeTransport struct {
|
||||
Err error
|
||||
Func func(*http.Request) (*http.Response, error)
|
||||
Resp *http.Response
|
||||
}
|
||||
|
||||
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if txp.Func != nil {
|
||||
return txp.Func(req)
|
||||
}
|
||||
if req.Body != nil {
|
||||
ioutil.ReadAll(req.Body)
|
||||
req.Body.Close()
|
||||
}
|
||||
if txp.Err != nil {
|
||||
return nil, txp.Err
|
||||
}
|
||||
txp.Resp.Request = req // non thread safe but it doesn't matter
|
||||
return txp.Resp, nil
|
||||
}
|
||||
|
||||
func (txp FakeTransport) CloseIdleConnections() {}
|
||||
|
||||
var _ netx.HTTPRoundTripper = FakeTransport{}
|
||||
|
||||
type FakeBody struct {
|
||||
Data []byte
|
||||
Err error
|
||||
}
|
||||
|
||||
func (fb *FakeBody) Read(p []byte) (int, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if fb.Err != nil {
|
||||
return 0, fb.Err
|
||||
}
|
||||
if len(fb.Data) <= 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, fb.Data)
|
||||
fb.Data = fb.Data[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (fb *FakeBody) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &FakeBody{}
|
||||
@@ -1,48 +0,0 @@
|
||||
// Command oohelper contains a simple command line
|
||||
// client for the Web Connectivity test helper.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"flag"
|
||||
"fmt"
|
||||
"net/http"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
var (
|
||||
ctx, cancel = context.WithCancel(context.Background())
|
||||
debug = flag.Bool("debug", false, "Toggle debug mode")
|
||||
httpClient *http.Client
|
||||
resolver netx.Resolver
|
||||
server = flag.String("server", "https://wcth.ooni.io/", "URL of the test helper")
|
||||
target = flag.String("target", "", "Target URL for the test helper")
|
||||
)
|
||||
|
||||
func init() {
|
||||
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
|
||||
httpClient = &http.Client{Transport: txp}
|
||||
resolver = netx.NewResolver(netx.Config{Logger: log.Log})
|
||||
}
|
||||
|
||||
func main() {
|
||||
logmap := map[bool]log.Level{
|
||||
true: log.DebugLevel,
|
||||
false: log.InfoLevel,
|
||||
}
|
||||
flag.Parse()
|
||||
log.SetLevel(logmap[*debug])
|
||||
clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver}
|
||||
config := internal.OOConfig{TargetURL: *target, ServerURL: *server}
|
||||
defer cancel()
|
||||
cresp, err := clnt.Do(ctx, config)
|
||||
runtimex.PanicOnError(err, "client.Do failed")
|
||||
data, err := json.MarshalIndent(cresp, "", " ")
|
||||
runtimex.PanicOnError(err, "json.MarshalIndent failed")
|
||||
fmt.Printf("%s\n", string(data))
|
||||
}
|
||||
@@ -1,8 +0,0 @@
|
||||
package main
|
||||
|
||||
import "testing"
|
||||
|
||||
func TestSmoke(t *testing.T) {
|
||||
*target = "http://www.example.com"
|
||||
main()
|
||||
}
|
||||
@@ -1,4 +0,0 @@
|
||||
# oohelperd
|
||||
|
||||
This directory contains the source code of the Web
|
||||
Connectivity test helper written in Go.
|
||||
@@ -1,32 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
|
||||
)
|
||||
|
||||
// newfailure is a convenience shortcut to save typing
|
||||
var newfailure = archival.NewFailure
|
||||
|
||||
// CtrlDNSResult is the result of the DNS check performed by
|
||||
// the Web Connectivity test helper.
|
||||
type CtrlDNSResult = webconnectivity.ControlDNSResult
|
||||
|
||||
// DNSConfig configures the DNS check.
|
||||
type DNSConfig struct {
|
||||
Domain string
|
||||
Out chan CtrlDNSResult
|
||||
Resolver netx.Resolver
|
||||
Wg *sync.WaitGroup
|
||||
}
|
||||
|
||||
// DNSDo performs the DNS check.
|
||||
func DNSDo(ctx context.Context, config *DNSConfig) {
|
||||
defer config.Wg.Done()
|
||||
addrs, err := config.Resolver.LookupHost(ctx, config.Domain)
|
||||
config.Out <- CtrlDNSResult{Failure: newfailure(err), Addrs: addrs}
|
||||
}
|
||||
@@ -1,127 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
type FakeResolver struct {
|
||||
NumFailures *atomicx.Int64
|
||||
Err error
|
||||
Result []string
|
||||
}
|
||||
|
||||
func NewFakeResolverThatFails() FakeResolver {
|
||||
return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
|
||||
}
|
||||
|
||||
func NewFakeResolverWithResult(r []string) FakeResolver {
|
||||
return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
|
||||
}
|
||||
|
||||
var ErrNotFound = &net.DNSError{
|
||||
Err: "no such host",
|
||||
}
|
||||
|
||||
func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if c.Err != nil {
|
||||
if c.NumFailures != nil {
|
||||
c.NumFailures.Add(1)
|
||||
}
|
||||
return nil, c.Err
|
||||
}
|
||||
return c.Result, nil
|
||||
}
|
||||
|
||||
func (c FakeResolver) Network() string {
|
||||
return "fake"
|
||||
}
|
||||
|
||||
func (c FakeResolver) Address() string {
|
||||
return ""
|
||||
}
|
||||
|
||||
var _ netx.Resolver = FakeResolver{}
|
||||
|
||||
type FakeTransport struct {
|
||||
Err error
|
||||
Func func(*http.Request) (*http.Response, error)
|
||||
Resp *http.Response
|
||||
}
|
||||
|
||||
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if txp.Func != nil {
|
||||
return txp.Func(req)
|
||||
}
|
||||
if req.Body != nil {
|
||||
ioutil.ReadAll(req.Body)
|
||||
req.Body.Close()
|
||||
}
|
||||
if txp.Err != nil {
|
||||
return nil, txp.Err
|
||||
}
|
||||
txp.Resp.Request = req // non thread safe but it doesn't matter
|
||||
return txp.Resp, nil
|
||||
}
|
||||
|
||||
func (txp FakeTransport) CloseIdleConnections() {}
|
||||
|
||||
var _ netx.HTTPRoundTripper = FakeTransport{}
|
||||
|
||||
type FakeBody struct {
|
||||
Data []byte
|
||||
Err error
|
||||
}
|
||||
|
||||
func (fb *FakeBody) Read(p []byte) (int, error) {
|
||||
time.Sleep(10 * time.Microsecond)
|
||||
if fb.Err != nil {
|
||||
return 0, fb.Err
|
||||
}
|
||||
if len(fb.Data) <= 0 {
|
||||
return 0, io.EOF
|
||||
}
|
||||
n := copy(p, fb.Data)
|
||||
fb.Data = fb.Data[n:]
|
||||
return n, nil
|
||||
}
|
||||
|
||||
func (fb *FakeBody) Close() error {
|
||||
return nil
|
||||
}
|
||||
|
||||
var _ io.ReadCloser = &FakeBody{}
|
||||
|
||||
type FakeResponseWriter struct {
|
||||
Body [][]byte
|
||||
HeaderMap http.Header
|
||||
StatusCode int
|
||||
}
|
||||
|
||||
func NewFakeResponseWriter() *FakeResponseWriter {
|
||||
return &FakeResponseWriter{HeaderMap: make(http.Header)}
|
||||
}
|
||||
|
||||
func (frw *FakeResponseWriter) Header() http.Header {
|
||||
return frw.HeaderMap
|
||||
}
|
||||
|
||||
func (frw *FakeResponseWriter) Write(b []byte) (int, error) {
|
||||
frw.Body = append(frw.Body, b)
|
||||
return len(b), nil
|
||||
}
|
||||
|
||||
func (frw *FakeResponseWriter) WriteHeader(statusCode int) {
|
||||
frw.StatusCode = statusCode
|
||||
}
|
||||
|
||||
var _ http.ResponseWriter = &FakeResponseWriter{}
|
||||
@@ -1,67 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
|
||||
)
|
||||
|
||||
// CtrlHTTPResponse is the result of the HTTP check performed by
|
||||
// the Web Connectivity test helper.
|
||||
type CtrlHTTPResponse = webconnectivity.ControlHTTPRequestResult
|
||||
|
||||
// HTTPConfig configures the HTTP check.
|
||||
type HTTPConfig struct {
|
||||
Client *http.Client
|
||||
Headers map[string][]string
|
||||
MaxAcceptableBody int64
|
||||
Out chan CtrlHTTPResponse
|
||||
URL string
|
||||
Wg *sync.WaitGroup
|
||||
}
|
||||
|
||||
// HTTPDo performs the HTTP check.
|
||||
func HTTPDo(ctx context.Context, config *HTTPConfig) {
|
||||
defer config.Wg.Done()
|
||||
req, err := http.NewRequestWithContext(ctx, "GET", config.URL, nil)
|
||||
if err != nil {
|
||||
config.Out <- CtrlHTTPResponse{Failure: newfailure(err)}
|
||||
return
|
||||
}
|
||||
// The original test helper failed with extra headers while here
|
||||
// we're implementing (for now?) a more liberal approach.
|
||||
for k, vs := range config.Headers {
|
||||
switch strings.ToLower(k) {
|
||||
case "user-agent":
|
||||
case "accept":
|
||||
case "accept-language":
|
||||
for _, v := range vs {
|
||||
req.Header.Add(k, v)
|
||||
}
|
||||
}
|
||||
}
|
||||
resp, err := config.Client.Do(req)
|
||||
if err != nil {
|
||||
config.Out <- CtrlHTTPResponse{Failure: newfailure(err)}
|
||||
return
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
headers := make(map[string]string)
|
||||
for k := range resp.Header {
|
||||
headers[k] = resp.Header.Get(k)
|
||||
}
|
||||
reader := &io.LimitedReader{R: resp.Body, N: config.MaxAcceptableBody}
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
config.Out <- CtrlHTTPResponse{
|
||||
BodyLength: int64(len(data)),
|
||||
Failure: newfailure(err),
|
||||
StatusCode: int64(resp.StatusCode),
|
||||
Headers: headers,
|
||||
Title: webconnectivity.GetTitle(string(data)),
|
||||
}
|
||||
}
|
||||
@@ -1,59 +0,0 @@
|
||||
package internal_test
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"sync"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
|
||||
)
|
||||
|
||||
func TestHTTPDoWithInvalidURL(t *testing.T) {
|
||||
ctx := context.Background()
|
||||
wg := new(sync.WaitGroup)
|
||||
httpch := make(chan internal.CtrlHTTPResponse, 1)
|
||||
wg.Add(1)
|
||||
go internal.HTTPDo(ctx, &internal.HTTPConfig{
|
||||
Client: http.DefaultClient,
|
||||
Headers: nil,
|
||||
MaxAcceptableBody: 1 << 24,
|
||||
Out: httpch,
|
||||
URL: "http://[::1]aaaa",
|
||||
Wg: wg,
|
||||
})
|
||||
// wait for measurement steps to complete
|
||||
wg.Wait()
|
||||
resp := <-httpch
|
||||
if resp.Failure == nil || !strings.HasSuffix(*resp.Failure, `invalid port "aaaa" after host`) {
|
||||
t.Fatal("not the failure we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestHTTPDoWithHTTPTransportFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
ctx := context.Background()
|
||||
wg := new(sync.WaitGroup)
|
||||
httpch := make(chan internal.CtrlHTTPResponse, 1)
|
||||
wg.Add(1)
|
||||
go internal.HTTPDo(ctx, &internal.HTTPConfig{
|
||||
Client: &http.Client{
|
||||
Transport: internal.FakeTransport{
|
||||
Err: expected,
|
||||
},
|
||||
},
|
||||
Headers: nil,
|
||||
MaxAcceptableBody: 1 << 24,
|
||||
Out: httpch,
|
||||
URL: "http://www.x.org",
|
||||
Wg: wg,
|
||||
})
|
||||
// wait for measurement steps to complete
|
||||
wg.Wait()
|
||||
resp := <-httpch
|
||||
if resp.Failure == nil || !strings.HasSuffix(*resp.Failure, "mocked error") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
@@ -1,61 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"fmt"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"net/http"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/version"
|
||||
)
|
||||
|
||||
// Handler implements the Web Connectivity test helper HTTP API.
|
||||
type Handler struct {
|
||||
Client *http.Client
|
||||
Dialer netx.Dialer
|
||||
MaxAcceptableBody int64
|
||||
Resolver netx.Resolver
|
||||
}
|
||||
|
||||
func (h Handler) ServeHTTP(w http.ResponseWriter, req *http.Request) {
|
||||
w.Header().Add("Server", fmt.Sprintf(
|
||||
"oohelperd/%s ooniprobe-engine/%s", version.Version, version.Version,
|
||||
))
|
||||
if req.Method != "POST" {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
if req.Header.Get("content-type") != "application/json" {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
reader := &io.LimitedReader{R: req.Body, N: h.MaxAcceptableBody}
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
var creq CtrlRequest
|
||||
if err := json.Unmarshal(data, &creq); err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
measureConfig := MeasureConfig{
|
||||
Client: h.Client,
|
||||
Dialer: h.Dialer,
|
||||
MaxAcceptableBody: h.MaxAcceptableBody,
|
||||
Resolver: h.Resolver,
|
||||
}
|
||||
cresp, err := Measure(req.Context(), measureConfig, &creq)
|
||||
if err != nil {
|
||||
w.WriteHeader(400)
|
||||
return
|
||||
}
|
||||
// We assume that the following call cannot fail because it's a
|
||||
// clearly serializable data structure.
|
||||
data, _ = json.Marshal(cresp)
|
||||
w.Header().Add("Content-Type", "application/json")
|
||||
w.Write(data)
|
||||
}
|
||||
@@ -1,160 +0,0 @@
|
||||
package internal_test
|
||||
|
||||
import (
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io/ioutil"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
|
||||
)
|
||||
|
||||
const simplerequest = `{
|
||||
"http_request": "https://dns.google",
|
||||
"http_request_headers": {
|
||||
"Accept": [
|
||||
"*/*"
|
||||
],
|
||||
"Accept-Language": [
|
||||
"en-US;q=0.8,en;q=0.5"
|
||||
],
|
||||
"User-Agent": [
|
||||
"Mozilla/5.0"
|
||||
]
|
||||
},
|
||||
"tcp_connect": [
|
||||
"8.8.8.8:443"
|
||||
]
|
||||
}`
|
||||
|
||||
const requestWithoutDomainName = `{
|
||||
"http_request": "https://8.8.8.8",
|
||||
"http_request_headers": {
|
||||
"Accept": [
|
||||
"*/*"
|
||||
],
|
||||
"Accept-Language": [
|
||||
"en-US;q=0.8,en;q=0.5"
|
||||
],
|
||||
"User-Agent": [
|
||||
"Mozilla/5.0"
|
||||
]
|
||||
},
|
||||
"tcp_connect": [
|
||||
"8.8.8.8:443"
|
||||
]
|
||||
}`
|
||||
|
||||
func TestWorkingAsIntended(t *testing.T) {
|
||||
handler := internal.Handler{
|
||||
Client: http.DefaultClient,
|
||||
Dialer: new(net.Dialer),
|
||||
MaxAcceptableBody: 1 << 24,
|
||||
Resolver: resolver.SystemResolver{},
|
||||
}
|
||||
srv := httptest.NewServer(handler)
|
||||
defer srv.Close()
|
||||
type expectationSpec struct {
|
||||
name string
|
||||
reqMethod string
|
||||
reqContentType string
|
||||
reqBody string
|
||||
respStatusCode int
|
||||
respContentType string
|
||||
parseBody bool
|
||||
}
|
||||
expectations := []expectationSpec{{
|
||||
name: "check for invalid method",
|
||||
reqMethod: "GET",
|
||||
respStatusCode: 400,
|
||||
}, {
|
||||
name: "check for invalid content-type",
|
||||
reqMethod: "POST",
|
||||
respStatusCode: 400,
|
||||
}, {
|
||||
name: "check for invalid request body",
|
||||
reqMethod: "POST",
|
||||
reqContentType: "application/json",
|
||||
reqBody: "{",
|
||||
respStatusCode: 400,
|
||||
}, {
|
||||
name: "with measurement failure",
|
||||
reqMethod: "POST",
|
||||
reqContentType: "application/json",
|
||||
reqBody: `{"http_request": "http://[::1]aaaa"}`,
|
||||
respStatusCode: 400,
|
||||
}, {
|
||||
name: "with reasonably good request",
|
||||
reqMethod: "POST",
|
||||
reqContentType: "application/json",
|
||||
reqBody: simplerequest,
|
||||
respStatusCode: 200,
|
||||
respContentType: "application/json",
|
||||
parseBody: true,
|
||||
}, {
|
||||
name: "when there's no domain name in the request",
|
||||
reqMethod: "POST",
|
||||
reqContentType: "application/json",
|
||||
reqBody: requestWithoutDomainName,
|
||||
respStatusCode: 200,
|
||||
respContentType: "application/json",
|
||||
parseBody: true,
|
||||
}}
|
||||
for _, expect := range expectations {
|
||||
t.Run(expect.name, func(t *testing.T) {
|
||||
body := strings.NewReader(expect.reqBody)
|
||||
req, err := http.NewRequest(expect.reqMethod, srv.URL, body)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %+v", expect.name, err)
|
||||
}
|
||||
if expect.reqContentType != "" {
|
||||
req.Header.Add("content-type", expect.reqContentType)
|
||||
}
|
||||
resp, err := http.DefaultClient.Do(req)
|
||||
if err != nil {
|
||||
t.Fatalf("%s: %+v", expect.name, err)
|
||||
}
|
||||
defer resp.Body.Close()
|
||||
if resp.StatusCode != expect.respStatusCode {
|
||||
t.Fatalf("unexpected status code: %+v", resp.StatusCode)
|
||||
}
|
||||
if v := resp.Header.Get("content-type"); v != expect.respContentType {
|
||||
t.Fatalf("unexpected content-type: %s", v)
|
||||
}
|
||||
data, err := ioutil.ReadAll(resp.Body)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !expect.parseBody {
|
||||
return
|
||||
}
|
||||
var v interface{}
|
||||
if err := json.Unmarshal(data, &v); err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
})
|
||||
}
|
||||
}
|
||||
|
||||
func TestHandlerWithRequestBodyReadingError(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
handler := internal.Handler{MaxAcceptableBody: 1 << 24}
|
||||
rw := internal.NewFakeResponseWriter()
|
||||
req := &http.Request{
|
||||
Method: "POST",
|
||||
Header: map[string][]string{
|
||||
"Content-Type": {"application/json"},
|
||||
"Content-Length": {"2048"},
|
||||
},
|
||||
Body: &internal.FakeBody{Err: expected},
|
||||
}
|
||||
handler.ServeHTTP(rw, req)
|
||||
if rw.StatusCode != 400 {
|
||||
t.Fatal("unexpected status code")
|
||||
}
|
||||
}
|
||||
@@ -1,88 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"net"
|
||||
"net/http"
|
||||
"net/url"
|
||||
"sync"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
type (
|
||||
// CtrlRequest is the request sent to the test helper
|
||||
CtrlRequest = webconnectivity.ControlRequest
|
||||
|
||||
// CtrlResponse is the response from the test helper
|
||||
CtrlResponse = webconnectivity.ControlResponse
|
||||
)
|
||||
|
||||
// MeasureConfig contains configuration for Measure.
|
||||
type MeasureConfig struct {
|
||||
Client *http.Client
|
||||
Dialer netx.Dialer
|
||||
MaxAcceptableBody int64
|
||||
Resolver netx.Resolver
|
||||
}
|
||||
|
||||
// Measure performs the measurement described by the request and
|
||||
// returns the corresponding response or an error.
|
||||
func Measure(ctx context.Context, config MeasureConfig, creq *CtrlRequest) (*CtrlResponse, error) {
|
||||
// parse input for correctness
|
||||
URL, err := url.Parse(creq.HTTPRequest)
|
||||
if err != nil {
|
||||
return nil, err
|
||||
}
|
||||
// dns: start
|
||||
wg := new(sync.WaitGroup)
|
||||
dnsch := make(chan CtrlDNSResult, 1)
|
||||
if net.ParseIP(URL.Hostname()) == nil {
|
||||
wg.Add(1)
|
||||
go DNSDo(ctx, &DNSConfig{
|
||||
Domain: URL.Hostname(),
|
||||
Out: dnsch,
|
||||
Resolver: config.Resolver,
|
||||
Wg: wg,
|
||||
})
|
||||
}
|
||||
// tcpconnect: start
|
||||
tcpconnch := make(chan TCPResultPair, len(creq.TCPConnect))
|
||||
for _, endpoint := range creq.TCPConnect {
|
||||
wg.Add(1)
|
||||
go TCPDo(ctx, &TCPConfig{
|
||||
Dialer: config.Dialer,
|
||||
Endpoint: endpoint,
|
||||
Out: tcpconnch,
|
||||
Wg: wg,
|
||||
})
|
||||
}
|
||||
// http: start
|
||||
httpch := make(chan CtrlHTTPResponse, 1)
|
||||
wg.Add(1)
|
||||
go HTTPDo(ctx, &HTTPConfig{
|
||||
Client: config.Client,
|
||||
Headers: creq.HTTPRequestHeaders,
|
||||
MaxAcceptableBody: config.MaxAcceptableBody,
|
||||
Out: httpch,
|
||||
URL: creq.HTTPRequest,
|
||||
Wg: wg,
|
||||
})
|
||||
// wait for measurement steps to complete
|
||||
wg.Wait()
|
||||
// assemble response
|
||||
cresp := new(CtrlResponse)
|
||||
select {
|
||||
case cresp.DNS = <-dnsch:
|
||||
default:
|
||||
// we land here when there's no domain name
|
||||
}
|
||||
cresp.HTTPRequest = <-httpch
|
||||
cresp.TCPConnect = make(map[string]CtrlTCPResult)
|
||||
for len(cresp.TCPConnect) < len(creq.TCPConnect) {
|
||||
tcpconn := <-tcpconnch
|
||||
cresp.TCPConnect[tcpconn.Endpoint] = tcpconn.Result
|
||||
}
|
||||
return cresp, nil
|
||||
}
|
||||
@@ -1,42 +0,0 @@
|
||||
package internal
|
||||
|
||||
import (
|
||||
"context"
|
||||
"sync"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
// CtrlTCPResult is the result of the TCP check performed by the test helper.
|
||||
type CtrlTCPResult = webconnectivity.ControlTCPConnectResult
|
||||
|
||||
// TCPResultPair contains the endpoint and the corresponding result.
|
||||
type TCPResultPair struct {
|
||||
Endpoint string
|
||||
Result CtrlTCPResult
|
||||
}
|
||||
|
||||
// TCPConfig configures the TCP connect check.
|
||||
type TCPConfig struct {
|
||||
Dialer netx.Dialer
|
||||
Endpoint string
|
||||
Out chan TCPResultPair
|
||||
Wg *sync.WaitGroup
|
||||
}
|
||||
|
||||
// TCPDo performs the TCP check.
|
||||
func TCPDo(ctx context.Context, config *TCPConfig) {
|
||||
defer config.Wg.Done()
|
||||
conn, err := config.Dialer.DialContext(ctx, "tcp", config.Endpoint)
|
||||
if conn != nil {
|
||||
conn.Close()
|
||||
}
|
||||
config.Out <- TCPResultPair{
|
||||
Endpoint: config.Endpoint,
|
||||
Result: CtrlTCPResult{
|
||||
Failure: newfailure(err),
|
||||
Status: err == nil,
|
||||
},
|
||||
}
|
||||
}
|
||||
@@ -1,67 +0,0 @@
|
||||
// Command oohelperd contains the Web Connectivity test helper.
|
||||
package main
|
||||
|
||||
import (
|
||||
"context"
|
||||
"flag"
|
||||
"net/http"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelperd/internal"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
const maxAcceptableBody = 1 << 24
|
||||
|
||||
var (
|
||||
dialer netx.Dialer
|
||||
endpoint = flag.String("endpoint", ":8080", "Endpoint where to listen")
|
||||
httpx *http.Client
|
||||
resolver netx.Resolver
|
||||
srvcancel context.CancelFunc
|
||||
srvctx context.Context
|
||||
srvwg = new(sync.WaitGroup)
|
||||
)
|
||||
|
||||
func init() {
|
||||
srvctx, srvcancel = context.WithCancel(context.Background())
|
||||
dialer = netx.NewDialer(netx.Config{Logger: log.Log})
|
||||
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
|
||||
httpx = &http.Client{Transport: txp}
|
||||
resolver = netx.NewResolver(netx.Config{Logger: log.Log})
|
||||
}
|
||||
|
||||
func shutdown(srv *http.Server) {
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 3*time.Second)
|
||||
defer cancel()
|
||||
srv.Shutdown(ctx)
|
||||
}
|
||||
|
||||
func main() {
|
||||
logmap := map[bool]log.Level{
|
||||
true: log.DebugLevel,
|
||||
false: log.InfoLevel,
|
||||
}
|
||||
debug := flag.Bool("debug", false, "Toggle debug mode")
|
||||
flag.Parse()
|
||||
log.SetLevel(logmap[*debug])
|
||||
testableMain()
|
||||
}
|
||||
|
||||
func testableMain() {
|
||||
mux := http.NewServeMux()
|
||||
mux.Handle("/", internal.Handler{
|
||||
Client: httpx,
|
||||
Dialer: dialer,
|
||||
MaxAcceptableBody: maxAcceptableBody,
|
||||
Resolver: resolver,
|
||||
})
|
||||
srv := &http.Server{Addr: *endpoint, Handler: mux}
|
||||
srvwg.Add(1)
|
||||
go srv.ListenAndServe()
|
||||
<-srvctx.Done()
|
||||
shutdown(srv)
|
||||
srvwg.Done()
|
||||
}
|
||||
@@ -1,15 +0,0 @@
|
||||
package main
|
||||
|
||||
import (
|
||||
"testing"
|
||||
)
|
||||
|
||||
func TestSmoke(t *testing.T) {
|
||||
// Just check whether we can start and then tear down the server, so
|
||||
// we have coverage of this code and when we see that some lines aren't
|
||||
// covered we know these are genuine places where we're not testing
|
||||
// the code rather than just places like this simple main.
|
||||
go testableMain()
|
||||
srvcancel() // kills the listener
|
||||
srvwg.Wait() // joined
|
||||
}
|
||||
@@ -16,7 +16,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/randx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/legacy/netxlogger"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/legacy/oonidatamodel"
|
||||
|
||||
@@ -10,7 +10,7 @@ import (
|
||||
"net/http/cookiejar"
|
||||
"net/url"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
|
||||
|
||||
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
func TestRunnerWithInvalidURLScheme(t *testing.T) {
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
|
||||
)
|
||||
|
||||
@@ -12,7 +12,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity/internal"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
|
||||
)
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
type avastResponse struct {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
func TestIPLookupWorksUsingAvast(t *testing.T) {
|
||||
|
||||
@@ -5,8 +5,8 @@ import (
|
||||
"net/http"
|
||||
"strings"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
func ipConfigIPLookup(
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
func TestIPLookupWorksUsingIPConfig(t *testing.T) {
|
||||
|
||||
@@ -4,8 +4,8 @@ import (
|
||||
"context"
|
||||
"net/http"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
type ipInfoResponse struct {
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
func TestIPLookupWorksUsingIPInfo(t *testing.T) {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/pion/stun"
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"encoding/xml"
|
||||
"net/http"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
type ubuntuResponse struct {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
func TestUbuntuParseError(t *testing.T) {
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
func TestFetchResourceIntegration(t *testing.T) {
|
||||
+1
-1
@@ -11,7 +11,7 @@ import (
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
|
||||
)
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"github.com/apex/log"
|
||||
"github.com/google/go-cmp/cmp"
|
||||
engine "github.com/ooni/probe-cli/v3/internal/engine"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
)
|
||||
|
||||
|
||||
@@ -6,7 +6,7 @@ import (
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/psiphonx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/torx"
|
||||
|
||||
@@ -8,7 +8,7 @@ import (
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
)
|
||||
|
||||
// DNSOverHTTPS is a DNS over HTTPS RoundTripper. Requests are submitted over
|
||||
|
||||
@@ -9,7 +9,7 @@ import (
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpheader"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
|
||||
)
|
||||
|
||||
|
||||
@@ -4,7 +4,7 @@ import (
|
||||
"context"
|
||||
"net/url"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
type checkReportIDResponse struct {
|
||||
|
||||
@@ -8,8 +8,8 @@ import (
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
|
||||
)
|
||||
|
||||
|
||||
@@ -5,7 +5,7 @@ import (
|
||||
"net/url"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
)
|
||||
|
||||
// MeasurementMetaConfig contains configuration for GetMeasurementMeta.
|
||||
|
||||
@@ -9,8 +9,8 @@ import (
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
|
||||
)
|
||||
|
||||
|
||||
@@ -29,7 +29,7 @@ import (
|
||||
"net/url"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
)
|
||||
|
||||
|
||||
@@ -7,7 +7,7 @@ import (
|
||||
"time"
|
||||
|
||||
"github.com/google/go-cmp/cmp"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
"os"
|
||||
"path/filepath"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/httpx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
)
|
||||
|
||||
|
||||
@@ -13,7 +13,7 @@ import (
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/geolocate"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/kvstore"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/platform"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/sessionresolver"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/tunnel"
|
||||
|
||||
Reference in New Issue
Block a user