c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse Source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso 2021-02-02 12:05:47 +01:00 committed by GitHub
parent b1ce300c8d
commit d57c78bc71
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
535 changed files with 66182 additions and 23 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
cmd/ooniprobe/internal/autorun/autorun_darwin.go
View File

@ -13,7 +13,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
"github.com/ooni/probe-engine/cmd/jafar/shellx"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
"golang.org/x/sys/unix"
)

2
cmd/ooniprobe/internal/enginex/enginex.go
View File

@ -11,7 +11,7 @@ var Logger = log.WithFields(log.Fields{
})
// LocationProvider is an interface that returns the current location. The
// github.com/ooni/probe-engine/session.Session implements it.
// github.com/ooni/probe-cli/v3/internal/engine/session.Session implements it.
type LocationProvider interface {
ProbeASN() uint
ProbeASNString() string

4
cmd/ooniprobe/internal/nettests/nettests.go
View File

@ -10,8 +10,8 @@ import (
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/ooni"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
engine "github.com/ooni/probe-engine"
"github.com/ooni/probe-engine/model"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/ooni/probe-cli/v3/internal/engine/model"
"github.com/pkg/errors"
)

2
cmd/ooniprobe/internal/nettests/web_connectivity.go
View File

@ -5,7 +5,7 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
engine "github.com/ooni/probe-engine"
engine "github.com/ooni/probe-cli/v3/internal/engine"
)
func lookupURLs(ctl *Controller, limit int64, categories []string) ([]string, map[int64]int64, error) {

2
cmd/ooniprobe/internal/ooni/ooni.go
View File

@ -13,7 +13,7 @@ import (
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/utils"
engine "github.com/ooni/probe-engine"
engine "github.com/ooni/probe-cli/v3/internal/engine"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)

27
go.mod
View File

@ -3,22 +3,45 @@ module github.com/ooni/probe-cli/v3
go 1.14
require (
git.torproject.org/pluggable-transports/goptlib.git v1.1.0
github.com/alecthomas/kingpin v2.2.6+incompatible
github.com/alecthomas/units v0.0.0-20201120081800-1786d5ef83d4 // indirect
github.com/apex/log v1.9.0
github.com/certifi/gocertifi v0.0.0-20200922220541-2c3bb06c6054 // indirect
github.com/cretz/bine v0.1.0
github.com/dchest/siphash v1.2.2 // indirect
github.com/fatih/color v1.10.0
github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea
github.com/golang/protobuf v1.4.3 // indirect
github.com/google/go-cmp v0.5.2
github.com/google/martian/v3 v3.1.0
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510
github.com/google/uuid v1.2.0
github.com/gorilla/websocket v1.4.2
github.com/iancoleman/strcase v0.1.3
github.com/lucas-clemente/quic-go v0.19.3
github.com/mattn/go-colorable v0.1.8
github.com/mattn/go-sqlite3 v1.14.6 // indirect
github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect
github.com/ooni/probe-engine v0.22.0
github.com/miekg/dns v1.1.35
github.com/montanaflynn/stats v0.6.4
github.com/ooni/psiphon v0.4.0
github.com/oschwald/geoip2-golang v1.4.0
github.com/oschwald/maxminddb-golang v1.8.0 // indirect
github.com/pborman/getopt/v2 v2.1.0
github.com/pion/stun v0.3.5
github.com/pkg/errors v0.9.1
github.com/rogpeppe/go-internal v1.7.0
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351
github.com/sirupsen/logrus v1.7.0 // indirect
gitlab.com/yawning/obfs4.git v0.0.0-20201217005658-f638c33f6c6f
golang.org/x/crypto v0.0.0-20201221181555-eec23a3978ad // indirect
golang.org/x/net v0.0.0-20210119194325-5f4716e94777
golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9 // indirect
golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c
golang.org/x/text v0.3.5 // indirect
golang.org/x/xerrors v0.0.0-20200804184101-5ec99f83aff1 // indirect
google.golang.org/protobuf v1.25.0 // indirect
gopkg.in/AlecAivazis/survey.v1 v1.8.8
gopkg.in/mgo.v2 v2.0.0-20190816093944-a6b53ec6cb22 // indirect
upper.io/db.v3 v3.8.0+incompatible

32
go.sum
View File

@ -179,7 +179,9 @@ github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=
github.com/google/go-querystring v1.0.0/go.mod h1:odCYkC5MyYFN7vkCjXpyrEuKhc/BUO6wN/zVPAxq5ck=
github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
github.com/google/martian v2.1.0+incompatible h1:/CP5g8u/VJHijgedC/Legn3BAbAaWPgecwXBIDzw5no=
github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
github.com/google/martian/v3 v3.1.0 h1:wCKgOCHuUEVfsaQLpPSJb7VdYCdTVZQAuOdYm1yc/60=
github.com/google/martian/v3 v3.1.0/go.mod h1:y5Zk1BBys9G+gd6Jrk0W3cC1+ELVxBWuIGO+w/tUAp0=
github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI=
@ -187,7 +189,8 @@ github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 h1:El6M4kTTCOh6aBiKaU
github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ=
github.com/google/uuid v1.0.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.1.4/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/google/uuid v1.2.0 h1:qJYtXnJRWmpe7m/3XlyhrsLrEURqHRM2kxzoxXqyUDs=
github.com/google/uuid v1.2.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk2eWhX2bMyUtNHzFKcPA9HY=
github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg=
github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8=
@ -229,8 +232,8 @@ github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW
github.com/hpcloud/tail v1.0.0 h1:nfCOvKYfkgYP8hkirhJocXT2+zOD8yUNjXaWfTlyFKI=
github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU=
github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg=
github.com/iancoleman/strcase v0.1.2 h1:gnomlvw9tnV3ITTAxzKSgTF+8kFWcU/f+TgttpXGz1U=
github.com/iancoleman/strcase v0.1.2/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
github.com/iancoleman/strcase v0.1.3 h1:dJBk1m2/qjL1twPLf68JND55vvivMupZ4wIzE8CTdBw=
github.com/iancoleman/strcase v0.1.3/go.mod h1:SK73tn/9oHe+/Y0h39VT4UCxmurVJkR5NA7kMEAOgSE=
github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8=
github.com/influxdata/influxdb1-client v0.0.0-20191209144304-8bf82d3c094d/go.mod h1:qj24IKcXYK6Iy9ceXlo3Tc+vtHo9lIhSX5JddghvEPo=
github.com/jellevandenhooff/dkim v0.0.0-20150330215556-f50fe3d243e1/go.mod h1:E0B/fFc00Y+Rasa88328GlI/XbtyysCtTHZS8h7IrBU=
@ -320,8 +323,8 @@ github.com/modern-go/concurrent v0.0.0-20180228061459-e0a39a4cb421/go.mod h1:6dJ
github.com/modern-go/concurrent v0.0.0-20180306012644-bacd9c7ef1dd/go.mod h1:6dJC0mAP4ikYIbvyc7fijjWJddQyLn8Ig3JB5CqoB9Q=
github.com/modern-go/reflect2 v0.0.0-20180701023420-4b7aa43c6742/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/modern-go/reflect2 v1.0.1/go.mod h1:bx2lNnkwVCuqBIxFjflWJWanXIb3RllmbCylyMrvgv0=
github.com/montanaflynn/stats v0.6.3 h1:F8446DrvIF5V5smZfZ8K9nrmmix0AFgevPdLruGOmzk=
github.com/montanaflynn/stats v0.6.3/go.mod h1:wL8QJuTMNUDYhXwkmfOly8iTdp5TEcJFWZD2D7SIkUc=
github.com/montanaflynn/stats v0.6.4 h1:ZaPgdYrxEyFUovAKlUKInQHcDhwvjq7HtjwREE7ys68=
github.com/montanaflynn/stats v0.6.4/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow=
github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U=
github.com/nats-io/jwt v0.3.0/go.mod h1:fRYCDE99xlTsqUzISS1Bi75UBJ6ljOJQOAAu5VglpSg=
github.com/nats-io/jwt v0.3.2/go.mod h1:/euKqTS1ZD+zzjYrY7pseZrTtWQSjujC7xjPc8wL6eU=
@ -349,10 +352,8 @@ github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1Cpa
github.com/onsi/gomega v1.7.1/go.mod h1:XdKZgCCFLUoM/7CFJVPcG8C1xQ1AJ0vpAezJrB7JYyY=
github.com/onsi/gomega v1.10.1 h1:o0+MgICZLuZ7xjH7Vx6zS/zcu93/BEp1VwkIW1mEXCE=
github.com/onsi/gomega v1.10.1/go.mod h1:iN09h71vgCQne3DLsj+A5owkum+a2tYe+TOCB1ybHNo=
github.com/ooni/probe-engine v0.22.0 h1:BVuYqJ4fSAEzordgPLvBG0LFutRv//e1MwAKZyRhELs=
github.com/ooni/probe-engine v0.22.0/go.mod h1:EpHBzce7lJoKIs7s0+AYEzSgkquegmjyJTj7fy6FXo0=
github.com/ooni/psiphon v0.3.0 h1:uFwtf7cIWidrwmljmyX4pIQRQL3H07GEfl00852Atxg=
github.com/ooni/psiphon v0.3.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU=
github.com/ooni/psiphon v0.4.0 h1:ZgqkAEJ8cTaP1EzINmn9qQzx39ApUbqjotHj9oSd0xo=
github.com/ooni/psiphon v0.4.0/go.mod h1:i1v6JweJtxDKaI0i1aEw2/Fr/CUi5BoQ75GYz5KmKwU=
github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
github.com/opentracing-contrib/go-observer v0.0.0-20170622124052-a52f23424492/go.mod h1:Ngi6UdF0k5OKD5t5wlmGhe/EDKPoUM3BXZSSfIuJbis=
github.com/opentracing/basictracer-go v1.0.0/go.mod h1:QfBfYuafItcjQuMwinw9GhYKwFXS9KnPs5lxoYwgW74=
@ -371,6 +372,7 @@ github.com/oschwald/maxminddb-golang v1.8.0 h1:Uh/DSnGoxsyp/KYbY1AuP0tYEwfs0sCph
github.com/oschwald/maxminddb-golang v1.8.0/go.mod h1:RXZtst0N6+FY/3qCNmZMBApR19cdQj43/NM9VkrNAis=
github.com/pact-foundation/pact-go v1.0.4/go.mod h1:uExwJY4kCzNPcHRj+hCR/HBbOOIwwtUjcrb0b5/5kLM=
github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc=
github.com/pborman/getopt/v2 v2.1.0 h1:eNfR+r+dWLdWmV8g5OlpyrTYHkhVNxHBdN2cCrJmOEA=
github.com/pborman/getopt/v2 v2.1.0/go.mod h1:4NtW75ny4eBw9fO1bhtNdYTlZKYX5/tBLtsOpwKIKd0=
github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@ -413,8 +415,8 @@ github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFR
github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
github.com/rogpeppe/go-internal v1.3.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.4.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.6.2 h1:aIihoIOHCiLZHxyoNQ+ABL4NKhFTgKLBdMLyEAh98m0=
github.com/rogpeppe/go-internal v1.6.2/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rogpeppe/go-internal v1.7.0 h1:3qqXGV8nn7GJT65debw77Dzrx9sfWYgP0DDo7xcMFRk=
github.com/rogpeppe/go-internal v1.7.0/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc=
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 h1:HXr/qUllAWv9riaI4zh2eXWKmCSDqVS/XH1MRHLKRwk=
github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351/go.mod h1:DCgfY80j8GYL7MLEfvcpSFvjD0L5yZq/aZUJmhZklyg=
github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g=
@ -578,8 +580,8 @@ golang.org/x/net v0.0.0-20190813141303-74dc4d7220e7/go.mod h1:z5CRVTTTmAJ677TzLL
golang.org/x/net v0.0.0-20190923162816-aa69164e4478/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s=
golang.org/x/net v0.0.0-20200520004742-59133d7f0dd7/go.mod h1:qpuaurCH72eLCgpAm/N6yyVIVM9cpaDIP3A8BGJEC5A=
golang.org/x/net v0.0.0-20200707034311-ab3426394381/go.mod h1:/O7V0waA8r7cgGh81Ro3o1hOxt32SMVPicZroKQ2sZA=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b h1:iFwSg7t5GZmB/Q5TjiEAsdoLDrdJRC1RiF2WhuV29Qw=
golang.org/x/net v0.0.0-20201224014010-6772e930b67b/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777 h1:003p0dJM77cxMSyCPFphvZf/Y5/NXf5fzg6ufd1/Oew=
golang.org/x/net v0.0.0-20210119194325-5f4716e94777/go.mod h1:m0MpNAwzfU5UDzcl9v0D8zg8gWTRqZa9RBIspLL5mdg=
golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181017192945-9dcd33a902f4/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
golang.org/x/oauth2 v0.0.0-20181203162652-d668ce993890/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U=
@ -627,8 +629,8 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w
golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20200519105757-fe76b779f299/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc h1:y0Og6AYdwus7SIAnKnDxjc4gJetRiYEWOx4AKbOeyEI=
golang.org/x/sys v0.0.0-20210112091331-59c308dcf3cc/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c h1:VwygUrnw9jn88c4u8GD3rZQbqrP/tgas88tPUbBxQrk=
golang.org/x/sys v0.0.0-20210124154548-22da62e12c0c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221 h1:/ZHdbVpdR/jk3g30/d4yUL0JU9kksj8+F/bnQUVLGDM=
golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1 h1:v+OssWQX+hTHEmOBgwxdZxK4zHq3yOs8F9J7mk0PY8E=

28
internal/engine/.github/ISSUE_TEMPLATE/bug_report.md vendored Normal file
View File

@ -0,0 +1,28 @@
---
name: Bug report
about: Create a report to help us improve
title: ''
labels: bug, triage
assignees: bassosimone
---
**Describe the bug**
A clear and concise description of what the bug is.
**To Reproduce**
Steps to reproduce the behavior.
**Expected behavior**
A clear and concise description of what you expected to happen.
**Screenshots**
If applicable, add screenshots to help explain your problem.
**System information (please complete the following information):**
- Device: [e.g. iPhone6]
- OS: [e.g. iOS8.1]
- Version [e.g. 22]
**Additional context**
Add any other context about the problem here.

29
internal/engine/.github/ISSUE_TEMPLATE/routine-sprint-releases.md vendored Normal file
View File

@ -0,0 +1,29 @@
---
name: Routine sprint releases
about: Bi-weekly releases of probe-engine, etc.
title: ''
labels: effort/M, priority/medium
assignees: bassosimone
---
- [ ] psiphon: run ./update.bash
- [ ] engine: update dependencies
- [ ] engine: update internal/httpheader/useragent.go
- [ ] engine: update version/version.go
- [ ] engine: update resources/assets.go
- [ ] engine: update bundled certs (using `go generate ./...`)
- [ ] engine: make sure all workflows are green
- [ ] engine: tag a new version
- [ ] engine: update again version.go to be alpha
- [ ] engine: create release at GitHub
- [ ] engine: update mobile-staging branch to create oonimkall
- [ ] cli: pin to latest engine
- [ ] cli: update internal/version/version.go
- [ ] cli: tag a new version
- [ ] cli: update internal/version/version.go again to be alpha
- [ ] android: pin to latest oonimkall
- [ ] ios: pin to latest oonimkall
- [ ] desktop: pin to latest cli
- [ ] engine: create issue for next routine release
- [ ] e2etesting: see whether we can remove legacy checks

16
internal/engine/.github/workflows/alltests.yml vendored Normal file
View File

@ -0,0 +1,16 @@
name: alltests
on:
push:
branches:
- 'release/**'
schedule:
- cron: "0 7 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: go test -race -tags integration,shaping -coverprofile=probe-engine.cov ./...

23
internal/engine/.github/workflows/android.yml vendored Normal file
View File

@ -0,0 +1,23 @@
name: android
on:
push:
branches:
- mobile-staging
- 'release/**'
jobs:
test:
runs-on: macos-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: brew install --cask android-sdk
- run: echo y | sdkmanager --install "platforms;android-29"
- run: echo y | sdkmanager --install "ndk;21.3.6528147"
- run: ./build-android.bash
env:
ANDROID_HOME: /usr/local/Caskroom/android-sdk/4333796
- run: ./publish-android.bash
env:
BINTRAY_API_KEY: ${{ secrets.BINTRAY_API_KEY }}

58
internal/engine/.github/workflows/codeql-analysis.yml vendored Normal file
View File

@ -0,0 +1,58 @@
# For most projects, this workflow file will not need changing; you simply need
# to commit it to your repository.
#
# You may wish to alter this file to override the set of languages analyzed,
# or to provide custom queries or build logic.
name: "CodeQL"
on:
push:
branches: [ master, mobile-staging ]
pull_request:
# The branches below must be a subset of the branches above
branches: [ master ]
schedule:
- cron: '35 10 * * 1'
jobs:
analyze:
name: Analyze
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
language: [ 'go' ]
steps:
- name: Checkout repository
uses: actions/checkout@v2
# Initializes the CodeQL tools for scanning.
- name: Initialize CodeQL
uses: github/codeql-action/init@v1
with:
languages: ${{ matrix.language }}
# If you wish to specify custom queries, you can do so here or in a config file.
# By default, queries listed here will override any specified in a config file.
# Prefix the list here with "+" to use these queries and those in the config file.
# queries: ./path/to/local/query, your-org/your-repo/queries@main
# Autobuild attempts to build any compiled languages (C/C++, C#, or Java).
# If this step fails, then you should remove it and run the build manually (see below)
- name: Autobuild
uses: github/codeql-action/autobuild@v1
# ℹ️ Command-line programs to run using the OS shell.
# 📚 https://git.io/JvXDl
# ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
# and modify them (or add more) to build your code if your project
# uses a compiled language
#- run: |
# make bootstrap
# make release
- name: Perform CodeQL Analysis
uses: github/codeql-action/analyze@v1

29
internal/engine/.github/workflows/coverage.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: coverage
on:
pull_request:
push:
branches:
- "master"
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
go: [ "1.14" ]
steps:
- uses: actions/setup-go@v1
with:
go-version: "${{ matrix.go }}"
- uses: actions/checkout@v2
- run: go test -short -race -tags shaping -coverprofile=probe-engine.cov ./...
- uses: shogo82148/actions-goveralls@v1
with:
path-to-profile: probe-engine.cov
parallel: true
finish:
needs: test
runs-on: ubuntu-latest
steps:
- uses: shogo82148/actions-goveralls@v1
with:
parallel-finished: true

16
internal/engine/.github/workflows/generate.yml vendored Normal file
View File

@ -0,0 +1,16 @@
name: generate
on:
push:
branches:
- 'release/**'
schedule:
- cron: "0 0 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: go generate ./...

18
internal/engine/.github/workflows/ios.yml vendored Normal file
View File

@ -0,0 +1,18 @@
name: ios
on:
push:
branches:
- mobile-staging
- 'release/**'
jobs:
test:
runs-on: macos-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./build-ios.bash
- run: ./publish-ios.bash
env:
BINTRAY_API_KEY: ${{ secrets.BINTRAY_API_KEY }}

14
internal/engine/.github/workflows/jafar.yml vendored Normal file
View File

@ -0,0 +1,14 @@
name: jafar
on:
schedule:
- cron: "0 5 * * 3"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: go build -v ./cmd/jafar
- run: sudo ./testjafar.bash

29
internal/engine/.github/workflows/libooniffi.yml vendored Normal file
View File

@ -0,0 +1,29 @@
name: libooniffi
on:
schedule:
- cron: "0 5 * * 3"
jobs:
darwin:
runs-on: macos-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./libooniffi/buildtest.bash darwin
linux:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./libooniffi/buildtest.bash linux
windows:
runs-on: windows-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: bash.exe ./libooniffi/buildtest.bash windows

47
internal/engine/.github/workflows/miniooni.yml vendored Normal file
View File

@ -0,0 +1,47 @@
name: miniooni
on:
push:
branches:
- 'release/**'
schedule:
- cron: "0 0 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./build-cli.sh linux
- run: ./CLI/linux/amd64/miniooni --yes -nNi https://example.com web_connectivity
- uses: actions/upload-artifact@v1
with:
name: miniooni-linux-386
path: ./CLI/linux/386/miniooni
- uses: actions/upload-artifact@v1
with:
name: miniooni-linux-amd64
path: ./CLI/linux/amd64/miniooni
- uses: actions/upload-artifact@v1
with:
name: miniooni-linux-armv7
path: ./CLI/linux/armv7/miniooni
- uses: actions/upload-artifact@v1
with:
name: miniooni-linux-arm64
path: ./CLI/linux/arm64/miniooni
- run: ./build-cli.sh darwin
- uses: actions/upload-artifact@v1
with:
name: miniooni-darwin-amd64
path: ./CLI/darwin/amd64/miniooni
- run: sudo apt install --yes mingw-w64
- run: ./build-cli.sh windows
- uses: actions/upload-artifact@v1
with:
name: miniooni-windows-amd64.exe
path: ./CLI/windows/amd64/miniooni.exe

16
internal/engine/.github/workflows/qafbmessenger.yml vendored Normal file
View File

@ -0,0 +1,16 @@
name: "qafbmessenger"
on:
push:
branches:
- 'release/**'
schedule:
- cron: "0 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "fbmessenger"

13
internal/engine/.github/workflows/qahhfm.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: "qahhfm"
on:
schedule:
- cron: "5 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "hhfm"

13
internal/engine/.github/workflows/qahirl.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: "qahirl"
on:
schedule:
- cron: "10 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "hirl"

13
internal/engine/.github/workflows/qatelegram.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: "qatelegram"
on:
schedule:
- cron: "15 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "telegram"

16
internal/engine/.github/workflows/qawebconnectivity.yml vendored Normal file
View File

@ -0,0 +1,16 @@
name: "qawebconnectivity"
on:
push:
branches:
- 'release/**'
schedule:
- cron: "20 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "webconnectivity"

13
internal/engine/.github/workflows/qawhatsapp.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: "qawhatsapp"
on:
schedule:
- cron: "25 3 * * */1"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./QA/rundocker.bash "whatsapp"

18
internal/engine/.github/workflows/shorttests.yml vendored Normal file
View File

@ -0,0 +1,18 @@
name: shorttests
on:
pull_request:
push:
branches:
- "master"
jobs:
test:
runs-on: ubuntu-latest
strategy:
matrix:
go: [ "1.14", "1.15" ]
steps:
- uses: actions/setup-go@v1
with:
go-version: "${{ matrix.go }}"
- uses: actions/checkout@v2
- run: go test -short -race -tags shaping ./...

13
internal/engine/.github/workflows/using.yml vendored Normal file
View File

@ -0,0 +1,13 @@
name: using
on:
schedule:
- cron: "0 5 * * 3"
jobs:
test:
runs-on: ubuntu-latest
steps:
- uses: actions/setup-go@v1
with:
go-version: "1.14"
- uses: actions/checkout@v2
- run: ./testusing.bash

17
internal/engine/.gitignore vendored Normal file
View File

@ -0,0 +1,17 @@
/*.jsonl
/.vscode
/apitool
/asn.mmdb
/badproxy.pem
/ca-bundle.pem
/cmd/jafar/badproxy.pem
/country.mmdb
/example.org
/jafar
/jafar.exe
/miniooni
/miniooni.exe
/oohelper
/oohelperd
/oonipsiphon/
.DS_Store

2
internal/engine/AUTHORS Normal file
View File

@ -0,0 +1,2 @@
Simone Basso
Arturo Filastò

3
internal/engine/CLI/README.md Normal file
View File

@ -0,0 +1,3 @@
# Directory github.com/ooni/probe-engine/CLI
We use this directory for building CLI binaries (e.g. miniooni).

1
internal/engine/CLI/darwin/amd64/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni

1
internal/engine/CLI/linux/386/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni

1
internal/engine/CLI/linux/amd64/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni

1
internal/engine/CLI/linux/arm64/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni

1
internal/engine/CLI/linux/armv7/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni

1
internal/engine/CLI/windows/386/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni.exe

1
internal/engine/CLI/windows/amd64/.gitignore vendored Normal file
View File

@ -0,0 +1 @@
/miniooni.exe

1
internal/engine/CODEOWNERS Normal file
View File

@ -0,0 +1 @@
* @bassosimone @hellais

255
internal/engine/CODE_OF_CONDUCT.md Normal file
View File

@ -0,0 +1,255 @@
# OONI Code of Conduct
Statement of Intent:
OONI is committed to fostering an inclusive environment and
community. OONI is a place where people should feel safe to engage, share their
point of view, and participate.
This code of conduct applies to OONI as a whole. It is intended to
provide guidelines for contributors. Employees and contractors of OONI
are also subject to company policies and procedures. Those
people should feel free to contact HR with questions or concerns.
This code of conduct is not exhaustive or complete. It is an ongoing effort to
summarize our shared understanding. We want to provide a welcoming, safe
environment, so we can work together to pursue powerful solutions. We reserve
the right to deviate from strictly enforcing this code. Any deviations must
produce an outcome which is fairer, and aligned with our values. We understand
that keeping a living document relevant and “patched” involves sustained
effort.
0. Summary: Don't be a jerk. Be awesome instead.
The OONI community should be a good place where people are glad to be.
DO: Be kind, thoughtful, and considerate.
DO: Make OONI a place where people are happy and comfortable.
DO: Remember: We are all contributing; we are all learning. Nobody was born
an expert.
DO: Yield the floor. Listen. Make sure everyone gets heard.
DON'T: Insult, harass, intimidate, or be a jerk.
DON'T: Treat honest mistakes as an excuse to hassle people. Mistakes are for
learning.
DON'T: Hunt for ways to uphold the letter of this code while violating its
spirit.
AND DO: Eagerly email: team@openobservatory.org with
questions or concerns.
1. Purpose
A primary goal of the OONI community is to be inclusive of many different
contributors. We want to include contributors from the most varied and diverse
backgrounds possible. As such, we are committed to providing a friendly, safe
and welcoming environment for all, regardless of their experience, gender
identity or expression, sexual orientation, family, relationships, ability
(whether bodily or mental), personal appearance, socioeconomic status, body
size, race, ethnicity, age, religion, nationality, or membership in a
disadvantaged and/or underrepresented group.
A safe environment means one free from abuse, whether isolated or systemic. We
explicitly acknowledge that tolerating abuse is a security problem. Allowing
abusers and malicious people to disrupt our community puts our software,
developers, and users at risk.
This code of conduct outlines our expectations for all those who participate in
our community, as well as the consequences for unacceptable behavior.
We invite all those who participate in OONI to help us create safe
and positive experiences for everyone.
2. Be your best self
The following behaviors are expected and requested of all community members:
* Participate in an honest and active way. In doing so, you contribute to
the health and longevity of this community.
* Exercise consideration and respect in your speech and actions.
* Attempt collaboration and dialog before engaging in conflict.
* Refrain from demeaning, discriminatory, or harassing behavior and speech.
* Be mindful of your surroundings and of your fellow participants. Alert
community leaders if you notice:
- a dangerous situation
- someone in distress
- violations of this code of conduct, even if they seem minor
* Remember that community event venues may be shared with members of the
public. Please be respectful to everyone using these locations.
* Respect the privacy of your fellow community members.
3. Unacceptable behaviors
The following behaviors are unacceptable within our community:
* Violence, sexual assault, threats of violence, or violent language
directed against another person, especially violence against a person or
group based on a protected characteristic. (Display of weapons may
constitute intimidation or a threat of violence.)
* Sexist, racist, homophobic, transphobic, ableist or otherwise
discriminatory jokes and language.
* Spontaneously posting or displaying sexually explicit or violent
material. (If it is necessary to share such material when working on
OONI's mission, do so with sensitivity. Be aware that many people don't
want to see it.)
* Posting or threatening to post other people’s personally identifying
information ("doxing") without their consent.
* Personal insults or attacks, particularly those related to:
- experience
- gender identity or expression
- sexual orientation
- family
- relationships
- ability (whether bodily or mental)
- personal appearance
- socioeconomic status
- body size
- race
- ethnicity
- age
- religion
- nationality
- membership in a disadvantaged and/or underrepresented group
* Inappropriate photography, audio recording, or recording of personal
information. You should have someone's consent before recording these
things, and before posting them publicly.
* Inappropriate physical contact. You should have someone’s consent before
touching them.
* Unwelcome sexual attention: this includes sexualized comments or jokes,
inappropriate touching, groping, and unwelcome sexual advances.
* Deliberate intimidation, stalking or following (online or in person).
* Deliberately undermining the spirit of this code while following the
letter.
* Sustained disruption of any community events, including talks,
presentations, and online conversations.
* Deliberately pushing against someone's stated boundaries.
* Advocating for, or encouraging, any of the above behavior.
4. Unacceptable behavior has consequences
We will not tolerate unacceptable behavior from any community member. We will
not make exceptions for sponsors and those with decision-making authority.
People in formal or informal leadership roles must model the highest standards
of behavior.
Anyone asked by another community member to stop unacceptable behavior is
expected to comply immediately. Please don't step in on someone else's behalf
without their consent.
5. Where to go for help
The OONI team can assist with intra-community conflict resolution. You can contact
the whole OONI team:
- if you have questions or concerns about the code of conduct, or
- if you feel that you have witnessed a code of conduct violation
However, if you feel that there is a conflict of interest with any team
member, you may contact members individually. See: https://ooni.org/about/.
6. What to do if you witness unacceptable behavior
If you are subject to or witness unacceptable behavior, or have any other
concerns, please notify the OONI team as soon as possible. You can
contact the OONI team in person, or at
team@openobservatory.org. Current team members are listed
on the [about page](https://ooni.org/about/). The OONI team's incident response will vary on a
case-by-case basis. We will make every effort to respond to the
incident immediately. We will prioritize the safety of the person who
has been harmed, or is at risk of harm. Person(s) who have been harmed or are
at risk of harm can withdraw the incident report at any time. We will never do
anything without the consent of the person who has been harmed or is at risk of
harm, except in situations where there is a threat of imminent danger or harm
to anyone.
7. What the person reporting a violation can expect
The OONI team prioritizes the safety and well-being of any person who
feels that they have been harmed or may be in danger of being harmed. Anyone
reporting an issue to the OONI team can expect that their report will
be taken seriously. Initial reports can be taken in written or verbal form.
The next steps in an incident response will vary on a case-by-case basis.
8. How the OONI team responds to incidents
If a community member engages in unacceptable behavior, the OONI team
may take any action they deem appropriate, including but not limited to a
warning, informal mediation, temporary ban or permanent expulsion from the
community.
9. Scope of this document
This code of conduct covers all community participants:
- paid and unpaid contributors
- sponsors
- other guests
when interacting:
- in all online and in-person community venues
- in one-on-one communications that relate to community work
This code of conduct and its related procedures also applies to unacceptable
behavior occurring outside the scope of community activities when such behavior
has the potential to adversely affect the safety and well-being of community
members. As members of the OONI community, we support and follow this Code while
we are working on OONI, and take care not to undermine it in the rest of our
time.
10. License and attribution
This code of conduct is shared under a Creative Commons CC-BY-SA 4.0
International license.
This code of conduct is adapted from The Tor Project code of conduct and uses
some language and framing from the [Citizen Code of Conduct](https://citizencodeofconduct.org),
which is shared under a CC-BY-SA license.

74
internal/engine/CONTRIBUTING.md Normal file
View File

@ -0,0 +1,74 @@
# Contributing to ooni/probe-engine
This is an open source project, and contributions are welcome! You are welcome
to open pull requests. An open pull request will be reviewed by a core
developer. The review may request you to apply changes. Once the assigned
reviewer is satisfied, they will merge the pull request.
## Opening issues
Please, before opening a new issue, check whether the issue or feature request
you want us to consider has not already been reported by someone else.
## PR requirements
Every pull request that introduces new functionality should feature
comprehensive test coverage. Any pull request that modifies existing
functionality should pass existing tests. What's more, any new pull
request that modifies existing functionality should not decrease the
existing code coverage.
Long-running tests should be skipped when running tests in short mode
using `go test -short`. We prefer external testing to internal
testing. We generally have a file called `foo_test.go` with tests
for every `foo.go` file. Sometimes we separate long running
integration tests in a `foo_integration_test.go` file.
If there is a top-level DESIGN.md document, make sure such document is
kept in sync with code changes you have applied.
Do not submit large PRs. A reviewer can best service your PR if the
code changes are around 200-600 lines. (It is okay to add more changes
afterwards, if the reviewer asks you to do more work; the key point
here is that the PR should be reasonably sized when the review starts.)
In this vein, we'd rather structure a complex issue as a sequence of
small PRs, than have a single large PR address it all.
As a general rule, a PR is reviewed by reading the whole diff. Let us
know if you want us to read each diff individually, if you think that's
functional to better understand your changes.
## Code style requirements
Please, use `go fmt`, `go vet`, and `golint` to check your code
contribution before submitting a pull request. Make sure your code
is documented. At the minimum document all the exported symbols.
Make sure you commit `go.mod` and `go.sum` changes. Make sure you
run `go mod tidy` to minimize such changes.
## Code testing requirements
Make sure all tests pass with `go test -race ./...` run from the
top-level directory of this repository.
## Writing a new OONI experiment
When you are implementing a new experiment (aka nettest), make sure
you have read the relevant spec from the [ooni/spec](
https://github.com/ooni/spec) repository. If the spec is missing,
please help the pull request reviewer to create it. If the spec is
not clear, please let us know during the review.
When you write a new experiment, keep the measurement phase and the
results analysis phases as separate functions. This helps us a lot
to write better unit tests for our code.
To get a sense of what we expect from an experiment, see:
- the experiment/example experiment
- the experiment/webconnectivity experiment
Thank you!

139
internal/engine/DESIGN.md Normal file
View File

@ -0,0 +1,139 @@
# Replacing Measurement Kit
| Author | Simone Basso |
|--------------|--------------|
| Last-Updated | 2020-07-09 |
| Status | historical |
*Abstract* We describe our plan of replacing Measurement Kit for OONI
Probe Android and iOS (in particular) and (also) the CLI.
## Introduction
We want to write experiments in Go. This reduces our burden
compared to writing them using C/C++ code.
Go consumers of probe-engine shall directly use its Go API. We
will discuss the Go API in a future revision of this spec.
For mobile apps, we want to replace these MK APIs:
- [measurement-kit/android-libs](https://github.com/measurement-kit/android-libs)
- [measurement-kit/mkall-ios](https://github.com/measurement-kit/mkall-ios)
We also want consumers of [measurement-kit's FFI API](https://git.io/Jv4Rv)
to be able to replace measurement-kit with probe-engine.
## APIs to replace
### Mobile APIs
We define a Go API that `gomobile` binds to a Java/ObjectiveC
API that is close enough to the MK's mobile APIs.
### FFI API
We define a CGO API such that `go build -buildmode=c-shared`
yields an API reasonably close to MK's FFI API.
## Running experiments
It seems the generic API for enabling running experiments both on
mobile devices and for FFI consumers is like:
```Go
type Task struct{ ... }
func StartTask(input string) (*Task, error)
func (t *Task) Interrupt()
func (t *Task) IsDone() bool
func (t *Task) WaitForNextEvent() string
```
This should be enough to generate a suitable mobile API when
using the `gomobile` Go subcommand.
We can likewise generate a FFI API as follows:
```Go
package main
import (
"C"
"sync"
"github.com/ooni/probe-engine/oonimkall"
)
var (
idx int64 = 1
m = make(map[int64]*oonimkall.Task)
mu sync.Mutex
)
//export ooni_task_start
func ooni_task_start(settings string) int64 {
tp, err := oonimkall.StartTask(settings)
if err != nil {
return 0
}
mu.Lock()
handle := idx
idx++
m[handle] = tp
mu.Unlock()
return handle
}
//export ooni_task_interrupt
func ooni_task_interrupt(handle int64) {
mu.Lock()
if tp := m[handle]; tp != nil {
tp.Interrupt()
}
mu.Unlock()
}
//export ooni_task_is_done
func ooni_task_is_done(handle int64) bool {
isdone := true
mu.Lock()
if tp := m[handle]; tp != nil {
isdone = tp.IsDone()
}
mu.Unlock()
return isdone
}
//export ooni_task_wait_for_next_event
func ooni_task_wait_for_next_event(handle int64) (event string) {
mu.Lock()
tp := m[handle]
mu.Unlock()
if tp != nil {
event = tp.WaitForNextEvent()
}
return
}
func main() {}
```
This is close enough to [measurement-kit's FFI API](https://git.io/Jv4Rv) that
a few lines of C allow to implement an ABI compatible replacement.
## Other APIs of interest
We currently don't have plans for replacing other MK APIs. We will introduce
new APIs specifically tailored for our OONI needs, but they will be out of
scope with respect to the main goal of this design document.
## History
[The initial version of this design document](
https://github.com/measurement-kit/engine/blob/master/DESIGN.md)
lived in the measurement-kit namespace at GitHub. It discussed
a bunch of broad, extra topics, e.g., code bloat that are not
discussed in this document. More details regarding the migration
from MK to probe-engine are at [measurement-kit/measurement-kit#1913](
https://github.com/measurement-kit/measurement-kit/issues/1913).

26
internal/engine/LICENSE Normal file
View File

@ -0,0 +1,26 @@
Copyright 2019 Open Observatory of Network Interference (OONI), The Tor Project
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions are met:
1. Redistributions of source code must retain the above copyright notice, this
list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright notice,
this list of conditions and the following disclaimer in the documentation
and/or other materials provided with the distribution.
3. Neither the name of the copyright holder nor the names of its contributors
may be used to endorse or promote products derived from this software
without specific prior written permission.
THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE
FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR
SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER
CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY,
OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.

3
internal/engine/MOBILE/README.md Normal file
View File

@ -0,0 +1,3 @@
# Directory github.com/ooni/probe-engine/MOBILE
This directory is used for building for Android and iOS.

13
internal/engine/MOBILE/template.podspec Normal file
View File

@ -0,0 +1,13 @@
Pod::Spec.new do |s|
s.name = "oonimkall"
s.version = "@VERSION@"
s.summary = "OONI Probe Engine for iOS"
s.author = "Simone Basso"
s.homepage = "https://github.com/ooni/probe-engine"
s.license = { :type => "BSD" }
s.source = {
:http => "https://dl.bintray.com/ooni/ios/oonimkall-@VERSION@.framework.zip"
}
s.platform = :ios, "9.0"
s.ios.vendored_frameworks = "oonimkall.framework"
end

33
internal/engine/MOBILE/template.pom Normal file
View File

@ -0,0 +1,33 @@
<?xml version="1.0" encoding="UTF-8"?>
<project xsi:schemaLocation="http://maven.apache.org/POM/4.0.0 http://maven.apache.org/xsd/maven-4.0.0.xsd" xmlns="http://maven.apache.org/POM/4.0.0"
xmlns:xsi="http://www.w3.org/2001/XMLSchema-instance">
<modelVersion>4.0.0</modelVersion>
<groupId>org.ooni</groupId>
<artifactId>oonimkall</artifactId>
<version>@VERSION@</version>
<packaging>aar</packaging>
<name>oonimkall</name>
<description>OONI Probe Engine for Android</description>
<url>https://github.com/ooni/probe-engine</url>
<licenses>
<license>
<name>The 3-Clause BSD License</name>
<url>https://opensource.org/licenses/BSD-3-Clause</url>
<distribution>repo</distribution>
</license>
</licenses>
<scm>
<url>https://github.com/ooni/probe-engine</url>
<connection>https://github.com/ooni/probe-engine.git</connection>
</scm>
<developers>
<developer>
<name>Simone Basso</name>
<email>simone@openobservatory.org</email>
<roles>
<role>developer</role>
</roles>
<timezone>Europe/Rome</timezone>
</developer>
</developers>
</project>

1
internal/engine/QA/.dockerignore Normal file
View File

@ -0,0 +1 @@
*

3
internal/engine/QA/.gitignore vendored Normal file
View File

@ -0,0 +1,3 @@
/GOPATH
/GOCACHE
/__pycache__

2
internal/engine/QA/Dockerfile Normal file
View File

@ -0,0 +1,2 @@
FROM golang:1.14-alpine
RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3

54
internal/engine/QA/README.md Normal file
View File

@ -0,0 +1,54 @@
# Quality Assurance scripts
This directory contains quality assurance scripts that use Jafar to
ensure that OONI implementations behave. These scripts take on the
command line as argument the path to a binary with a OONI Probe v2.x
like command line interface. We do not care about full compatibility
but rather about having enough similar flags that running these tools
in parallel is not too much of a burden for us.
Tools with this shallow-compatible CLI are:
1. `github.com/ooni/probe-legacy`
2. `github.com/measurement-kit/measurement-kit/src/measurement_kit`
3. `github.com/ooni/probe-engine/cmd/miniooni`
## Run QA on a Linux system
These scripts assume you're on a Linux system with `iptables`, `bash`,
`python3`, and possibly a bunch of other tools installed.
To start the QA script, run this command:
```bash
sudo ./QA/$nettest.py $ooni_exe
```
where `$nettest` is the nettest name (e.g. `telegram`) and `$ooni_exe`
is the OONI Probe v2.x compatible binary to test.
The Python script needs to run as root. Note however that sudo will also
be used to run `$ooni_exe` with the privileges of the `nobody` user.
## Run QA using a docker container
Run test in a suitable Docker container using:
```bash
./QA/rundocker.sh $nettest
```
Note that this will run a `--privileged` docker container. This will
eventually run the Python script you would run on Linux.
For now, the docker scripts only perform QA of `miniooni`.
## Diagnosing issues
The Python script that performs the QA runs a specific OONI test under
different failure conditions and stops at the first unexpected value found
in the resulting JSONL report. You can infer what went wrong by reading
the output of the `$ooni_exe` command itself, which should be above the point
where the Python script stopped, as well as by inspecting the JSONL file on
disk. By convention such file is named `$nettest.jsonl` and only contains
the result of the last run of `$nettest`.

72
internal/engine/QA/common.py Normal file
View File

@ -0,0 +1,72 @@
""" ./QA/common.py - common code for QA """
import contextlib
import json
import os
import shlex
import shutil
import socket
import subprocess
import sys
import time
import urllib.parse
def execute(args):
""" Execute a specified command """
subprocess.run(args)
def execute_jafar_and_miniooni(ooni_exe, outfile, experiment, tag, args):
""" Executes jafar and miniooni. Returns the test keys. """
tmpoutfile = "/tmp/{}".format(outfile)
with contextlib.suppress(FileNotFoundError):
os.remove(tmpoutfile) # just in case
execute(
[
"./jafar",
"-main-command",
"./QA/minioonilike.py {} -n -o '{}' --home /tmp {}".format(
ooni_exe, tmpoutfile, experiment
),
"-main-user",
"nobody", # should be present on Unix
"-tag",
tag,
]
+ args
)
shutil.copy(tmpoutfile, outfile)
result = read_result(outfile)
assert isinstance(result, dict)
assert isinstance(result["test_keys"], dict)
return result["test_keys"]
def read_result(outfile):
""" Reads the result of an experiment """
return json.load(open(outfile, "rb"))
def test_keys(result):
""" Returns just the test keys of a specific result """
return result["test_keys"]
def check_maybe_binary_value(value):
""" Make sure a maybe binary value is correct """
assert isinstance(value, str) or (
isinstance(value, dict)
and value["format"] == "base64"
and isinstance(value["data"], str)
)
def with_free_port(func):
""" This function executes |func| passing it a port number on localhost
which is bound but not listening for new connections """
# See <https://stackoverflow.com/a/45690594>
with contextlib.closing(socket.socket(socket.AF_INET, socket.SOCK_STREAM)) as sock:
sock.bind(("127.0.0.1", 0))
sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
func(sock.getsockname()[1])

286
internal/engine/QA/fbmessenger.py Executable file
View File

@ -0,0 +1,286 @@
#!/usr/bin/env python3
""" ./QA/fbmessenger.py - main QA script for fbmessenger
This script performs a bunch of fbmessenger tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
services = {
"stun": "stun.fbsbx.com",
"b_api": "b-api.facebook.com",
"b_graph": "b-graph.facebook.com",
"edge": "edge-mqtt.facebook.com",
"external_cdn": "external.xx.fbcdn.net",
"scontent_cdn": "scontent.xx.fbcdn.net",
"star": "star.c10r.facebook.com",
}
def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(
ooni_exe, outfile, "facebook_messenger", tag, args
)
assert tk["requests"] is None
if tk["tcp_connect"] is not None:
assert isinstance(tk["tcp_connect"], list)
assert len(tk["tcp_connect"]) > 0
for entry in tk["tcp_connect"]:
assert isinstance(entry, dict)
assert isinstance(entry["ip"], str)
assert isinstance(entry["port"], int)
assert isinstance(entry["status"], dict)
failure = entry["status"]["failure"]
success = entry["status"]["success"]
assert isinstance(failure, str) or failure is None
assert isinstance(success, bool)
return tk
def helper_for_blocking_services_via_dns(service):
""" Helper for hijacking a service via dns """
args = []
args.append("-iptables-hijack-dns-to")
args.append("127.0.0.1:53")
args.append("-dns-proxy-block")
args.append(service)
return args
def helper_for_hijacking_services_via_dns(service):
""" Helper for hijacking a service via dns """
args = []
args.append("-iptables-hijack-dns-to")
args.append("127.0.0.1:53")
args.append("-dns-proxy-hijack")
args.append(service)
return args
def helper_for_blocking_services_via_tcp(service):
""" Helper for blocking a service via tcp """
args = []
args.append("-iptables-reset-ip")
args.append(service)
return args
def fbmessenger_dns_hijacked_for_all(ooni_exe, outfile):
""" Test case where everything we measure is DNS hijacked """
args = []
for _, value in services.items():
args.extend(helper_for_hijacking_services_via_dns(value))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_dns_hijacked_for_all", args,
)
assert tk["facebook_b_api_dns_consistent"] == False
assert tk["facebook_b_api_reachable"] == None
assert tk["facebook_b_graph_dns_consistent"] == False
assert tk["facebook_b_graph_reachable"] == None
assert tk["facebook_edge_dns_consistent"] == False
assert tk["facebook_edge_reachable"] == None
assert tk["facebook_external_cdn_dns_consistent"] == False
assert tk["facebook_external_cdn_reachable"] == None
assert tk["facebook_scontent_cdn_dns_consistent"] == False
assert tk["facebook_scontent_cdn_reachable"] == None
assert tk["facebook_star_dns_consistent"] == False
assert tk["facebook_star_reachable"] == None
assert tk["facebook_stun_dns_consistent"] == False
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == True
assert tk["facebook_tcp_blocking"] == False
def fbmessenger_dns_hijacked_for_some(ooni_exe, outfile):
""" Test case where some endpoints are DNS hijacked """
args = []
args.extend(helper_for_hijacking_services_via_dns(services["star"]))
args.extend(helper_for_hijacking_services_via_dns(services["edge"]))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_dns_hijacked_for_some", args,
)
assert tk["facebook_b_api_dns_consistent"] == True
assert tk["facebook_b_api_reachable"] == True
assert tk["facebook_b_graph_dns_consistent"] == True
assert tk["facebook_b_graph_reachable"] == True
assert tk["facebook_edge_dns_consistent"] == False
assert tk["facebook_edge_reachable"] == None
assert tk["facebook_external_cdn_dns_consistent"] == True
assert tk["facebook_external_cdn_reachable"] == True
assert tk["facebook_scontent_cdn_dns_consistent"] == True
assert tk["facebook_scontent_cdn_reachable"] == True
assert tk["facebook_star_dns_consistent"] == False
assert tk["facebook_star_reachable"] == None
assert tk["facebook_stun_dns_consistent"] == True
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == True
assert tk["facebook_tcp_blocking"] == False
def fbmessenger_dns_blocked_for_all(ooni_exe, outfile):
""" Test case where everything we measure is DNS blocked """
args = []
for _, value in services.items():
args.extend(helper_for_blocking_services_via_dns(value))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_dns_blocked_for_all", args,
)
assert tk["facebook_b_api_dns_consistent"] == False
assert tk["facebook_b_api_reachable"] == None
assert tk["facebook_b_graph_dns_consistent"] == False
assert tk["facebook_b_graph_reachable"] == None
assert tk["facebook_edge_dns_consistent"] == False
assert tk["facebook_edge_reachable"] == None
assert tk["facebook_external_cdn_dns_consistent"] == False
assert tk["facebook_external_cdn_reachable"] == None
assert tk["facebook_scontent_cdn_dns_consistent"] == False
assert tk["facebook_scontent_cdn_reachable"] == None
assert tk["facebook_star_dns_consistent"] == False
assert tk["facebook_star_reachable"] == None
assert tk["facebook_stun_dns_consistent"] == False
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == True
assert tk["facebook_tcp_blocking"] == False
def fbmessenger_dns_blocked_for_some(ooni_exe, outfile):
""" Test case where some endpoints are DNS blocked """
args = []
args.extend(helper_for_blocking_services_via_dns(services["b_graph"]))
args.extend(helper_for_blocking_services_via_dns(services["stun"]))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_dns_blocked_for_some", args,
)
assert tk["facebook_b_api_dns_consistent"] == True
assert tk["facebook_b_api_reachable"] == True
assert tk["facebook_b_graph_dns_consistent"] == False
assert tk["facebook_b_graph_reachable"] == None
assert tk["facebook_edge_dns_consistent"] == True
assert tk["facebook_edge_reachable"] == True
assert tk["facebook_external_cdn_dns_consistent"] == True
assert tk["facebook_external_cdn_reachable"] == True
assert tk["facebook_scontent_cdn_dns_consistent"] == True
assert tk["facebook_scontent_cdn_reachable"] == True
assert tk["facebook_star_dns_consistent"] == True
assert tk["facebook_star_reachable"] == True
assert tk["facebook_stun_dns_consistent"] == False
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == True
assert tk["facebook_tcp_blocking"] == False
def fbmessenger_tcp_blocked_for_all(ooni_exe, outfile):
""" Test case where everything we measure is TCP blocked """
args = []
for _, value in services.items():
args.extend(helper_for_blocking_services_via_tcp(value))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_tcp_blocked_for_all", args,
)
assert tk["facebook_b_api_dns_consistent"] == True
assert tk["facebook_b_api_reachable"] == False
assert tk["facebook_b_graph_dns_consistent"] == True
assert tk["facebook_b_graph_reachable"] == False
assert tk["facebook_edge_dns_consistent"] == True
assert tk["facebook_edge_reachable"] == False
assert tk["facebook_external_cdn_dns_consistent"] == True
assert tk["facebook_external_cdn_reachable"] == False
assert tk["facebook_scontent_cdn_dns_consistent"] == True
assert tk["facebook_scontent_cdn_reachable"] == False
assert tk["facebook_star_dns_consistent"] == True
assert tk["facebook_star_reachable"] == False
assert tk["facebook_stun_dns_consistent"] == True
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == False
assert tk["facebook_tcp_blocking"] == True
def fbmessenger_tcp_blocked_for_some(ooni_exe, outfile):
""" Test case where only some endpoints are TCP blocked """
args = []
args.extend(helper_for_blocking_services_via_tcp(services["edge"]))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args,
)
assert tk["facebook_b_api_dns_consistent"] == True
assert tk["facebook_b_api_reachable"] == True
assert tk["facebook_b_graph_dns_consistent"] == True
assert tk["facebook_b_graph_reachable"] == True
assert tk["facebook_edge_dns_consistent"] == True
assert tk["facebook_edge_reachable"] == False
assert tk["facebook_external_cdn_dns_consistent"] == True
assert tk["facebook_external_cdn_reachable"] == True
assert tk["facebook_scontent_cdn_dns_consistent"] == True
assert tk["facebook_scontent_cdn_reachable"] == True
assert tk["facebook_star_dns_consistent"] == True
assert tk["facebook_star_reachable"] == True
assert tk["facebook_stun_dns_consistent"] == True
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == False
assert tk["facebook_tcp_blocking"] == True
def fbmessenger_mixed_results(ooni_exe, outfile):
""" Test case where only some endpoints are TCP blocked """
args = []
args.extend(helper_for_blocking_services_via_tcp(services["edge"]))
args.extend(helper_for_blocking_services_via_dns(services["b_api"]))
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "fbmessenger_tcp_blocked_for_some", args,
)
assert tk["facebook_b_api_dns_consistent"] == False
assert tk["facebook_b_api_reachable"] == None
assert tk["facebook_b_graph_dns_consistent"] == True
assert tk["facebook_b_graph_reachable"] == True
assert tk["facebook_edge_dns_consistent"] == True
assert tk["facebook_edge_reachable"] == False
assert tk["facebook_external_cdn_dns_consistent"] == True
assert tk["facebook_external_cdn_reachable"] == True
assert tk["facebook_scontent_cdn_dns_consistent"] == True
assert tk["facebook_scontent_cdn_reachable"] == True
assert tk["facebook_star_dns_consistent"] == True
assert tk["facebook_star_reachable"] == True
assert tk["facebook_stun_dns_consistent"] == True
assert tk["facebook_stun_reachable"] == None
assert tk["facebook_dns_blocking"] == True
assert tk["facebook_tcp_blocking"] == True
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "fbmessenger.jsonl"
ooni_exe = sys.argv[1]
tests = [
fbmessenger_dns_hijacked_for_all,
fbmessenger_dns_hijacked_for_some,
fbmessenger_dns_blocked_for_all,
fbmessenger_dns_blocked_for_some,
fbmessenger_tcp_blocked_for_all,
fbmessenger_tcp_blocked_for_some,
fbmessenger_mixed_results,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

66
internal/engine/QA/hhfm.py Executable file
View File

@ -0,0 +1,66 @@
#!/usr/bin/env python3
""" ./QA/hhfm.py - main QA script for hhfm
This script performs a bunch of hhfm tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(
ooni_exe, outfile, "http_header_field_manipulation", tag, args
)
# TODO(bassosimone): what checks to put here?
return tk
def hhfm_transparent_proxy(ooni_exe, outfile):
""" Test case where we're passing through a transparent proxy """
args = ["-iptables-hijack-http-to", "127.0.0.1:80"]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "hhfm_transparent_proxy", args,
)
# The proxy sees a domain that does not make any sense and does not
# otherwise know where to connect to. Hence the most likely result is
# a `dns_nxdomain_error` with total tampering.
assert tk["tampering"]["header_field_name"] == False
assert tk["tampering"]["header_field_number"] == False
assert tk["tampering"]["header_field_value"] == False
assert tk["tampering"]["header_name_capitalization"] == False
assert tk["tampering"]["header_name_diff"] == []
assert tk["tampering"]["request_line_capitalization"] == False
assert tk["tampering"]["total"] == True
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "hhfm.jsonl"
ooni_exe = sys.argv[1]
tests = [
hhfm_transparent_proxy,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

67
internal/engine/QA/hirl.py Executable file
View File

@ -0,0 +1,67 @@
#!/usr/bin/env python3
""" ./QA/hirl.py - main QA script for hirl
This script performs a bunch of hirl tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(
ooni_exe, outfile, "http_invalid_request_line", tag, args
)
# TODO(bassosimone): what checks to put here?
return tk
def hirl_transparent_proxy(ooni_exe, outfile):
""" Test case where we're passing through a transparent proxy """
args = ["-iptables-hijack-http-to", "127.0.0.1:80"]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "hirl_transparent_proxy", args,
)
count = 0
for entry in tk["failure_list"]:
if entry is None:
count += 1
elif entry == "eof_error":
count += 1e03
else:
count += 1e06
assert count == 3002
assert tk["tampering_list"] == [True, True, True, True, True]
assert tk["tampering"] == True
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "hirl.jsonl"
ooni_exe = sys.argv[1]
tests = [
hirl_transparent_proxy,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

88
internal/engine/QA/minioonilike.py Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env python3
""" This script takes in input the name of the tool to run followed by
arguments and followed by the nettest name. The format recognized is
the same of miniooni. Depending on the tool that we want to run, we
reorder arguments so that they make sense for the tool.
This is necessary because, albeit miniooni, MK, and OONI v2.x have
more or less the same arguments, there are some differences. We could
modify other tools to match miniooni, but this seems useless. """
import argparse
import os
import shlex
import sys
sys.path.insert(0, ".")
import common
def file_must_exist(pathname):
""" Throws an exception if the given file does not actually exist. """
if not os.path.isfile(pathname):
raise RuntimeError("missing {}: please run miniooni first".format(pathname))
return pathname
def main():
apa = argparse.ArgumentParser()
apa.add_argument("command", nargs=1, help="command to execute")
# subset of arguments accepted by miniooni
apa.add_argument(
"-n", "--no-collector", action="count", help="don't submit measurement"
)
apa.add_argument("-o", "--reportfile", help="specify report file to use")
apa.add_argument("-i", "--input", help="input for nettests taking an input")
apa.add_argument("--home", help="override home directory")
apa.add_argument("nettest", nargs=1, help="nettest to run")
out = apa.parse_args()
command, nettest = out.command[0], out.nettest[0]
if "miniooni" not in command and "measurement_kit" not in command:
raise RuntimeError("unrecognized tool")
args = []
args.append(command)
if "miniooni" in command:
args.extend(["--yes"]) # make sure we have informed consent
if "measurement_kit" in command:
args.extend(
[
"--ca-bundle-path",
file_must_exist("{}/.miniooni/assets/ca-bundle.pem".format(out.home)),
]
)
args.extend(
[
"--geoip-country-path",
file_must_exist("{}/.miniooni/assets/country.mmdb".format(out.home)),
]
)
args.extend(
[
"--geoip-asn-path",
file_must_exist("{}/.miniooni/assets/asn.mmdb".format(out.home)),
]
)
if out.home and "miniooni" in command:
args.extend(["--home", out.home]) # home applies to miniooni only
if out.input:
if "miniooni" in command:
args.extend(["-i", out.input]) # input is -i for miniooni
if out.no_collector:
args.append("-n")
if out.reportfile:
args.extend(["-o", out.reportfile])
args.append(nettest)
if out.input and "measurement_kit" in command:
if nettest == "web_connectivity":
args.extend(["-u", out.input]) # MK's Web Connectivity uses -u for input
sys.stderr.write("minioonilike.py: {}\n".format(shlex.join(args)))
common.execute(args)
if __name__ == "__main__":
main()

88
internal/engine/QA/probeasn.py Executable file
View File

@ -0,0 +1,88 @@
#!/usr/bin/env python3
""" ./QA/probeasn.py - QA script for the -g miniooni option. """
import contextlib
import json
import os
import shlex
import shutil
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
def execute_miniooni(ooni_exe, outfile, arguments):
""" Executes miniooni and returns the whole measurement. """
if "miniooni" not in ooni_exe:
return None
tmpoutfile = "/tmp/{}".format(outfile)
with contextlib.suppress(FileNotFoundError):
os.remove(tmpoutfile) # just in case
cmdline = [
ooni_exe,
arguments,
"-o",
tmpoutfile,
"--home",
"/tmp",
"example",
]
print("exec: {}".format(cmdline))
common.execute(cmdline)
shutil.copy(tmpoutfile, outfile)
result = common.read_result(outfile)
assert isinstance(result, dict)
assert isinstance(result["test_keys"], dict)
return result
def probeasn_without_g_option(ooni_exe, outfile):
""" Test case where we're not passing to miniooni the -g option """
m = execute_miniooni(ooni_exe, outfile, "-n")
if m is None:
return
assert m["probe_cc"] != "ZZ"
assert m["probe_ip"] == "127.0.0.1"
assert m["probe_asn"] != "AS0"
assert m["probe_network_name"] != ""
assert m["resolver_ip"] == "127.0.0.2"
assert m["resolver_asn"] != "AS0"
assert m["resolver_network_name"] != ""
def probeasn_with_g_option(ooni_exe, outfile):
""" Test case where we're passing the -g option """
m = execute_miniooni(ooni_exe, outfile, "-gn")
if m is None:
return
assert m["probe_cc"] != "ZZ"
assert m["probe_ip"] == "127.0.0.1"
assert m["probe_asn"] == "AS0"
assert m["probe_network_name"] == ""
assert m["resolver_ip"] == "127.0.0.2"
assert m["resolver_asn"] == "AS0"
assert m["resolver_network_name"] == ""
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "probeasn.jsonl"
ooni_exe = sys.argv[1]
tests = [
probeasn_with_g_option,
probeasn_without_g_option,
]
for test in tests:
test(ooni_exe, outfile)
if __name__ == "__main__":
main()

6
internal/engine/QA/pyrun.sh Executable file
View File

@ -0,0 +1,6 @@
#!/bin/sh
set -ex
export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on
go build -v ./cmd/miniooni
go build -v ./cmd/jafar
sudo ./QA/$1.py ./miniooni

5
internal/engine/QA/rundocker.bash Executable file
View File

@ -0,0 +1,5 @@
#!/bin/sh
set -ex
DOCKER=${DOCKER:-docker}
$DOCKER build -t jafar-qa ./QA/
$DOCKER run --privileged -v`pwd`:/jafar -w/jafar jafar-qa ./QA/pyrun.sh "$@"

218
internal/engine/QA/telegram.py Executable file
View File

@ -0,0 +1,218 @@
#!/usr/bin/env python3
""" ./QA/telegram.py - main QA script for telegram
This script performs a bunch of telegram tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
ALL_POP_IPS = (
"149.154.175.50",
"149.154.167.51",
"149.154.175.100",
"149.154.167.91",
"149.154.171.5",
)
def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "telegram", tag, args)
assert isinstance(tk["requests"], list)
assert len(tk["requests"]) > 0
for entry in tk["requests"]:
assert isinstance(entry, dict)
failure = entry["failure"]
assert isinstance(failure, str) or failure is None
assert isinstance(entry["request"], dict)
req = entry["request"]
common.check_maybe_binary_value(req["body"])
assert isinstance(req["headers"], dict)
for key, value in req["headers"].items():
assert isinstance(key, str)
common.check_maybe_binary_value(value)
assert isinstance(req["method"], str)
assert isinstance(entry["response"], dict)
resp = entry["response"]
common.check_maybe_binary_value(resp["body"])
assert isinstance(resp["code"], int)
if resp["headers"] is not None:
for key, value in resp["headers"].items():
assert isinstance(key, str)
common.check_maybe_binary_value(value)
assert isinstance(tk["tcp_connect"], list)
assert len(tk["tcp_connect"]) > 0
for entry in tk["tcp_connect"]:
assert isinstance(entry, dict)
assert isinstance(entry["ip"], str)
assert isinstance(entry["port"], int)
assert isinstance(entry["status"], dict)
failure = entry["status"]["failure"]
success = entry["status"]["success"]
assert isinstance(failure, str) or failure is None
assert isinstance(success, bool)
return tk
def args_for_blocking_all_pop_ips():
""" Returns the arguments useful for blocking all POPs IPs """
args = []
for ip in ALL_POP_IPS:
args.append("-iptables-reset-ip")
args.append(ip)
return args
def args_for_blocking_web_telegram_org_http():
""" Returns arguments for blocking web.telegram.org over http """
return ["-iptables-reset-keyword", "Host: web.telegram.org"]
def args_for_blocking_web_telegram_org_https():
""" Returns arguments for blocking web.telegram.org over https """
#
# 00 00 <SNI extension ID>
# 00 15 <full extension length>
# 00 13 <first entry length>
# 00 <DNS hostname type>
# 00 10 <string length>
# 77 65 ... 67 web.telegram.org
#
return [
"-iptables-reset-keyword-hex",
"|00 00 00 15 00 13 00 00 10 77 65 62 2e 74 65 6c 65 67 72 61 6d 2e 6f 72 67|",
]
def telegram_block_everything(ooni_exe, outfile):
""" Test case where everything we measure is blocked """
args = []
args.extend(args_for_blocking_all_pop_ips())
args.extend(args_for_blocking_web_telegram_org_https())
args.extend(args_for_blocking_web_telegram_org_http())
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_block_everything", args,
)
assert tk["telegram_tcp_blocking"] == True
assert tk["telegram_http_blocking"] == True
assert tk["telegram_web_failure"] == "connection_reset"
assert tk["telegram_web_status"] == "blocked"
def telegram_tcp_blocking_all(ooni_exe, outfile):
""" Test case where all POPs are TCP/IP blocked """
args = args_for_blocking_all_pop_ips()
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_tcp_blocking_all", args
)
assert tk["telegram_tcp_blocking"] == True
assert tk["telegram_http_blocking"] == True
assert tk["telegram_web_failure"] == None
assert tk["telegram_web_status"] == "ok"
def telegram_tcp_blocking_some(ooni_exe, outfile):
""" Test case where some POPs are TCP/IP blocked """
args = [
"-iptables-reset-ip",
ALL_POP_IPS[0],
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_tcp_blocking_some", args
)
assert tk["telegram_tcp_blocking"] == False
assert tk["telegram_http_blocking"] == False
assert tk["telegram_web_failure"] == None
assert tk["telegram_web_status"] == "ok"
def telegram_http_blocking_all(ooni_exe, outfile):
""" Test case where all POPs are HTTP blocked """
args = []
for ip in ALL_POP_IPS:
args.append("-iptables-reset-keyword")
args.append(ip)
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_http_blocking_all", args,
)
assert tk["telegram_tcp_blocking"] == False
assert tk["telegram_http_blocking"] == True
assert tk["telegram_web_failure"] == None
assert tk["telegram_web_status"] == "ok"
def telegram_http_blocking_some(ooni_exe, outfile):
""" Test case where some POPs are HTTP blocked """
args = [
"-iptables-reset-keyword",
ALL_POP_IPS[0],
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_http_blocking_some", args,
)
assert tk["telegram_tcp_blocking"] == False
assert tk["telegram_http_blocking"] == False
assert tk["telegram_web_failure"] == None
assert tk["telegram_web_status"] == "ok"
def telegram_web_failure_http(ooni_exe, outfile):
""" Test case where the web HTTP endpoint is blocked """
args = args_for_blocking_web_telegram_org_http()
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_web_failure_http", args,
)
assert tk["telegram_tcp_blocking"] == False
assert tk["telegram_http_blocking"] == False
assert tk["telegram_web_failure"] == "connection_reset"
assert tk["telegram_web_status"] == "blocked"
def telegram_web_failure_https(ooni_exe, outfile):
""" Test case where the web HTTPS endpoint is blocked """
args = args_for_blocking_web_telegram_org_https()
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "telegram_web_failure_https", args,
)
assert tk["telegram_tcp_blocking"] == False
assert tk["telegram_http_blocking"] == False
assert tk["telegram_web_failure"] == "connection_reset"
assert tk["telegram_web_status"] == "blocked"
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "telegram.jsonl"
ooni_exe = sys.argv[1]
tests = [
telegram_block_everything,
telegram_tcp_blocking_all,
telegram_tcp_blocking_some,
telegram_http_blocking_all,
telegram_http_blocking_some,
telegram_web_failure_http,
telegram_web_failure_https,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

855
internal/engine/QA/webconnectivity.py Executable file
View File

@ -0,0 +1,855 @@
#!/usr/bin/env python3
""" ./QA/webconnectivity.py - main QA script for webconnectivity
This script performs a bunch of webconnectivity tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
def execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, experiment_args, tag, args
):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(
ooni_exe, outfile, experiment_args, tag, args
)
return tk
def assert_status_flags_are(ooni_exe, tk, desired):
""" Checks whether the status flags are what we expect them to
be when we're running miniooni. This check only makes sense
with miniooni b/c status flags are a miniooni extension. """
if "miniooni" not in ooni_exe:
return
assert tk["x_status"] == desired
def webconnectivity_https_ok_with_control_failure(ooni_exe, outfile):
""" Successful HTTPS measurement but control failure. """
args = [
"-iptables-reset-keyword",
"wcth.ooni.io",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.com/ web_connectivity",
"webconnectivity_https_ok_with_control_failure",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == None
assert tk["control_failure"] == "connection_reset"
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
if "miniooni" in ooni_exe:
assert tk["blocking"] == False
assert tk["accessible"] == True
else:
assert tk["blocking"] == None
assert tk["accessible"] == None
assert_status_flags_are(ooni_exe, tk, 1)
def webconnectivity_http_ok_with_control_failure(ooni_exe, outfile):
""" Successful HTTP measurement but control failure. """
args = [
"-iptables-reset-keyword",
"wcth.ooni.io",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://example.org/ web_connectivity",
"webconnectivity_http_ok_with_control_failure",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == None
assert tk["control_failure"] == "connection_reset"
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == None
assert tk["accessible"] == None
assert_status_flags_are(ooni_exe, tk, 8)
def webconnectivity_transparent_http_proxy(ooni_exe, outfile):
""" Test case where we pass through a transparent HTTP proxy """
args = []
args.append("-iptables-hijack-https-to")
args.append("127.0.0.1:443")
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.org web_connectivity",
"webconnectivity_transparent_http_proxy",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == True
assert tk["body_proportion"] == 1
assert tk["status_code_match"] == True
assert tk["headers_match"] == True
assert tk["title_match"] == True
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 1)
def webconnectivity_dns_hijacking(ooni_exe, outfile):
""" Test case where there is DNS hijacking towards a transparent proxy. """
args = []
args.append("-iptables-hijack-dns-to")
args.append("127.0.0.1:53")
args.append("-dns-proxy-hijack")
args.append("example.org")
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.org web_connectivity",
"webconnectivity_dns_hijacking",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == True
assert tk["body_proportion"] == 1
assert tk["status_code_match"] == True
assert tk["headers_match"] == True
assert tk["title_match"] == True
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 1)
def webconnectivity_control_unreachable_and_using_http(ooni_exe, outfile):
""" Test case where the control is unreachable and we're using the
plaintext HTTP protocol rather than HTTPS """
args = []
args.append("-iptables-reset-keyword")
args.append("wcth.ooni.io")
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://example.org web_connectivity",
"webconnectivity_control_unreachable_and_using_http",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == None
assert tk["control_failure"] == "connection_reset"
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == None
assert tk["accessible"] == None
assert_status_flags_are(ooni_exe, tk, 8)
def webconnectivity_nonexistent_domain(ooni_exe, outfile):
""" Test case where the domain does not exist """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://antani.ooni.io web_connectivity",
"webconnectivity_nonexistent_domain",
args,
)
# TODO(bassosimone): Debateable result. We need to do better here.
# See <https://github.com/ooni/probe-engine/issues/579>.
#
# Note that MK is not doing it right here because it's suppressing the
# dns_nxdomain_error that instead is very informative. Yet, it is reporting
# a failure in HTTP, which miniooni does not because it does not make
# sense to perform HTTP when there are no IP addresses.
#
# The following seems indeed a bug in MK where we don't properly record the
# actual error that occurred when performing the DNS experiment.
#
# See <https://github.com/measurement-kit/measurement-kit/issues/1931>.
if "miniooni" in ooni_exe:
assert tk["dns_experiment_failure"] == "dns_nxdomain_error"
else:
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == None
else:
assert tk["http_experiment_failure"] == "dns_lookup_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 2052)
def webconnectivity_tcpip_blocking_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's TCP/IP blocking w/ consistent DNS """
ip = socket.gethostbyname("nexa.polito.it")
args = [
"-iptables-drop-ip",
ip,
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://nexa.polito.it web_connectivity",
"webconnectivity_tcpip_blocking_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "generic_timeout_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "tcp_ip"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 4224)
def webconnectivity_tcpip_blocking_with_inconsistent_dns(ooni_exe, outfile):
""" Test case where there's TCP/IP blocking w/ inconsistent DNS """
def runner(port):
args = [
"-dns-proxy-hijack",
"nexa.polito.it",
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-iptables-hijack-http-to",
"127.0.0.1:{}".format(port),
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://nexa.polito.it web_connectivity",
"webconnectivity_tcpip_blocking_with_inconsistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "connection_refused"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 4256)
common.with_free_port(runner)
def webconnectivity_http_connection_refused_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's TCP/IP blocking w/ consistent DNS that occurs
while we're following the chain of redirects. """
# We use a bit.ly link redirecting to nexa.polito.it. We block the IP address
# used by nexa.polito.it. So the error should happen in the redirect chain.
ip = socket.gethostbyname("nexa.polito.it")
args = [
"-iptables-reset-ip",
ip,
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://bit.ly/3h9EJR3 web_connectivity",
"webconnectivity_http_connection_refused_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "connection_refused"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "http-failure"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8320)
def webconnectivity_http_connection_reset_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's RST-based blocking blocking w/ consistent DNS that
occurs while we're following the chain of redirects. """
# We use a bit.ly link redirecting to nexa.polito.it. We block the Host header
# used for nexa.polito.it. So the error should happen in the redirect chain.
args = [
"-iptables-reset-keyword",
"Host: nexa",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://bit.ly/3h9EJR3 web_connectivity",
"webconnectivity_http_connection_reset_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "connection_reset"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "http-failure"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8448)
def webconnectivity_http_nxdomain_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's a redirection and the redirected request cannot
continue because a NXDOMAIN error occurs. """
# We use a bit.ly link redirecting to nexa.polito.it. We block the DNS request
# for nexa.polito.it. So the error should happen in the redirect chain.
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-block",
"nexa.polito.it",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://bit.ly/3h9EJR3 web_connectivity",
"webconnectivity_http_nxdomain_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert (
tk["http_experiment_failure"] == "dns_nxdomain_error" # miniooni
or tk["http_experiment_failure"] == "dns_lookup_error" # MK
)
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8224)
def webconnectivity_http_eof_error_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's a redirection and the redirected request cannot
continue because an eof_error error occurs. """
# We use a bit.ly link redirecting to nexa.polito.it. We block the HTTP request
# for nexa.polito.it using the cleartext bad proxy. So the error should happen in
# the redirect chain and should be EOF.
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-hijack",
"nexa.polito.it",
"-iptables-hijack-http-to",
"127.0.0.1:7117", # this is badproxy's cleartext endpoint
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://bit.ly/3h9EJR3 web_connectivity", # bit.ly uses https
"webconnectivity_http_eof_error_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "eof_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "http-failure"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8448)
def webconnectivity_http_generic_timeout_error_with_consistent_dns(ooni_exe, outfile):
""" Test case where there's a redirection and the redirected request cannot
continue because a generic_timeout_error error occurs. """
# We use a bit.ly link redirecting to nexa.polito.it. We block the HTTP request
# for nexa.polito.it by dropping packets using DPI. So the error should happen in
# the redirect chain and should be timeout.
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-hijack",
"nexa.polito.it",
"-iptables-drop-keyword",
"Host: nexa",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://bit.ly/3h9EJR3 web_connectivity",
"webconnectivity_http_generic_timeout_error_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "generic_timeout_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "http-failure"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8704)
def webconnectivity_http_connection_reset_with_inconsistent_dns(ooni_exe, outfile):
""" Test case where there's inconsistent DNS and the connection is RST when
we're executing HTTP code. """
args = [
"-iptables-reset-keyword",
"nexa.polito.it",
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-hijack",
"polito",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://nexa.polito.it/ web_connectivity",
"webconnectivity_http_connection_reset_with_inconsistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == "connection_reset"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 8480)
def webconnectivity_http_successful_website(ooni_exe, outfile):
""" Test case where we succeed with an HTTP only webpage """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://example.org/ web_connectivity",
"webconnectivity_http_successful_website",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == True
assert tk["body_proportion"] == 1
assert tk["status_code_match"] == True
assert tk["headers_match"] == True
assert tk["title_match"] == True
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 2)
def webconnectivity_https_successful_website(ooni_exe, outfile):
""" Test case where we succeed with an HTTPS only webpage """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.com/ web_connectivity",
"webconnectivity_https_successful_website",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == True
assert tk["body_proportion"] == 1
assert tk["status_code_match"] == True
assert tk["headers_match"] == True
assert tk["title_match"] == True
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 1)
def webconnectivity_http_diff_with_inconsistent_dns(ooni_exe, outfile):
""" Test case where we get an http-diff and the DNS is inconsistent """
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-hijack",
"example.org",
"-http-proxy-block",
"example.org",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://example.org/ web_connectivity",
"webconnectivity_http_diff_with_inconsistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == False
assert tk["body_proportion"] < 1
assert tk["status_code_match"] == False
assert tk["headers_match"] == False
assert tk["title_match"] == False
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 96)
def webconnectivity_http_diff_with_consistent_dns(ooni_exe, outfile):
""" Test case where we get an http-diff and the DNS is consistent """
args = [
"-iptables-hijack-http-to",
"127.0.0.1:80",
"-http-proxy-block",
"example.org",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i http://example.org/ web_connectivity",
"webconnectivity_http_diff_with_consistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
assert tk["http_experiment_failure"] == None
assert tk["body_length_match"] == False
assert tk["body_proportion"] < 1
assert tk["status_code_match"] == False
assert tk["headers_match"] == False
assert tk["title_match"] == False
assert tk["blocking"] == "http-diff"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 64)
def webconnectivity_https_expired_certificate(ooni_exe, outfile):
""" Test case where the domain's certificate is expired """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://expired.badssl.com/ web_connectivity",
"webconnectivity_https_expired_certificate",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == "ssl_invalid_certificate"
else:
assert "certificate verify failed" in tk["http_experiment_failure"]
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
# The following strikes me as a measurement_kit bug. We are saying
# that all is good with a domain where actually we don't know why the
# control is failed and that is clearly not accessible according to
# our measurement of the domain (certificate expired).
#
# See <https://github.com/ooni/probe-engine/issues/858>.
if "miniooni" in ooni_exe:
assert tk["blocking"] == None
assert tk["accessible"] == None
else:
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 16)
def webconnectivity_https_wrong_host(ooni_exe, outfile):
""" Test case where the hostname is wrong for the certificate """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://wrong.host.badssl.com/ web_connectivity",
"webconnectivity_https_wrong_host",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == "ssl_invalid_hostname"
else:
assert "certificate verify failed" in tk["http_experiment_failure"]
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
# The following strikes me as a measurement_kit bug. We are saying
# that all is good with a domain where actually we don't know why the
# control is failed and that is clearly not accessible according to
# our measurement of the domain (wrong host for certificate).
#
# See <https://github.com/ooni/probe-engine/issues/858>.
if "miniooni" in ooni_exe:
assert tk["blocking"] == None
assert tk["accessible"] == None
else:
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 16)
def webconnectivity_https_self_signed(ooni_exe, outfile):
""" Test case where the certificate is self signed """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://self-signed.badssl.com/ web_connectivity",
"webconnectivity_https_self_signed",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == "ssl_unknown_authority"
else:
assert "certificate verify failed" in tk["http_experiment_failure"]
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
# The following strikes me as a measurement_kit bug. We are saying
# that all is good with a domain where actually we don't know why the
# control is failed and that is clearly not accessible according to
# our measurement of the domain (self signed certificate).
#
# See <https://github.com/ooni/probe-engine/issues/858>.
if "miniooni" in ooni_exe:
assert tk["blocking"] == None
assert tk["accessible"] == None
else:
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 16)
def webconnectivity_https_untrusted_root(ooni_exe, outfile):
""" Test case where the certificate has an untrusted root """
args = []
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://untrusted-root.badssl.com/ web_connectivity",
"webconnectivity_https_untrusted_root",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "consistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == "ssl_unknown_authority"
else:
assert "certificate verify failed" in tk["http_experiment_failure"]
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
# The following strikes me as a measurement_kit bug. We are saying
# that all is good with a domain where actually we don't know why the
# control is failed and that is clearly not accessible according to
# our measurement of the domain (untrusted root certificate).
#
# See <https://github.com/ooni/probe-engine/issues/858>.
if "miniooni" in ooni_exe:
assert tk["blocking"] == None
assert tk["accessible"] == None
else:
assert tk["blocking"] == False
assert tk["accessible"] == True
assert_status_flags_are(ooni_exe, tk, 16)
def webconnectivity_dns_blocking_nxdomain(ooni_exe, outfile):
""" Test case where there is blocking using NXDOMAIN """
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-block",
"example.com",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.com/ web_connectivity",
"webconnectivity_dns_blocking_nxdomain",
args,
)
# The following seems a bug in MK where we don't properly record the
# actual error that occurred when performing the DNS experiment.
#
# See <https://github.com/measurement-kit/measurement-kit/issues/1931>.
if "miniooni" in ooni_exe:
assert tk["dns_experiment_failure"] == "dns_nxdomain_error"
else:
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == None
else:
assert tk["http_experiment_failure"] == "dns_lookup_error"
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 2080)
def webconnectivity_https_unknown_authority_with_inconsistent_dns(ooni_exe, outfile):
""" Test case where the DNS is sending us towards a website where
we're served an invalid certificate """
args = [
"-iptables-hijack-dns-to",
"127.0.0.1:53",
"-dns-proxy-hijack",
"example.org",
"-bad-proxy-address-tls",
"127.0.0.1:443",
"-tls-proxy-address",
"127.0.0.1:4114",
]
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe,
outfile,
"-i https://example.org/ web_connectivity",
"webconnectivity_https_unknown_authority_with_inconsistent_dns",
args,
)
assert tk["dns_experiment_failure"] == None
assert tk["dns_consistency"] == "inconsistent"
assert tk["control_failure"] == None
if "miniooni" in ooni_exe:
assert tk["http_experiment_failure"] == "ssl_unknown_authority"
else:
assert "certificate verify failed" in tk["http_experiment_failure"]
assert tk["body_length_match"] == None
assert tk["body_proportion"] == 0
assert tk["status_code_match"] == None
assert tk["headers_match"] == None
assert tk["title_match"] == None
assert tk["blocking"] == "dns"
assert tk["accessible"] == False
assert_status_flags_are(ooni_exe, tk, 9248)
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "webconnectivity.jsonl"
ooni_exe = sys.argv[1]
tests = [
webconnectivity_https_ok_with_control_failure,
webconnectivity_http_ok_with_control_failure,
webconnectivity_transparent_http_proxy,
webconnectivity_dns_hijacking,
webconnectivity_control_unreachable_and_using_http,
webconnectivity_nonexistent_domain,
webconnectivity_tcpip_blocking_with_consistent_dns,
webconnectivity_tcpip_blocking_with_inconsistent_dns,
webconnectivity_http_connection_refused_with_consistent_dns,
webconnectivity_http_connection_reset_with_consistent_dns,
webconnectivity_http_nxdomain_with_consistent_dns,
webconnectivity_http_eof_error_with_consistent_dns,
webconnectivity_http_generic_timeout_error_with_consistent_dns,
webconnectivity_http_connection_reset_with_inconsistent_dns,
webconnectivity_http_successful_website,
webconnectivity_https_successful_website,
webconnectivity_http_diff_with_inconsistent_dns,
webconnectivity_http_diff_with_consistent_dns,
webconnectivity_https_expired_certificate,
webconnectivity_https_wrong_host,
webconnectivity_https_self_signed,
webconnectivity_https_untrusted_root,
webconnectivity_dns_blocking_nxdomain,
webconnectivity_https_unknown_authority_with_inconsistent_dns,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

235
internal/engine/QA/whatsapp.py Executable file
View File

@ -0,0 +1,235 @@
#!/usr/bin/env python3
""" ./QA/whatsapp.py - main QA script for whatsapp
This script performs a bunch of whatsapp tests under censored
network conditions and verifies that the measurement is consistent
with the expectations, by parsing the resulting JSONL. """
import contextlib
import json
import os
import shlex
import socket
import subprocess
import sys
import time
import urllib.parse
sys.path.insert(0, ".")
import common
def execute_jafar_and_return_validated_test_keys(ooni_exe, outfile, tag, args):
""" Executes jafar and returns the validated parsed test keys, or throws
an AssertionError if the result is not valid. """
tk = common.execute_jafar_and_miniooni(ooni_exe, outfile, "whatsapp", tag, args)
assert isinstance(tk["requests"], list)
assert len(tk["requests"]) > 0
for entry in tk["requests"]:
assert isinstance(entry, dict)
failure = entry["failure"]
assert isinstance(failure, str) or failure is None
assert isinstance(entry["request"], dict)
req = entry["request"]
common.check_maybe_binary_value(req["body"])
assert isinstance(req["headers"], dict)
for key, value in req["headers"].items():
assert isinstance(key, str)
common.check_maybe_binary_value(value)
assert isinstance(req["method"], str)
assert isinstance(entry["response"], dict)
resp = entry["response"]
common.check_maybe_binary_value(resp["body"])
assert isinstance(resp["code"], int)
if resp["headers"] is not None:
for key, value in resp["headers"].items():
assert isinstance(key, str)
common.check_maybe_binary_value(value)
assert isinstance(tk["tcp_connect"], list)
assert len(tk["tcp_connect"]) > 0
for entry in tk["tcp_connect"]:
assert isinstance(entry, dict)
assert isinstance(entry["ip"], str)
assert isinstance(entry["port"], int)
assert isinstance(entry["status"], dict)
failure = entry["status"]["failure"]
success = entry["status"]["success"]
assert isinstance(failure, str) or failure is None
assert isinstance(success, bool)
return tk
def helper_for_blocking_endpoints(start, stop):
""" Helper function for generating args for blocking endpoints """
args = []
for num in range(start, stop):
args.append("-iptables-reset-ip")
args.append("e{}.whatsapp.net".format(num))
return args
def args_for_blocking_all_endpoints():
""" Returns the arguments useful for blocking all endpoints """
return helper_for_blocking_endpoints(1, 17)
def args_for_blocking_some_endpoints():
""" Returns the arguments useful for blocking some endpoints """
# Implementation note: apparently all the endpoints are now using just
# four IP addresses, hence here we block some endpoints via DNS.
#
# TODO(bassosimone): this fact calls for creating an issue for making
# the whatsapp experiment implementation more efficient.
args = []
args.append("-iptables-hijack-dns-to")
args.append("127.0.0.1:53")
for n in range(1, 7):
args.append("-dns-proxy-block")
args.append("e{}.whatsapp.net".format(n))
return args
def args_for_blocking_v_whatsapp_net_https():
""" Returns arguments for blocking v.whatsapp.net over https """
#
# 00 00 <SNI extension ID>
# 00 13 <full extension length>
# 00 11 <first entry length>
# 00 <DNS hostname type>
# 00 0e <string length>
# 76 2e ... 74 v.whatsapp.net
#
return [
"-iptables-reset-keyword-hex",
"|00 00 00 13 00 11 00 00 0e 76 2e 77 68 61 74 73 61 70 70 2e 6e 65 74|",
]
def args_for_blocking_web_whatsapp_com_http():
""" Returns arguments for blocking web.whatsapp.com over http """
return ["-iptables-reset-keyword", "Host: web.whatsapp.com"]
def args_for_blocking_web_whatsapp_com_https():
""" Returns arguments for blocking web.whatsapp.com over https """
#
# 00 00 <SNI extension ID>
# 00 15 <full extension length>
# 00 13 <first entry length>
# 00 <DNS hostname type>
# 00 10 <string length>
# 77 65 ... 6d web.whatsapp.com
#
return [
"-iptables-reset-keyword-hex",
"|00 00 00 15 00 13 00 00 10 77 65 62 2e 77 68 61 74 73 61 70 70 2e 63 6f 6d|",
]
def whatsapp_block_everything(ooni_exe, outfile):
""" Test case where everything we measure is blocked """
args = []
args.extend(args_for_blocking_all_endpoints())
args.extend(args_for_blocking_v_whatsapp_net_https())
args.extend(args_for_blocking_web_whatsapp_com_https())
args.extend(args_for_blocking_web_whatsapp_com_http())
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_everything", args,
)
assert tk["registration_server_failure"] == "connection_reset"
assert tk["registration_server_status"] == "blocked"
assert tk["whatsapp_endpoints_status"] == "blocked"
assert tk["whatsapp_web_failure"] == "connection_reset"
assert tk["whatsapp_web_status"] == "blocked"
def whatsapp_block_all_endpoints(ooni_exe, outfile):
""" Test case where we only block whatsapp endpoints """
args = args_for_blocking_all_endpoints()
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_all_endpoints", args
)
assert tk["registration_server_failure"] == None
assert tk["registration_server_status"] == "ok"
assert tk["whatsapp_endpoints_status"] == "blocked"
assert tk["whatsapp_web_failure"] == None
assert tk["whatsapp_web_status"] == "ok"
def whatsapp_block_some_endpoints(ooni_exe, outfile):
""" Test case where we block some whatsapp endpoints """
args = args_for_blocking_some_endpoints()
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_some_endpoints", args
)
assert tk["registration_server_failure"] == None
assert tk["registration_server_status"] == "ok"
assert tk["whatsapp_endpoints_status"] == "ok"
assert tk["whatsapp_web_failure"] == None
assert tk["whatsapp_web_status"] == "ok"
def whatsapp_block_registration_server(ooni_exe, outfile):
""" Test case where we block the registration server """
args = []
args.extend(args_for_blocking_v_whatsapp_net_https())
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_registration_server", args,
)
assert tk["registration_server_failure"] == "connection_reset"
assert tk["registration_server_status"] == "blocked"
assert tk["whatsapp_endpoints_status"] == "ok"
assert tk["whatsapp_web_failure"] == None
assert tk["whatsapp_web_status"] == "ok"
def whatsapp_block_web_http(ooni_exe, outfile):
""" Test case where we block the HTTP web chat """
args = []
args.extend(args_for_blocking_web_whatsapp_com_http())
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_web_http", args,
)
assert tk["registration_server_failure"] == None
assert tk["registration_server_status"] == "ok"
assert tk["whatsapp_endpoints_status"] == "ok"
assert tk["whatsapp_web_failure"] == "connection_reset"
assert tk["whatsapp_web_status"] == "blocked"
def whatsapp_block_web_https(ooni_exe, outfile):
""" Test case where we block the HTTPS web chat """
args = []
args.extend(args_for_blocking_web_whatsapp_com_https())
tk = execute_jafar_and_return_validated_test_keys(
ooni_exe, outfile, "whatsapp_block_web_https", args,
)
assert tk["registration_server_failure"] == None
assert tk["registration_server_status"] == "ok"
assert tk["whatsapp_endpoints_status"] == "ok"
assert tk["whatsapp_web_failure"] == "connection_reset"
assert tk["whatsapp_web_status"] == "blocked"
def main():
if len(sys.argv) != 2:
sys.exit("usage: %s /path/to/ooniprobelegacy-like/binary" % sys.argv[0])
outfile = "whatsapp.jsonl"
ooni_exe = sys.argv[1]
tests = [
whatsapp_block_everything,
whatsapp_block_all_endpoints,
whatsapp_block_some_endpoints,
whatsapp_block_registration_server,
whatsapp_block_web_http,
whatsapp_block_web_https,
]
for test in tests:
test(ooni_exe, outfile)
time.sleep(7)
if __name__ == "__main__":
main()

83
internal/engine/README.md Normal file
View File

@ -0,0 +1,83 @@
# OONI probe measurement engine
[![GoDoc](https://godoc.org/github.com/ooni/probe-engine?status.svg)](https://godoc.org/github.com/ooni/probe-engine) [![Short Tests Status](https://github.com/ooni/probe-engine/workflows/shorttests/badge.svg)](https://github.com/ooni/probe-engine/actions?query=workflow%3Ashorttests) [![All Tests Status](https://github.com/ooni/probe-engine/workflows/alltests/badge.svg)](https://github.com/ooni/probe-engine/actions?query=workflow%3Aalltests) [![Coverage Status](https://coveralls.io/repos/github/ooni/probe-engine/badge.svg?branch=master)](https://coveralls.io/github/ooni/probe-engine?branch=master) [![Go Report Card](https://goreportcard.com/badge/github.com/ooni/probe-engine)](https://goreportcard.com/report/github.com/ooni/probe-engine)
This repository contains OONI probe's [measurement engine](
https://github.com/ooni/spec/tree/master/probe#engine). That is, the
piece of software that implements OONI nettests as well as all the
required functionality to run such nettests.
We expect you to use the Go version indicated in [go.mod](go.mod).
## Integrating ooni/probe-engine
We recommend pinning to a specific version of probe-engine:
```bash
go get -v github.com/ooni/probe-engine@VERSION
```
See also the [workflows/using.yml](.github/workflows/using.yml) test
where we check that the latest commit can be imported by a third party.
We do not provide any API stability guarantee.
## Building miniooni
[miniooni](cmd/miniooni) is a small command line client used for
research and quality assurance testing. Build using:
```bash
go build -v ./cmd/miniooni/
```
See also `./build-cli.bash` for more advanced builds (e.g. to create
statically linked and/or stripped binaries).
We don't provide any `miniooni` command line flags stability guarantee.
See
```bash
./miniooni --help
```
for more help.
## Building Android bindings
```bash
./build-android.bash
```
We automatically build Android bindings whenever commits are pushed to the
`mobile-staging` branch. Such builds could be integrated by using:
```Groovy
implementation "org.ooni:oonimkall:VERSION"
```
Where VERSION is like `2020.03.30-231914` corresponding to the
time when the build occurred.
## Building iOS bindings
```bash
./build-ios.bash
```
We automatically build iOS bindings whenever commits are pushed to the
`mobile-staging` branch. Such builds could be integrated by using:
```ruby
pod 'oonimkall', :podspec => 'https://dl.bintray.com/ooni/ios/oonimkall-VERSION.podspec'
```
Where VERSION is like `2020.03.30-231914` corresponding to the
time when the build occurred.
## Updating dependencies
```
go get -u -v ./... && go mod tidy
```

323
internal/engine/allexperiments.go Normal file
View File

@ -0,0 +1,323 @@
package engine
import (
"time"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/dash"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/dnscheck"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/example"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/fbmessenger"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/hhfm"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/hirl"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/httphostheader"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/ndt7"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/psiphon"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/riseupvpn"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/run"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/sniblocking"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/stunreachability"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/telegram"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tlstool"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/tor"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/whatsapp"
)
var experimentsByName = map[string]func(*Session) *ExperimentBuilder{
"dash": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, dash.NewExperimentMeasurer(
*config.(*dash.Config),
))
},
config: &dash.Config{},
interruptible: true,
inputPolicy: InputNone,
}
},
"dnscheck": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, dnscheck.NewExperimentMeasurer(
*config.(*dnscheck.Config),
))
},
config: &dnscheck.Config{},
inputPolicy: InputStrictlyRequired,
}
},
"example": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, example.NewExperimentMeasurer(
*config.(*example.Config), "example",
))
},
config: &example.Config{
Message: "Good day from the example experiment!",
SleepTime: int64(time.Second),
},
interruptible: true,
inputPolicy: InputNone,
}
},
"example_with_input": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, example.NewExperimentMeasurer(
*config.(*example.Config), "example_with_input",
))
},
config: &example.Config{
Message: "Good day from the example with input experiment!",
SleepTime: int64(time.Second),
},
interruptible: true,
inputPolicy: InputStrictlyRequired,
}
},
// TODO(bassosimone): when we can set experiment options using the JSON
// we need to get rid of all these multiple experiments.
//
// See https://github.com/ooni/probe-cli/v3/internal/engine/issues/413
"example_with_input_non_interruptible": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, example.NewExperimentMeasurer(
*config.(*example.Config), "example_with_input_non_interruptible",
))
},
config: &example.Config{
Message: "Good day from the example with input experiment!",
SleepTime: int64(time.Second),
},
interruptible: false,
inputPolicy: InputStrictlyRequired,
}
},
"example_with_failure": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, example.NewExperimentMeasurer(
*config.(*example.Config), "example_with_failure",
))
},
config: &example.Config{
Message: "Good day from the example with failure experiment!",
ReturnError: true,
SleepTime: int64(time.Second),
},
interruptible: true,
inputPolicy: InputNone,
}
},
"facebook_messenger": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, fbmessenger.NewExperimentMeasurer(
*config.(*fbmessenger.Config),
))
},
config: &fbmessenger.Config{},
inputPolicy: InputNone,
}
},
"http_header_field_manipulation": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, hhfm.NewExperimentMeasurer(
*config.(*hhfm.Config),
))
},
config: &hhfm.Config{},
inputPolicy: InputNone,
}
},
"http_host_header": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, httphostheader.NewExperimentMeasurer(
*config.(*httphostheader.Config),
))
},
config: &httphostheader.Config{},
inputPolicy: InputOrQueryTestLists,
}
},
"http_invalid_request_line": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, hirl.NewExperimentMeasurer(
*config.(*hirl.Config),
))
},
config: &hirl.Config{},
inputPolicy: InputNone,
}
},
"ndt": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, ndt7.NewExperimentMeasurer(
*config.(*ndt7.Config),
))
},
config: &ndt7.Config{},
interruptible: true,
inputPolicy: InputNone,
}
},
"psiphon": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, psiphon.NewExperimentMeasurer(
*config.(*psiphon.Config),
))
},
config: &psiphon.Config{},
inputPolicy: InputOptional,
}
},
"riseupvpn": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, riseupvpn.NewExperimentMeasurer(
*config.(*riseupvpn.Config),
))
},
config: &riseupvpn.Config{},
inputPolicy: InputNone,
}
},
"run": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, run.NewExperimentMeasurer(
*config.(*run.Config),
))
},
config: &run.Config{},
inputPolicy: InputStrictlyRequired,
}
},
"sni_blocking": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, sniblocking.NewExperimentMeasurer(
*config.(*sniblocking.Config),
))
},
config: &sniblocking.Config{},
inputPolicy: InputOrQueryTestLists,
}
},
"stun_reachability": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, stunreachability.NewExperimentMeasurer(
*config.(*stunreachability.Config),
))
},
config: &stunreachability.Config{},
inputPolicy: InputOptional,
}
},
"telegram": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, telegram.NewExperimentMeasurer(
*config.(*telegram.Config),
))
},
config: &telegram.Config{},
inputPolicy: InputNone,
}
},
"tlstool": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, tlstool.NewExperimentMeasurer(
*config.(*tlstool.Config),
))
},
config: &tlstool.Config{},
inputPolicy: InputOrQueryTestLists,
}
},
"tor": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, tor.NewExperimentMeasurer(
*config.(*tor.Config),
))
},
config: &tor.Config{},
inputPolicy: InputNone,
}
},
"urlgetter": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, urlgetter.NewExperimentMeasurer(
*config.(*urlgetter.Config),
))
},
config: &urlgetter.Config{},
inputPolicy: InputStrictlyRequired,
}
},
"web_connectivity": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, webconnectivity.NewExperimentMeasurer(
*config.(*webconnectivity.Config),
))
},
config: &webconnectivity.Config{},
inputPolicy: InputOrQueryTestLists,
}
},
"whatsapp": func(session *Session) *ExperimentBuilder {
return &ExperimentBuilder{
build: func(config interface{}) *Experiment {
return NewExperiment(session, whatsapp.NewExperimentMeasurer(
*config.(*whatsapp.Config),
))
},
config: &whatsapp.Config{},
inputPolicy: InputNone,
}
},
}
// AllExperiments returns the name of all experiments
func AllExperiments() []string {
var names []string
for key := range experimentsByName {
names = append(names, key)
}
return names
}

3
internal/engine/atomicx/README.md Normal file
View File

@ -0,0 +1,3 @@
# Package github.com/ooni/probe-engine/atomicx
Atomic int64/float64 that works also on 32 bit platforms.

68
internal/engine/atomicx/atomicx.go Normal file
View File

@ -0,0 +1,68 @@
// Package atomicx contains atomic int64/float64 that work also on 32 bit
// platforms. The main reason for rolling out this package is to avoid potential
// crashes when using 32 bit devices where we are atomically accessing a 64 bit
// variable that is not aligned. The solution to this issue is rather crude: use
// a normal variable and protect it using a normal mutex. While this could be
// disappointing in general, it seems fine to be done in our context where
// we mainly use atomic semantics for counting.
package atomicx
import (
"sync"
)
// Int64 is an int64 with atomic semantics.
type Int64 struct {
mu sync.Mutex
v int64
}
// NewInt64 creates a new int64 with atomic semantics.
func NewInt64() *Int64 {
return new(Int64)
}
// Add behaves like atomic.AddInt64
func (i64 *Int64) Add(delta int64) (newvalue int64) {
i64.mu.Lock()
i64.v += delta
newvalue = i64.v
i64.mu.Unlock()
return
}
// Load behaves like atomic.LoadInt64
func (i64 *Int64) Load() (v int64) {
i64.mu.Lock()
v = i64.v
i64.mu.Unlock()
return
}
// Float64 is an float64 with atomic semantics.
type Float64 struct {
mu sync.Mutex
v float64
}
// NewFloat64 creates a new float64 with atomic semantics.
func NewFloat64() *Float64 {
return new(Float64)
}
// Add behaves like AtomicInt64.Add but for float64
func (f64 *Float64) Add(delta float64) (newvalue float64) {
f64.mu.Lock()
f64.v += delta
newvalue = f64.v
f64.mu.Unlock()
return
}
// Load behaves like LoadInt64.Load buf for float64
func (f64 *Float64) Load() (v float64) {
f64.mu.Lock()
v = f64.v
f64.mu.Unlock()
return
}

50
internal/engine/atomicx/atomicx_test.go Normal file
View File

@ -0,0 +1,50 @@
package atomicx_test
import (
"testing"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
)
func TestInt64(t *testing.T) {
// TODO(bassosimone): how to write tests with race conditions
// and be confident that they're WAI? Here I hope this test is
// run with `-race` and I'm doing something that AFAICT will
// be flagged as race if we were not be using mutexes.
v := atomicx.NewInt64()
go func() {
v.Add(17)
}()
go func() {
v.Add(14)
}()
time.Sleep(1 * time.Second)
if v.Add(3) != 34 {
t.Fatal("unexpected result")
}
if v.Load() != 34 {
t.Fatal("unexpected result")
}
}
func TestFloat64(t *testing.T) {
// TODO(bassosimone): how to write tests with race conditions
// and be confident that they're WAI? Here I hope this test is
// run with `-race` and I'm doing something that AFAICT will
// be flagged as race if we were not be using mutexes.
v := atomicx.NewFloat64()
go func() {
v.Add(17.0)
}()
go func() {
v.Add(14.0)
}()
time.Sleep(1 * time.Second)
if r := v.Add(3); r < 33.9 && r > 34.1 {
t.Fatal("unexpected result")
}
if v.Load() < 33.9 && v.Load() > 34.1 {
t.Fatal("unexpected result")
}
}

55
internal/engine/build-android.bash Executable file
View File

@ -0,0 +1,55 @@
#!/bin/bash
set -e
if [ -z "$ANDROID_HOME" -o "$1" = "--help" ]; then
echo ""
echo "usage: $0"
echo ""
echo "Please set ANDROID_HOME. We assume you have installed"
echo "the Android SDK. You can do that on macOS using:"
echo ""
echo " brew install --cask android-sdk"
echo ""
echo "Then make sure you install the required packages:"
echo ""
echo "sdkmanager --install 'build-tools;29.0.3' 'ndk;21.3.6528147'"
echo ""
echo "or, if you already installed, that you're up to date:"
echo ""
echo "sdkmanager --update"
echo ""
echo "Once you have done that, please export ANDROID_HOME to"
echo "point to /usr/local/Caskroom/android-sdk/<version>."
echo ""
exit 1
fi
if [ -d $ANDROID_HOME/ndk-bundle ]; then
echo ""
echo "FATAL: currently we need 'ndk;21.3.6528147' instead of ndk-bundle"
echo ""
echo "See https://github.com/ooni/probe-engine/issues/1179."
echo ""
echo "To fix: sdkmanager --uninstall ndk-bundle"
echo ""
exit 1
fi
export ANDROID_NDK_HOME=$ANDROID_HOME/ndk/21.3.6528147
if [ ! -d $ANDROID_NDK_HOME ]; then
echo ""
echo "FATAL: currently we need 'ndk;21.3.6528147'"
echo ""
echo "See https://github.com/ooni/probe-engine/issues/1179."
echo ""
echo "To fix: sdkmanager --install 'ndk;21.3.6528147'"
echo ""
exit 1
fi
topdir=$(cd $(dirname $0) && pwd -P)
set -x
export PATH=$(go env GOPATH)/bin:$PATH
export GO111MODULE=off
go get -u golang.org/x/mobile/cmd/gomobile
gomobile init
export GO111MODULE=on
output=MOBILE/android/oonimkall.aar
gomobile bind -target=android -o $output -ldflags="-s -w" ./oonimkall

31
internal/engine/build-cli.sh Executable file
View File

@ -0,0 +1,31 @@
#!/bin/sh
set -e
case $1 in
macos|darwin)
export GOOS=darwin GOARCH=amd64
go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./cmd/miniooni
echo "Binary ready at ./CLI/darwin/amd64/miniooni";;
linux)
export GOOS=linux GOARCH=386
go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni
echo "Binary ready at ./CLI/linux/386/miniooni"
export GOOS=linux GOARCH=amd64
go build -o ./CLI/linux/amd64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni
echo "Binary ready at ./CLI/linux/amd64/miniooni"
export GOOS=linux GOARCH=arm GOARM=7
go build -o ./CLI/linux/armv7 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni
echo "Binary ready at ./CLI/linux/armv7/miniooni"
export GOOS=linux GOARCH=arm64
go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./cmd/miniooni
echo "Binary ready at ./CLI/linux/arm64/miniooni";;
windows)
export GOOS=windows GOARCH=386
go build -o ./CLI/windows/386 -ldflags="-s -w" ./cmd/miniooni
echo "Binary ready at ./CLI/windows/386/miniooni.exe"
export GOOS=windows GOARCH=amd64
go build -o ./CLI/windows/amd64 -ldflags="-s -w" ./cmd/miniooni
echo "Binary ready at ./CLI/windows/amd64/miniooni.exe";;
*)
echo "usage: $0 darwin|linux|windows" 1>&2
exit 1
esac

11
internal/engine/build-ios.bash Executable file
View File

@ -0,0 +1,11 @@
#!/bin/bash
set -e
topdir=$(cd $(dirname $0) && pwd -P)
set -x
export PATH=$(go env GOPATH)/bin:$PATH
export GO111MODULE=off
go get -u golang.org/x/mobile/cmd/gomobile
gomobile init
export GO111MODULE=on
output=MOBILE/ios/oonimkall.framework
gomobile bind -target=ios -o $output -ldflags="-s -w" ./oonimkall

3
internal/engine/cmd/README.md Normal file
View File

@ -0,0 +1,3 @@
# Directory github.com/ooni/probe-engine/cmd
This directory contains the source code for the CLI tools we build.

8
internal/engine/cmd/apitool/README.md Normal file
View File

@ -0,0 +1,8 @@
# apitool
This directory contains a tool to fetch measurements. This tool is
intended to sporadically fetch measurements, not for batch downloading.
Please, see https://ooni.org/data for information pertaining how to
access OONI data in bulk. Please see https://explorer.ooni.org if your
intent is to navigate and explore OONI data.

115
internal/engine/cmd/apitool/main.go Normal file
View File

@ -0,0 +1,115 @@
// Command apitool is a simple tool to fetch individual OONI measurements.
//
// This tool IS NOT intended for batch downloading.
//
// Please, see https://ooni.org/data for information pertaining how to
// access OONI data in bulk. Please see https://explorer.ooni.org if your
// intent is to navigate and explore OONI data
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"net/http"
"os"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/kvstore"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/probeservices"
"github.com/ooni/probe-cli/v3/internal/engine/version"
)
func newclient() probeservices.Client {
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
ua := fmt.Sprintf("apitool/%s ooniprobe-engine/%s", version.Version, version.Version)
return probeservices.Client{
Client: httpx.Client{
BaseURL: "https://ams-pg.ooni.org/",
HTTPClient: &http.Client{Transport: txp},
Logger: log.Log,
UserAgent: ua,
},
LoginCalls: atomicx.NewInt64(),
RegisterCalls: atomicx.NewInt64(),
StateFile: probeservices.NewStateFile(kvstore.NewMemoryKeyValueStore()),
}
}
var osExit = os.Exit
func fatalOnError(err error, message string) {
if err != nil {
log.WithError(err).Error(message)
osExit(1) // overridable from tests
}
}
var (
debug = flag.Bool("v", false, "Enable verbose mode")
input = flag.String("input", "", "Input of the measurement")
mode = flag.String("mode", "", "One of: check, meta, raw")
reportid = flag.String("report-id", "", "Report ID of the measurement")
)
var logmap = map[bool]log.Level{
true: log.DebugLevel,
false: log.InfoLevel,
}
func main() {
flag.Parse()
log.SetLevel(logmap[*debug])
client := newclient()
switch *mode {
case "check":
check(client)
case "meta":
meta(client)
case "raw":
raw(client)
default:
fatalOnError(fmt.Errorf("invalid -mode flag value: %s", *mode), "usage error")
}
}
func check(c probeservices.Client) {
found, err := c.CheckReportID(context.Background(), *reportid)
fatalOnError(err, "c.CheckReportID failed")
fmt.Printf("%+v\n", found)
}
func meta(c probeservices.Client) {
pprint(mmeta(c, false))
}
func raw(c probeservices.Client) {
m := mmeta(c, true)
rm := []byte(m.RawMeasurement)
var opaque interface{}
err := json.Unmarshal(rm, &opaque)
fatalOnError(err, "json.Unmarshal failed")
pprint(opaque)
}
func pprint(opaque interface{}) {
data, err := json.MarshalIndent(opaque, "", " ")
fatalOnError(err, "json.MarshalIndent failed")
fmt.Printf("%s\n", data)
}
func mmeta(c probeservices.Client, full bool) *probeservices.MeasurementMeta {
config := probeservices.MeasurementMetaConfig{
ReportID: *reportid,
Full: full,
Input: *input,
}
ctx := context.Background()
m, err := c.GetMeasurementMeta(ctx, config)
fatalOnError(err, "client.GetMeasurementMeta failed")
return m
}

39
internal/engine/cmd/apitool/main_test.go Normal file
View File

@ -0,0 +1,39 @@
package main
import (
"fmt"
"testing"
)
func init() {
*reportid = `20201209T052225Z_urlgetter_IT_30722_n1_E1VUhMz08SEkgYFU`
*input = `https://www.example.org`
}
func TestCheck(t *testing.T) {
*mode = "check"
main()
}
func TestRaw(t *testing.T) {
*mode = "raw"
main()
}
func TestMeta(t *testing.T) {
*mode = "meta"
main()
}
func TestInvalidMode(t *testing.T) {
defer func() {
if recover() == nil {
t.Fatal("the code did not panic")
}
}()
osExit = func(code int) {
panic(fmt.Errorf("%d", code))
}
*mode = "antani"
main()
}

258
internal/engine/cmd/jafar/README.md Normal file
View File

@ -0,0 +1,258 @@
# Jafar
> We stepped up the game of simulating censorship upgrading from the
> evil genius to the evil grand vizier.
Jafar is a censorship simulation tool used for testing OONI. It builds on
any system but it really on works on Linux.
## Building
We use Go >= 1.14. Jafar also needs the C library headers,
iptables installed, and root permissions.
With Linux Alpine edge, you can compile Jafar with:
```bash
apk add go git musl-dev iptables
go build -v .
```
Otherwise, using Docker:
```bash
docker build -t jafar-runner .
docker run -it --privileged -v`pwd`:/jafar -w/jafar jafar-runner
go build -v .
```
## Usage
You need to run Jafar as root. You can get a complete list
of all flags using `./jafar -help`. Jafar is composed of modules. Each
module is controllable via flags. We describe modules below.
### main
The main module starts all the other modules. If you don't provide the
`-main-command <command>` flag, the code will run until interrupted. If
instead you use the `-main-command` flag, you can specify a command to
run inside the censored environment. In such case, the main module
will exit when the specified command terminates. Note that the main
module will propagate the child exit code, if the child fails.
The command can also include arguments. Make sure you quote the arguments
such that your shell passes the whole string to the specified option, as
in `-main-command 'ls -lha'`. This will execute the `ls -lha` command line
inside the censored Jafar context. You can also combine that with quoting
and variables interpolation, e.g., `-main-command "echo '$USER is the
walrus'"`. The `$USER` variable will be expanded by your shell. Assuming
your user name is `paul`, then Jafar will lex the main command as `echo
"paul is the walrus"` and will execute it.
Use the `-main-user <username>` flag to select the user to use for
running child commands. By default, we use the `nobody` user for this
purpose. We implement this feature using `sudo`, therefore you need
to make sure that `sudo` is installed.
### iptables
The iptables module is only available on Linux. It exports these flags:
```bash
-iptables-drop-ip value
Drop traffic to the specified IP address
-iptables-drop-keyword-hex value
Drop traffic containing the specified hex keyword
-iptables-drop-keyword value
Drop traffic containing the specified keyword
-iptables-hijack-dns-to string
Hijack all DNS UDP traffic to the specified endpoint
-iptables-hijack-https-to string
Hijack all HTTPS traffic to the specified endpoint
-iptables-hijack-http-to string
Hijack all HTTP traffic to the specified endpoint
-iptables-reset-ip value
Reset TCP/IP traffic to the specified IP address
-iptables-reset-keyword-hex value
Reset TCP/IP traffic containing the specified hex keyword
-iptables-reset-keyword value
Reset TCP/IP traffic containing the specified keyword
```
The difference between `drop` and `reset` is that in the former case
a packet is dropped, in the latter case a RST is sent.
The difference between `ip` and `keyword` flags is that the former
match an outgoing IP, the latter uses DPI.
The `drop` and `reset` rules allow you to simulate, respectively, when
operations timeout and when a connection cannot be established (with
`reset` and `ip`) or is reset after a keyword is seen (with `keyword`).
Hijacking DNS traffic is useful, for example, to redirect all DNS UDP
traffic from the box to the `dns-proxy` module.
Hijacking HTTP and HTTPS traffic actually hijacks based on ports rather
than on DPI. As a known bug, when hijacking HTTP or HTTPS traffic, we
do not hijack traffic owned by root. This is because Jafar runs as root
and therefore its traffic must not match the hijack rule.
When matching keywords, the simplest option is to use ASCII strings as
in `-iptables-drop-keyword ooni`. However, you can also specify a sequence
of hex bytes, as in `-iptables-drop-keyword-hex |6f 6f 6e 69|`.
Note that with `-iptables-drop-keyword`, DNS queries containing such
keyword will fail returning `EPERM`. For a more realistic approach to
dropping specific DNS packets, combine DNS traffic hijacking with
`-dns-proxy-ignore`, to "drop" packets at the DNS proxy.
### dns-proxy (aka resolver)
The DNS proxy or resolver allows to manipulate DNS. Unless you use DNS
hijacking, you will need to configure your application explicitly to use
the proxy with application specific command line flags.
```bash
-dns-proxy-address string
Address where the DNS proxy should listen (default "127.0.0.1:53")
-dns-proxy-block value
Register keyword triggering NXDOMAIN censorship
-dns-proxy-hijack value
Register keyword triggering redirection to 127.0.0.1
-dns-proxy-ignore value
Register keyword causing the proxy to ignore the query
```
The `-dns-proxy-address` flag controls the endpoint where the proxy is
listening.
The `-dns-proxy-block` tells the resolver that every incoming request whose
query contains the specifed string shall receive an `NXDOMAIN` reply.
The `-dns-proxy-hijack` is similar but instead lies and returns to the
client that the requested domain is at `127.0.0.1`. This is an opportunity
to redirect traffic to the HTTP and TLS proxies.
The `-dns-proxy-ignore` is similar but instead just ignores the query.
### http-proxy
The HTTP proxy is an HTTP proxy that may refuse to forward some
specific requests. It's controlled by these flags:
```bash
-http-proxy-address string
Address where the HTTP proxy should listen (default "127.0.0.1:80")
-http-proxy-block value
Register keyword triggering HTTP 451 censorship
```
The `-http-proxy-address` flag has the same semantics it has for the DNS
proxy.
The `-http-proxy-block` flag tells the proxy that it should return a `451`
response for every request whose `Host` contains the specified string.
### tls-proxy
TLS proxy is a proxy that routes traffic to specific servers depending
on their SNI value. It is controlled by the following flags:
```bash
-tls-proxy-address string
Address where the HTTP proxy should listen (default "127.0.0.1:443")
-tls-proxy-block value
Register keyword triggering TLS censorship
```
The `-tls-proxy-address` flags has the same semantics it has for the DNS
proxy.
The `-tls-proxy-block` specifies which string or strings should cause the
proxy to return an internal-erorr alert when the incoming ClientHello's SNI
contains one of the strings provided with this option.
### bad-proxy
```bash
-bad-proxy-address string
Address where to listen for TCP connections (default "127.0.0.1:7117")
-bad-proxy-address-tls string
Address where to listen for TLS connections (default "127.0.0.1:4114")
-bad-proxy-tls-output-ca string
File where to write the CA used by the bad proxy (default "badproxy.pem")
```
The bad proxy is a proxy that reads some bytes from any incoming connection
and then closes the connection without replying anything. This simulates a
proxy that is not working properly, hence the name of the module.
When connecting using TLS, the above behaviour happens after the handshake.
We write the CA on the file specified using `-bad-proxy-tls-output-ca` such that
tools like curl(1) can use such CA to avoid TLS handshake errors. The code will
generate on the fly a certificate for the provided SNI. Not providing any SNI in
the client Hello message will cause the TLS handshake to fail.
### uncensored
```bash
-uncensored-resolver-url string
URL of an hopefully uncensored resolver (default "dot://1.1.1.1:853")
```
The HTTP, DNS, and TLS proxies need to resolve domain names. If you setup DNS
censorship, they may be affected as well. To avoid this issue, we use a different
resolver for them, which by default is `dot://1.1.1.1:853`. You can change such
default by using the `-uncensored-resolver-url` command line flag. The input
URL is `<transport>://<domain>[:<port>][/<path>]`. Here are some examples:
* `system:///` uses the system resolver (i.e. `getaddrinfo`)
* `udp://8.8.8.8:53` uses DNS over UDP
* `tcp://8.8.8.8:53` used DNS over TCP
* `dot://8.8.8.8:853` uses DNS over TLS
* `https://dns.google/dns-query` uses DNS over HTTPS
So, for example, if you are using Jafar to censor `1.1.1.1:853`, then you
most likely want to use `-uncensored-resolver-url`.
## Examples
Block `play.google.com` with RST injection, force DNS traffic to use the our
DNS proxy, and force it to censor `play.google.com` with `NXDOMAIN`.
```bash
# ./jafar -iptables-reset-keyword play.google.com \
-iptables-hijack-dns-to 127.0.0.1:5353 \
-dns-proxy-address 127.0.0.1:5353 \
-dns-proxy-block play.google.com
```
Force all traffic through the HTTP and TLS proxy and use them to censor
`play.google.com` using HTTP 451 and responding with TLS alerts:
```bash
# ./jafar -iptables-hijack-dns-to 127.0.0.1:5353 \
-dns-proxy-address 127.0.0.1:5353 \
-dns-proxy-hijack play.google.com \
-http-proxy-block play.google.com \
-tls-proxy-block play.google.com
```
Run `ping` in a censored environment:
```bash
# ./jafar -iptables-drop-ip 8.8.8.8 -main-command 'ping -c3 8.8.8.8'
```
Run `curl` in a censored environment where it cannot connect to
`play.google.com` using `https`:
```bash
# ./jafar -iptables-hijack-https-to 127.0.0.1:443 \
-tls-proxy-block play.google.com \
-main-command 'curl -Lv http://play.google.com'
```
For more usage examples, see `../../testjafar.bash`.

113
internal/engine/cmd/jafar/badproxy/badproxy.go Normal file
View File

@ -0,0 +1,113 @@
// Package badproxy implements misbehaving proxies. We have a single
// CensoringProxy that exports two misbehaving endpoints. Each endpoint
// implements a different proxy-censorsing technique. The first one
// reads some bytes from the connection then closes the connection. The
// other instead replies with a self signed x509 certificate.
package badproxy
import (
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"io"
"io/ioutil"
"net"
"strings"
"time"
"github.com/google/martian/v3/mitm"
)
// CensoringProxy is a proxy that does not behave correctly.
type CensoringProxy struct {
mitmNewAuthority func(
name string, organization string,
validity time.Duration,
) (*x509.Certificate, *rsa.PrivateKey, error)
mitmNewConfig func(
ca *x509.Certificate, privateKey interface{},
) (*mitm.Config, error)
tlsListen func(
network string, laddr string, config *tls.Config,
) (net.Listener, error)
}
// NewCensoringProxy creates a new instance of a misbehaving proxy.
func NewCensoringProxy() *CensoringProxy {
return &CensoringProxy{
mitmNewAuthority: mitm.NewAuthority,
mitmNewConfig: mitm.NewConfig,
tlsListen: tls.Listen,
}
}
func (p *CensoringProxy) serve(conn net.Conn) {
deadline := time.Now().Add(250 * time.Millisecond)
conn.SetDeadline(deadline)
// To simulate the case where the proxy isn't willing to forward our
// traffic, we close the connection (1) right after the handshake for
// TLS connections and (2) reasonably after we've received the HTTP
// request for cleartext connections. This may break in several cases
// but is good enough approximation of these bad proxies for now.
if tlsconn, ok := conn.(*tls.Conn); ok {
tlsconn.Handshake()
} else {
const maxread = 1 << 17
reader := io.LimitReader(conn, maxread)
ioutil.ReadAll(reader)
}
conn.Close()
}
func (p *CensoringProxy) run(listener net.Listener) {
for {
conn, err := listener.Accept()
if err != nil && strings.Contains(
err.Error(), "use of closed network connection") {
return
}
if err == nil {
// It's difficult to make accept fail, so restructure
// the code such that we enter into the happy path
go p.serve(conn)
}
}
}
// Start starts the misbehaving proxy for TCP. This endpoint will read some
// bytes from the request and then close the connection. This behaviour is
// implemented by a bunch of censoring proxy around the world. Usually such
// proxies only close the connection with offending SNIs/Host headers.
func (p *CensoringProxy) Start(address string) (net.Listener, error) {
listener, err := net.Listen("tcp", address)
if err != nil {
return nil, err
}
go p.run(listener)
return listener, nil
}
// StartTLS starts the misbehaving proxy for TLS. This endpoint will return
// to the client a self signed certificate. Thus, it models the case where a
// MITM forces users to accept a rogue certificate. After sending such a
// certificate, this proxy will close the TCP connection.
func (p *CensoringProxy) StartTLS(address string) (net.Listener, *x509.Certificate, error) {
cert, privkey, err := p.mitmNewAuthority(
"jafar", "OONI", 24*time.Hour,
)
if err != nil {
return nil, nil, err
}
config, err := p.mitmNewConfig(cert, privkey)
if err != nil {
return nil, nil, err
}
listener, err := p.tlsListen("tcp", address, config.TLS())
if err != nil {
return nil, nil, err
}
go p.run(listener)
return listener, cert, nil
}

157
internal/engine/cmd/jafar/badproxy/badproxy_test.go Normal file
View File

@ -0,0 +1,157 @@
package badproxy
import (
"crypto/rsa"
"crypto/tls"
"crypto/x509"
"errors"
"net"
"testing"
"time"
"github.com/google/martian/v3/mitm"
)
func TestCleartext(t *testing.T) {
listener := newproxy(t)
checkdial(t, listener.Addr().String(), nil, net.Dial)
killproxy(t, listener)
}
func TestTLS(t *testing.T) {
listener := newproxytls(t)
checkdial(t, listener.Addr().String(), nil,
func(network, address string) (net.Conn, error) {
conn, err := tls.Dial(network, address, &tls.Config{
InsecureSkipVerify: true,
ServerName: "antani.local",
})
if err != nil {
return nil, err
}
if err = conn.Handshake(); err != nil {
conn.Close()
return nil, err
}
return conn, nil
})
killproxy(t, listener)
}
func TestListenError(t *testing.T) {
proxy := NewCensoringProxy()
listener, err := proxy.Start("8.8.8.8:80")
if err == nil {
t.Fatal("expected an error here")
}
if listener != nil {
t.Fatal("expected nil listener here")
}
}
func TestStarTLS(t *testing.T) {
expected := errors.New("mocked error")
t.Run("when we cannot create a new authority", func(t *testing.T) {
proxy := NewCensoringProxy()
proxy.mitmNewAuthority = func(
name string, organization string,
validity time.Duration,
) (*x509.Certificate, *rsa.PrivateKey, error) {
return nil, nil, expected
}
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if cert != nil {
t.Fatal("expected nil cert")
}
if privkey != nil {
t.Fatal("expected nil privkey")
}
})
t.Run("when we cannot create a new config", func(t *testing.T) {
proxy := NewCensoringProxy()
proxy.mitmNewConfig = func(
ca *x509.Certificate, privateKey interface{},
) (*mitm.Config, error) {
return nil, expected
}
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if cert != nil {
t.Fatal("expected nil cert")
}
if privkey != nil {
t.Fatal("expected nil privkey")
}
})
t.Run("when we cannot listen", func(t *testing.T) {
proxy := NewCensoringProxy()
proxy.tlsListen = func(
network string, laddr string, config *tls.Config,
) (net.Listener, error) {
return nil, expected
}
cert, privkey, err := proxy.StartTLS("127.0.0.1:0")
if !errors.Is(err, expected) {
t.Fatal("not the error we expected")
}
if cert != nil {
t.Fatal("expected nil cert")
}
if privkey != nil {
t.Fatal("expected nil privkey")
}
})
}
func newproxy(t *testing.T) net.Listener {
proxy := NewCensoringProxy()
listener, err := proxy.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
return listener
}
func newproxytls(t *testing.T) net.Listener {
proxy := NewCensoringProxy()
listener, _, err := proxy.StartTLS("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
return listener
}
func killproxy(t *testing.T, listener net.Listener) {
err := listener.Close()
if err != nil {
t.Fatal(err)
}
}
func checkdial(
t *testing.T, proxyAddr string, expectErr error,
dial func(network, address string) (net.Conn, error),
) {
conn, err := dial("tcp", proxyAddr)
if err != expectErr {
t.Fatal("not the result we expected")
}
if conn == nil && expectErr == nil {
t.Fatal("expected actionable conn")
}
if conn != nil && expectErr != nil {
t.Fatal("expected nil conn")
}
if conn != nil {
conn.Write([]byte("123454321"))
conn.Close()
}
}

46
internal/engine/cmd/jafar/flagx/stringarray.go Normal file
View File

@ -0,0 +1,46 @@
// Package flagx contains extensions for the standard library
// flag package. The code is adapted from github.com/m-lab/go and more
// specifically from <https://git.io/JJ8UA>. This file is licensed under
// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
package flagx
import (
"fmt"
"strings"
)
// StringArray is a new flag type. It appends the flag parameter to an
// `[]string` allowing the parameter to be specified multiple times or using ","
// separated items. Unlike other Flag types, the default argument should almost
// always be the empty array, because there is no way to remove an element, only
// to add one.
type StringArray []string
// Get retrieves the value contained in the flag.
func (sa StringArray) Get() interface{} {
return sa
}
// Set accepts a string parameter and appends it to the associated StringArray.
// Set attempts to split the given string on commas "," and appends each element
// to the StringArray.
func (sa *StringArray) Set(s string) error {
f := strings.Split(s, ",")
*sa = append(*sa, f...)
return nil
}
// String reports the StringArray as a Go value.
func (sa StringArray) String() string {
return fmt.Sprintf("%#v", []string(sa))
}
// Contains returns true when the given value equals one of the StringArray values.
func (sa StringArray) Contains(value string) bool {
for _, v := range sa {
if v == value {
return true
}
}
return false
}

73
internal/engine/cmd/jafar/flagx/stringarray_test.go Normal file
View File

@ -0,0 +1,73 @@
package flagx_test
// The code in this file is adapted from github.com/m-lab/go and more
// specifically from <https://git.io/JJ8UA>. This file is licensed under
// version 2.0 of the Apache License <https://git.io/JJ8Ux>.
import (
"flag"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
)
func TestStringArray(t *testing.T) {
tests := []struct {
name string
args []string
expt flagx.StringArray
repr string
contains string
wantErr bool
}{
{
name: "okay",
args: []string{"a", "b"},
expt: flagx.StringArray{"a", "b"},
repr: `[]string{"a", "b"}`,
contains: "b",
},
{
name: "okay-split-commas",
args: []string{"a", "b", "c,d"},
expt: flagx.StringArray{"a", "b", "c", "d"},
repr: `[]string{"a", "b", "c", "d"}`,
contains: "d",
},
{
name: "empty",
args: []string{},
expt: flagx.StringArray{},
repr: `[]string{}`,
contains: "a",
wantErr: true,
},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
sa := &flagx.StringArray{}
for i := range tt.args {
if err := sa.Set(tt.args[i]); err != nil {
t.Errorf("StringArray.Set() error = %v, want nil", err)
}
}
v := (sa.Get().(flagx.StringArray))
if diff := cmp.Diff(v, tt.expt); diff != "" {
t.Errorf("StringArray.Get() unexpected differences %v", diff)
}
if tt.repr != sa.String() {
t.Errorf("StringArray.String() want = %q, got %q", tt.repr, sa.String())
}
if sa.Contains(tt.contains) == tt.wantErr {
t.Errorf("StringArray.Contains() want = %q, got %t", tt.repr, sa.Contains(tt.contains))
}
})
}
}
// Successful compilation of this function means that StringArray implements the
// flag.Getter interface. The function need not be called.
func assertFlagGetterStringArray(b flagx.StringArray) {
func(in flag.Getter) {}(&b)
}

80
internal/engine/cmd/jafar/httpproxy/httpproxy.go Normal file
View File

@ -0,0 +1,80 @@
// Package httpproxy contains a censoring HTTP proxy. This proxy will
// vet all the traffic and reply with 451 responses for a configurable
// set of offending Host headers in incoming requests.
package httpproxy
import (
"net"
"net/http"
"net/http/httputil"
"net/url"
"strings"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
const product = "jafar/0.1.0"
// CensoringProxy is a censoring HTTP proxy
type CensoringProxy struct {
keywords []string
transport http.RoundTripper
}
// NewCensoringProxy creates a new CensoringProxy instance using
// the specified list of keywords to censor. keywords is the list
// of keywords that trigger censorship if any of them appears in
// the Host header of a request. dnsNetwork and dnsAddress are
// settings to configure the upstream, non censored DNS.
func NewCensoringProxy(
keywords []string, uncensored netx.HTTPRoundTripper,
) *CensoringProxy {
return &CensoringProxy{keywords: keywords, transport: uncensored}
}
var blockpage = []byte(`<html><head>
<title>451 Unavailable For Legal Reasons</title>
</head><body>
<center><h1>451 Unavailable For Legal Reasons</h1></center>
<p>This content is not available in your jurisdiction.</p>
</body></html>
`)
// ServeHTTP serves HTTP requests
func (p *CensoringProxy) ServeHTTP(w http.ResponseWriter, r *http.Request) {
// Implementation note: use Via header to detect in a loose way
// requests originated by us and directed to us
if r.Header.Get("Via") != "" || r.Host == "" {
w.WriteHeader(http.StatusBadRequest)
return
}
for _, pattern := range p.keywords {
if strings.Contains(r.Host, pattern) {
w.WriteHeader(http.StatusUnavailableForLegalReasons)
w.Write(blockpage)
return
}
}
r.Header.Add("Via", product) // see above
proxy := httputil.NewSingleHostReverseProxy(&url.URL{
Host: r.Host,
Scheme: "http",
})
proxy.ModifyResponse = func(resp *http.Response) error {
resp.Header.Add("Via", product) // see above
return nil
}
proxy.Transport = p.transport
proxy.ServeHTTP(w, r)
}
// Start starts the censoring proxy.
func (p *CensoringProxy) Start(address string) (*http.Server, net.Addr, error) {
server := &http.Server{Handler: p}
listener, err := net.Listen("tcp", address)
if err != nil {
return nil, nil, err
}
go server.Serve(listener)
return server, listener.Addr(), nil
}

130
internal/engine/cmd/jafar/httpproxy/httpproxy_test.go Normal file
View File

@ -0,0 +1,130 @@
package httpproxy
import (
"bytes"
"context"
"io/ioutil"
"net"
"net/http"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
)
func TestPass(t *testing.T) {
server, addr := newproxy(t, "ooni.io")
// We're filtering ooni.io, so we expect example.com to pass
// through the proxy with 200 and we also expect to see the
// Via header in the responses we receive, of course.
checkrequest(t, addr.String(), "example.com", 200, true)
killproxy(t, server)
}
func TestBlock(t *testing.T) {
server, addr := newproxy(t, "ooni.io")
// Here we're filtering any domain containing ooni.io, so we
// expect the proxy to send 451 without actually proxing, thus
// there should not be any Via header in the output.
checkrequest(t, addr.String(), "mia-ps.ooni.io", 451, false)
killproxy(t, server)
}
func TestLoop(t *testing.T) {
server, addr := newproxy(t, "ooni.io")
// Here we're forcing the proxy to connect to itself. It does
// does that and recognizes itself because of the Via header
// being set in the request generated by the connection to itself,
// which should cause a 400. The response should have the Via
// header set because the 400 is received by the connection that
// this code has made to the proxy.
checkrequest(t, addr.String(), addr.String(), 400, true)
killproxy(t, server)
}
func TestListenError(t *testing.T) {
proxy := NewCensoringProxy([]string{""}, uncensored.DefaultClient)
server, addr, err := proxy.Start("8.8.8.8:80")
if err == nil {
t.Fatal("expected an error here")
}
if server != nil {
t.Fatal("expected nil server here")
}
if addr != nil {
t.Fatal("expected nil addr here")
}
}
func newproxy(t *testing.T, blocked string) (*http.Server, net.Addr) {
proxy := NewCensoringProxy([]string{blocked}, uncensored.DefaultClient)
server, addr, err := proxy.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
return server, addr
}
func killproxy(t *testing.T, server *http.Server) {
err := server.Shutdown(context.Background())
if err != nil {
t.Fatal(err)
}
}
func checkrequest(
t *testing.T, proxyAddr, host string,
expectStatus int, expectVia bool,
) {
req, err := http.NewRequest("GET", "http://"+proxyAddr, nil)
if err != nil {
t.Fatal(err)
}
req.Host = host
resp, err := http.DefaultClient.Do(req)
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != expectStatus {
t.Fatal("unexpected value of status code")
}
t.Log(resp)
values, _ := resp.Header["Via"]
var foundProduct bool
for _, value := range values {
if value == product {
foundProduct = true
}
}
if foundProduct && !expectVia {
t.Fatal("unexpectedly found Via header")
}
if !foundProduct && expectVia {
t.Fatal("Via header not found")
}
proxiedData, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if expectStatus == 200 {
checkbody(t, proxiedData, host)
}
}
func checkbody(t *testing.T, proxiedData []byte, host string) {
resp, err := http.Get("http://" + host)
if err != nil {
t.Fatal(err)
}
if resp.StatusCode != 200 {
t.Fatal("unexpected status code")
}
defer resp.Body.Close()
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if bytes.Equal(data, proxiedData) == false {
t.Fatal("body mismatch")
}
}

98
internal/engine/cmd/jafar/iptables/iptables.go Normal file
View File

@ -0,0 +1,98 @@
// Package iptables contains code for managing firewall rules. This package
// really only works reliably on Linux. In all other systems the functionality
// in here is just a set of stubs returning errors.
package iptables
import (
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
)
type shell interface {
createChains() error
dropIfDestinationEquals(ip string) error
rstIfDestinationEqualsAndIsTCP(ip string) error
dropIfContainsKeywordHex(keyword string) error
dropIfContainsKeyword(keyword string) error
rstIfContainsKeywordHexAndIsTCP(keyword string) error
rstIfContainsKeywordAndIsTCP(keyword string) error
hijackDNS(address string) error
hijackHTTPS(address string) error
hijackHTTP(address string) error
waive() error
}
// CensoringPolicy implements a censoring policy.
type CensoringPolicy struct {
DropIPs []string // drop IP traffic to these IPs
DropKeywordsHex []string // drop IP packets with these hex keywords
DropKeywords []string // drop IP packets with these keywords
HijackDNSAddress string // where to hijack DNS to
HijackHTTPSAddress string // where to hijack HTTPS to
HijackHTTPAddress string // where to hijack HTTP to
ResetIPs []string // RST TCP/IP traffic to these IPs
ResetKeywordsHex []string // RST TCP/IP flows with these hex keywords
ResetKeywords []string // RST TCP/IP flows with these keywords
sh shell
}
// NewCensoringPolicy returns a new censoring policy.
func NewCensoringPolicy() *CensoringPolicy {
return &CensoringPolicy{
sh: newShell(),
}
}
// Apply applies the censorship policy
func (c *CensoringPolicy) Apply() (err error) {
defer func() {
if recover() != nil {
// JUST KNOW WE'VE BEEN HERE
}
}()
err = c.sh.createChains()
runtimex.PanicOnError(err, "c.sh.createChains failed")
// Implementation note: we want the RST rules to be first such
// that we end up enforcing them before the drop rules.
for _, keyword := range c.ResetKeywordsHex {
err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword)
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed")
}
for _, keyword := range c.ResetKeywords {
err = c.sh.rstIfContainsKeywordAndIsTCP(keyword)
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed")
}
for _, ip := range c.ResetIPs {
err = c.sh.rstIfDestinationEqualsAndIsTCP(ip)
runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed")
}
for _, keyword := range c.DropKeywordsHex {
err = c.sh.dropIfContainsKeywordHex(keyword)
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed")
}
for _, keyword := range c.DropKeywords {
err = c.sh.dropIfContainsKeyword(keyword)
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed")
}
for _, ip := range c.DropIPs {
err = c.sh.dropIfDestinationEquals(ip)
runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed")
}
if c.HijackDNSAddress != "" {
err = c.sh.hijackDNS(c.HijackDNSAddress)
runtimex.PanicOnError(err, "c.sh.hijackDNS failed")
}
if c.HijackHTTPSAddress != "" {
err = c.sh.hijackHTTPS(c.HijackHTTPSAddress)
runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed")
}
if c.HijackHTTPAddress != "" {
err = c.sh.hijackHTTP(c.HijackHTTPAddress)
runtimex.PanicOnError(err, "c.sh.hijackHTTP failed")
}
return
}
// Waive removes any censorship policy
func (c *CensoringPolicy) Waive() error {
return c.sh.waive()
}

345
internal/engine/cmd/jafar/iptables/iptables_integration_test.go Normal file
View File

@ -0,0 +1,345 @@
package iptables
import (
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os/exec"
"runtime"
"strings"
"testing"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
)
func init() {
log.SetLevel(log.ErrorLevel)
}
func newCensoringPolicy() *CensoringPolicy {
policy := NewCensoringPolicy()
policy.Waive() // start over to allow for repeated tests on failure
return policy
}
func TestCannotApplyPolicy(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropIPs = []string{"antani"}
if err := policy.Apply(); err == nil {
t.Fatal("expected an error here")
}
}
func TestCreateChainsError(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
// you should not be able to apply the policy when there is
// already a policy, you need to waive it first
if err := policy.Apply(); err == nil {
t.Fatal("expected an error here")
}
}
func TestDropIP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropIPs = []string{"1.1.1.1"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853")
if err == nil {
t.Fatalf("expected an error here")
}
if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" {
t.Fatal("unexpected error occurred")
}
if conn != nil {
t.Fatal("expected nil connection here")
}
}
func TestDropKeyword(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropKeywords = []string{"ooni.io"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err == nil {
t.Fatal("expected an error here")
}
if !strings.HasSuffix(err.Error(), "context deadline exceeded") {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestDropKeywordHex(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err == nil {
t.Fatal("expected an error here")
}
// the error we see with GitHub Actions is different from the error
// we see when testing locally on Fedora
if !strings.HasSuffix(err.Error(), "operation not permitted") &&
!strings.HasSuffix(err.Error(), "Temporary failure in name resolution") &&
!strings.HasSuffix(err.Error(), "no such host") {
t.Fatalf("unexpected error occurred: %+v", err)
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestResetIP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetIPs = []string{"1.1.1.1"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853")
if err == nil {
t.Fatalf("expected an error here")
}
if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" {
t.Fatal("unexpected error occurred")
}
if conn != nil {
t.Fatal("expected nil connection here")
}
}
func TestResetKeyword(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetKeywords = []string{"ooni.io"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
resp, err := http.Get("http://www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestResetKeywordHex(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
resp, err := http.Get("http://www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestHijackDNS(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
resolver := resolver.NewCensoringResolver(
[]string{"ooni.io"}, nil, nil,
uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")),
)
server, err := resolver.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer server.Shutdown()
policy := newCensoringPolicy()
defer policy.Waive()
policy.HijackDNSAddress = server.PacketConn.LocalAddr().String()
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
addrs, err := net.LookupHost("www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "no such host") == false {
t.Fatal("unexpected error occurred")
}
if addrs != nil {
t.Fatal("expected nil addrs here")
}
}
func TestHijackHTTP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
// Implementation note: this test is complicated by the fact
// that we are running as root and so we're whitelisted.
server := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(451)
}),
)
defer server.Close()
policy := newCensoringPolicy()
defer policy.Waive()
pu, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
policy.HijackHTTPAddress = pu.Host
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
err = shellx.Run("sudo", "-u", "nobody", "--",
"curl", "-sf", "http://example.com")
if err == nil {
t.Fatal("expected an error here")
}
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatal("not the error type we expected")
}
if exitErr.ExitCode() != 22 {
t.Fatal("not the exit code we expected")
}
}
func TestHijackHTTPS(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
// Implementation note: this test is complicated by the fact
// that we are running as root and so we're whitelisted.
server := httptest.NewTLSServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(451)
}),
)
defer server.Close()
policy := newCensoringPolicy()
defer policy.Waive()
pu, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
policy.HijackHTTPSAddress = pu.Host
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
err = shellx.Run("sudo", "-u", "nobody", "--",
"curl", "-sf", "https://example.com")
if err == nil {
t.Fatal("expected an error here")
}
t.Log(err)
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatal("not the error type we expected")
}
if exitErr.ExitCode() != 60 {
t.Fatal("not the exit code we expected")
}
}

117
internal/engine/cmd/jafar/iptables/iptables_linux.go Normal file
View File

@ -0,0 +1,117 @@
// +build linux
package iptables
import (
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
)
type linuxShell struct{}
func (s *linuxShell) createChains() (err error) {
defer func() {
if recover() != nil {
// JUST KNOW WE'VE BEEN HERE
}
}()
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_INPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain")
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_OUTPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain")
err = shellx.Run("sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain")
err = shellx.Run("sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT")
err = shellx.Run("sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT")
err = shellx.Run("sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT")
return nil
}
func (s *linuxShell) dropIfDestinationEquals(ip string) error {
return shellx.Run("sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP")
}
func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip,
"-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
"--hex-string", keyword, "-j", "DROP",
)
}
func (s *linuxShell) dropIfContainsKeyword(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
"--string", keyword, "-j", "DROP",
)
}
func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
"kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
"kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) hijackDNS(address string) error {
// Hijack any DNS query, like the Vodafone station does when using the
// secure network feature. Our transparent proxies will use DoT, in order
// to bypass this restriction and avoid routing loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp",
"--dport", "53", "-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) hijackHTTPS(address string) error {
// We need to whitelist root otherwise the traffic sent by Jafar
// itself will match the rule and loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
"--dport", "443", "-m", "owner", "!", "--uid-owner", "0",
"-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) hijackHTTP(address string) error {
// We need to whitelist root otherwise the traffic sent by Jafar
// itself will match the rule and loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
"--dport", "80", "-m", "owner", "!", "--uid-owner", "0",
"-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) waive() error {
shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT")
return nil
}
func newShell() *linuxShell {
return &linuxShell{}
}

45
internal/engine/cmd/jafar/iptables/iptables_unsupported.go Normal file
View File

@ -0,0 +1,45 @@
// +build !linux
package iptables
import "errors"
type otherwiseShell struct{}
func (*otherwiseShell) createChains() error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfDestinationEquals(ip string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfContainsKeyword(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackDNS(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackHTTPS(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackHTTP(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) waive() error {
return errors.New("not implemented")
}
func newShell() *otherwiseShell {
return &otherwiseShell{}
}

286
internal/engine/cmd/jafar/main.go Normal file
View File

@ -0,0 +1,286 @@
// Jafar is a censorship simulation tool used for testing OONI.
package main
import (
"encoding/pem"
"errors"
"flag"
"fmt"
"io/ioutil"
"net"
"net/http"
"os"
"os/exec"
"os/signal"
"strings"
"syscall"
"github.com/apex/log"
"github.com/apex/log/handlers/cli"
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/badproxy"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/flagx"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/httpproxy"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/resolver"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/tlsproxy"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
)
var (
badProxyAddress *string
badProxyAddressTLS *string
badProxyTLSOutputCA *string
dnsProxyAddress *string
dnsProxyBlock flagx.StringArray
dnsProxyHijack flagx.StringArray
dnsProxyIgnore flagx.StringArray
httpProxyAddress *string
httpProxyBlock flagx.StringArray
iptablesDropIP flagx.StringArray
iptablesDropKeywordHex flagx.StringArray
iptablesDropKeyword flagx.StringArray
iptablesHijackDNSTo *string
iptablesHijackHTTPSTo *string
iptablesHijackHTTPTo *string
iptablesResetIP flagx.StringArray
iptablesResetKeywordHex flagx.StringArray
iptablesResetKeyword flagx.StringArray
mainCh chan os.Signal
mainCommand *string
mainUser *string
tag *string
tlsProxyAddress *string
tlsProxyBlock flagx.StringArray
uncensoredResolverURL *string
)
func init() {
// badProxy
badProxyAddress = flag.String(
"bad-proxy-address", "127.0.0.1:7117",
"Address where to listen for TCP connections",
)
badProxyAddressTLS = flag.String(
"bad-proxy-address-tls", "127.0.0.1:4114",
"Address where to listen for TLS connections",
)
badProxyTLSOutputCA = flag.String(
"bad-proxy-tls-output-ca", "badproxy.pem",
"File where to write the CA used by the bad proxy",
)
// dnsProxy
dnsProxyAddress = flag.String(
"dns-proxy-address", "127.0.0.1:53",
"Address where the DNS proxy should listen",
)
flag.Var(
&dnsProxyBlock, "dns-proxy-block",
"Register keyword triggering NXDOMAIN censorship",
)
flag.Var(
&dnsProxyHijack, "dns-proxy-hijack",
"Register keyword triggering redirection to 127.0.0.1",
)
flag.Var(
&dnsProxyIgnore, "dns-proxy-ignore",
"Register keyword causing the proxy to ignore the query",
)
// httpProxy
httpProxyAddress = flag.String(
"http-proxy-address", "127.0.0.1:80",
"Address where the HTTP proxy should listen",
)
flag.Var(
&httpProxyBlock, "http-proxy-block",
"Register keyword triggering HTTP 451 censorship",
)
// iptables
flag.Var(
&iptablesDropIP, "iptables-drop-ip",
"Drop traffic to the specified IP address",
)
flag.Var(
&iptablesDropKeywordHex, "iptables-drop-keyword-hex",
"Drop traffic containing the specified keyword in hex",
)
flag.Var(
&iptablesDropKeyword, "iptables-drop-keyword",
"Drop traffic containing the specified keyword",
)
iptablesHijackDNSTo = flag.String(
"iptables-hijack-dns-to", "",
"Hijack all DNS UDP traffic to the specified endpoint",
)
iptablesHijackHTTPSTo = flag.String(
"iptables-hijack-https-to", "",
"Hijack all HTTPS traffic to the specified endpoint",
)
iptablesHijackHTTPTo = flag.String(
"iptables-hijack-http-to", "",
"Hijack all HTTP traffic to the specified endpoint",
)
flag.Var(
&iptablesResetIP, "iptables-reset-ip",
"Reset TCP/IP traffic to the specified IP address",
)
flag.Var(
&iptablesResetKeywordHex, "iptables-reset-keyword-hex",
"Reset TCP/IP traffic containing the specified keyword in hex",
)
flag.Var(
&iptablesResetKeyword, "iptables-reset-keyword",
"Reset TCP/IP traffic containing the specified keyword",
)
// main
mainCh = make(chan os.Signal, 1)
signal.Notify(
mainCh, syscall.SIGHUP, syscall.SIGINT, syscall.SIGTERM, syscall.SIGQUIT,
)
mainCommand = flag.String("main-command", "", "Optional command to execute")
mainUser = flag.String("main-user", "nobody", "Run command as user")
// tag
tag = flag.String("tag", "", "Add tag to a specific run")
// tlsProxy
tlsProxyAddress = flag.String(
"tls-proxy-address", "127.0.0.1:443",
"Address where the HTTP proxy should listen",
)
flag.Var(
&tlsProxyBlock, "tls-proxy-block",
"Register keyword triggering TLS censorship",
)
// uncensored
uncensoredResolverURL = flag.String(
"uncensored-resolver-url", "dot://1.1.1.1:853",
"URL of an hopefully uncensored resolver",
)
}
func badProxyStart() net.Listener {
proxy := badproxy.NewCensoringProxy()
listener, err := proxy.Start(*badProxyAddress)
runtimex.PanicOnError(err, "proxy.Start failed")
return listener
}
func badProxyStartTLS() net.Listener {
proxy := badproxy.NewCensoringProxy()
listener, cert, err := proxy.StartTLS(*badProxyAddressTLS)
runtimex.PanicOnError(err, "proxy.StartTLS failed")
err = ioutil.WriteFile(*badProxyTLSOutputCA, pem.EncodeToMemory(&pem.Block{
Type: "CERTIFICATE",
Bytes: cert.Raw,
}), 0644)
runtimex.PanicOnError(err, "ioutil.WriteFile failed")
return listener
}
func dnsProxyStart(uncensored *uncensored.Client) *dns.Server {
proxy := resolver.NewCensoringResolver(
dnsProxyBlock, dnsProxyHijack, dnsProxyIgnore, uncensored,
)
server, err := proxy.Start(*dnsProxyAddress)
runtimex.PanicOnError(err, "proxy.Start failed")
return server
}
func httpProxyStart(uncensored *uncensored.Client) *http.Server {
proxy := httpproxy.NewCensoringProxy(httpProxyBlock, uncensored)
server, _, err := proxy.Start(*httpProxyAddress)
runtimex.PanicOnError(err, "proxy.Start failed")
return server
}
func iptablesStart() *iptables.CensoringPolicy {
policy := iptables.NewCensoringPolicy()
// For robustness waive the policy so we start afresh
policy.Waive()
policy.DropIPs = iptablesDropIP
policy.DropKeywordsHex = iptablesDropKeywordHex
policy.DropKeywords = iptablesDropKeyword
policy.HijackDNSAddress = *iptablesHijackDNSTo
policy.HijackHTTPSAddress = *iptablesHijackHTTPSTo
policy.HijackHTTPAddress = *iptablesHijackHTTPTo
policy.ResetIPs = iptablesResetIP
policy.ResetKeywordsHex = iptablesResetKeywordHex
policy.ResetKeywords = iptablesResetKeyword
err := policy.Apply()
runtimex.PanicOnError(err, "policy.Apply failed")
return policy
}
func tlsProxyStart(uncensored *uncensored.Client) net.Listener {
proxy := tlsproxy.NewCensoringProxy(tlsProxyBlock, uncensored)
listener, err := proxy.Start(*tlsProxyAddress)
runtimex.PanicOnError(err, "proxy.Start failed")
return listener
}
func newUncensoredClient() *uncensored.Client {
clnt, err := uncensored.NewClient(*uncensoredResolverURL)
runtimex.PanicOnError(err, "uncensored.NewClient failed")
return clnt
}
func mustx(err error, message string, osExit func(int)) {
if err != nil {
var (
exitcode = 1
exiterr *exec.ExitError
)
if errors.As(err, &exiterr) {
exitcode = exiterr.ExitCode()
}
log.Errorf("%s", message)
osExit(exitcode)
}
}
func main() {
flag.Parse()
// TODO(bassosimone): we may want a verbose flag
log.SetLevel(log.InfoLevel)
log.SetHandler(cli.Default)
log.Infof("jafar command line: [%s]", strings.Join(os.Args, ", "))
log.Infof("jafar tag: %s", *tag)
uncensoredClient := newUncensoredClient()
defer uncensoredClient.CloseIdleConnections()
badlistener := badProxyStart()
defer badlistener.Close()
badtlslistener := badProxyStartTLS()
defer badtlslistener.Close()
dnsproxy := dnsProxyStart(uncensoredClient)
defer dnsproxy.Shutdown()
httpproxy := httpProxyStart(uncensoredClient)
defer httpproxy.Close()
tlslistener := tlsProxyStart(uncensoredClient)
defer tlslistener.Close()
policy := iptablesStart()
var err error
if *mainCommand != "" {
err = shellx.RunCommandline(fmt.Sprintf(
"sudo -u '%s' -- %s", *mainUser, *mainCommand,
))
} else {
<-mainCh
}
policy.Waive()
mustx(err, "subcommand failed", os.Exit)
}

89
internal/engine/cmd/jafar/main_test.go Normal file
View File

@ -0,0 +1,89 @@
package main
import (
"errors"
"os"
"runtime"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/iptables"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/shellx"
)
func ensureWeStartOverWithIPTables() {
iptables.NewCensoringPolicy().Waive()
}
func TestNoCommand(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skip test on non Linux systems")
}
ensureWeStartOverWithIPTables()
*dnsProxyAddress = "127.0.0.1:0"
*httpProxyAddress = "127.0.0.1:0"
*tlsProxyAddress = "127.0.0.1:0"
go func() {
mainCh <- os.Interrupt
}()
main()
}
func TestWithCommand(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("skip test on non Linux systems")
}
ensureWeStartOverWithIPTables()
*dnsProxyAddress = "127.0.0.1:0"
*httpProxyAddress = "127.0.0.1:0"
*tlsProxyAddress = "127.0.0.1:0"
*mainCommand = "whoami"
defer func() {
*mainCommand = ""
}()
main()
}
func TestMustx(t *testing.T) {
t.Run("with no error", func(t *testing.T) {
var called int
mustx(nil, "", func(int) {
called++
})
if called != 0 {
t.Fatal("should not happen")
}
})
t.Run("with non-exit-code error", func(t *testing.T) {
var (
called int
exitcode int
)
mustx(errors.New("antani"), "", func(ec int) {
called++
exitcode = ec
})
if called != 1 {
t.Fatal("not called?!")
}
if exitcode != 1 {
t.Fatal("unexpected exitcode value")
}
})
t.Run("with exit-code error", func(t *testing.T) {
var (
called int
exitcode int
)
err := shellx.Run("curl", "-sf", "") // cause exitcode == 3
mustx(err, "", func(ec int) {
called++
exitcode = ec
})
if called != 1 {
t.Fatal("not called?!")
}
if exitcode != 3 {
t.Fatal("unexpected exitcode value")
}
})
}

130
internal/engine/cmd/jafar/resolver/resolver.go Normal file
View File

@ -0,0 +1,130 @@
// Package resolver contains a censoring DNS resolver. Most queries are
// answered without censorship, but selected queries could either be
// discarded or replied to with a bogon or NXDOMAIN answer.
package resolver
import (
"context"
"net"
"strings"
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
// CensoringResolver is a censoring resolver.
type CensoringResolver struct {
blocked []string
hijacked []string
ignored []string
lookupHost func(ctx context.Context, host string) ([]string, error)
}
// NewCensoringResolver creates a new CensoringResolver instance using
// the specified list of keywords to censor. blocked is the list of
// keywords that trigger NXDOMAIN if they appear in a query. hijacked
// is similar but redirects to 127.0.0.1, where the transparent HTTP
// and TLS proxies will pick them up. dnsNetwork and dnsAddress are the
// settings to configure the upstream, non censored DNS.
func NewCensoringResolver(
blocked, hijacked, ignored []string, uncensored netx.Resolver,
) *CensoringResolver {
return &CensoringResolver{
blocked: blocked,
hijacked: hijacked,
ignored: ignored,
lookupHost: uncensored.LookupHost,
}
}
func (r *CensoringResolver) roundtrip(rw dns.ResponseWriter, req *dns.Msg) {
name := req.Question[0].Name
addrs, err := r.lookupHost(context.Background(), name)
var ips []net.IP
if err == nil {
for _, addr := range addrs {
if ip := net.ParseIP(addr); ip != nil {
ips = append(ips, ip)
}
}
}
r.reply(rw, req, ips)
}
func (r *CensoringResolver) reply(
rw dns.ResponseWriter, req *dns.Msg, ips []net.IP,
) {
m := new(dns.Msg)
m.Compress = true
m.MsgHdr.RecursionAvailable = true
m.SetReply(req)
for _, ip := range ips {
ipv6 := strings.Contains(ip.String(), ":")
if !ipv6 && req.Question[0].Qtype == dns.TypeA {
m.Answer = append(m.Answer, &dns.A{
Hdr: dns.RR_Header{
Name: req.Question[0].Name,
Rrtype: dns.TypeA,
Class: dns.ClassINET,
Ttl: 0,
},
A: ip,
})
}
}
if m.Answer == nil {
m.SetRcode(req, dns.RcodeNameError)
}
rw.WriteMsg(m)
}
func (r *CensoringResolver) failure(rw dns.ResponseWriter, req *dns.Msg) {
m := new(dns.Msg)
m.Compress = true
m.MsgHdr.RecursionAvailable = true
m.SetRcode(req, dns.RcodeServerFailure)
rw.WriteMsg(m)
}
// ServeDNS serves a DNS request
func (r *CensoringResolver) ServeDNS(rw dns.ResponseWriter, req *dns.Msg) {
if len(req.Question) < 1 {
r.failure(rw, req)
return
}
name := req.Question[0].Name
for _, pattern := range r.blocked {
if strings.Contains(name, pattern) {
r.reply(rw, req, nil)
return
}
}
for _, pattern := range r.hijacked {
if strings.Contains(name, pattern) {
r.reply(rw, req, []net.IP{net.IPv4(127, 0, 0, 1)})
return
}
}
for _, pattern := range r.ignored {
if strings.Contains(name, pattern) {
return
}
}
r.roundtrip(rw, req)
}
// Start starts the DNS resolver
func (r *CensoringResolver) Start(address string) (*dns.Server, error) {
packetconn, err := net.ListenPacket("udp", address)
if err != nil {
return nil, err
}
server := &dns.Server{
Addr: address,
Handler: r,
Net: "udp",
PacketConn: packetconn,
}
go server.ActivateAndServe()
return server, nil
}

173
internal/engine/cmd/jafar/resolver/resolver_test.go Normal file
View File

@ -0,0 +1,173 @@
package resolver
import (
"strings"
"testing"
"github.com/miekg/dns"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
)
func TestPass(t *testing.T) {
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
checkrequest(t, server, "example.com", "success", nil)
killserver(t, server)
}
func TestBlock(t *testing.T) {
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
checkrequest(t, server, "mia-ps.ooni.io", "blocked", nil)
killserver(t, server)
}
func TestRedirect(t *testing.T) {
server := newresolver(t, []string{"ooni.io"}, []string{"ooni.nu"}, nil)
checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", nil)
killserver(t, server)
}
func TestIgnore(t *testing.T) {
server := newresolver(t, nil, nil, []string{"ooni.nu"})
iotimeout := "i/o timeout"
checkrequest(t, server, "hkgmetadb.ooni.nu", "hijacked", &iotimeout)
killserver(t, server)
}
func TestLookupFailure(t *testing.T) {
server := newresolver(t, nil, nil, nil)
// we should receive same response as when we're blocked
checkrequest(t, server, "example.antani", "blocked", nil)
killserver(t, server)
}
func TestFailureNoQuestion(t *testing.T) {
resolver := NewCensoringResolver(
nil, nil, nil, uncensored.DefaultClient,
)
resolver.ServeDNS(&fakeResponseWriter{t: t}, new(dns.Msg))
}
func TestListenFailure(t *testing.T) {
resolver := NewCensoringResolver(
nil, nil, nil, uncensored.DefaultClient,
)
server, err := resolver.Start("8.8.8.8:53")
if err == nil {
t.Fatal("expected an error here")
}
if server != nil {
t.Fatal("expected nil server here")
}
}
func newresolver(t *testing.T, blocked, hijacked, ignored []string) *dns.Server {
resolver := NewCensoringResolver(
blocked, hijacked, ignored,
// using faster dns because dot here causes miekg/dns's
// dns.Exchange to timeout and I don't want more complexity
uncensored.Must(uncensored.NewClient("system:///")),
)
server, err := resolver.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
return server
}
func killserver(t *testing.T, server *dns.Server) {
err := server.Shutdown()
if err != nil {
t.Fatal(err)
}
}
func checkrequest(
t *testing.T, server *dns.Server, host string, expectStatus string,
expectErrorSuffix *string,
) {
address := server.PacketConn.LocalAddr().String()
query := newquery(host)
reply, err := dns.Exchange(query, address)
if err != nil {
if expectErrorSuffix != nil &&
strings.HasSuffix(err.Error(), *expectErrorSuffix) {
return
}
t.Fatal(err)
}
switch expectStatus {
case "success":
checksuccess(t, reply)
case "hijacked":
checkhijacked(t, reply)
case "blocked":
checkblocked(t, reply)
default:
panic("unexpected value")
}
}
func checksuccess(t *testing.T, reply *dns.Msg) {
if reply.Rcode != dns.RcodeSuccess {
t.Fatal("unexpected rcode")
}
if len(reply.Answer) < 1 {
t.Fatal("too few answers")
}
for _, answer := range reply.Answer {
if rr, ok := answer.(*dns.A); ok {
if rr.A.String() == "127.0.0.1" {
t.Fatal("unexpected hijacked response here")
}
}
}
}
func checkhijacked(t *testing.T, reply *dns.Msg) {
if reply.Rcode != dns.RcodeSuccess {
t.Fatal("unexpected rcode")
}
if len(reply.Answer) < 1 {
t.Fatal("too few answers")
}
for _, answer := range reply.Answer {
if rr, ok := answer.(*dns.A); ok {
if rr.A.String() != "127.0.0.1" {
t.Fatal("unexpected non-hijacked response here")
}
}
}
}
func checkblocked(t *testing.T, reply *dns.Msg) {
if reply.Rcode != dns.RcodeNameError {
t.Fatal("unexpected rcode")
}
if len(reply.Answer) >= 1 {
t.Fatal("too many answers")
}
}
func newquery(name string) *dns.Msg {
query := new(dns.Msg)
query.Id = dns.Id()
query.RecursionDesired = true
query.Question = append(query.Question, dns.Question{
Name: dns.Fqdn(name),
Qclass: dns.ClassINET,
Qtype: dns.TypeA,
})
return query
}
type fakeResponseWriter struct {
dns.ResponseWriter
t *testing.T
}
func (rw *fakeResponseWriter) WriteMsg(m *dns.Msg) error {
if m.Rcode != dns.RcodeServerFailure {
rw.t.Fatal("unexpected rcode")
}
return nil
}

66
internal/engine/cmd/jafar/shellx/shellx.go Normal file
View File

@ -0,0 +1,66 @@
// Package shellx contains utilities to run external commands.
package shellx
import (
"errors"
"os"
"os/exec"
"strings"
"github.com/apex/log"
"github.com/google/shlex"
"github.com/ooni/probe-cli/v3/internal/engine/model"
)
type runconfig struct {
args []string
loginfof func(format string, v ...interface{})
name string
stdout *os.File
stderr *os.File
}
func run(config runconfig) error {
config.loginfof("exec: %s %s", config.name, strings.Join(config.args, " "))
cmd := exec.Command(config.name, config.args...)
cmd.Stdout = config.stdout
cmd.Stderr = config.stderr
err := cmd.Run()
config.loginfof("exec result: %+v", err)
return err
}
// Run executes the specified command with the specified args
func Run(name string, arg ...string) error {
return run(runconfig{
args: arg,
loginfof: log.Log.Infof,
name: name,
stdout: os.Stdout,
stderr: os.Stderr,
})
}
// RunQuiet is like Run but it does not emit any output.
func RunQuiet(name string, arg ...string) error {
return run(runconfig{
args: arg,
loginfof: model.DiscardLogger.Infof,
name: name,
stdout: nil,
stderr: nil,
})
}
// RunCommandline is like Run but its only argument is a command
// line that will be splitted using the google/shlex package
func RunCommandline(cmdline string) error {
args, err := shlex.Split(cmdline)
if err != nil {
return err
}
if len(args) < 1 {
return errors.New("shellx: no command to execute")
}
return Run(args[0], args[1:]...)
}

44
internal/engine/cmd/jafar/shellx/shellx_test.go Normal file
View File

@ -0,0 +1,44 @@
package shellx
import "testing"
func TestRun(t *testing.T) {
if err := Run("whoami"); err != nil {
t.Fatal(err)
}
if err := Run("./nonexistent/command"); err == nil {
t.Fatal("expected an error here")
}
}
func TestRunQuiet(t *testing.T) {
if err := RunQuiet("true"); err != nil {
t.Fatal(err)
}
if err := RunQuiet("./nonexistent/command"); err == nil {
t.Fatal("expected an error here")
}
}
func TestRunCommandline(t *testing.T) {
t.Run("when the command does not parse", func(t *testing.T) {
if err := RunCommandline(`"foobar`); err == nil {
t.Fatal("expected an error here")
}
})
t.Run("when we have no arguments", func(t *testing.T) {
if err := RunCommandline(""); err == nil {
t.Fatal("expected an error here")
}
})
t.Run("when we have a single argument", func(t *testing.T) {
if err := RunCommandline("ls"); err != nil {
t.Fatal(err)
}
})
t.Run("when we have more than one argument", func(t *testing.T) {
if err := RunCommandline("ls ."); err != nil {
t.Fatal(err)
}
})
}

194
internal/engine/cmd/jafar/tlsproxy/tlsproxy.go Normal file
View File

@ -0,0 +1,194 @@
// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed
// through using the SNI to choose the hostname to connect to. Specific offending
// SNIs are censored by returning a TLS alert to the client.
package tlsproxy
import (
"context"
"crypto/tls"
"errors"
"net"
"strings"
"sync"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
// CensoringProxy is a censoring TLS proxy
type CensoringProxy struct {
keywords []string
dial func(network, address string) (net.Conn, error)
}
// NewCensoringProxy creates a new CensoringProxy instance using
// the specified list of keywords to censor. keywords is the list
// of keywords that trigger censorship if any of them appears in
// the SNII record of a ClientHello. dnsNetwork and dnsAddress are
// settings to configure the upstream, non censored DNS.
func NewCensoringProxy(
keywords []string, uncensored netx.Dialer,
) *CensoringProxy {
return &CensoringProxy{
keywords: keywords,
dial: func(network, address string) (net.Conn, error) {
return uncensored.DialContext(context.Background(), network, address)
},
}
}
// handshakeReader is a hack to perform the initial part of the
// TLS handshake so to know the SNI and then replay the bytes of
// this initial part of the handshake with the server.
type handshakeReader struct {
net.Conn
incoming []byte
}
// Read saves the initial bytes of the handshake such that later
// we can replay the handshake with the real TLS server.
func (c *handshakeReader) Read(b []byte) (int, error) {
count, err := c.Conn.Read(b)
if err == nil {
c.incoming = append(c.incoming, b[:count]...)
}
return count, err
}
// Write prevents writing on the real connection
func (c *handshakeReader) Write(b []byte) (int, error) {
return 0, errors.New("cannot write on this connection")
}
// forward forwards left traffic to right
func forward(wg *sync.WaitGroup, left, right net.Conn) {
data := make([]byte, 1<<18)
for {
n, err := left.Read(data)
if err != nil {
break
}
if _, err = right.Write(data[:n]); err != nil {
break
}
}
wg.Done()
}
// reset closes the connection with a RST segment
func reset(conn net.Conn) {
if tc, ok := conn.(*net.TCPConn); ok {
tc.SetLinger(0)
}
conn.Close()
}
// alertclose sends a TLS alert and then closes the connection
func alertclose(conn net.Conn) {
alertdata := []byte{
21, // alert
3, // version[0]
3, // version[1]
0, // length[0]
2, // length[1]
2, // fatal
80, // internal error
}
conn.Write(alertdata)
conn.Close()
}
// getsni attempts the handshakeReader hack to obtain the SNI by reading
// the beginning of the TLS handshake. On success a nonempty SNI string
// is returned. Otherwise we cannot distinguish between the absence of a
// SNI and any other reading network error that may have occurred.
func getsni(conn *handshakeReader) string {
var (
sni string
mutex sync.Mutex // just for safety
)
tls.Server(conn, &tls.Config{
GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) {
mutex.Lock()
sni = info.ServerName
mutex.Unlock()
return nil, errors.New("tlsproxy: we can't really continue handshake")
},
}).Handshake()
return sni
}
func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool {
local := conn.LocalAddr().String()
localAddr, _, localErr := net.SplitHostPort(local)
remote := conn.RemoteAddr().String()
remoteAddr, _, remoteErr := net.SplitHostPort(remote)
return localErr != nil || remoteErr != nil || localAddr == remoteAddr
}
// handle implements the TLS SNI proxy
func (p *CensoringProxy) handle(clientconn net.Conn) {
hr := &handshakeReader{Conn: clientconn}
sni := getsni(hr)
if sni == "" {
log.Warn("tlsproxy: network failure or SNI not provided")
reset(clientconn)
return
}
for _, pattern := range p.keywords {
if strings.Contains(sni, pattern) {
log.Warnf("tlsproxy: reject SNI by policy: %s", sni)
alertclose(clientconn)
return
}
}
serverconn, err := p.dial("tcp", net.JoinHostPort(sni, "443"))
if err != nil {
log.WithError(err).Warn("tlsproxy: p.dial failed")
alertclose(clientconn)
return
}
if p.connectingToMyself(serverconn) {
log.Warn("tlsproxy: connecting to myself")
alertclose(clientconn)
return
}
if _, err := serverconn.Write(hr.incoming); err != nil {
log.WithError(err).Warn("tlsproxy: serverconn.Write failed")
alertclose(clientconn)
return
}
log.Debugf("tlsproxy: routing for %s", sni)
defer clientconn.Close()
defer serverconn.Close()
var wg sync.WaitGroup
wg.Add(2)
go forward(&wg, clientconn, serverconn)
go forward(&wg, serverconn, clientconn)
wg.Wait()
}
func (p *CensoringProxy) run(listener net.Listener) {
for {
conn, err := listener.Accept()
if err != nil && strings.Contains(
err.Error(), "use of closed network connection") {
return
}
if err == nil {
// It's difficult to make accept fail, so restructure
// the code such that we enter into the happy path
go p.handle(conn)
}
}
}
// Start starts the censoring proxy.
func (p *CensoringProxy) Start(address string) (net.Listener, error) {
listener, err := net.Listen("tcp", address)
if err != nil {
return nil, err
}
go p.run(listener)
return listener, nil
}

183
internal/engine/cmd/jafar/tlsproxy/tlsproxy_test.go Normal file
View File

@ -0,0 +1,183 @@
package tlsproxy
import (
"crypto/tls"
"errors"
"net"
"sync"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/jafar/uncensored"
)
func TestPass(t *testing.T) {
listener := newproxy(t, "ooni.io")
checkdialtls(t, listener.Addr().String(), true, &tls.Config{
ServerName: "example.com",
})
killproxy(t, listener)
}
func TestBlock(t *testing.T) {
listener := newproxy(t, "ooni.io")
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "mia-ps.ooni.io",
})
killproxy(t, listener)
}
func TestNoSNI(t *testing.T) {
listener := newproxy(t, "ooni.io")
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "",
})
killproxy(t, listener)
}
func TestInvalidDomain(t *testing.T) {
listener := newproxy(t, "ooni.io")
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "antani.local",
})
killproxy(t, listener)
}
func TestFailHandshake(t *testing.T) {
listener := newproxy(t, "ooni.io")
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "expired.badssl.com",
})
killproxy(t, listener)
}
func TestFailConnectingToSelf(t *testing.T) {
proxy := &CensoringProxy{
dial: func(network string, address string) (net.Conn, error) {
return &mockedConnWriteError{}, nil
},
}
listener, err := proxy.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
if listener == nil {
t.Fatal("expected non nil listener here")
}
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "www.google.com",
})
killproxy(t, listener)
}
func TestFailWriteAfterConnect(t *testing.T) {
proxy := &CensoringProxy{
dial: func(network string, address string) (net.Conn, error) {
return &mockedConnWriteError{
// must be different or it refuses connecting to self
localIP: net.IPv4(127, 0, 0, 1),
remoteIP: net.IPv4(127, 0, 0, 2),
}, nil
},
}
listener, err := proxy.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
if listener == nil {
t.Fatal("expected non nil listener here")
}
checkdialtls(t, listener.Addr().String(), false, &tls.Config{
ServerName: "www.google.com",
})
killproxy(t, listener)
}
func TestListenError(t *testing.T) {
proxy := NewCensoringProxy(
[]string{""}, uncensored.DefaultClient,
)
listener, err := proxy.Start("8.8.8.8:80")
if err == nil {
t.Fatal("expected an error here")
}
if listener != nil {
t.Fatal("expected nil listener here")
}
}
func newproxy(t *testing.T, blocked string) net.Listener {
proxy := NewCensoringProxy(
[]string{blocked}, uncensored.DefaultClient,
)
listener, err := proxy.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
return listener
}
func killproxy(t *testing.T, listener net.Listener) {
err := listener.Close()
if err != nil {
t.Fatal(err)
}
}
func checkdialtls(
t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config,
) {
conn, err := tls.Dial("tcp", proxyAddr, config)
if err != nil && expectSuccess {
t.Fatal(err)
}
if err == nil && !expectSuccess {
t.Fatal("expected failure here")
}
if conn == nil && expectSuccess {
t.Fatal("expected actionable conn")
}
if conn != nil && !expectSuccess {
t.Fatal("expected nil conn")
}
if conn != nil {
conn.Close()
}
}
type mockedConnWriteError struct {
net.Conn
localIP net.IP
remoteIP net.IP
}
func (c *mockedConnWriteError) Write(b []byte) (int, error) {
return 0, errors.New("cannot write sorry")
}
func (c *mockedConnWriteError) LocalAddr() net.Addr {
return &net.TCPAddr{
IP: c.localIP,
}
}
func (c *mockedConnWriteError) RemoteAddr() net.Addr {
return &net.TCPAddr{
IP: c.remoteIP,
}
}
func TestForwardWriteError(t *testing.T) {
var wg sync.WaitGroup
wg.Add(1)
forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{})
}
type mockedConnReadOkay struct {
net.Conn
localIP net.IP
remoteIP net.IP
}
func (c *mockedConnReadOkay) Read(b []byte) (int, error) {
return len(b), nil
}

86
internal/engine/cmd/jafar/uncensored/uncensored.go Normal file
View File

@ -0,0 +1,86 @@
// Package uncensored contains code used by Jafar to evade its own
// censorship efforts by taking alternate routes.
package uncensored
import (
"context"
"net"
"net/http"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
// Client is DNS, HTTP, and TCP client.
type Client struct {
dnsClient *netx.DNSClient
httpTransport netx.HTTPRoundTripper
dialer netx.Dialer
}
// NewClient creates a new Client.
func NewClient(resolverURL string) (*Client, error) {
configuration, err := urlgetter.Configurer{
Config: urlgetter.Config{
ResolverURL: resolverURL,
},
Logger: log.Log,
}.NewConfiguration()
if err != nil {
return nil, err
}
return &Client{
dnsClient: &configuration.DNSClient,
httpTransport: netx.NewHTTPTransport(configuration.HTTPConfig),
dialer: netx.NewDialer(configuration.HTTPConfig),
}, nil
}
// Must panics if it's not possible to create a Client. Usually you should
// use it like `uncensored.Must(uncensored.NewClient(URL))`.
func Must(client *Client, err error) *Client {
runtimex.PanicOnError(err, "cannot create uncensored client")
return client
}
// DefaultClient is the default client for DNS, HTTP, and TCP.
var DefaultClient = Must(NewClient(""))
var _ netx.Resolver = DefaultClient
// Address implements netx.Resolver.Address
func (c *Client) Address() string {
return c.dnsClient.Address()
}
// LookupHost implements netx.Resolver.LookupHost
func (c *Client) LookupHost(ctx context.Context, domain string) ([]string, error) {
return c.dnsClient.LookupHost(ctx, domain)
}
// Network implements netx.Resolver.Network
func (c *Client) Network() string {
return c.dnsClient.Network()
}
var _ netx.Dialer = DefaultClient
// DialContext implements netx.Dialer.DialContext
func (c *Client) DialContext(ctx context.Context, network, address string) (net.Conn, error) {
return c.dialer.DialContext(ctx, network, address)
}
var _ netx.HTTPRoundTripper = DefaultClient
// CloseIdleConnections implement netx.HTTPRoundTripper.CloseIdleConnections
func (c *Client) CloseIdleConnections() {
c.dnsClient.CloseIdleConnections()
c.httpTransport.CloseIdleConnections()
}
// RoundTrip implement netx.HTTPRoundTripper.RoundTrip
func (c *Client) RoundTrip(req *http.Request) (*http.Response, error) {
return c.httpTransport.RoundTrip(req)
}

75
internal/engine/cmd/jafar/uncensored/uncensored_test.go Normal file
View File

@ -0,0 +1,75 @@
package uncensored
import (
"bytes"
"context"
"io/ioutil"
"net/http"
"net/url"
"testing"
)
func TestGood(t *testing.T) {
client, err := NewClient("dot://1.1.1.1:853")
if err != nil {
t.Fatal(err)
}
defer client.CloseIdleConnections()
if client.Address() != "1.1.1.1:853" {
t.Fatal("invalid address")
}
if client.Network() != "dot" {
t.Fatal("invalid network")
}
ctx := context.Background()
addrs, err := client.LookupHost(ctx, "dns.google")
if err != nil {
t.Fatal(err)
}
var quad8, two8two4 bool
for _, addr := range addrs {
quad8 = quad8 || (addr == "8.8.8.8")
two8two4 = two8two4 || (addr == "8.8.4.4")
}
if quad8 != true && two8two4 != true {
t.Fatal("invalid response")
}
conn, err := client.DialContext(ctx, "tcp", "8.8.8.8:853")
if err != nil {
t.Fatal(err)
}
defer conn.Close()
resp, err := client.RoundTrip(&http.Request{
Method: "GET",
URL: &url.URL{
Scheme: "https",
Host: "www.google.com",
Path: "/humans.txt",
},
Header: http.Header{},
})
if err != nil {
t.Fatal(err)
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
t.Fatal("invalid status-code")
}
data, err := ioutil.ReadAll(resp.Body)
if err != nil {
t.Fatal(err)
}
if !bytes.HasPrefix(data, []byte("Google is built by a large team")) {
t.Fatal("not the expected body")
}
}
func TestNewClientFailure(t *testing.T) {
clnt, err := NewClient("antani:///")
if err == nil {
t.Fatal("expected an error here")
}
if clnt != nil {
t.Fatal("expected nil client here")
}
}

8
internal/engine/cmd/miniooni/README.md Normal file
View File

@ -0,0 +1,8 @@
# miniooni
This directory contains the source code of a simple CLI client that we
use for research as well as for running QA scripts. We designed this tool
to have a CLI similar to MK and OONI Probe v2.x to ease running Jafar
scripts that check whether these tools behave similarly.
See also libminiooni.

21
internal/engine/cmd/miniooni/main.go Normal file
View File

@ -0,0 +1,21 @@
// Command miniooni is a simple binary for research and QA purposes
// with a CLI interface similar to MK and OONI Probe v2.x.
//
// See also libminiooni, which is where we implement this CLI.
package main
import (
"fmt"
"os"
"github.com/ooni/probe-cli/v3/internal/engine/libminiooni"
)
func main() {
defer func() {
if s := recover(); s != nil {
fmt.Fprintf(os.Stderr, "%s", s)
}
}()
libminiooni.Main()
}

4
internal/engine/cmd/oohelper/README.md Normal file
View File

@ -0,0 +1,4 @@
# oohelper
This directory contains the source code of a simple client
for the Web Connectivity test helper.

138
internal/engine/cmd/oohelper/internal/client.go Normal file
View File

@ -0,0 +1,138 @@
package internal
import (
"bytes"
"context"
"encoding/json"
"errors"
"fmt"
"io/ioutil"
"net"
"net/http"
"net/url"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity"
"github.com/ooni/probe-cli/v3/internal/engine/internal/httpheader"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
"github.com/ooni/probe-cli/v3/internal/engine/version"
)
type (
// CtrlResponse is the type of response returned by the test helper.
CtrlResponse = webconnectivity.ControlResponse
// ctrlRequest is the type of the request sent to the test helper.
ctrlRequest = webconnectivity.ControlRequest
)
// The following errors may be returned by this implementation.
var (
ErrHTTPStatusCode = errors.New("oohelper: http status code indicates failure")
ErrUnsupportedURLScheme = errors.New("oohelper: unsupported URL scheme")
ErrUnsupportedExplicitPort = errors.New("oohelper: unsupported explicit port")
ErrEmptyURL = errors.New("oohelper: empty server and/or target URL")
ErrInvalidURL = errors.New("oohelper: cannot parse URL")
ErrCannotCreateRequest = errors.New("oohelper: cannot create HTTP request")
ErrCannotParseJSONReply = errors.New("oohelper: cannot parse JSON reply")
)
// OOClient is a client for the OONI Web Connectivity test helper.
type OOClient struct {
// HTTPClient is the HTTP client to use.
HTTPClient *http.Client
// Resolver is the resolver to user.
Resolver netx.Resolver
}
// OOConfig contains configuration for the client.
type OOConfig struct {
// ServerURL is the URL of the test helper server.
ServerURL string
// TargetURL is the URL that we want to measure.
TargetURL string
}
// MakeTCPEndpoints constructs the list of TCP endpoints to send
// to the Web Connectivity test helper.
func MakeTCPEndpoints(URL *url.URL, addrs []string) ([]string, error) {
var (
port string
out []string
)
if URL.Host != URL.Hostname() {
return nil, ErrUnsupportedExplicitPort
}
switch URL.Scheme {
case "https":
port = "443"
case "http":
port = "80"
default:
return nil, ErrUnsupportedURLScheme
}
for _, addr := range addrs {
out = append(out, net.JoinHostPort(addr, port))
}
return out, nil
}
// Do sends a measurement request to the Web Connectivity test
// helper and receives the corresponding response.
func (oo OOClient) Do(ctx context.Context, config OOConfig) (*CtrlResponse, error) {
if config.TargetURL == "" || config.ServerURL == "" {
return nil, ErrEmptyURL
}
targetURL, err := url.Parse(config.TargetURL)
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrInvalidURL, err.Error())
}
addrs, err := oo.Resolver.LookupHost(ctx, targetURL.Hostname())
if err != nil {
return nil, err
}
endpoints, err := MakeTCPEndpoints(targetURL, addrs)
if err != nil {
return nil, err
}
creq := ctrlRequest{
HTTPRequest: config.TargetURL,
HTTPRequestHeaders: map[string][]string{
"Accept": {httpheader.Accept()},
"Accept-Language": {httpheader.AcceptLanguage()},
"User-Agent": {httpheader.UserAgent()},
},
TCPConnect: endpoints,
}
data, err := json.Marshal(creq)
runtimex.PanicOnError(err, "oohelper: cannot marshal control request")
log.Debugf("out: %s", string(data))
req, err := http.NewRequestWithContext(ctx, "POST", config.ServerURL, bytes.NewReader(data))
if err != nil {
return nil, fmt.Errorf("%w: %s", ErrCannotCreateRequest, err.Error())
}
req.Header.Add("user-agent", fmt.Sprintf(
"oohelper/%s ooniprobe-engine/%s", version.Version, version.Version,
))
req.Header.Add("content-type", "application/json")
resp, err := oo.HTTPClient.Do(req)
if err != nil {
return nil, err
}
defer resp.Body.Close()
if resp.StatusCode != 200 {
return nil, ErrHTTPStatusCode
}
data, err = ioutil.ReadAll(resp.Body)
if err != nil {
return nil, err
}
var cresp CtrlResponse
if err := json.Unmarshal(data, &cresp); err != nil {
return nil, fmt.Errorf("%w: %s", ErrCannotParseJSONReply, err.Error())
}
return &cresp, nil
}

338
internal/engine/cmd/oohelper/internal/client_test.go Normal file
View File

@ -0,0 +1,338 @@
package internal_test
import (
"context"
"errors"
"net/http"
"net/url"
"reflect"
"testing"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
)
func TestMakeTCPEndpoints(t *testing.T) {
type args struct {
URL *url.URL
addrs []string
}
tests := []struct {
name string
args args
want []string
err error
}{{
name: "with host != hostname",
args: args{URL: &url.URL{Host: "127.0.0.1:8080"}},
err: internal.ErrUnsupportedExplicitPort,
}, {
name: "with unsupported URL scheme",
args: args{URL: &url.URL{Host: "127.0.0.1", Scheme: "imap"}},
err: internal.ErrUnsupportedURLScheme,
}, {
name: "with http scheme",
args: args{
URL: &url.URL{Host: "www.kernel.org", Scheme: "http"},
addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
},
want: []string{"1.1.1.1:80", "2.2.2.2:80", "[::1]:80"},
}, {
name: "with https scheme",
args: args{
URL: &url.URL{Host: "www.kernel.org", Scheme: "https"},
addrs: []string{"1.1.1.1", "2.2.2.2", "::1"},
},
want: []string{"1.1.1.1:443", "2.2.2.2:443", "[::1]:443"},
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := internal.MakeTCPEndpoints(tt.args.URL, tt.args.addrs)
if !errors.Is(err, tt.err) {
t.Errorf("MakeTCPEndpoints() error = %v, wantErr %v", err, tt.err)
return
}
if !reflect.DeepEqual(got, tt.want) {
t.Errorf("MakeTCPEndpoints() = %v, want %v", got, tt.want)
}
})
}
}
func TestOOClientDoWithEmptyTargetURL(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{}
clnt := internal.OOClient{}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrEmptyURL) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithEmptyServerURL(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{TargetURL: "http://www.example.com"}
clnt := internal.OOClient{}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrEmptyURL) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithInvalidTargetURL(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{TargetURL: "\t", ServerURL: "https://wcth.ooni.io"}
clnt := internal.OOClient{}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrInvalidURL) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithResolverFailure(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverThatFails(),
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrNotFound) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithUnsupportedExplicitPort(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com:8080",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrUnsupportedExplicitPort) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithInvalidServerURL(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "\t",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrCannotCreateRequest) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithRoundTripError(t *testing.T) {
expected := errors.New("mocked error")
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
HTTPClient: &http.Client{
Transport: internal.FakeTransport{Err: expected},
},
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithInvalidStatusCode(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
HTTPClient: &http.Client{Transport: internal.FakeTransport{
Resp: &http.Response{
StatusCode: 400,
Body: &internal.FakeBody{},
},
}},
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrHTTPStatusCode) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithBodyReadError(t *testing.T) {
expected := errors.New("mocked error")
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
HTTPClient: &http.Client{Transport: internal.FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: &internal.FakeBody{
Err: expected,
},
},
}},
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
func TestOOClientDoWithInvalidJSON(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
HTTPClient: &http.Client{Transport: internal.FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: &internal.FakeBody{
Data: []byte("{"),
},
},
}},
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, internal.ErrCannotParseJSONReply) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp != nil {
t.Fatal("expected nil response")
}
}
const goodresponse = `{
"tcp_connect": {
"172.217.21.68:80": {
"status": true,
"failure": null
}
},
"http_request": {
"body_length": 207878,
"failure": null,
"title": "Google",
"headers": {
"Content-Type": "text/html"
},
"status_code": 200
},
"dns": {
"failure": null,
"addrs": [
"172.217.17.68"
]
}
}`
func TestOOClientDoWithParseableJSON(t *testing.T) {
ctx := context.Background()
config := internal.OOConfig{
TargetURL: "http://www.example.com",
ServerURL: "https://wcth.ooni.io",
}
clnt := internal.OOClient{
Resolver: internal.NewFakeResolverWithResult([]string{"1.1.1.1"}),
HTTPClient: &http.Client{Transport: internal.FakeTransport{
Resp: &http.Response{
StatusCode: 200,
Body: &internal.FakeBody{
Data: []byte(goodresponse),
},
},
}},
}
cresp, err := clnt.Do(ctx, config)
if !errors.Is(err, nil) {
t.Fatalf("not the error we expected: %+v", err)
}
if cresp.DNS.Failure != nil {
t.Fatal("unexpected Failure value")
}
if len(cresp.DNS.Addrs) != 1 {
t.Fatal("unexpected number of DNS entries")
}
if cresp.DNS.Addrs[0] != "172.217.17.68" {
t.Fatal("unexpected DNS addrs [0]")
}
if cresp.HTTPRequest.BodyLength != 207878 {
t.Fatal("invalid http body length")
}
if cresp.HTTPRequest.Failure != nil {
t.Fatal("invalid http failure")
}
if cresp.HTTPRequest.Title != "Google" {
t.Fatal("invalid http title")
}
if len(cresp.HTTPRequest.Headers) != 1 {
t.Fatal("invalid http headers length")
}
if cresp.HTTPRequest.Headers["Content-Type"] != "text/html" {
t.Fatal("invalid http content-type header")
}
if cresp.HTTPRequest.StatusCode != 200 {
t.Fatal("invalid http status code")
}
if len(cresp.TCPConnect) != 1 {
t.Fatal("invalid tcp connect length")
}
entry, ok := cresp.TCPConnect["172.217.21.68:80"]
if !ok {
t.Fatal("cannot find expected TCP connect entry")
}
if entry.Status != true {
t.Fatal("unexpected TCP connect entry status")
}
if entry.Failure != nil {
t.Fatal("unexpected TCP connect entry failure value")
}
}

102
internal/engine/cmd/oohelper/internal/fake_test.go Normal file
View File

@ -0,0 +1,102 @@
package internal
import (
"context"
"io"
"io/ioutil"
"net"
"net/http"
"time"
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
type FakeResolver struct {
NumFailures *atomicx.Int64
Err error
Result []string
}
func NewFakeResolverThatFails() FakeResolver {
return FakeResolver{NumFailures: atomicx.NewInt64(), Err: ErrNotFound}
}
func NewFakeResolverWithResult(r []string) FakeResolver {
return FakeResolver{NumFailures: atomicx.NewInt64(), Result: r}
}
var ErrNotFound = &net.DNSError{
Err: "no such host",
}
func (c FakeResolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
time.Sleep(10 * time.Microsecond)
if c.Err != nil {
if c.NumFailures != nil {
c.NumFailures.Add(1)
}
return nil, c.Err
}
return c.Result, nil
}
func (c FakeResolver) Network() string {
return "fake"
}
func (c FakeResolver) Address() string {
return ""
}
var _ netx.Resolver = FakeResolver{}
type FakeTransport struct {
Err error
Func func(*http.Request) (*http.Response, error)
Resp *http.Response
}
func (txp FakeTransport) RoundTrip(req *http.Request) (*http.Response, error) {
time.Sleep(10 * time.Microsecond)
if txp.Func != nil {
return txp.Func(req)
}
if req.Body != nil {
ioutil.ReadAll(req.Body)
req.Body.Close()
}
if txp.Err != nil {
return nil, txp.Err
}
txp.Resp.Request = req // non thread safe but it doesn't matter
return txp.Resp, nil
}
func (txp FakeTransport) CloseIdleConnections() {}
var _ netx.HTTPRoundTripper = FakeTransport{}
type FakeBody struct {
Data []byte
Err error
}
func (fb *FakeBody) Read(p []byte) (int, error) {
time.Sleep(10 * time.Microsecond)
if fb.Err != nil {
return 0, fb.Err
}
if len(fb.Data) <= 0 {
return 0, io.EOF
}
n := copy(p, fb.Data)
fb.Data = fb.Data[n:]
return n, nil
}
func (fb *FakeBody) Close() error {
return nil
}
var _ io.ReadCloser = &FakeBody{}

48
internal/engine/cmd/oohelper/oohelper.go Normal file
View File

@ -0,0 +1,48 @@
// Command oohelper contains a simple command line
// client for the Web Connectivity test helper.
package main
import (
"context"
"encoding/json"
"flag"
"fmt"
"net/http"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/engine/cmd/oohelper/internal"
"github.com/ooni/probe-cli/v3/internal/engine/internal/runtimex"
"github.com/ooni/probe-cli/v3/internal/engine/netx"
)
var (
ctx, cancel = context.WithCancel(context.Background())
debug = flag.Bool("debug", false, "Toggle debug mode")
httpClient *http.Client
resolver netx.Resolver
server = flag.String("server", "https://wcth.ooni.io/", "URL of the test helper")
target = flag.String("target", "", "Target URL for the test helper")
)
func init() {
txp := netx.NewHTTPTransport(netx.Config{Logger: log.Log})
httpClient = &http.Client{Transport: txp}
resolver = netx.NewResolver(netx.Config{Logger: log.Log})
}
func main() {
logmap := map[bool]log.Level{
true: log.DebugLevel,
false: log.InfoLevel,
}
flag.Parse()
log.SetLevel(logmap[*debug])
clnt := internal.OOClient{HTTPClient: httpClient, Resolver: resolver}
config := internal.OOConfig{TargetURL: *target, ServerURL: *server}
defer cancel()
cresp, err := clnt.Do(ctx, config)
runtimex.PanicOnError(err, "client.Do failed")
data, err := json.MarshalIndent(cresp, "", " ")
runtimex.PanicOnError(err, "json.MarshalIndent failed")
fmt.Printf("%s\n", string(data))
}

8
internal/engine/cmd/oohelper/oohelper_test.go Normal file
View File

@ -0,0 +1,8 @@
package main
import "testing"
func TestSmoke(t *testing.T) {
*target = "http://www.example.com"
main()
}

Some files were not shown because too many files have changed in this diff Show More