From b9b555ba68a8c2f843d72bcb7c1810984785ac09 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Thu, 15 Aug 2019 18:08:43 +0200 Subject: [PATCH] Integrate further with ooni/probe-engine: episode two (#46) * utils/geoip.go: use github.com/ooni/probe-engine Let's start using the engine by rewriting utils/geoip.go to be just a thin wrapper around the engine functionality. * Ready for review * Checkpoint: the im tests are converted Still have some doubts with respect to the variables that are passed to MK via probe-engine. Will double check. * fix(i/c/r/run.go): write the correct logic * nettests: one more comment and also fix a format string * Tweak previous * progress * Fix doofus * better comment * XXX => actionable comment * Add glue to simplify test keys management Making the concept of measurement more abstract in the engine is not feasible because, when submitting a measurement, we need to modify it to update the report ID and the measurement ID. Therefore, returning a serialized measurement is not a good idea. We will keep using a model.Measurement in the engine. Changing model.Measurement.TestKeys's type from a `interface{}` pointing to a well defined data structure to `map[string]interface{}` is a regression because means that we are moving from code that has a clear and defined structure to code that is more complicated to parse and validate. Since we're already suffering havily from the lack of a good schema, I'm not going to make the situation worst by worsening the engine. At least for ndt7 and psiphon, we now have a good schema and I don't want to lose that. However, the current code in this repository is expecting the test keys to be a `map[string]interface{}`. This choice was dictated by the fact that we receive a JSON from Measurement Kit and by the fact that there's not a clear schema. To solve this tension, in this commit I am going to write glue adapter code that makes sure that the TestKeys of a Measurement are converted to `map[string]interface{}`. This will be done using a type cast where possible and JSON serialization and parsing otherwise. In a perfect world, glue is not a good idea, but in a real world it may actually be useful. When all tests in the engine will have a clear Go data structure, we'll then remove the glue and just cast to the proper data structure from `interface{}` where required. * nettests/performance: use probe-engine * go.{mod,sum}: upgrade to latest probe-engine * nettests/middlebox: use ooni/probe-engine * Update to the latest probe-engine * web_connectivity: rewrite to use probe-engine * Cosmetic change suggested by @hellais * nettests/nettests.go: remove unused code * nettests/nettests.go: fix progress * nettests/nettests.go: remove go-measurement-kit code * We don't depend on go-measurement-kit anymore * Improve non-verbose output where possible See also: https://github.com/measurement-kit/measurement-kit/issues/1856 * Make web_connectivity output pleasant * Update to the latest probe-engine * nettests/nettests.go: honour sharing settings * Update to the latest probe-engine * Use log.WithFields for probe-engine * Update go.mod go.sum * Revert "Update go.mod go.sum" This reverts commit 5ecd38d8236f4a4e9b77ddb8e8a0d1e3cdd4b818. * Revert "Revert "Update go.mod go.sum"" This reverts commit 6114b31eca98826112032776bd0feff02d763ecd. * Upgrade ooni/probe-engine * Unset GOPATH before running go build commands * Dockefile: fix linux build by using latest * Update to the latest ooni/probe-engine ``` go get -u github.com/ooni/probe-engine go mod tidy ``` * Repair build --- Dockerfile | 2 +- Makefile | 2 +- go.mod | 16 +- go.sum | 85 +++-- internal/cli/run/run.go | 17 + internal/enginex/enginex.go | 38 ++ internal/log/handlers/cli/cli.go | 3 + nettests/im/facebook_messenger.go | 9 +- nettests/im/telegram.go | 9 +- nettests/im/whatsapp.go | 9 +- .../http_header_field_manipulation.go | 9 +- .../middlebox/http_invalid_request_line.go | 9 +- nettests/nettests.go | 339 +++++++----------- nettests/performance/dash.go | 9 +- nettests/performance/ndt.go | 9 +- nettests/websites/web_connectivity.go | 68 +--- ooni.go | 17 +- 17 files changed, 315 insertions(+), 335 deletions(-) create mode 100644 internal/enginex/enginex.go diff --git a/Dockerfile b/Dockerfile index 2c26371..db8b1c6 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,3 +1,3 @@ -FROM openobservatory/mk-alpine:20190509 +FROM openobservatory/mk-alpine:latest RUN apk add --no-progress git go ADD . /oonibuild diff --git a/Makefile b/Makefile index c338dc6..b0767ea 100644 --- a/Makefile +++ b/Makefile @@ -1,4 +1,4 @@ -GO ?= go +GO ?= GOPATH="" go install-dev-deps: @$(GO) get golang.org/x/tools/cmd/cover diff --git a/go.mod b/go.mod index 7dd4571..03f3321 100644 --- a/go.mod +++ b/go.mod @@ -4,24 +4,24 @@ go 1.12 require ( github.com/alecthomas/kingpin v2.2.6+incompatible - github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect - github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect - github.com/apex/log v1.1.0 + github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 // indirect + github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 // indirect + github.com/apex/log v1.1.1 github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect github.com/fatih/color v1.7.0 github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea github.com/go-sql-driver/mysql v1.4.1 // indirect github.com/gobuffalo/packr v1.25.0 // indirect + github.com/kr/pty v1.1.8 // indirect github.com/lib/pq v1.1.1 // indirect - github.com/mattn/go-colorable v0.0.9 - github.com/mattn/go-isatty v0.0.7 // indirect + github.com/mattn/go-colorable v0.1.2 github.com/mattn/go-sqlite3 v1.10.0 // indirect - github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d - github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea + github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632 + github.com/oschwald/maxminddb-golang v1.3.1 // indirect github.com/pkg/errors v0.8.1 github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051 github.com/ziutek/mymysql v1.5.4 // indirect - google.golang.org/appengine v1.5.0 // indirect + google.golang.org/appengine v1.6.1 // indirect gopkg.in/AlecAivazis/survey.v1 v1.8.4 gopkg.in/gorp.v1 v1.7.2 // indirect upper.io/db.v3 v3.5.7+incompatible diff --git a/go.sum b/go.sum index 1f1dca1..33e717a 100644 --- a/go.sum +++ b/go.sum @@ -14,22 +14,29 @@ github.com/Yawning/chacha20 v0.0.0-20170904085104-e3b1f968fc63/go.mod h1:nf+Komq github.com/agl/ed25519 v0.0.0-20170116200512-5312a6153412/go.mod h1:WPjqKcmVOxf0XSf3YxCJs6N6AOSrOx3obionmG7T0y0= github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= -github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= -github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= -github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= -github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= +github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4 h1:Hs82Z41s6SdL1CELW+XaDYmOH4hkBN4/N9og/AsOv7E= +github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apex/log v1.1.1 h1:BwhRZ0qbjYtTob0I+2M+smavV0kOC8XgcnGZcyL9liA= +github.com/apex/log v1.1.1/go.mod h1:Ls949n1HFtXfbDcjiTTFQqkVUrte0puoIBfO3SVgwOA= +github.com/aphistic/golf v0.0.0-20180712155816-02c07f170c5a/go.mod h1:3NqKYiepwy8kCu4PNA+aP7WUV72eXWJeP9/r3/K9aLE= +github.com/aphistic/sweet v0.2.0/go.mod h1:fWDlIh/isSE9n6EPsRmC0det+whmX6dJid3stzu0Xys= github.com/aristanetworks/goarista v0.0.0-20190514202536-8f808a500156/go.mod h1:D/tb0zPVXnP7fmsLZjtdUhSsumbK/ij54UXjjVgMGxQ= github.com/armon/go-proxyproto v0.0.0-20190211145416-68259f75880e/go.mod h1:QmP9hvJ91BbJmGVGSbutW19IC0Q9phDCLGaomwTJbgU= github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5/go.mod h1:wHh0iHkYZB8zMSxRWpUBQtwG5a7fFgvEO+odwuTv2gs= +github.com/avast/retry-go v2.4.1+incompatible h1:WMHc0mwoz20UVmBYK89mUB/KFRlxO0p+s+sgpmJMviY= +github.com/avast/retry-go v2.4.1+incompatible/go.mod h1:XtSnn+n/sHqQIpZ10K1qAevBhOOCWBLXXy3hyiqqBrY= +github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= +github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= github.com/bifurcation/mint v0.0.0-20180715133206-93c51c6ce115/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= github.com/boltdb/bolt v1.3.1/go.mod h1:clJnj/oiGkjum5o1McbSZDSLxVThjynRyGBgiAx27Ps= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= github.com/cheekybits/genny v0.0.0-20170328200008-9127e812e1e9/go.mod h1:+tQajlRqAUrPI7DOSpB0XAqZYtQakVtB7wXkRAgjxjQ= github.com/creack/goselect v0.1.0/go.mod h1:gHrIcH/9UZDn2qgeTUeW5K9eZsVYCH6/60J/FHysWyE= -github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/creack/pty v1.1.7 h1:6pwm8kMQKCmgUg0ZHTm5+/YvRK0s3THD/28+T6/kk4A= +github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= @@ -44,6 +51,7 @@ github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5Kwzbycv github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea h1:vpAHg3H71YFc5TqRSqbhMq8Wd0kdPoQMMAeQSgnUTpg= github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= @@ -71,8 +79,11 @@ github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY9 github.com/gobwas/glob v0.2.3/go.mod h1:d3Ez4x06l9bZtSvzIay5+Yzi0fmZzPgnTbPcKjJAkT8= github.com/golang/mock v1.2.0/go.mod h1:oTYuIxOrZwtPieC+H1uAHpcLFnEyAGVDL/k47Jfbm0A= github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.1/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/golang/protobuf v1.3.2/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/gxui v0.0.0-20151028112939-f85e0a97b3a4/go.mod h1:Pw1H1OjSNHiqeuxAduB1BKYXIwFtsyrY47nEqSgEiCM= +github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/websocket v1.4.0/go.mod h1:E7qHFY5m1UJ88s3WnNqhKjPHQ0heANvMoAMk2YaljkQ= github.com/grafov/m3u8 v0.6.1/go.mod h1:PdjzaU/pJUo4jTIn2rcgMFs+HqBGl/sPJLr8BI0Xq/I= @@ -81,18 +92,23 @@ github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDG github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/jpillora/backoff v0.0.0-20180909062703-3050d21c67d7/go.mod h1:2iMrUgbbvHEiQClaW2NsSzMyGHqN+rDFqY705q49KG0= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ratelimit v1.0.1/go.mod h1:qapgC/Gy+xNh9UxzV13HGGl/6UXNN+ct+vwSgWNm/qk= github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/pty v1.1.8 h1:AkaSdXYQOWeaO3neb8EM634ahkXXe3jYbVh/F9lq+GI= +github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= @@ -101,32 +117,36 @@ github.com/lucas-clemente/aes12 v0.0.0-20171027163421-cd47fb39b79f/go.mod h1:JpH github.com/lucas-clemente/quic-clients v0.1.0/go.mod h1:y5xVIEoObKqULIKivu+gD/LU90pL73bTdtQjPBvtCBk= github.com/lucas-clemente/quic-go v0.10.2/go.mod h1:hvaRS9IHjFLMq76puFJeWNfmn+H70QZ/CXoxqw9bzao= github.com/lucas-clemente/quic-go-certificates v0.0.0-20160823095156-d2f86524cced/go.mod h1:NCcRLrOTZbzhZvixZLlERbJtDtYsmMw8Jc4vS8Z0g58= -github.com/m-lab/ndt7-client-go v0.0.0-20190516113520-f8aec6e1ef2e/go.mod h1:nD9WTkxP4/mV2ph5uUpVPD2ZOJdZ9QKPTKu3tJGkM9o= +github.com/m-lab/ndt7-client-go v0.0.0-20190724152841-ad7eefc52fe1/go.mod h1:nD9WTkxP4/mV2ph5uUpVPD2ZOJdZ9QKPTKu3tJGkM9o= github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= github.com/marten-seemann/qtls v0.2.3/go.mod h1:xzjG7avBwGGbdZ8dTGxlBnLArsVKLvwmjgmPuiQEcYk= github.com/marusama/semaphore v0.0.0-20190110074507-6952cef993b2/go.mod h1:TmeOqAKoDinfPfSohs14CO3VcEf7o+Bem6JiNe05yrQ= github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= +github.com/mattn/go-colorable v0.1.2 h1:/bC9yWikZXAL9uJdulbSfyVNIR3n3trXl+v8+1sx8mU= +github.com/mattn/go-colorable v0.1.2/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= -github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= -github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= +github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= -github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d h1:w8l3iUS/zDNcNQwios6icSyH3qXrDE+D6k1kSob0RnU= -github.com/measurement-kit/go-measurement-kit v0.0.0-20190521082856-635e836bbb9d/go.mod h1:n4Mki43BUXopDPfUkgry9vmSzwZ7dVL5Mx5mRUCWKEw= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= github.com/miekg/dns v1.1.12/go.mod h1:W1PPwlIAgtquWBMBEV9nkV9Cazfe8ScdGz/Lj7v3Nrg= github.com/onsi/ginkgo v1.6.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/ginkgo v1.8.0/go.mod h1:lLunBs/Ym6LB5Z9jYTR76FiuTmxDTDusOGeTQH+WWjE= github.com/onsi/gomega v1.5.0/go.mod h1:ex+gbHU/CVuBBDIJjb2X0qEXbFg53c61hWP/1CpauHY= -github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea h1:Dx1uyp66YkHDR42r66cA1GlCp4y5hF2fuW+apL9/oV0= -github.com/ooni/probe-engine v0.0.0-20190522105151-0c60545f09ea/go.mod h1:o6RJkZwHVn6GP7xMLVdRsd4pOxXeo1tJKK29TKKiRhA= +github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632 h1:58EgLL9v2729NkY0U/lixOAGr9UsvRH67use+iJvz+c= +github.com/ooni/probe-engine v0.0.0-20190814091928-473884e88632/go.mod h1:iq63Xvw89LLS+2Q9vdqNpmuGQLntyomd9K5w49hU+Ks= github.com/oschwald/geoip2-golang v1.3.0 h1:D+Hsdos1NARPbzZ2aInUHZL+dApIzo8E0ErJVsWcku8= github.com/oschwald/geoip2-golang v1.3.0/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE= github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE= github.com/oschwald/maxminddb-golang v1.3.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/oschwald/maxminddb-golang v1.3.1 h1:kPc5+ieL5CC/Zn0IaXJPxDFlUxKTQEU8QBTtmfQDAIo= +github.com/oschwald/maxminddb-golang v1.3.1/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= github.com/patrickmn/go-cache v2.1.0+incompatible/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/pborman/getopt v0.0.0-20190409184431-ee0cd42419d3/go.mod h1:85jBQOZwpVEaDAr341tbn15RS4fCAsIst0qp7i8ex1o= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= @@ -136,6 +156,7 @@ github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZb github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= github.com/redjack/marionette v0.0.0-20180930054334-5d0d25fc4084/go.mod h1:yJd0pT0e04p+VSmLGjce8BoPlRDlrGrdfXf2En7oq9A= github.com/refraction-networking/utls v0.0.0-20190415193640-32987941ebd3/go.mod h1:tz9gX959MEFfFN5whTIocCLUG57WiILqtdVxI8c6Wj0= +github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc= github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= @@ -146,21 +167,29 @@ github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051/go.mod h1:WS0rl github.com/ryanuber/go-glob v1.0.0/go.mod h1:807d1WSdnB0XRJzKNil9Om6lcp/3a0v4qIHxIXzX/Yc= github.com/sergeyfrolov/bsbuffer v0.0.0-20180903213811-94e85abb8507/go.mod h1:DbI1gxrXI2jRGw7XGEUZQOOMd6PsnKzRrCKabvvMrwM= github.com/sergeyfrolov/gotapdance v0.0.0-20190321212638-ca457d7d62c3/go.mod h1:iQJhqHl49y9CULep5DOD+0W5dKgky8GQOioe3pUd/FY= +github.com/sergi/go-diff v1.0.0/go.mod h1:0CfEIISq7TuYL3j771MWULgwwjU+GofnZX9QAmXWZgo= github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/assertions v1.0.1/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= +github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= github.com/smartystreets/goconvey v0.0.0-20190330032615-68dc04aab96a/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= +github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= -github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= +github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= +github.com/tj/go-elastic v0.0.0-20171221160941-36157cbbebc2/go.mod h1:WjeM0Oo1eNAjXGDx2yma7uG2XoyRZTq1uv3M/o7imD0= +github.com/tj/go-kinesis v0.0.0-20171128231115-08b17f58cb1b/go.mod h1:/yhzCV0xPfx6jb1bBgRFjl5lytqVqZXEaeqWP8lTEao= +github.com/tj/go-spin v1.1.0/go.mod h1:Mg1mzmePZm4dva8Qz60H2lHwmJ2loum4VIrLgVnKwh4= github.com/zach-klippenstein/goregen v0.0.0-20160303162051-795b5e3961ea/go.mod h1:eNr558nEUjP8acGw8FFjTeWvSgU1stO7FAO6eknhHe4= github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= @@ -168,33 +197,44 @@ go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= go.uber.org/zap v1.10.0/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= -golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b h1:Elez2XeF2p9uyVj0yEUDqQ56NFcDtcBNkYP7yv8YbUE= golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190228161510-8dd112bcdc25/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= -golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/crypto v0.0.0-20190426145343-a29dc8fdc734/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190605123033-f99c8df09eb5/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4 h1:HuIa8hRrWRSrqYzx1qI49NNxhdi2PrY7gxVSq1JjLDc= +golang.org/x/crypto v0.0.0-20190701094942-4def268fd1a4/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= -golang.org/x/net v0.0.0-20190509222800-a4d6f7feada5/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/net v0.0.0-20190603091049-60506f45cf65/go.mod h1:HSz+uSET+XFnRR8LxR5pz3Of3rY3CfYBVs4xY44aLks= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190724013045-ca1201d0de80/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190228124157-a34e9553db1e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190412213103-97732733099d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20190422165155-953cdadca894/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= -golang.org/x/sys v0.0.0-20190516110030-61b9204099cb h1:k07iPOt0d6nEnwXF+kHB+iEg+WSuKe/SOQuFM2QoD+E= -golang.org/x/sys v0.0.0-20190516110030-61b9204099cb/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190606165138-5da285871e9c/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a h1:aYOabOQFp6Vj6W1F80affTUvO9UxmJRx8K0gsfABByQ= +golang.org/x/sys v0.0.0-20190813064441-fde4db37ae7a/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/text v0.3.2/go.mod h1:bEr9sfX3Q8Zfm5fL9x+3itogRgK3+ptLWKqgva+5dAk= +golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20190328211700-ab21143f2384/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190404132500-923d25813098/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= -google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= -google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +golang.org/x/tools v0.0.0-20190606124116-d0a3d012864b/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +google.golang.org/appengine v1.6.1 h1:QzqyMA1tlu6CgqCDUtU9V+ZKhLFT2dkJuANu5QaxI3I= +google.golang.org/appengine v1.6.1/go.mod h1:i06prIuMbXzDqacNJfV5OdTW448YApPu5ww/cMBSeb0= gopkg.in/AlecAivazis/survey.v1 v1.8.4 h1:10xXXN3wgIhPheb5NI58zFgZv32Ana7P3Tl4shW+0Qc= gopkg.in/AlecAivazis/survey.v1 v1.8.4/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= gopkg.in/check.v1 v0.0.0-20161208181325-20d25e280405/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= @@ -206,5 +246,6 @@ gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWDmTeBkI65Dw0HsyUHuEVlX15mw= gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= upper.io/db.v3 v3.5.7+incompatible h1:3MJSnJQ+NMxBxuNwO+gOKFiugwv+f61LbyuZYSPzoi4= upper.io/db.v3 v3.5.7+incompatible/go.mod h1:FgTdD24eBjJAbPKsQSiHUNgXjOR4Lub3u1UMHSIh82Y= diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 2e08221..7c7bd08 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -1,6 +1,7 @@ package run import ( + "context" "errors" "fmt" "strings" @@ -95,6 +96,22 @@ func init() { log.WithError(err).Error("Failed to create the network row") return err } + if ctx.Config.Advanced.BouncerURL != "" { + ctx.Session.AddAvailableHTTPSBouncer(ctx.Config.Advanced.BouncerURL) + } + if err := ctx.Session.MaybeLookupBackends(context.Background()); err != nil { + log.WithError(err).Warn("Failed to discover available test helpers") + // Rationale for falling through: some tests may be able to complete + // with no test helpers, so stopping may be excessive here. + } + if ctx.Config.Sharing.UploadResults { + if ctx.Config.Advanced.CollectorURL != "" { + ctx.Session.AddAvailableHTTPSCollector(ctx.Config.Advanced.CollectorURL) + } else if err := ctx.Session.MaybeLookupCollectors(context.Background()); err != nil { + log.WithError(err).Error("Failed to discover available collectors") + return err + } + } if *nettestGroup == "" { log.Infof("Running %s tests", color.BlueString("all")) diff --git a/internal/enginex/enginex.go b/internal/enginex/enginex.go new file mode 100644 index 0000000..5c90b51 --- /dev/null +++ b/internal/enginex/enginex.go @@ -0,0 +1,38 @@ +// Package enginex contains ooni/probe-engine extensions. +package enginex + +import ( + "encoding/json" + + "github.com/apex/log" + "github.com/ooni/probe-engine/model" +) + +// Logger is the logger used by the engine. +var Logger = log.WithFields(log.Fields{ + "type": "engine", +}) + +// MakeGenericTestKeys casts the m.TestKeys to a map[string]interface{}. +// +// Ideally, all tests should have a clear Go structure, well defined, that +// will be stored in m.TestKeys as an interface. This is not already the +// case and it's just valid for tests written in Go. Until all tests will +// be written in Go, we'll keep this glue here to make sure we convert from +// the engine format to the cli format. +// +// This function will first attempt to cast directly to map[string]interface{}, +// which is possible for MK tests, and then use JSON serialization and +// de-serialization only if that's required. +func MakeGenericTestKeys(m model.Measurement) (map[string]interface{}, error) { + if result, ok := m.TestKeys.(map[string]interface{}); ok { + return result, nil + } + data, err := json.Marshal(m.TestKeys) + if err != nil { + return nil, err + } + var result map[string]interface{} + err = json.Unmarshal(data, &result) + return result, err +} diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index a478a7f..b826db5 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -104,6 +104,9 @@ func logTable(w io.Writer, f log.Fields) error { // TypedLog is used for handling special "typed" logs to the CLI func (h *Handler) TypedLog(t string, e *log.Entry) error { switch t { + case "engine": + fmt.Fprintf(h.Writer, "[engine] %s\n", e.Message) + return nil case "progress": perc := e.Fields.Get("percentage").(float64) * 100 s := fmt.Sprintf(" %s %-25s", diff --git a/nettests/im/facebook_messenger.go b/nettests/im/facebook_messenger.go index c0c237c..f513886 100644 --- a/nettests/im/facebook_messenger.go +++ b/nettests/im/facebook_messenger.go @@ -1,8 +1,8 @@ package im import ( - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/fbmessenger" ) // FacebookMessenger test implementation @@ -11,9 +11,10 @@ type FacebookMessenger struct { // Run starts the test func (h FacebookMessenger) Run(ctl *nettests.Controller) error { - mknt := mk.NewNettest("FacebookMessenger") - ctl.Init(mknt) - return mknt.Run() + experiment := fbmessenger.NewExperiment(ctl.Ctx.Session, fbmessenger.Config{ + LogLevel: "INFO", + }) + return ctl.Run(experiment, []string{""}) } // FacebookMessengerTestKeys for the test diff --git a/nettests/im/telegram.go b/nettests/im/telegram.go index 10cbf00..520d41b 100644 --- a/nettests/im/telegram.go +++ b/nettests/im/telegram.go @@ -1,8 +1,8 @@ package im import ( - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/telegram" ) // Telegram test implementation @@ -11,9 +11,10 @@ type Telegram struct { // Run starts the test func (h Telegram) Run(ctl *nettests.Controller) error { - mknt := mk.NewNettest("Telegram") - ctl.Init(mknt) - return mknt.Run() + experiment := telegram.NewExperiment(ctl.Ctx.Session, telegram.Config{ + LogLevel: "INFO", + }) + return ctl.Run(experiment, []string{""}) } // TelegramTestKeys for the test diff --git a/nettests/im/whatsapp.go b/nettests/im/whatsapp.go index 10f9f44..f3f387c 100644 --- a/nettests/im/whatsapp.go +++ b/nettests/im/whatsapp.go @@ -1,8 +1,8 @@ package im import ( - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/whatsapp" ) // WhatsApp test implementation @@ -11,9 +11,10 @@ type WhatsApp struct { // Run starts the test func (h WhatsApp) Run(ctl *nettests.Controller) error { - mknt := mk.NewNettest("Whatsapp") - ctl.Init(mknt) - return mknt.Run() + experiment := whatsapp.NewExperiment(ctl.Ctx.Session, whatsapp.Config{ + LogLevel: "INFO", + }) + return ctl.Run(experiment, []string{""}) } // WhatsAppTestKeys for the test diff --git a/nettests/middlebox/http_header_field_manipulation.go b/nettests/middlebox/http_header_field_manipulation.go index 6b6b335..e6af0c8 100644 --- a/nettests/middlebox/http_header_field_manipulation.go +++ b/nettests/middlebox/http_header_field_manipulation.go @@ -3,8 +3,8 @@ package middlebox import ( "errors" - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/hhfm" ) // HTTPHeaderFieldManipulation test implementation @@ -13,9 +13,10 @@ type HTTPHeaderFieldManipulation struct { // Run starts the test func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error { - mknt := mk.NewNettest("HttpHeaderFieldManipulation") - ctl.Init(mknt) - return mknt.Run() + experiment := hhfm.NewExperiment(ctl.Ctx.Session, hhfm.Config{ + LogLevel: "INFO", + }) + return ctl.Run(experiment, []string{""}) } // HTTPHeaderFieldManipulationTestKeys for the test diff --git a/nettests/middlebox/http_invalid_request_line.go b/nettests/middlebox/http_invalid_request_line.go index 0d7cc7e..b12afc0 100644 --- a/nettests/middlebox/http_invalid_request_line.go +++ b/nettests/middlebox/http_invalid_request_line.go @@ -3,8 +3,8 @@ package middlebox import ( "errors" - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/hirl" ) // HTTPInvalidRequestLine test implementation @@ -13,9 +13,10 @@ type HTTPInvalidRequestLine struct { // Run starts the test func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error { - mknt := mk.NewNettest("HttpInvalidRequestLine") - ctl.Init(mknt) - return mknt.Run() + experiment := hirl.NewExperiment(ctl.Ctx.Session, hirl.Config{ + LogLevel: "INFO", + }) + return ctl.Run(experiment, []string{""}) } // HTTPInvalidRequestLineTestKeys for the test diff --git a/nettests/nettests.go b/nettests/nettests.go index bc6beaa..ae21cc4 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -1,22 +1,23 @@ package nettests import ( + "context" "database/sql" - "encoding/json" "fmt" "path/filepath" "time" "github.com/apex/log" "github.com/fatih/color" - "github.com/measurement-kit/go-measurement-kit" ooni "github.com/ooni/probe-cli" - "github.com/ooni/probe-cli/internal/crashreport" "github.com/ooni/probe-cli/internal/database" + "github.com/ooni/probe-cli/internal/enginex" "github.com/ooni/probe-cli/internal/output" "github.com/ooni/probe-cli/utils" - "github.com/ooni/probe-cli/utils/strcase" - "github.com/ooni/probe-cli/version" + "github.com/ooni/probe-engine/experiment" + "github.com/ooni/probe-engine/experiment/handler" + "github.com/ooni/probe-engine/model" + "github.com/pkg/errors" ) // Nettest interface. Every Nettest should implement this. @@ -50,6 +51,12 @@ type Controller struct { msmts map[int64]*database.Measurement msmtPath string // XXX maybe we can drop this and just use a temporary file inputIdxMap map[int64]int64 // Used to map mk idx to database id + + // numInputs is the total number of inputs + numInputs int + + // curInputIdx is the current input index + curInputIdx int } // SetInputIdxMap is used to set the mapping of index into input. This mapping @@ -67,252 +74,148 @@ func (c *Controller) SetNettestIndex(i, n int) { c.ntIndex = i } -// Init should be called once to initialise the nettest -func (c *Controller) Init(nt *mk.Nettest) error { - log.Debugf("Init: %v", nt) - err := c.Ctx.MaybeLocationLookup() - if err != nil { - return err - } +// Run runs the selected nettest using the related experiment +// with the specified inputs. +// +// This function will continue to run in most cases but will +// immediately halt if something's wrong with the file system. +func (c *Controller) Run(exp *experiment.Experiment, inputs []string) error { + ctx := context.Background() + + // This will configure the controller as handler for the callbacks + // called by ooni/probe-engine/experiment.Experiment. + exp.Callbacks = handler.Callbacks(c) + c.numInputs = len(inputs) c.msmts = make(map[int64]*database.Measurement) // These values are shared by every measurement - reportID := sql.NullString{String: "", Valid: false} - testName := strcase.ToSnake(nt.Name) + var reportID sql.NullString resultID := c.res.ID - reportFilePath := c.msmtPath - geoIPCountryPath := c.Ctx.Session.CountryDatabasePath() - geoIPASNPath := c.Ctx.Session.ASNDatabasePath() - msmtPath := c.msmtPath - log.Debugf("OutputPath: %s", msmtPath) - nt.Options = mk.NettestOptions{ - IncludeIP: c.Ctx.Config.Sharing.IncludeIP, - IncludeASN: c.Ctx.Config.Sharing.IncludeASN, - IncludeCountry: c.Ctx.Config.Sharing.IncludeCountry, - LogLevel: "INFO", + log.Debug(color.RedString("status.queued")) + log.Debug(color.RedString("status.started")) + log.Debugf("OutputPath: %s", c.msmtPath) - ProbeCC: c.Ctx.Session.ProbeCC(), - ProbeASN: c.Ctx.Session.ProbeASNString(), - ProbeIP: c.Ctx.Session.ProbeIP(), - - DisableReportFile: false, - DisableCollector: !c.Ctx.Config.Sharing.UploadResults, - RandomizeInput: false, // It's important to disable input randomization to ensure the URLs are written in sync to the DB - SoftwareName: "ooniprobe-desktop", - SoftwareVersion: version.Version, - CollectorBaseURL: c.Ctx.Config.Advanced.CollectorURL, - BouncerBaseURL: c.Ctx.Config.Advanced.BouncerURL, - - OutputPath: msmtPath, - GeoIPCountryPath: geoIPCountryPath, - GeoIPASNPath: geoIPASNPath, - CaBundlePath: c.Ctx.Session.CABundlePath(), + if c.Ctx.Config.Sharing.UploadResults { + if err := exp.OpenReport(ctx); err != nil { + log.Debugf( + "%s: %s", color.RedString("failure.report_create"), err.Error(), + ) + } else { + defer exp.CloseReport(ctx) + log.Debugf(color.RedString("status.report_create")) + reportID = sql.NullString{String: exp.ReportID(), Valid: true} + } } - log.Debugf("CaBundlePath: %s", nt.Options.CaBundlePath) - log.Debugf("GeoIPASNPath: %s", nt.Options.GeoIPASNPath) - log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) - - nt.On("log", func(e mk.Event) { - level := e.Value.LogLevel - msg := e.Value.Message - - switch level { - case "ERROR": - log.Errorf("%v: %s", color.RedString("mklog"), msg) - case "INFO": - log.Infof("%v: %s", color.BlueString("mklog"), msg) - default: - log.Debugf("%v: %s", color.WhiteString("mklog"), msg) - } - - }) - - nt.On("status.queued", func(e mk.Event) { - log.Debugf("%s", e.Key) - }) - - nt.On("status.started", func(e mk.Event) { - log.Debugf("%s", e.Key) - }) - - nt.On("status.report_create", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - reportID = sql.NullString{String: e.Value.ReportID, Valid: true} - }) - - nt.On("failure.report_create", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - - nt.On("status.geoip_lookup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - }) - - nt.On("status.measurement_start", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - idx := e.Value.Idx - urlID := sql.NullInt64{Int64: 0, Valid: false} + for idx, input := range inputs { + c.curInputIdx = idx // allow for precise progress + idx64 := int64(idx) + log.Debug(color.RedString("status.measurement_start")) + var urlID sql.NullInt64 if c.inputIdxMap != nil { - urlID = sql.NullInt64{Int64: c.inputIdxMap[idx], Valid: true} + urlID = sql.NullInt64{Int64: c.inputIdxMap[idx64], Valid: true} } - msmt, err := database.CreateMeasurement(c.Ctx.DB, reportID, testName, resultID, reportFilePath, urlID) + msmt, err := database.CreateMeasurement( + c.Ctx.DB, reportID, exp.TestName, resultID, c.msmtPath, urlID, + ) if err != nil { - log.WithError(err).Error("Failed to create measurement") - return + return errors.Wrap(err, "failed to create measurement") } - c.msmts[idx] = msmt - }) + c.msmts[idx64] = msmt - nt.On("status.progress", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - perc := e.Value.Percentage - if c.ntCount > 0 { - perc = float64(c.ntIndex)/float64(c.ntCount) + perc/float64(c.ntCount) - } - c.OnProgress(perc, e.Value.Message) - }) - - nt.On("status.update.*", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - }) - - // XXX should these be made into permanent failures? - nt.On("failure.asn_lookup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - nt.On("failure.cc_lookup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - nt.On("failure.ip_lookup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - - nt.On("failure.resolver_lookup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - - nt.On("failure.report_close", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - log.Debugf("%v", e.Value) - }) - - nt.On("failure.startup", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure) - }) - - nt.On("failure.measurement", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - c.msmts[e.Value.Idx].Failed(c.Ctx.DB, e.Value.Failure) - }) - - nt.On("failure.measurement_submission", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - failure := e.Value.Failure - c.msmts[e.Value.Idx].UploadFailed(c.Ctx.DB, failure) - }) - - nt.On("status.measurement_submission", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - // XXX maybe this should change once MK is aligned with the spec - if c.Ctx.Config.Sharing.UploadResults == true { - if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil { - log.WithError(err).Error("failed to mark msmt as uploaded") + measurement, err := exp.Measure(ctx, input) + if err != nil { + log.WithError(err).Debug(color.RedString("failure.measurement")) + if err := c.msmts[idx64].Failed(c.Ctx.DB, err.Error()); err != nil { + return errors.Wrap(err, "failed to mark measurement as failed") } + continue } - }) - nt.On("status.measurement_done", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - - if err := c.msmts[e.Value.Idx].Done(c.Ctx.DB); err != nil { - log.WithError(err).Error("failed to mark msmt as done") + // Make sure we share what the user wants us to share. + if c.Ctx.Config.Sharing.IncludeIP == false { + measurement.ProbeIP = model.DefaultProbeIP + } + if c.Ctx.Config.Sharing.IncludeASN == false { + measurement.ProbeASN = fmt.Sprintf("AS%d", model.DefaultProbeASN) + } + if c.Ctx.Config.Sharing.IncludeCountry == false { + measurement.ProbeCC = model.DefaultProbeCC } - }) - nt.On("measurement", func(e mk.Event) { - log.Debugf("status.end") - - crashreport.CapturePanicAndWait(func() { - c.OnEntry(e.Value.Idx, e.Value.JSONStr) - }, nil) - }) - - nt.On("status.end", func(e mk.Event) { - log.Debugf("status.end") - - for idx, msmt := range c.msmts { - log.Debugf("adding msmt#%d to result", idx) - if err := msmt.AddToResult(c.Ctx.DB, c.res); err != nil { - log.WithError(err).Error("failed to add to result") + if c.Ctx.Config.Sharing.UploadResults { + // Implementation note: SubmitMeasurement will fail here if we did fail + // to open the report but we still want to continue. There will be a + // bit of a spew in the logs, perhaps, but stopping seems less efficient. + if err := exp.SubmitMeasurement(ctx, &measurement); err != nil { + log.Debug(color.RedString("failure.measurement_submission")) + if err := c.msmts[idx64].UploadFailed(c.Ctx.DB, err.Error()); err != nil { + return errors.Wrap(err, "failed to mark upload as failed") + } + } else if err := c.msmts[idx64].UploadSucceeded(c.Ctx.DB); err != nil { + return errors.Wrap(err, "failed to mark upload as succeeded") } } - if e.Value.Failure != "" { - log.Errorf("Failure in status.end: %s", e.Value.Failure) + if err := exp.SaveMeasurement(measurement, c.msmtPath); err != nil { + return errors.Wrap(err, "failed to save measurement on disk") + } + if err := c.msmts[idx64].Done(c.Ctx.DB); err != nil { + return errors.Wrap(err, "failed to mark measurement as done") } - c.res.DataUsageDown += e.Value.DownloadedKB - c.res.DataUsageUp += e.Value.UploadedKB - }) + // We're not sure whether it's enough to log the error or we should + // instead also mark the measurement as failed. Strictly speaking this + // is an inconsistency between the code that generate the measurement + // and the code that process the measurement. We do have some data + // but we're not gonna have a summary. To be reconsidered. + genericTk, err := enginex.MakeGenericTestKeys(measurement) + if err != nil { + log.WithError(err).Error("failed to cast the test keys") + continue + } + tk, err := c.nt.GetTestKeys(genericTk) + if err != nil { + log.WithError(err).Error("failed to obtain testKeys") + continue + } + log.Debugf("Fetching: %d %v", idx, c.msmts[idx64]) + if err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx64], tk); err != nil { + return errors.Wrap(err, "failed to add test keys to summary") + } + } - log.Debugf("Registered all the handlers") + log.Debugf("status.end") + for idx, msmt := range c.msmts { + log.Debugf("adding msmt#%d to result", idx) + if err := msmt.AddToResult(c.Ctx.DB, c.res); err != nil { + return errors.Wrap(err, "failed to add to result") + } + } return nil } // OnProgress should be called when a new progress event is available. func (c *Controller) OnProgress(perc float64, msg string) { log.Debugf("OnProgress: %f - %s", perc, msg) - + if c.numInputs >= 1 { + // make the percentage relative to the current input over all inputs + floor := (float64(c.curInputIdx) / float64(c.numInputs)) + step := 1.0 / float64(c.numInputs) + perc = floor + perc * step + } + if c.ntCount > 0 { + // make the percentage relative to the current nettest over all nettests + perc = float64(c.ntIndex)/float64(c.ntCount) + perc/float64(c.ntCount) + } key := fmt.Sprintf("%T", c.nt) output.Progress(key, perc, msg) } -// Entry is an opaque measurement entry -type Entry struct { - TestKeys map[string]interface{} `json:"test_keys"` -} - -// OnEntry should be called every time there is a new entry -func (c *Controller) OnEntry(idx int64, jsonStr string) { - log.Debugf("OnEntry") - - var entry Entry - if err := json.Unmarshal([]byte(jsonStr), &entry); err != nil { - log.WithError(err).Error("failed to parse onEntry") - return - } - // XXX is it correct to just log the error instead of marking the whole - // measurement as failed? - tk, err := c.nt.GetTestKeys(entry.TestKeys) - if err != nil { - log.WithError(err).Error("failed to obtain testKeys") - } - - log.Debugf("Fetching: %s %v", idx, c.msmts[idx]) - err = database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk) - if err != nil { - log.WithError(err).Error("failed to add test keys to summary") - } -} - -// MKStart is the interface for the mk.Nettest Start() function -type MKStart func(name string) (chan bool, error) - -// Start should be called to start the test -func (c *Controller) Start(f MKStart) { - log.Debugf("MKStart: %s", f) +// OnDataUsage should be called when we have a data usage update. +func (c *Controller) OnDataUsage(dloadKiB, uploadKiB float64) { + c.res.DataUsageDown += dloadKiB + c.res.DataUsageUp += uploadKiB } diff --git a/nettests/performance/dash.go b/nettests/performance/dash.go index f1f5a18..dd4b88e 100644 --- a/nettests/performance/dash.go +++ b/nettests/performance/dash.go @@ -3,8 +3,8 @@ package performance import ( "github.com/pkg/errors" - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/dash" ) // Dash test implementation @@ -13,9 +13,10 @@ type Dash struct { // Run starts the test func (d Dash) Run(ctl *nettests.Controller) error { - dash := mk.NewNettest("Dash") - ctl.Init(dash) - return dash.Run() + experiment := dash.NewExperiment( + ctl.Ctx.Session, dash.Config{}, + ) + return ctl.Run(experiment, []string{""}) } // DashTestKeys for the test diff --git a/nettests/performance/ndt.go b/nettests/performance/ndt.go index 9d0ae00..bb85307 100644 --- a/nettests/performance/ndt.go +++ b/nettests/performance/ndt.go @@ -1,8 +1,8 @@ package performance import ( - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/nettests" + "github.com/ooni/probe-engine/experiment/ndt" "github.com/pkg/errors" ) @@ -12,9 +12,10 @@ type NDT struct { // Run starts the test func (n NDT) Run(ctl *nettests.Controller) error { - nt := mk.NewNettest("Ndt") - ctl.Init(nt) - return nt.Run() + experiment := ndt.NewExperiment( + ctl.Ctx.Session, ndt.Config{}, + ) + return ctl.Run(experiment, []string{""}) } // NDTTestKeys for the test diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 5494ae8..1291777 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -1,63 +1,32 @@ package websites import ( - "encoding/json" - "fmt" - "io/ioutil" - "net/http" + "context" "github.com/apex/log" - "github.com/measurement-kit/go-measurement-kit" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" - "github.com/pkg/errors" + "github.com/ooni/probe-engine/experiment/web_connectivity" + "github.com/ooni/probe-engine/orchestra/testlists" ) -// URLInfo contains the URL and the citizenlab category code for that URL -type URLInfo struct { - URL string `json:"url"` - CountryCode string `json:"country_code"` - CategoryCode string `json:"category_code"` -} - -// URLResponse is the orchestrate url response containing a list of URLs -type URLResponse struct { - Results []URLInfo `json:"results"` -} - -const orchestrateBaseURL = "https://events.proteus.test.ooni.io" - func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { - var ( - parsed = new(URLResponse) - urls []string - ) + var urls []string urlIDMap := make(map[int64]int64) - log.Debug("Looking up URLs") - // XXX pass in the configuration for category codes - reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", - orchestrateBaseURL, - ctl.Ctx.Session.ProbeCC()) - - resp, err := http.Get(reqURL) + testlist, err := testlists.NewClient(ctl.Ctx.Session).Do( + context.Background(), ctl.Ctx.Session.ProbeCC(), + ) if err != nil { - return urls, urlIDMap, errors.Wrap(err, "failed to perform request") + return nil, nil, err } - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - return urls, urlIDMap, errors.Wrap(err, "failed to read response body") - } - err = json.Unmarshal([]byte(body), &parsed) - if err != nil { - return urls, urlIDMap, errors.Wrap(err, "failed to parse json") - } - - for idx, url := range parsed.Results { + for idx, url := range testlist { log.Debugf("Going over URL %d", idx) - urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode) + urlID, err := database.CreateOrUpdateURL( + ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode, + ) if err != nil { log.Error("failed to add to the URL table") + return nil, nil, err } log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID) urlIDMap[int64(idx)] = urlID @@ -72,17 +41,16 @@ type WebConnectivity struct { // Run starts the test func (n WebConnectivity) Run(ctl *nettests.Controller) error { - nt := mk.NewNettest("WebConnectivity") - ctl.Init(nt) - urls, urlIDMap, err := lookupURLs(ctl) if err != nil { return err } ctl.SetInputIdxMap(urlIDMap) - nt.Options.Inputs = urls - - return nt.Run() + experiment := web_connectivity.NewExperiment( + ctl.Ctx.Session, + web_connectivity.Config{LogLevel: "INFO"}, + ) + return ctl.Run(experiment, urls) } // WebConnectivityTestKeys for the test diff --git a/ooni.go b/ooni.go index c56d217..20dbf33 100644 --- a/ooni.go +++ b/ooni.go @@ -9,6 +9,7 @@ import ( "github.com/ooni/probe-cli/config" "github.com/ooni/probe-cli/internal/bindata" "github.com/ooni/probe-cli/internal/database" + "github.com/ooni/probe-cli/internal/enginex" "github.com/ooni/probe-cli/internal/legacy" "github.com/ooni/probe-cli/internal/onboard" "github.com/ooni/probe-cli/utils" @@ -20,10 +21,10 @@ import ( // Context for OONI Probe type Context struct { - Config *config.Config - DB sqlbuilder.Database - IsBatch bool - Session *session.Session + Config *config.Config + DB sqlbuilder.Database + IsBatch bool + Session *session.Session Home string TempDir string @@ -34,7 +35,7 @@ type Context struct { // MaybeLocationLookup will lookup the location of the user unless it's already cached func (c *Context) MaybeLocationLookup() error { - return c.Session.LookupLocation(context.Background()) + return c.Session.MaybeLookupLocation(context.Background()) } // MaybeOnboarding will run the onboarding process only if the informed consent @@ -97,11 +98,13 @@ func NewContext(configPath string, homePath string) *Context { Home: homePath, Config: &config.Config{}, configPath: configPath, - Session: session.New( - log.Log, + Session: session.New( + enginex.Logger, "ooniprobe-desktop", version.Version, utils.AssetsDir(homePath), + nil, // explicit proxy url.URL + nil, // explicit tls.Config ), } }