diff --git a/go.mod b/go.mod index 64cbb7f..60f5494 100644 --- a/go.mod +++ b/go.mod @@ -8,6 +8,8 @@ require ( git.torproject.org/pluggable-transports/snowflake.git/v2 v2.3.0 github.com/AlecAivazis/survey/v2 v2.3.5 github.com/alecthomas/kingpin v2.2.6+incompatible + github.com/anacrolix/dht/v2 v2.19.1 + github.com/anacrolix/torrent v1.47.0 github.com/apex/log v1.9.0 github.com/armon/go-socks5 v0.0.0-20160902184237-e75332964ef5 github.com/cretz/bine v0.2.0 @@ -44,9 +46,48 @@ require ( ) require ( + crawshaw.io/sqlite v0.3.3-0.20210127221821-98b1f83c5508 // indirect + github.com/RoaringBitmap/roaring v1.2.1 // indirect + github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 // indirect + github.com/alecthomas/atomic v0.1.0-alpha2 // indirect + github.com/anacrolix/chansync v0.3.0 // indirect + github.com/anacrolix/envpprof v1.2.1 // indirect + github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 // indirect + github.com/anacrolix/go-libutp v1.2.0 // indirect + github.com/anacrolix/log v0.13.2-0.20220711050817-613cb738ef30 // indirect + github.com/anacrolix/missinggo v1.3.0 // indirect + github.com/anacrolix/missinggo/perf v1.0.0 // indirect + github.com/anacrolix/missinggo/v2 v2.7.0 // indirect + github.com/anacrolix/mmsg v1.0.0 // indirect + github.com/anacrolix/multiless v0.3.0 // indirect + github.com/anacrolix/stm v0.4.0 // indirect + github.com/anacrolix/sync v0.4.0 // indirect + github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 // indirect + github.com/anacrolix/utp v0.1.0 // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect + github.com/benbjohnson/immutable v0.3.0 // indirect + github.com/bits-and-blooms/bitset v1.2.2 // indirect + github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/edsrzf/mmap-go v1.1.0 // indirect + github.com/go-logr/logr v1.2.3 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/google/btree v1.1.2 // indirect + github.com/huandu/xstrings v1.3.2 // indirect github.com/inconshreveable/mousetrap v1.0.0 // indirect + github.com/lispad/go-generics-tools v1.1.0 // indirect + github.com/mattn/go-sqlite3 v1.14.14 // indirect + github.com/mschoch/smat v0.2.0 // indirect + github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 // indirect github.com/segmentio/fasthash v1.0.3 // indirect github.com/spf13/pflag v1.0.5 // indirect + github.com/tidwall/btree v1.3.1 // indirect + go.etcd.io/bbolt v1.3.6 // indirect + go.opentelemetry.io/otel v1.8.0 // indirect + go.opentelemetry.io/otel/trace v1.8.0 // indirect + golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d // indirect + golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 // indirect + golang.org/x/time v0.0.0-20220609170525-579cf78fd858 // indirect ) require ( @@ -84,7 +125,6 @@ require ( github.com/marten-seemann/qtls-go1-18 v0.1.2 // indirect github.com/marten-seemann/qtls-go1-19 v0.1.0 // indirect github.com/mattn/go-isatty v0.0.16 // indirect - github.com/mattn/go-sqlite3 v1.14.15 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.1 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mroth/weightedrand v0.4.1 // indirect diff --git a/go.sum b/go.sum index aea655d..8b1fbfa 100644 --- a/go.sum +++ b/go.sum @@ -38,6 +38,11 @@ cloud.google.com/go/storage v1.5.0/go.mod h1:tpKbwo567HUNpVclU5sGELwQWBDZ8gh0Zeo cloud.google.com/go/storage v1.6.0/go.mod h1:N7U0C8pVQ/+NIKOBQyamJIeKQKkZ+mxpohlUTyfDhBk= cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RXyy7KQOVs= cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= +crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797 h1:yDf7ARQc637HoxDho7xjqdvO5ZA2Yb+xzv/fOnnvZzw= +crawshaw.io/iox v0.0.0-20181124134642-c51c3df30797/go.mod h1:sXBiorCo8c46JlQV3oXPKINnZ8mcqnye1EkVkqsectk= +crawshaw.io/sqlite v0.3.2/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= +crawshaw.io/sqlite v0.3.3-0.20210127221821-98b1f83c5508 h1:fILCBBFnjnrQ0whVJlGhfv1E/QiaFDNtGFBObEVRnYg= +crawshaw.io/sqlite v0.3.3-0.20210127221821-98b1f83c5508/go.mod h1:igAO5JulrQ1DbdZdtVq48mnZUBAPOeFzer7VhDWNtW4= dmitri.shuralyov.com/app/changes v0.0.0-20180602232624-0a106ad413e3/go.mod h1:Yl+fi1br7+Rr3LqpNJf1/uxUdtRUV+Tnj0o93V2B9MU= dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU= dmitri.shuralyov.com/html/belt v0.0.0-20180602232347-f7d459c86be0/go.mod h1:JLBrvjyP0v+ecvNYvCpyZgu5/xkfAUhi6wJj28eUfSU= @@ -70,12 +75,23 @@ github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2 h1:+vx7roKuyA63n github.com/Netflix/go-expect v0.0.0-20220104043353-73e0943537d2/go.mod h1:HBCaDeC1lPdgDeDbhX8XFpy1jqjK0IBG8W5K+xYqA0w= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/Psiphon-Inc/rotate-safe-writer v0.0.0-20210303140923-464a7a37606e h1:NPfqIbzmijrl0VclX2t8eO5EPBhqe47LLGKpRrcVjXk= +github.com/RoaringBitmap/roaring v0.4.7/go.mod h1:8khRDP4HmeXns4xIj9oGrKSz7XTQiJx2zgh7AcNke4w= +github.com/RoaringBitmap/roaring v0.4.17/go.mod h1:D3qVegWTmfCaX4Bl5CrBE9hfrSrrXIr8KVNvRsDi1NI= +github.com/RoaringBitmap/roaring v0.4.23/go.mod h1:D0gp8kJQgE1A4LQ5wFLggQEyvDi06Mq5mKs52e1TwOo= +github.com/RoaringBitmap/roaring v1.2.1 h1:58/LJlg/81wfEHd5L9qsHduznOIhyv4qb1yWcSvVq9A= +github.com/RoaringBitmap/roaring v1.2.1/go.mod h1:icnadbWcNyfEHlYdr+tDlOTih1Bf/h+rzPpv4sbomAA= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= github.com/VividCortex/gohistogram v1.0.0/go.mod h1:Pf5mBqqDxYaXu3hDrrU+w6nw50o/4+TcAqDqk/vUH7g= github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia65gfNATL8TAiHDNxPzPdmEL5uirI2Uyuz6c= +github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0 h1:byYvvbfSo3+9efR4IeReh77gVs4PnNDR3AMOE9NJ7a0= +github.com/ajwerner/btree v0.0.0-20211221152037-f427b3e689c0/go.mod h1:q37NoqncT41qKc048STsifIt69LfUJ8SrWWcz/yam5k= +github.com/alecthomas/assert/v2 v2.0.0-alpha3 h1:pcHeMvQ3OMstAWgaeaXIAL8uzB9xMm2zlxt+/4ml8lk= +github.com/alecthomas/atomic v0.1.0-alpha2 h1:dqwXmax66gXvHhsOS4pGPZKqYOlTkapELkLb3MNdlH8= +github.com/alecthomas/atomic v0.1.0-alpha2/go.mod h1:zD6QGEyw49HIq19caJDc2NMXAy8rNi9ROrxtMXATfyI= github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/repr v0.0.0-20210801044451-80ca428c5142 h1:8Uy0oSf5co/NZXje7U1z8Mpep++QJOldL2hs/sBQf48= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751 h1:JYp7IbQjafoB+tBA3gMyHYHrpOtNuDiK/uB5uXxq5wM= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= @@ -84,6 +100,61 @@ github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137 h1:s6gZFSlWYmbqAuRjVTiNNhvNRfY2Wxp9nhfyel4rklc= github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= +github.com/anacrolix/chansync v0.3.0 h1:lRu9tbeuw3wl+PhMu/r+JJCRu5ArFXIluOgdF0ao6/U= +github.com/anacrolix/chansync v0.3.0/go.mod h1:DZsatdsdXxD0WiwcGl0nJVwyjCKMDv+knl1q2iBjA2k= +github.com/anacrolix/dht/v2 v2.19.1 h1:V/UUGBASGYqYkSnmHJwX8uQmzkyhbgwE6jqcHKnNTD8= +github.com/anacrolix/dht/v2 v2.19.1/go.mod h1:3TU93c1s/oA8I/VH4m3CNP/BeKsiOGmo6HwfZBMTKUs= +github.com/anacrolix/envpprof v0.0.0-20180404065416-323002cec2fa/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= +github.com/anacrolix/envpprof v1.0.0/go.mod h1:KgHhUaQMc8cC0+cEflSgCFNFbKwi5h54gqtVn8yhP7c= +github.com/anacrolix/envpprof v1.1.0/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= +github.com/anacrolix/envpprof v1.2.1 h1:25TJe6t/i0AfzzldiGFKCpD+s+dk8lONBcacJZB2rdE= +github.com/anacrolix/envpprof v1.2.1/go.mod h1:My7T5oSqVfEn4MD4Meczkw/f5lSIndGAKu/0SM/rkf4= +github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60 h1:k4/h2B1gGF+PJGyGHxs8nmHHt1pzWXZWBj6jn4OBlRc= +github.com/anacrolix/generics v0.0.0-20220618083756-f99e35403a60/go.mod h1:ff2rHB/joTV03aMSSn/AZNnaIpUw0h3njetGsaXcMy8= +github.com/anacrolix/go-libutp v1.2.0 h1:sjxoB+/ARiKUR7IK/6wLWyADIBqGmu1fm0xo+8Yy7u0= +github.com/anacrolix/go-libutp v1.2.0/go.mod h1:RrJ3KcaDcf9Jqp33YL5V/5CBEc6xMc7aJL8wXfuWL50= +github.com/anacrolix/log v0.3.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= +github.com/anacrolix/log v0.6.0/go.mod h1:lWvLTqzAnCWPJA08T2HCstZi0L1y2Wyvm3FJgwU9jwU= +github.com/anacrolix/log v0.10.0/go.mod h1:s5yBP/j046fm9odtUTbHOfDUq/zh1W8OkPpJtnX0oQI= +github.com/anacrolix/log v0.10.1-0.20220123034749-3920702c17f8/go.mod h1:GmnE2c0nvz8pOIPUSC9Rawgefy1sDXqposC2wgtBZE4= +github.com/anacrolix/log v0.13.2-0.20220711050817-613cb738ef30 h1:bAgFzUxN1K3U8KwOzqCOhiygOr5NqYO3kNlV9tvp2Rc= +github.com/anacrolix/log v0.13.2-0.20220711050817-613cb738ef30/go.mod h1:D4+CvN8SnruK6zIFS/xPoRJmtvtnxs+CSfDQ+BFxZ68= +github.com/anacrolix/lsan v0.0.0-20211126052245-807000409a62 h1:P04VG6Td13FHMgS5ZBcJX23NPC/fiC4cp9bXwYujdYM= +github.com/anacrolix/lsan v0.0.0-20211126052245-807000409a62/go.mod h1:66cFKPCO7Sl4vbFnAaSq7e4OXtdMhRSBagJGWgmpJbM= +github.com/anacrolix/missinggo v0.0.0-20180725070939-60ef2fbf63df/go.mod h1:kwGiTUTZ0+p4vAz3VbAI5a30t2YbvemcmspjKwrAz5s= +github.com/anacrolix/missinggo v1.1.0/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= +github.com/anacrolix/missinggo v1.1.2-0.20190815015349-b888af804467/go.mod h1:MBJu3Sk/k3ZfGYcS7z18gwfu72Ey/xopPFJJbTi5yIo= +github.com/anacrolix/missinggo v1.2.1/go.mod h1:J5cMhif8jPmFoC3+Uvob3OXXNIhOUikzMt+uUjeM21Y= +github.com/anacrolix/missinggo v1.3.0 h1:06HlMsudotL7BAELRZs0yDZ4yVXsHXGi323QBjAVASw= +github.com/anacrolix/missinggo v1.3.0/go.mod h1:bqHm8cE8xr+15uVfMG3BFui/TxyB6//H5fwlq/TeqMc= +github.com/anacrolix/missinggo/perf v1.0.0 h1:7ZOGYziGEBytW49+KmYGTaNfnwUqP1HBsy6BqESAJVw= +github.com/anacrolix/missinggo/perf v1.0.0/go.mod h1:ljAFWkBuzkO12MQclXzZrosP5urunoLS0Cbvb4V0uMQ= +github.com/anacrolix/missinggo/v2 v2.2.0/go.mod h1:o0jgJoYOyaoYQ4E2ZMISVa9c88BbUBVQQW4QeRkNCGY= +github.com/anacrolix/missinggo/v2 v2.5.1/go.mod h1:WEjqh2rmKECd0t1VhQkLGTdIWXO6f6NLjp5GlMZ+6FA= +github.com/anacrolix/missinggo/v2 v2.5.2/go.mod h1:yNvsLrtZYRYCOI+KRH/JM8TodHjtIE/bjOGhQaLOWIE= +github.com/anacrolix/missinggo/v2 v2.7.0 h1:4fzOAAn/VCvfWGviLmh64MPMttrlYew81JdPO7nSHvI= +github.com/anacrolix/missinggo/v2 v2.7.0/go.mod h1:2IZIvmRTizALNYFYXsPR7ofXPzJgyBpKZ4kMqMEICkI= +github.com/anacrolix/mmsg v0.0.0-20180515031531-a4a3ba1fc8bb/go.mod h1:x2/ErsYUmT77kezS63+wzZp8E3byYB0gzirM/WMBLfw= +github.com/anacrolix/mmsg v1.0.0 h1:btC7YLjOn29aTUAExJiVUhQOuf/8rhm+/nWCMAnL3Hg= +github.com/anacrolix/mmsg v1.0.0/go.mod h1:x8kRaJY/dCrY9Al0PEcj1mb/uFHwP6GCJ9fLl4thEPc= +github.com/anacrolix/multiless v0.3.0 h1:5Bu0DZncjE4e06b9r1Ap2tUY4Au0NToBP5RpuEngSis= +github.com/anacrolix/multiless v0.3.0/go.mod h1:TrCLEZfIDbMVfLoQt5tOoiBS/uq4y8+ojuEVVvTNPX4= +github.com/anacrolix/stm v0.2.0/go.mod h1:zoVQRvSiGjGoTmbM0vSLIiaKjWtNPeTvXUSdJQA4hsg= +github.com/anacrolix/stm v0.4.0 h1:tOGvuFwaBjeu1u9X1eIh9TX8OEedEiEQ1se1FjhFnXY= +github.com/anacrolix/stm v0.4.0/go.mod h1:GCkwqWoAsP7RfLW+jw+Z0ovrt2OO7wRzcTtFYMYY5t8= +github.com/anacrolix/sync v0.0.0-20180808010631-44578de4e778/go.mod h1:s735Etp3joe/voe2sdaXLcqDdJSay1O0OPnM0ystjqk= +github.com/anacrolix/sync v0.3.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= +github.com/anacrolix/sync v0.4.0 h1:T+MdO/u87ir/ijWsTFsPYw5jVm0SMm4kVpg8t4KF38o= +github.com/anacrolix/sync v0.4.0/go.mod h1:BbecHL6jDSExojhNtgTFSBcdGerzNc64tz3DCOj/I0g= +github.com/anacrolix/tagflag v0.0.0-20180109131632-2146c8d41bf0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anacrolix/tagflag v1.0.0/go.mod h1:1m2U/K6ZT+JZG0+bdMK6qauP49QT4wE5pmhJXOKKCHw= +github.com/anacrolix/tagflag v1.1.0/go.mod h1:Scxs9CV10NQatSmbyjqmqmeQNwGzlNe0CMUMIxqHIG8= +github.com/anacrolix/torrent v1.47.0 h1:aDUnhQZ8+kfStLICHiXOGGYVFgDENK+kz4q96linyRg= +github.com/anacrolix/torrent v1.47.0/go.mod h1:SYPxEUjMwqhDr3kWGzyQLkFMuAb1bgJ57JRMpuD3ZzE= +github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96 h1:QAVZ3pN/J4/UziniAhJR2OZ9Ox5kOY2053tBbbqUPYA= +github.com/anacrolix/upnp v0.1.3-0.20220123035249-922794e51c96/go.mod h1:Wa6n8cYIdaG35x15aH3Zy6d03f7P728QfdcDeD/IEOs= +github.com/anacrolix/utp v0.1.0 h1:FOpQOmIwYsnENnz7tAGohA+r6iXpRjrq8ssKSre2Cp4= +github.com/anacrolix/utp v0.1.0/go.mod h1:MDwc+vsGEq7RMw6lr2GKOEqjWny5hO5OZXRVNaBJ2Dk= github.com/anmitsu/go-shlex v0.0.0-20161002113705-648efa622239/go.mod h1:2FmKhYUyUczH0OGQWaF5ceTx0UBShxjsH6f8oGKYe2c= github.com/antihax/optional v1.0.0/go.mod h1:uupD/76wgC+ih3iEmQUL+0Ugr19nfwCT1kdvxnR2qWY= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= @@ -107,6 +178,11 @@ github.com/aws/aws-sdk-go v1.20.6/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= +github.com/benbjohnson/immutable v0.2.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= +github.com/benbjohnson/immutable v0.3.0 h1:TVRhuZx2wG9SZ0LRdqlbs9S5BZ6Y24hJEHTCgWHZEIw= +github.com/benbjohnson/immutable v0.3.0/go.mod h1:uc6OHo6PN2++n98KHLxW8ef4W42ylHiQSENghE1ezxI= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -114,8 +190,15 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61 h1:BU+NxuoaYPIvvp8NNkNlLr8aA0utGyuunf4Q3LJ0bh0= github.com/bifurcation/mint v0.0.0-20180306135233-198357931e61/go.mod h1:zVt7zX3K/aDCk9Tj+VM7YymsX66ERvzCJzw8rFCX2JU= +github.com/bits-and-blooms/bitset v1.2.0/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= +github.com/bits-and-blooms/bitset v1.2.2 h1:J5gbX05GpMdBjCvQ9MteIg2KKDExr7DrgK+Yc15FvIk= +github.com/bits-and-blooms/bitset v1.2.2/go.mod h1:gIdJ4wp64HaoK2YrL1Q5/N7Y16edYb8uY+O0FJTyyDA= github.com/bketelsen/crypt v0.0.4/go.mod h1:aI6NrJ0pMGgvZKL1iVgXLnfIFJtfV+bKCoqOes/6LfM= github.com/bradfitz/go-smtpd v0.0.0-20170404230938-deb6d6237625/go.mod h1:HYsPBTaaSFSlLx/70C2HPIMNZpVV8+vt/A+FMnYP11g= +github.com/bradfitz/iter v0.0.0-20140124041915-454541ec3da2/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/bradfitz/iter v0.0.0-20190303215204-33e6a9893b0c/go.mod h1:PyRFw1Lt2wKX4ZVSQ2mk+PeDa1rxyObEDlApuIsUKuo= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8 h1:GKTyiRCL6zVf5wWaqKnf+7Qs6GbEPfd4iMOitWzXJx8= +github.com/bradfitz/iter v0.0.0-20191230175014-e8f45d346db8/go.mod h1:spo1JLcs67NmW1aVLEgtA8Yy1elc+X8y5SRW1sFW4Og= github.com/buger/jsonparser v0.0.0-20181115193947-bf1c66bbce23/go.mod h1:bbYlZJ7hK1yFx9hf58LP0zeX7UjIGs20ufpu3evjr+s= github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/ccding/go-stun v0.1.5-0.20220419042218-44e89cab7805 h1:AkTX8U06UIH//16PyxKLOPjlGoqcTEYpjipeCNsASfQ= @@ -157,6 +240,7 @@ github.com/cpuguy83/go-md2man/v2 v2.0.0-20190314233015-f79a8a8ca69d/go.mod h1:ma github.com/cpuguy83/go-md2man/v2 v2.0.0/go.mod h1:maD7wRr/U5Z6m/iR4s+kqSMx2CaBsrgA7czyZG/E6dU= github.com/cpuguy83/go-md2man/v2 v2.0.2/go.mod h1:tgQtvFlXSQOSOSIRvRPT7W67SCa46tRHOmNcaadrF8o= github.com/creack/pty v1.1.7/go.mod h1:lj5s0c3V2DBrqTV7llrYr5NG6My20zk30Fl46Y7DoTY= +github.com/creack/pty v1.1.9/go.mod h1:oKZEueFk5CKHvIhNR5MUki03XCEU+Q6VDXinZuGJ33E= github.com/creack/pty v1.1.17 h1:QeVUsEDNrLBW4tMgZHvxy18sKtr6VI492kBhUfhDJNI= github.com/creack/pty v1.1.17/go.mod h1:MOBLtS5ELjhRRrroQr9kyvTxUAFNvYEK993ew/Vr4O4= github.com/cretz/bine v0.1.0/go.mod h1:6PF6fWAvYtwjRGkAuDEJeWNOv3a2hUouSP/yRYXmvHw= @@ -179,16 +263,20 @@ github.com/dgraph-io/ristretto v0.1.0/go.mod h1:fux0lOrBhrVCJd3lcTHsIJhq1T2rokOu github.com/dgrijalva/jwt-go v3.2.0+incompatible/go.mod h1:E3ru+11k8xSBh+hMPgOLZmtrrCbhqsmaPHjLKYnJCaQ= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2 h1:tdlZCpZ/P9DhczCTSixgIKmwPv6+wP5DGjqLYw5SUiA= github.com/dgryski/go-farm v0.0.0-20190423205320-6a90982ecee2/go.mod h1:SqUrOPUnsFjfmXRMNPybcSiG0BgUW2AuFH8PAnS2iTw= +github.com/docopt/docopt-go v0.0.0-20180111231733-ee0de3bc6815/go.mod h1:WwZ+bS3ebgob9U8Nd0kOddGdZWjyMGR8Wziv+TBNwSE= github.com/dsnet/compress v0.0.1 h1:PlZu0n3Tuv04TzpfPbrnI0HW/YwodEXDS+oPKahKF0Q= github.com/dsnet/compress v0.0.1/go.mod h1:Aw8dCMJ7RioblQeTqt88akK31OvO8Dhf5JflhBbQEHo= github.com/dsnet/golib v0.0.0-20171103203638-1ea166775780/go.mod h1:Lj+Z9rebOhdfkVLjJ8T6VcRQv3SXugXy999NBtR9aFY= github.com/dustin/go-humanize v0.0.0-20171111073723-bb3d318650d4/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/dustin/go-humanize v0.0.0-20180421182945-02af3965c54e/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/dustin/go-humanize v1.0.0 h1:VSnTsYCnlFHaM2/igO1h6X3HA71jcobQuxemgkq4zYo= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= github.com/eapache/go-resiliency v1.1.0/go.mod h1:kFI+JgMyC7bLPUVY133qvEBtVayf5mFgVsvEsIPBvNs= github.com/eapache/go-xerial-snappy v0.0.0-20180814174437-776d5712da21/go.mod h1:+020luEh2TKB4/GOp8oxxtq0Daoen/Cii55CzbTV6DU= github.com/eapache/queue v1.1.0/go.mod h1:6eCeP0CKFpHLu8blIFXhExK/dRa7WDZfr6jVFPTqq+I= github.com/edsrzf/mmap-go v1.0.0/go.mod h1:YO35OhQPt3KJa3ryjFM5Bs14WD66h8eGKpfaBNrHW5M= +github.com/edsrzf/mmap-go v1.1.0 h1:6EUwBLQ/Mcr1EYLE4Tn1VdW1A4ckqCQWZBw8Hr0kjpQ= +github.com/edsrzf/mmap-go v1.1.0/go.mod h1:19H/e8pUPLicwkyNgOykDXkJ9F0MHE+Z52B8EIth78Q= github.com/elazarl/goproxy v0.0.0-20200809112317-0581fc3aee2d h1:rtM8HsT3NG37YPjz8sYSbUSdElP9lUsQENYzJDZDUBE= github.com/elazarl/goproxy/ext v0.0.0-20200809112317-0581fc3aee2d h1:st1tmvy+4duoRj+RaeeJoECWCWM015fBtf/4aR+hhqk= github.com/envoyproxy/go-control-plane v0.6.9/go.mod h1:SBwIajubJHhxtWwsL9s8ss4safvEdbitLhGGK48rN6g= @@ -209,12 +297,20 @@ github.com/flynn/noise v1.0.0/go.mod h1:xbMo+0i6+IGbYdJhF31t2eR1BIU0CYc12+BNAKwU github.com/francoispqt/gojay v1.2.13/go.mod h1:ehT5mTG4ua4581f1++1WLG0vPdaA9HaiDsoyrBGkyDY= github.com/franela/goblin v0.0.0-20200105215937-c9ffbefa60db/go.mod h1:7dvUGVsVBjqR7JHJk0brhHOZYGmfBYOrK0ZhYMEtBr4= github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2rbfLwlschooIH4+wKKDR4Pdxhh+TRoA20= +github.com/frankban/quicktest v1.9.0/go.mod h1:ui7WezCLWMWxVWr1GETZY3smRy0G4KWq9vcPtJmFl7Y= +github.com/frankban/quicktest v1.14.3 h1:FJKSZTDHjyhriyC81FLQ0LY93eSai0ZyR/ZIkd3ZUKE= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/fsnotify/fsnotify v1.4.9/go.mod h1:znqG4EE+3YCdAaPaxE2ZRY/06pZUdp0tY4IgpuI1SZQ= github.com/fsnotify/fsnotify v1.5.4 h1:jRbGcIw6P2Meqdwuo0H1p6JVLbL5DHKAKlYndzMwVZI= github.com/fsnotify/fsnotify v1.5.4/go.mod h1:OVB6XrOHzAwXMpEM7uPOzcehqUV2UqJxmVXmkdnm1bU= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= github.com/gliderlabs/ssh v0.1.1/go.mod h1:U7qILu1NlMHj9FlMhZLlkCdDnU1DBEAqr0aevW3Awn0= +github.com/glycerine/go-unsnap-stream v0.0.0-20180323001048-9f0cb55181dd/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20181221182339-f9677308dec2/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/go-unsnap-stream v0.0.0-20190901134440-81cf024a9e0a/go.mod h1:/20jfyN9Y5QPEAprSgKAUr+glWDY39ZiUEAYOEv5dsE= +github.com/glycerine/goconvey v0.0.0-20180728074245-46e3a41ad493/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/glycerine/goconvey v0.0.0-20190315024820-982ee783a72e/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= +github.com/glycerine/goconvey v0.0.0-20190410193231-58a59202ab31/go.mod h1:Ogl1Tioa0aV7gstGFO7KhffUsb9M4ydbEbbxpcEDc24= github.com/go-errors/errors v1.0.1/go.mod h1:f4zRHt4oKfwPJE5k8C9vpYG+aDHdBFUsgrm6/TyX73Q= github.com/go-gl/glfw v0.0.0-20190409004039-e6da0acd62b1/go.mod h1:vR7hzQXu2zJy9AVAgeJqvqgH9Q5CA+iKCZ2gyEVpxRU= github.com/go-gl/glfw/v3.3/glfw v0.0.0-20191125211704-12ad95a8df72/go.mod h1:tQ2UAYgL5IevRw8kRxooKSPJfGvJ9fJQFa0TUsXzTg8= @@ -230,6 +326,11 @@ github.com/go-logfmt/logfmt v0.3.0/go.mod h1:Qt1PoO58o5twSAckw1HlFXLmHsOX5/0LbT9 github.com/go-logfmt/logfmt v0.4.0/go.mod h1:3RMwSq7FuexP4Kalkev3ejPJsZTpXXBr9+V4qmtdjCk= github.com/go-logfmt/logfmt v0.5.0/go.mod h1:wCYkCAKZfumFQihp8CzCvQ3paCTfi41vtzG1KdI/P7A= github.com/go-logfmt/logfmt v0.5.1/go.mod h1:WYhtIu8zTZfxdn5+rREduYbwxfcBr/Vr6KEVveWlfTs= +github.com/go-logr/logr v1.2.2/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/logr v1.2.3 h1:2DntVwHkVopvECVRSlL5PSo9eG+cAkDCuckLubN+rq0= +github.com/go-logr/logr v1.2.3/go.mod h1:jdQByPbusPIv2/zmleS9BjJVeZ6kBagPoEUsqbVz/1A= +github.com/go-logr/stdr v1.2.2 h1:hSWxHoqTgW2S2qGc0LTAI563KZ5YKYRhT3MFKZMbjag= +github.com/go-logr/stdr v1.2.2/go.mod h1:mMo/vtBO5dYbehREoey6XUKy/eSumjCCveDpRre4VKE= github.com/go-sql-driver/mysql v1.4.0/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= @@ -293,10 +394,14 @@ github.com/golang/protobuf v1.5.2/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiu github.com/golang/protobuf v1.5.3-0.20210916003710-5d5e8c018a13 h1:yztvEbaW/qZGubeP7+Lug7PXl7NBfilUK6mw3jq25gQ= github.com/golang/protobuf v1.5.3-0.20210916003710-5d5e8c018a13/go.mod h1:XVQd3VNwM+JqD3oG2Ue2ip4fOMUkwXdXDdiuN0vRsmY= github.com/golang/snappy v0.0.0-20180518054509-2e65f85255db/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/golang/snappy v0.0.1/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v0.0.3 h1:fHPg5GQYlCeLIPB9BZqMVR5nR9A+IM5zcgeTdjMYmLA= github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/google/btree v0.0.0-20180124185431-e89373fe6b4a/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= +github.com/google/btree v1.1.2 h1:xf4v41cLI2Z6FxbKm+8Bu+m8ifhj15JuZ9sa0jZCMUU= +github.com/google/btree v1.1.2/go.mod h1:qOPhT0dTNdNzV6Z/lhRX0YXUafgPLFUh+gZMl761Gm4= github.com/google/go-cmp v0.2.0/go.mod h1:oXzfMopK8JAjlY9xF4vHSVASa0yLyX7SntLO5aqRK0M= github.com/google/go-cmp v0.3.0/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= github.com/google/go-cmp v0.3.1/go.mod h1:8QqcDgzrUqlUb/G2PQTWiueGozuR1884gddMywk6iLU= @@ -343,8 +448,11 @@ github.com/googleapis/gax-go v2.0.0+incompatible/go.mod h1:SFVmujtThgffbyetf+mdk github.com/googleapis/gax-go/v2 v2.0.3/go.mod h1:LLvjysVCY1JZeum8Z6l8qUty8fiNwE08qbEPm1M08qg= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= -github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1 h1:EGx4pi6eqNxGaHF6qqu48+N2wcFQ5qg5FXgOdqsJ5d8= github.com/gopherjs/gopherjs v0.0.0-20181017120253-0766667cb4d1/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20181103185306-d547d1d9531e/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190309154008-847fc94819f9/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99 h1:twflg0XRTjwKpxb/jFExr4HGq6on2dEOmnL6FV+fgPw= +github.com/gopherjs/gopherjs v0.0.0-20190910122728-9d188e94fb99/go.mod h1:wJfORRmW1u3UXTncJ5qlYoELFm8eSnnEO6hX4iZ3EWY= github.com/gorilla/context v1.1.1/go.mod h1:kBGZzfjB9CEq2AlWe17Uuf7NDRt0dE0s8S51q0aT7Yg= github.com/gorilla/mux v1.6.2/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= github.com/gorilla/mux v1.7.3/go.mod h1:1lud6UwP+6orDFRuTfBEV8e9/aOM/c4fVVCaMa2zaAs= @@ -391,7 +499,11 @@ github.com/hexops/gotextdiff v1.0.3/go.mod h1:pSWU5MAI3yDq+fZBTazCSJysOMbxWL1BSo github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec h1:qv2VnGeEQHchGaZ/u7lxST/RaJw+cv273q79D81Xbog= github.com/hinshun/vt10x v0.0.0-20220119200601-820417d04eec/go.mod h1:Q48J4R4DvxnHolD5P8pOtXigYlRuPLGl6moFx3ulM68= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= +github.com/huandu/xstrings v1.0.0/go.mod h1:4qWG/gcEcfX4z/mBDHJ++3ReCw9ibxbsNJbcucJdbSo= +github.com/huandu/xstrings v1.2.0/go.mod h1:DvyZB1rfVYsBIigL8HwpZgxHwXozlTgGqn63UyNX5k4= +github.com/huandu/xstrings v1.3.0/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= +github.com/huandu/xstrings v1.3.2 h1:L18LIDzqlW6xN2rEkpdV8+oL/IXWJ1APd+vsdYy4Wdw= github.com/huandu/xstrings v1.3.2/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/iancoleman/strcase v0.2.0 h1:05I4QRnGpI0m37iZQRuskXh+w77mr6Z41lwQzuHLwW0= @@ -452,11 +564,13 @@ github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.7/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.8/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= +github.com/json-iterator/go v1.1.9/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.11/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= github.com/json-iterator/go v1.1.12/go.mod h1:e30LSqwooZae/UwlEbR2852Gd8hjQvJoHmT4TnhNGBo= github.com/jstemmer/go-junit-report v0.0.0-20190106144839-af01ea7f8024/go.mod h1:6v2b51hI/fHJwM22ozAgKL4VKDeJcHhJFhtBdhmNjmU= github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/XSXhF0NWZEnDohbsk= +github.com/jtolds/gls v4.2.1+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/jtolds/gls v4.20.0+incompatible h1:xdiiI2gbIgH/gLH7ADydsJ1uDOEzR8yvV7C0MuV77Wo= github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU= github.com/juju/ratelimit v1.0.2 h1:sRxmtRiajbvrcLQT7S+JbqU0ntsb9W2yhSdNN8tWfaI= @@ -490,13 +604,15 @@ github.com/kr/fs v0.1.0/go.mod h1:FFnZGqtBN9Gxj7eW1uZ42v5BccTP0vu6NEaFoC2HwRg= github.com/kr/logfmt v0.0.0-20140226030751-b84e30acd515/go.mod h1:+0opPa2QZZtGFBFZlji/RkVcI2GknAs/DXo4wKdlNEc= github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= -github.com/kr/pretty v0.2.1 h1:Fmg33tUaq4/8ym9TJN1x7sLJnHVwhP33CNkpYV/7rwI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= +github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= +github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.3/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/pty v1.1.8/go.mod h1:O1sed60cT9XZ5uDucP5qwvh+TE3NnUj51EiZO/lmSfw= -github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= +github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= github.com/kylelemons/godebug v0.0.0-20170820004349-d65d576e9348/go.mod h1:B69LEHPfb2qLo0BaaOLcbitczOKLWTsrBG9LczfCD4k= github.com/lib/pq v1.0.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= @@ -507,6 +623,8 @@ github.com/lib/pq v1.10.4 h1:SO9z7FRPzA03QhHKJrH5BXA6HU1rS4V2nIVrrNC1iYk= github.com/lib/pq v1.10.4/go.mod h1:AlVN5x4E4T544tWzH6hKfbfQvm3HdbOxrmggDNAPY9o= github.com/lightstep/lightstep-tracer-common/golang/gogo v0.0.0-20190605223551-bc2310a04743/go.mod h1:qklhhLq1aX+mtWk9cPHPzaBjWImj5ULL6C7HFJtXQMM= github.com/lightstep/lightstep-tracer-go v0.18.1/go.mod h1:jlF1pusYV4pidLvZ+XD0UBX0ZE6WURAspgAczcDHrL4= +github.com/lispad/go-generics-tools v1.1.0 h1:mbSgcxdFVmpoyso1X/MJHXbSbSL3dD+qhRryyxk+/XY= +github.com/lispad/go-generics-tools v1.1.0/go.mod h1:2csd1EJljo/gy5qG4khXol7ivCPptNjG5Uv2X8MgK84= github.com/lucas-clemente/quic-go v0.28.1 h1:Uo0lvVxWg5la9gflIF9lwa39ONq85Xq2D91YNEIslzU= github.com/lucas-clemente/quic-go v0.28.1/go.mod h1:oGz5DKK41cJt5+773+BSO9BXDsREY4HLf7+0odGAPO0= github.com/lunixbochs/vtclean v1.0.0/go.mod h1:pHhQNgMf3btfWnGBVipUOjRYhoOsdGqdm/+2c2E2WMI= @@ -553,9 +671,8 @@ github.com/mattn/go-runewidth v0.0.2/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzp github.com/mattn/go-runewidth v0.0.9/go.mod h1:H031xJmbD/WCDINGzjvQ9THkh0rPKHF+m2gUSrubnMI= github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/go-sqlite3 v1.14.9/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= +github.com/mattn/go-sqlite3 v1.14.14 h1:qZgc/Rwetq+MtyE18WhzjokPD93dNqLGNT3QJuLvBGw= github.com/mattn/go-sqlite3 v1.14.14/go.mod h1:NyWgC/yNuGj7Q9rpYnZvas74GogHl5/Z4A/KQRfk6bU= -github.com/mattn/go-sqlite3 v1.14.15 h1:vfoHhTN1af61xCRSWzFIWzx2YskyMTwHLrExkBOjvxI= -github.com/mattn/go-sqlite3 v1.14.15/go.mod h1:2eHXhiwb8IkHr+BDWZGa96P6+rkvnG63S2DGjv9HUNg= github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mdlayher/netlink v1.4.2-0.20210930205308-a81a8c23d40a h1:yk5OmRew64lWdeNanQ3l0hDgUt1E8MfipPhh/GO9Tuw= @@ -592,6 +709,9 @@ github.com/montanaflynn/stats v0.6.6 h1:Duep6KMIDpY4Yo11iFsvyqJDyfzLF9+sndUKT+v6 github.com/montanaflynn/stats v0.6.6/go.mod h1:etXPPgVO6n31NxCd9KQUMvCM+ve0ruNzt6R8Bnaayow= github.com/mroth/weightedrand v0.4.1 h1:rHcbUBopmi/3x4nnrvwGJBhX9d0vk+KgoLUZeDP6YyI= github.com/mroth/weightedrand v0.4.1/go.mod h1:3p2SIcC8al1YMzGhAIoXD+r9olo/g/cdJgAD905gyNE= +github.com/mschoch/smat v0.0.0-20160514031455-90eadee771ae/go.mod h1:qAyveg+e4CE+eKJXWVjKXM4ck2QobLqTDytGJbLLhJg= +github.com/mschoch/smat v0.2.0 h1:8imxQsjDm8yFEAVBe7azKmKSgzSkZXDuKkSq9374khM= +github.com/mschoch/smat v0.2.0/go.mod h1:kc9mz7DoBKqDyiRL7VZN8KvXQMWeTaVnttLRXOlotKw= github.com/mwitkow/go-conntrack v0.0.0-20161129095857-cc309e4a2223/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f/go.mod h1:qRWi+5nqEBWmkhHvq77mSJWrCKwh8bxhgT7d/eI7P4U= github.com/naoina/go-stringutil v0.1.0/go.mod h1:XJ2SJL9jCtBh+P9q5btrd/Ylo8XwT/h1USek5+NqSA0= @@ -662,6 +782,7 @@ github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/9 github.com/pelletier/go-toml v1.9.3/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/performancecopilot/speed v3.0.0+incompatible/go.mod h1:/CLtqpZ5gBg1M9iaPbIdPPGyKcA8hKdoy6hAWba7Yac= +github.com/philhofer/fwd v1.0.0/go.mod h1:gk3iGcWd9+svBvR0sR+KPcfE+RNWozjowpeBVG3ZVNU= github.com/pierrec/lz4 v1.0.2-0.20190131084431-473cd7ce01a1/go.mod h1:3/3N9NVKO0jef7pBehbT1qWhCMrIgbYNnFAZCqQ5LRc= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pion/datachannel v1.5.2 h1:piB93s8LGmbECrpO84DnkIVWasRMk3IimbcXkTQLE6E= @@ -727,6 +848,7 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod h1:p2iRAGwDERtqlqzRXnrOVns+ignqQo//hLXqYxZYVNs= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.5.1/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.10.0/go.mod h1:WJM3cc3yu7XKBKa/I8WeZm+V3eltZnBwfENSU7mdogU= github.com/prometheus/client_golang v1.11.0/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= @@ -744,6 +866,7 @@ github.com/prometheus/common v0.0.0-20180801064454-c7de2306084e/go.mod h1:daVV7q github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.18.0/go.mod h1:U+gB1OBLb1lF3O42bTCL+FK18tX9Oar16Clt/msog/s= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -755,6 +878,7 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= +github.com/prometheus/procfs v0.0.11/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= @@ -771,9 +895,12 @@ github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6So github.com/rogpeppe/fastuuid v1.1.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/rogpeppe/go-internal v1.9.0 h1:73kH8U+JUqXU8lRuOHeVHaa/SZPifC7BkcraZVejAe8= github.com/rogpeppe/go-internal v1.9.0/go.mod h1:WtVeX8xhTBvf0smdhujwtBcq4Qrzq/fJaraNFVN+nFs= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417 h1:Lt9DzQALzHoDwMBGJ6v8ObDPR0dzr2a6sXTB1Fq7IHs= +github.com/rs/dnscache v0.0.0-20211102005908-e0241e321417/go.mod h1:qe5TWALJ8/a1Lqznoc5BDHpYX/8HU60Hm2AwRmqzxqA= github.com/rs/xid v1.2.1/go.mod h1:+uKXf+4Djp6Md1KODXJxgGQPKngRmWyn10oCKFzNHOQ= github.com/rs/zerolog v1.13.0/go.mod h1:YbFCdg8HfsridGWAh22vktObvhZbQsZXe4/zB0OKkWU= github.com/rs/zerolog v1.15.0/go.mod h1:xYTKnLHcpfU2225ny5qZjxnj9NvkumZYjJHlAThCjNc= @@ -784,6 +911,8 @@ github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQD github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/go-glob v0.0.0-20170128012129-256dc444b735 h1:7YvPJVmEeFHR1Tj9sZEYsmarJEQfMVYpd/Vyy/A8dqE= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46 h1:GHRpF1pTW19a8tTFrMLUcfWwyC0pnifVo2ClaLq+hP8= +github.com/ryszard/goskiplist v0.0.0-20150312221310-2dfbae5fcf46/go.mod h1:uAQ5PCi+MFsC7HjREoAz1BU+Mq60+05gifQSsHSDG/8= github.com/samuel/go-zookeeper v0.0.0-20190923202752-2cc03de413da/go.mod h1:gi+0XIa01GRL2eRQVjQkKGqKF3SF9vZR/HnPullcV2E= github.com/satori/go.uuid v1.2.0/go.mod h1:dA0hQrYB0VpLJoorglMZABFdXlWrHn1NEOzdhQKdks0= github.com/sclevine/agouti v3.0.0+incompatible/go.mod h1:b4WX9W9L1sfQKXeJf1mUTLZKJ48R1S7H23Ji7oFO5Bw= @@ -826,9 +955,12 @@ github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= +github.com/smartystreets/assertions v0.0.0-20190215210624-980c5ac6f3ac/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/assertions v1.0.0 h1:UVQPSSmc3qtTi+zPPkCXvZX9VvW/xT/NsRvKfwY81a8= github.com/smartystreets/assertions v1.0.0/go.mod h1:kHHU4qYBaI3q23Pp3VPrmWhuIUrLW/7eUrw0BU5VaoM= github.com/smartystreets/go-aws-auth v0.0.0-20180515143844-0c1422d1fdb9/go.mod h1:SnhjPscd9TpLiy1LpzGSKh3bXCfxxXuqd9xmQJy3slM= +github.com/smartystreets/goconvey v0.0.0-20181108003508-044398e4856c/go.mod h1:XDJAKZRPZ1CvBcN2aX5YOUTYGHki24fSF0Iv48Ibg0s= +github.com/smartystreets/goconvey v0.0.0-20190306220146-200a235640ff/go.mod h1:KSQcGKpxUMHk3nbYzs/tIBAM2iDooCn0BmttHOJEbLs= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/smartystreets/goconvey v1.7.2 h1:9RBaZCeXEQ3UselpuwUQHltGVXvdwm6cv1hgR6gDIPg= github.com/smartystreets/gunit v1.0.0/go.mod h1:qwPWnhz6pn0NnRBP++URONOVyNkPyr4SauJk4cUOwJs= @@ -861,6 +993,7 @@ github.com/streadway/handy v0.0.0-20190108123426-d5acb3125c2a/go.mod h1:qNTQ5P5J github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0/go.mod h1:qt09Ya8vawLte6SNmTgCsAVtYtaKzEcn8ATUoHMkEqE= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= @@ -879,6 +1012,11 @@ github.com/templexxx/cpu v0.0.9 h1:cGGLK8twbc1J1S/fHnZW7BylXYaFP+0fR2s+nzsFDiU= github.com/templexxx/cpu v0.0.9/go.mod h1:w7Tb+7qgcAlIyX4NhLuDKt78AHA5SzPmq0Wj6HiEnnk= github.com/templexxx/xorsimd v0.4.1 h1:iUZcywbOYDRAZUasAs2eSCUW8eobuZDy0I9FJiORkVg= github.com/templexxx/xorsimd v0.4.1/go.mod h1:W+ffZz8jJMH2SXwuKu9WhygqBMbFnp14G2fqEr8qaNo= +github.com/tidwall/btree v1.3.1 h1:636+tdVDs8Hjcf35Di260W2xCW4KuoXOKyk9QWOvCpA= +github.com/tidwall/btree v1.3.1/go.mod h1:LGm8L/DZjPLmeWGjv5kFrY8dL4uVhMmzmmLYmsObdKE= +github.com/tinylib/msgp v1.0.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.0/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= +github.com/tinylib/msgp v1.1.2/go.mod h1:+d+yLhGm8mzTaHzB+wgMYrodPfmZrzkirds8fDWklFE= github.com/tj/assert v0.0.0-20171129193455-018094318fb0/go.mod h1:mZ9/Rh9oLWpLLDRpvE+3b7gP/C2YyLFYxNmcLnPTMe0= github.com/tj/assert v0.0.3 h1:Df/BlaZ20mq6kuai7f5z2TvPFiwC3xaWJSDQNiIS3Rk= github.com/tj/assert v0.0.3/go.mod h1:Ne6X72Q+TB1AteidzQncjw9PabbMp4PBMZ1k+vd1Pvk= @@ -900,6 +1038,8 @@ github.com/viant/assertly v0.4.8/go.mod h1:aGifi++jvCrUaklKEKT0BU95igDNaqkvz+49u github.com/viant/toolbox v0.24.0/go.mod h1:OxMCG57V0PXuIP2HNQrtJf2CjqdmbrOx5EkMILuUhzM= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78 h1:9sreu9e9KOihf2Y0NbpyfWhd1XFDcL4GTkPYL4IvMrg= github.com/wader/filtertransport v0.0.0-20200316221534-bdd9e61eee78/go.mod h1:HazXTRLhXFyq80TQp7PUXi6BKE6mS+ydEdzEqNBKopQ= +github.com/willf/bitset v1.1.9/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= +github.com/willf/bitset v1.1.10/go.mod h1:RjeCKbqT1RxIR/KWY6phxZiaY1IyutSBfGjNPySAYV4= github.com/xiang90/probing v0.0.0-20190116061207-43a291ad63a2/go.mod h1:UETIi67q53MR2AWcXfiuqkDkRtnGDLqkBTpCHuJHxtU= github.com/xordataexchange/crypt v0.0.3-0.20170626215501-b2862e3d0a77/go.mod h1:aYKd//L2LvnjZzWKhF00oedf4jCCReLcmhLdhm1A27Q= github.com/xtaci/kcp-go/v5 v5.6.1 h1:Pwn0aoeNSPF9dTS7IgiPXn0HEtaIlVb6y5UKWPsx8bI= @@ -932,6 +1072,8 @@ gitlab.com/yawning/utls.git v0.0.12-1 h1:RL6O0MP2YI0KghuEU/uGN6+8b4183eqNWoYgx7C gitlab.com/yawning/utls.git v0.0.12-1/go.mod h1:3ONKiSFR9Im/c3t5RKmMJTVdmZN496FNyk3mjrY1dyo= gitlab.torproject.org/tpo/anti-censorship/geoip v0.0.0-20210928150955-7ce4b3d98d01/go.mod h1:K3LOI4H8fa6j+7E10ViHeGEQV10304FG4j94ypmKLjY= go.etcd.io/bbolt v1.3.3/go.mod h1:IbVyRI1SCnLcuJnV2u8VeU0CEYM7e686BmAb1XKL+uU= +go.etcd.io/bbolt v1.3.6 h1:/ecaJf0sk1l4l6V4awd65v2C3ILy7MSj+s/x1ADCIMU= +go.etcd.io/bbolt v1.3.6/go.mod h1:qXsaaIqmgQH0T+OPdb99Bf+PKfBBQVAdyD6TY9G8XM4= go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg= go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs= go.etcd.io/etcd/client/pkg/v3 v3.5.0/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= @@ -946,6 +1088,10 @@ go.opencensus.io v0.22.3/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.4/go.mod h1:yxeiOL68Rb0Xd1ddK5vPZ/oVn4vY4Ynel7k9FzqtOIw= go.opencensus.io v0.22.5/go.mod h1:5pWMHQbX5EPX2/62yrJeAkowc+lfs/XD7Uxpq3pI6kk= go.opencensus.io v0.23.0/go.mod h1:XItmlyltB5F7CS4xOC1DcqMoFqwtC6OG2xF7mCv7P7E= +go.opentelemetry.io/otel v1.8.0 h1:zcvBFizPbpa1q7FehvFiHbQwGzmPILebO0tyqIR5Djg= +go.opentelemetry.io/otel v1.8.0/go.mod h1:2pkj+iMj0o03Y+cW6/m8Y4WkRdYN3AvCXCnzRMp9yvM= +go.opentelemetry.io/otel/trace v1.8.0 h1:cSy0DF9eGI5WIfNwZ1q2iUyGj00tGzP24dE1lOlHrfY= +go.opentelemetry.io/otel/trace v1.8.0/go.mod h1:0Bt3PXY8w+3pheS3hQUt+wow8b1ojPaTBoTCh2zIFI4= go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= go.uber.org/atomic v1.5.0/go.mod h1:sABNBOSYdrvTF6hTgEIbc7YasKWGhgEQZyfxyTvoXHQ= @@ -1008,6 +1154,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d h1:vtUKgx8dahOomfFzLREU8nSv25YHnTgLBn4rDnWZdU0= +golang.org/x/exp v0.0.0-20220613132600-b0d781184e0d/go.mod h1:Kr81I6Kryrl9sr8s2FK3vxD90NdsKWRuOIl2O4CvYbA= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= @@ -1036,6 +1184,7 @@ golang.org/x/mod v0.4.1/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.4.2/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4 h1:6zppjxzCulZykYSLyVDYbneBfbaBIQPYMevg0bEwv2s= golang.org/x/mod v0.6.0-dev.0.20220419223038-86c51ed26bb4/go.mod h1:jJ57K6gSWd91VN4djpZkiMVwK6gcyfeH4XE8wZrZaV4= +golang.org/x/net v0.0.0-20180524181706-dfa909b99c79/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180906233101-161cd47e91fd/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -1129,6 +1278,7 @@ golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4 h1:uVc8UZUe6tr40fFVnUP5Oj+veunVezqYl9z7DYw9xzw= +golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -1173,6 +1323,7 @@ golang.org/x/sys v0.0.0-20200223170610-d5e6a3e2c0ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200302150141-5c8b2ff67527/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200331124033-c3d80250170d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200413165638-669c56c373c4/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200501052902-10377860bb8e/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200511232937-7e40ca221e25/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200515095857-1151b9dac4a9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1183,6 +1334,7 @@ golang.org/x/sys v0.0.0-20200625212154-ddb9806d33ae/go.mod h1:h1NjWce9XRLGQEsW7w golang.org/x/sys v0.0.0-20200803210538-64077c9b5642/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200808120158-1030fc2bf1d9/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200905004654-be1d3432aa8f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/sys v0.0.0-20200923182605-d9f96fdee20d/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20200930185726-fdedc70b468f/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201119102817-f84b799fce68/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= golang.org/x/sys v0.0.0-20201201145000-ef89a241ccb3/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= @@ -1236,6 +1388,8 @@ golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxb golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858 h1:Dpdu/EMxGMFgq0CeYMh4fazTD2vtlZRYE7wyynxJb9U= +golang.org/x/time v0.0.0-20220609170525-579cf78fd858/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/tools v0.0.0-20180221164845-07fd8470d635/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180828015842-6cd1fcedba52/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= golang.org/x/tools v0.0.0-20180917221912-90fa682c2a6e/go.mod h1:n7NCudcB/nEzxVGmLbDWY5pfWTLqBcC2KZ6jyYvM4mQ= @@ -1389,8 +1543,8 @@ google.golang.org/genproto v0.0.0-20210303154014-9728d6b83eeb/go.mod h1:FWY/as6D google.golang.org/genproto v0.0.0-20210310155132-4ce2db91004e/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210319143718-93e7006c17a6/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no= google.golang.org/genproto v0.0.0-20210402141018-6c239bbf2bb1/go.mod h1:9lPAdzaEmUacj36I+k7YKbEc5CXzPIeORRgDAUOu28A= -google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c h1:wtujag7C+4D6KMoulW9YauvK2lgdvCMS260jsqqBXr0= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= +google.golang.org/genproto v0.0.0-20211118181313-81c1377c94b1 h1:b9mVrqYfq3P4bCdaLg1qtBnPzUYgglsIdjZkL/fQVOE= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.16.0/go.mod h1:0JHn/cJsOMiMfNA9+DeHDlAU7KAAB5GDlYFpa9MZMio= google.golang.org/grpc v1.17.0/go.mod h1:6QZJwpn2B+Zp71q/5VxRsJ6NXXVCE5NRUHRo+f3cWCs= @@ -1418,8 +1572,8 @@ google.golang.org/grpc v1.35.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAG google.golang.org/grpc v1.36.0/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.36.1/go.mod h1:qjiiYl8FncCW8feJPdyg3v6XW24KsRHe+dy9BAGRRjU= google.golang.org/grpc v1.37.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.38.0 h1:/9BgsAsa5nWe26HqOlvlgJnqBuktYOLCgjCPqsa56W0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= +google.golang.org/grpc v1.46.2 h1:u+MLGgVf7vRdjEYZ8wDFhAVNmhkbJ5hmrA1LMWK1CAQ= google.golang.org/grpc/cmd/protoc-gen-go-grpc v1.1.0/go.mod h1:6Kw0yEErY5E/yWrBtf03jp27GLLJujG4z/JK95pnjjw= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= diff --git a/internal/cmd/jafar/tcpproxy/tcpproxy.go b/internal/cmd/jafar/tcpproxy/tcpproxy.go new file mode 100644 index 0000000..77f8726 --- /dev/null +++ b/internal/cmd/jafar/tcpproxy/tcpproxy.go @@ -0,0 +1,210 @@ +// Package tlsproxy contains a censoring TLS proxy. Most traffic is passed +// through using the SNI to choose the hostname to connect to. Specific offending +// SNIs are censored by returning a TLS alert to the client. +package tlsproxy + +import ( + "context" + "crypto/tls" + "errors" + "net" + "strings" + "sync" + + "github.com/apex/log" +) + +// Dialer establishes network connections +type Dialer interface { + DialContext(ctx context.Context, network, address string) (net.Conn, error) +} + +// CensoringProxy is a censoring TLS proxy +type CensoringProxy struct { + keywords []string + dial func(network, address string) (net.Conn, error) +} + +// NewCensoringProxy creates a new CensoringProxy instance using +// the specified list of keywords to censor. keywords is the list +// of keywords that trigger censorship if any of them appears in +// the SNII record of a ClientHello. dnsNetwork and dnsAddress are +// settings to configure the upstream, non censored DNS. +func NewCensoringProxy( + keywords []string, to string, uncensored Dialer, +) *CensoringProxy { + return &CensoringProxy{ + keywords: keywords, + to: to, + dial: func(network, address string) (net.Conn, error) { + return uncensored.DialContext(context.Background(), network, address) + }, + } +} + +// handshakeReader is a hack to perform the initial part of the +// TLS handshake so to know the SNI and then replay the bytes of +// this initial part of the handshake with the server. +//type handshakeReader struct { +// net.Conn +// incoming []byte +//} +// +//// Read saves the initial bytes of the handshake such that later +//// we can replay the handshake with the real TLS server. +//func (c *handshakeReader) Read(b []byte) (int, error) { +// count, err := c.Conn.Read(b) +// if err == nil { +// c.incoming = append(c.incoming, b[:count]...) +// } +// return count, err +//} +// +//// Write prevents writing on the real connection +//func (c *handshakeReader) Write(b []byte) (int, error) { +// return 0, errors.New("cannot write on this connection") +//} +type censoredReader struct { + net.Conn + outgoing []byte +} + +// forward forwards left traffic to right +func forward(wg *sync.WaitGroup, left, right net.Conn) { + data := make([]byte, 1<<18) + for { + n, err := left.Read(data) + if err != nil { + break + } + if _, err = right.Write(data[:n]); err != nil { + break + } + } + wg.Done() +} + +// reset closes the connection with a RST segment +func reset(conn net.Conn) { + if tc, ok := conn.(*net.TCPConn); ok { + tc.SetLinger(0) + } + conn.Close() +} + +// alertclose sends a TLS alert and then closes the connection +func alertclose(conn net.Conn) { + alertdata := []byte{ + 21, // alert + 3, // version[0] + 3, // version[1] + 0, // length[0] + 2, // length[1] + 2, // fatal + 80, // internal error + } + conn.Write(alertdata) + conn.Close() +} + +// getsni attempts the handshakeReader hack to obtain the SNI by reading +// the beginning of the TLS handshake. On success a nonempty SNI string +// is returned. Otherwise we cannot distinguish between the absence of a +// SNI and any other reading network error that may have occurred. +func getsni(conn *handshakeReader) string { + var ( + sni string + mutex sync.Mutex // just for safety + ) + tls.Server(conn, &tls.Config{ + GetCertificate: func(info *tls.ClientHelloInfo) (*tls.Certificate, error) { + mutex.Lock() + sni = info.ServerName + mutex.Unlock() + return nil, errors.New("tlsproxy: we can't really continue handshake") + }, + }).Handshake() + return sni +} + +func (p *CensoringProxy) connectingToMyself(conn net.Conn) bool { + local := conn.LocalAddr().String() + localAddr, _, localErr := net.SplitHostPort(local) + remote := conn.RemoteAddr().String() + remoteAddr, _, remoteErr := net.SplitHostPort(remote) + return localErr != nil || remoteErr != nil || localAddr == remoteAddr +} + +// handle implements the TLS SNI proxy +func (p *CensoringProxy) handle(clientconn net.Conn, to string) { + lr := &ougGoingReader{Conn: clientconn } + line := readline(lr) + //hr := &handshakeReader{Conn: clientconn} + //sni := getsni(hr) +// if sni == "" { +// log.Warn("tlsproxy: network failure or SNI not provided") +// reset(clientconn) +// return +// } + // TODO + for _, pattern := range p.keywords { + if strings.Contains(line, pattern) { + log.Warnf("tlsproxy: reject SNI by policy: %s", sni) + alertclose(clientconn) + return + } + } + serverconn, err := p.dial("tcp", to) + if err != nil { + log.WithError(err).Warn("tlsproxy: p.dial failed") + alertclose(clientconn) + return + } + if p.connectingToMyself(serverconn) { + log.Warn("tlsproxy: connecting to myself") + alertclose(clientconn) + return + } + if _, err := serverconn.Write(hr.incoming); err != nil { + log.WithError(err).Warn("tlsproxy: serverconn.Write failed") + alertclose(clientconn) + return + } + log.Debugf("tlsproxy: routing for %s", sni) + defer clientconn.Close() + defer serverconn.Close() + var wg sync.WaitGroup + wg.Add(2) + go forward(&wg, clientconn, serverconn) + go forward(&wg, serverconn, clientconn) + wg.Wait() +} + +func (p *CensoringProxy) run(listener net.Listener, outboundPort string) { + for { + conn, err := listener.Accept() + if err != nil && strings.Contains( + err.Error(), "use of closed network connection") { + return + } + if err == nil { + // It's difficult to make accept fail, so restructure + // the code such that we enter into the happy path + go p.handle(conn, outboundPort) + } + } +} + +// Start starts the censoring proxy. +func (p *CensoringProxy) Start(address string, to string) (net.Listener, error) { + _, port, err := net.SplitHostPort(address) + if err != nil { + return nil, err + } + listener, err := net.Listen("tcp", address) + if err != nil { + return nil, err + } + go p.run(listener, to) + return listener, nil +} diff --git a/internal/cmd/jafar/tcpproxy/tlsproxy_test.go b/internal/cmd/jafar/tcpproxy/tlsproxy_test.go new file mode 100644 index 0000000..86a21d8 --- /dev/null +++ b/internal/cmd/jafar/tcpproxy/tlsproxy_test.go @@ -0,0 +1,181 @@ +package tlsproxy + +import ( + "crypto/tls" + "errors" + "net" + "sync" + "testing" + + "github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored" +) + +func TestPass(t *testing.T) { + listener := newproxy(t, "ooni.io") + checkdialtls(t, listener.Addr().String(), true, &tls.Config{ + ServerName: "example.com", + }) + killproxy(t, listener) +} + +func TestBlock(t *testing.T) { + listener := newproxy(t, "ooni.io") + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "api.ooni.io", + }) + killproxy(t, listener) +} + +func TestNoSNI(t *testing.T) { + listener := newproxy(t, "ooni.io") + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "", + }) + killproxy(t, listener) +} + +func TestInvalidDomain(t *testing.T) { + listener := newproxy(t, "ooni.io") + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "antani.local", + }) + killproxy(t, listener) +} + +func TestFailHandshake(t *testing.T) { + listener := newproxy(t, "ooni.io") + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "expired.badssl.com", + }) + killproxy(t, listener) +} + +func TestFailConnectingToSelf(t *testing.T) { + proxy := &CensoringProxy{ + dial: func(network string, address string) (net.Conn, error) { + return &mockedConnWriteError{}, nil + }, + } + listener, err := proxy.Start("127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + if listener == nil { + t.Fatal("expected non nil listener here") + } + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "www.google.com", + }) + killproxy(t, listener) +} + +func TestFailWriteAfterConnect(t *testing.T) { + proxy := &CensoringProxy{ + dial: func(network string, address string) (net.Conn, error) { + return &mockedConnWriteError{ + // must be different or it refuses connecting to self + localIP: net.IPv4(127, 0, 0, 1), + remoteIP: net.IPv4(127, 0, 0, 2), + }, nil + }, + } + listener, err := proxy.Start("127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + if listener == nil { + t.Fatal("expected non nil listener here") + } + checkdialtls(t, listener.Addr().String(), false, &tls.Config{ + ServerName: "www.google.com", + }) + killproxy(t, listener) +} + +func TestListenError(t *testing.T) { + proxy := NewCensoringProxy( + []string{""}, uncensored.NewClient("https://1.1.1.1/dns-query"), + ) + listener, err := proxy.Start("8.8.8.8:80") + if err == nil { + t.Fatal("expected an error here") + } + if listener != nil { + t.Fatal("expected nil listener here") + } +} + +func newproxy(t *testing.T, blocked string) net.Listener { + proxy := NewCensoringProxy( + []string{blocked}, uncensored.NewClient("https://1.1.1.1/dns-query"), + ) + listener, err := proxy.Start("127.0.0.1:0") + if err != nil { + t.Fatal(err) + } + return listener +} + +func killproxy(t *testing.T, listener net.Listener) { + err := listener.Close() + if err != nil { + t.Fatal(err) + } +} + +func checkdialtls( + t *testing.T, proxyAddr string, expectSuccess bool, config *tls.Config, +) { + conn, err := tls.Dial("tcp", proxyAddr, config) + if err != nil && expectSuccess { + t.Fatal(err) + } + if err == nil && !expectSuccess { + t.Fatal("expected failure here") + } + if conn == nil && expectSuccess { + t.Fatal("expected actionable conn") + } + if conn != nil && !expectSuccess { + t.Fatal("expected nil conn") + } + if conn != nil { + conn.Close() + } +} + +type mockedConnWriteError struct { + net.Conn + localIP net.IP + remoteIP net.IP +} + +func (c *mockedConnWriteError) Write(b []byte) (int, error) { + return 0, errors.New("cannot write sorry") +} + +func (c *mockedConnWriteError) LocalAddr() net.Addr { + return &net.TCPAddr{ + IP: c.localIP, + } +} + +func (c *mockedConnWriteError) RemoteAddr() net.Addr { + return &net.TCPAddr{ + IP: c.remoteIP, + } +} + +func TestForwardWriteError(t *testing.T) { + var wg sync.WaitGroup + wg.Add(1) + forward(&wg, &mockedConnReadOkay{}, &mockedConnWriteError{}) +} + +type mockedConnReadOkay struct { + net.Conn +} + +func (c *mockedConnReadOkay) Read(b []byte) (int, error) { + return len(b), nil +} diff --git a/internal/engine/experiment.go b/internal/engine/experiment.go index f40f4c3..325bb6f 100644 --- a/internal/engine/experiment.go +++ b/internal/engine/experiment.go @@ -92,7 +92,12 @@ func (eaw *experimentAsyncWrapper) RunAsync( out := make(chan *model.ExperimentAsyncTestKeys) measurement := eaw.experiment.newMeasurement(input) start := time.Now() - err := eaw.experiment.measurer.Run(ctx, eaw.session, measurement, eaw.callbacks) + args := &model.ExperimentArgs{ + Callbacks: eaw.callbacks, + Measurement: measurement, + Session: eaw.session, + } + err := eaw.experiment.measurer.Run(ctx, args) stop := time.Now() if err != nil { return nil, err diff --git a/internal/engine/experiment/bittorrent/bittorrent.go b/internal/engine/experiment/bittorrent/bittorrent.go new file mode 100644 index 0000000..c98dd0f --- /dev/null +++ b/internal/engine/experiment/bittorrent/bittorrent.go @@ -0,0 +1,240 @@ +package bittorrent + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "net" + "net/url" + "os" + "time" + + "github.com/anacrolix/torrent" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +var ( + // errNoInputProvided indicates no input was passed + errNoInputProvided = errors.New("no input provided") + + // errInputIsNotAnURL indicates that input is not an URL + errInputIsNotAnURL = errors.New("input is not an URL") + + // errInvalidScheme indicates that the scheme is invalid + // golint is stupid and does not let us end erorr with ":" + errInvalidScheme = errors.New("scheme must be magnet") +) + +const ( + testName = "bittorrent" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +type runtimeConfig struct { + magnet string +} + +func config(input model.MeasurementTarget) (*runtimeConfig, error) { + if input == "" { + return nil, errNoInputProvided + } + + parsed, err := url.Parse(string(input)) + if err != nil { + return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) + } + if parsed.Scheme != "magnet" { + return nil, errInvalidScheme + } + + validConfig := runtimeConfig{ + magnet: string(input), + } + + return &validConfig, nil +} + +// TestKeys contains the experiment results +type TestKeys struct { + // DNS queries when resolving trackers + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + // Indicates any kind of failure + Failure string `json:"failure"` + // The total number of peers contacted about the requested magnet + PeersNum int `json:"peers_num"` + // The complete list of peers contacted + Peers []string `json:"peers"` + // The total number of bytes received by the client + TotalBytesRead int64 `json:"total_bytes_received"` + // The total number of bad pieces (failed verification) received by the client + TotalBadPieces int64 `json:"total_bad_pieces"` +} + +func (tk *TestKeys) failure(err error) { + tk.Failure = *tracex.NewFailure(err) +} + +// Measurer performs the measurement. +type Measurer struct { + // Config contains the experiment settings. If empty we + // will be using default settings. + Config Config + + // Getter is an optional getter to be used for testing. + Getter urlgetter.MultiGetter +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +func torrentStats(torrent *torrent.Torrent, client *torrent.Client, tk *TestKeys) { + stats := torrent.Stats() + tk.PeersNum = len(tk.Peers) + tk.TotalBytesRead = stats.ConnStats.BytesRead.Int64() + tk.TotalBadPieces = stats.ConnStats.PiecesDirtiedBad.Int64() +} + +func timeoutStats(torrent *torrent.Torrent, client *torrent.Client, tk *TestKeys) { + torrentStats(torrent, client, tk) + tk.Failure = "download_timeout" +} + +// Run implements ExperimentMeasurer.Run +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + //ctx context.Context, sess model.ExperimentSession, + //measurement *model.Measurement, callbacks model.ExperimentCallbacks, + //) error { + sess := args.Session + measurement := args.Measurement + log := sess.Logger() + trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) + resolver := trace.NewStdlibResolver(log) + + config, err := config(measurement.Input) + if err != nil { + // Invalid input data, we don't even generate report + return err + } + + tk := new(TestKeys) + measurement.TestKeys = tk + + ctx, cancel := context.WithTimeout(ctx, 120*time.Second) + defer cancel() + + tmpdir, err := os.MkdirTemp("", "ooni") + if err != nil { + log.Warnf(*tracex.NewFailure(err)) + return nil + } + log.Infof("Using temporary directory %s", tmpdir) + defer os.RemoveAll(tmpdir) + + conf := torrent.NewDefaultClientConfig() + conf.DataDir = tmpdir + conf.NoUpload = true + + // Lookup tracker IPs via ooni utils + conf.LookupTrackerIp = func(u *url.URL) ([]net.IP, error) { + + log.Infof("Resolving DNS for %s", u.Hostname()) + resolvedAddrs, err := resolver.LookupHost(ctx, u.Hostname()) + addrs := []net.IP{} + if err != nil { + return addrs, nil + } + log.Infof("Finished DNS for %s: %v", u.Hostname(), resolvedAddrs) + for _, addr := range resolvedAddrs { + addrs = append(addrs, net.ParseIP(addr)) + } + tk.Queries = append(tk.Queries, trace.DNSLookupsFromRoundTrip()...) + return addrs, err + } + + // We want to test Bittorrent connectivity, not HTTPS/websockets + conf.DisableWebtorrent = true + conf.DisableWebseeds = true + + // Register new peers to the test keys + clientCallbacks := new(torrent.Callbacks) + clientCallbacks.NewPeer = append(clientCallbacks.NewPeer, + func(peer *torrent.Peer) { + log.Debugf("Found new peer: %s", peer.RemoteAddr.String()) + tk.Peers = append(tk.Peers, peer.RemoteAddr.String()) + }, + ) + conf.Callbacks = *clientCallbacks + + client, err := torrent.NewClient(conf) + if err != nil { + log.Warnf(*tracex.NewFailure(err)) + return nil + } + defer client.Close() + + torrent, err := client.AddMagnet(config.magnet) + if err != nil { + log.Warnf(*tracex.NewFailure(err)) + return nil + } + + select { + case <-ctx.Done(): + tk.Failure = "metainfo_timeout" + return nil + case <-torrent.GotInfo(): + } + + torrent.DownloadAll() + + // Setup a new chan to know when the torrent is finished... allows to apply timeout + finished := make(chan bool) + + go func() { + client.WaitAll() + finished <- true + }() + select { + case <-ctx.Done(): + timeoutStats(torrent, client, tk) + case <-finished: + torrentStats(torrent, client, tk) + } + return nil +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return &Measurer{Config: config} +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{IsAnomaly: false} + _, ok := measurement.TestKeys.(*TestKeys) + if !ok { + return sk, errors.New("invalid test keys type") + } + return sk, nil +} diff --git a/internal/engine/experiment/bittorrent/bittorrent_test.go b/internal/engine/experiment/bittorrent/bittorrent_test.go new file mode 100644 index 0000000..7423671 --- /dev/null +++ b/internal/engine/experiment/bittorrent/bittorrent_test.go @@ -0,0 +1,54 @@ +package bittorrent + +import ( + "context" + "errors" + "testing" + + "github.com/ooni/probe-cli/v3/internal/engine/mockable" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func TestMeasurer_run(t *testing.T) { + // runHelper is an helper function to run this set of tests. + runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { + measurer := NewExperimentMeasurer(Config{}) + ctx := context.Background() + measurement := &model.Measurement{ + Input: model.MeasurementTarget(input), + } + session := &mockable.Session{ + MockableLogger: model.DiscardLogger, + } + + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: measurement, + Session: session, + } + + err := measurer.Run(ctx, args) + return measurement, measurer, err + } + + t.Run("with empty input", func(t *testing.T) { + _, _, err := runHelper("") + if !errors.Is(err, errNoInputProvided) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid URL", func(t *testing.T) { + _, _, err := runHelper("\t") + if !errors.Is(err, errInputIsNotAnURL) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid scheme", func(t *testing.T) { + _, _, err := runHelper("https://8.8.8.8:443/") + if !errors.Is(err, errInvalidScheme) { + t.Fatal("unexpected error", err) + } + }) +} diff --git a/internal/engine/experiment/dash/dash.go b/internal/engine/experiment/dash/dash.go index a44021d..8489286 100644 --- a/internal/engine/experiment/dash/dash.go +++ b/internal/engine/experiment/dash/dash.go @@ -249,10 +249,10 @@ func (m Measurer) ExperimentVersion() string { } // Run implements model.ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session tk := new(TestKeys) measurement.TestKeys = tk saver := &tracex.Saver{} diff --git a/internal/engine/experiment/dash/dash_test.go b/internal/engine/experiment/dash/dash_test.go index 7b5addc..b2a910c 100644 --- a/internal/engine/experiment/dash/dash_test.go +++ b/internal/engine/experiment/dash/dash_test.go @@ -270,15 +270,15 @@ func TestMeasureWithCancelledContext(t *testing.T) { cancel() // cause failure measurement := new(model.Measurement) m := &Measurer{} - err := m.Run( - ctx, - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableHTTPClient: http.DefaultClient, MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := m.Run(ctx, args) // See corresponding comment in Measurer.Run implementation to // understand why here it's correct to return nil. if !errors.Is(err, nil) { diff --git a/internal/engine/experiment/dht/dht.go b/internal/engine/experiment/dht/dht.go new file mode 100644 index 0000000..8c7526e --- /dev/null +++ b/internal/engine/experiment/dht/dht.go @@ -0,0 +1,349 @@ +package dht + +import ( + "context" + "fmt" + "github.com/pkg/errors" + "net" + "net/url" + "time" + + "github.com/anacrolix/dht/v2" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" + "github.com/ooni/probe-cli/v3/internal/measurexlite" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/tracex" +) + +var ( + // errNoInputProvided indicates no input was passed + errNoInputProvided = errors.New("no input provided") + + // errInputIsNotAnURL indicates that input is not an URL + errInputIsNotAnURL = errors.New("input is not an URL") + + // errInvalidScheme indicates that the scheme is invalid + errInvalidScheme = errors.New("scheme must be dht://") + + // errMissingPort indicates that no port was provided + errMissingPort = errors.New("no port was provided but dht:// requires explicit port") +) + +const ( + testName = "dht" + testVersion = "0.0.1" +) + +// Config contains the experiment config. +type Config struct{} + +type runtimeConfig struct { + // nodeaddr IP or domain name + dhtnode string + port string + infohash string +} + +func config(input model.MeasurementTarget) (*runtimeConfig, error) { + // Bittorrent v2 hybrid test torrent: https://blog.libtorrent.org/2020/09/bittorrent-v2/ + // Has good chances of being seeded years from now + hash := "631a31dd0a46257d5078c0dee4e66e26f73e42ac" + + if input == "" { + return nil, errNoInputProvided + } + + // TODO: static input from defaultDHTBoostrapNodes() + // input == "" triggers runtime error from the experiment runner + if input == "DUMMY" { + // No requested DHT bootstrap node, let the DHT library try all it knows + return &runtimeConfig{ + dhtnode: "", + port: "", + infohash: hash, + }, nil + } + + parsed, err := url.Parse(string(input)) + if err != nil { + return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) + } + if parsed.Scheme != "dht" { + return nil, errInvalidScheme + } + + if parsed.Port() == "" { + // Port is mandatory because DHT bootstrap nodes use different ports + return nil, errMissingPort + } + + validConfig := runtimeConfig{ + dhtnode: parsed.Hostname(), + port: parsed.Port(), + infohash: hash, + } + + return &validConfig, nil +} + +// TestKeys contains the experiment results +type TestKeys struct { + Queries []*model.ArchivalDNSLookupResult `json:"queries"` + Runs []*IndividualTestKeys `json:"runs"` + // Used for global failure (DNS resolution) + Failure string `json:"failure"` +} + +func (tk *TestKeys) failure(err error) { + tk.Failure = *tracex.NewFailure(err) +} + +func (tk *TestKeys) computeFailure() { + if tk.Failure != "" { + return + } + for _, itk := range tk.Runs { + if itk.Failure != "" { + tk.Failure = itk.Failure + return + } + } +} + +// IndividualTestKeys indicate results for a single IP/port combo DHT bootstrap node +// in case the DNS resolves to several IPs, or multiple bootstrap domains were used +type IndividualTestKeys struct { + // Logger, not exported to JSON + logger model.Logger + + // List of IP/port combos tried to boostrap DHT + BootstrapNodes []string `json:"bootstrap_nodes"` + // Number of DHT bootsrap nodes + BootstrapNum int `json:"bootstrap_num"` + // Number of DHT peers contacted + PeersTriedNum uint32 `json:"peers_tried_num"` + // Number of DHT peers who answered + PeersRespondedNum uint32 `json:"peers_responded_num"` + // Number of DHT peers found for specific requested infohash + InfohashPeersNum int `json:"infohash_peers_num"` + // Actual DHT peers found for requested infohash + InfohashPeers []string `json:"infohash_peers"` + // Individual failure aborting the test run for this address/port combo + Failure string `json:"failure"` +} + +func (itk *IndividualTestKeys) error(err error) { + itk.Failure = *tracex.NewFailure(err) + itk.logger.Warn(itk.Failure) +} + +func newITK(tk *TestKeys, log model.Logger) *IndividualTestKeys { + itk := new(IndividualTestKeys) + itk.logger = log + tk.Runs = append(tk.Runs, itk) + return itk +} + +// Measurer performs the measurement. +type Measurer struct { + // Config contains the experiment settings. If empty we + // will be using default settings. + Config Config + + // Getter is an optional getter to be used for testing. + Getter urlgetter.MultiGetter +} + +// ExperimentName implements ExperimentMeasurer.ExperimentName +func (m Measurer) ExperimentName() string { + return testName +} + +// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion +func (m Measurer) ExperimentVersion() string { + return testVersion +} + +func defaultDHTBoostrapNodes() []string { + return []string{ + "router.utorrent.com:6881", + "router.bittorrent.com:6881", + "dht.transmissionbt.com:6881", + "dht.aelitis.com:6881", + "router.silotis.us:6881", + "dht.libtorrent.org:25401", + "dht.anacrolix.link:42069", + "router.bittorrent.cloud:42069", + } +} + +// Server starts a DHT server with a list of bootstrap nodes and stores +// failure cases inside a IndividualTestKeys +func Server(bootstrapNodes []string, itk *IndividualTestKeys) (*dht.Server, bool) { + itk.BootstrapNodes = bootstrapNodes + itk.BootstrapNum = len(bootstrapNodes) + + // Starting new DHT client + dhtconf := dht.NewDefaultServerConfig() + dhtconf.QueryResendDelay = func() time.Duration { + return 10 * time.Second + } + + dhtconf.StartingNodes = func() (addrs []dht.Addr, err error) { + for _, addrport := range bootstrapNodes { + udpAddr, err := net.ResolveUDPAddr("udp", addrport) + if err != nil { + return nil, err + } + addrs = append(addrs, dht.NewAddr(udpAddr)) + } + return addrs, nil + } + + dhtsrv, err := dht.NewServer(dhtconf) + if err != nil { + itk.error(err) + return nil, false + } + itk.logger.Infof("Finished starting DHT server with bootstrap nodes: %v", bootstrapNodes) + return dhtsrv, true +} + +func testServer(dht *dht.Server, infohash [20]byte, itk *IndividualTestKeys) bool { + announce, err := dht.AnnounceTraversal(infohash) + if err != nil { + itk.error(err) + return false + } + defer announce.Close() + + counter := 0 + for entry := range announce.Peers { + counter++ + itk.InfohashPeers = append(itk.InfohashPeers, entry.NodeInfo.Addr.String()) + itk.logger.Debugf("peer %d: %s", counter, entry.NodeInfo.Addr) + } + + stats := announce.TraversalStats() + itk.PeersTriedNum = stats.NumAddrsTried + itk.PeersRespondedNum = stats.NumResponses + itk.InfohashPeersNum = counter + + if itk.PeersRespondedNum == 0 { + itk.error(errors.New("No DHT peers were found")) + return false + } + + itk.logger.Infof("Tried %d peers obtained from %d bootstrap nodes. Got response from %d. %d have requested infohash.", itk.PeersTriedNum, itk.BootstrapNum, itk.PeersRespondedNum, itk.InfohashPeersNum) + + return true + +} + +// Run implements ExperimentMeasurer.Run +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + //ctx context.Context, sess model.ExperimentSession, + //measurement *model.Measurement, callbacks model.ExperimentCallbacks, + //) error { + sess := args.Session + measurement := args.Measurement + log := sess.Logger() + trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) + resolver := trace.NewStdlibResolver(log) + + config, err := config(measurement.Input) + if err != nil { + // Invalid input data, we don't even generate report + return err + } + + tk := new(TestKeys) + measurement.TestKeys = tk + + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) + defer cancel() + + // Turn string infohash into 20-bytes array + var infohash [20]byte + copy(infohash[:], config.infohash) + + if config.dhtnode != "" { + // Specific node provided: resolve it + log.Infof("Resolving DNS for %s", config.dhtnode) + resolvedAddrs, err := resolver.LookupHost(ctx, config.dhtnode) + tk.Queries = append(tk.Queries, trace.DNSLookupsFromRoundTrip()...) + if err != nil { + tk.failure(err) + return nil + } + log.Infof("Finished DNS for %s: %v", config.dhtnode, resolvedAddrs) + + for _, addr := range resolvedAddrs { + + nodeAddrport := net.JoinHostPort(addr, config.port) + log.Infof("Trying DHT bootstrap node %s", nodeAddrport) + nodeAddrports := []string{nodeAddrport} + + itk := newITK(tk, log) + + dht, success := Server(nodeAddrports, itk) + if !success { + continue + } + + testServer(dht, infohash, itk) + } + } else { + // Use default DHT bootstrap nodes because none was given by input + resolvedAddrports := []string{} + for _, bootstrapDomain := range defaultDHTBoostrapNodes() { + // Ignore error because we use static input so panic chance is 0 + host, port, _ := net.SplitHostPort(bootstrapDomain) + log.Infof("Resolving DNS for %s", host) + resolvedAddrs, err := resolver.LookupHost(ctx, host) + tk.Queries = append(tk.Queries, trace.DNSLookupsFromRoundTrip()...) + if err != nil { + tk.failure(err) + return nil + } + log.Infof("Finished DNS for %s: %v", host, resolvedAddrs) + for _, resolvedAddr := range resolvedAddrs { + resolvedAddrports = append(resolvedAddrports, net.JoinHostPort(resolvedAddr, port)) + } + } + log.Infof("Resolved the following bootstrap nodes: %v", resolvedAddrports) + + itk := newITK(tk, log) + dht, success := Server(resolvedAddrports, itk) + if success { + testServer(dht, infohash, itk) + } + } + + tk.computeFailure() + + return nil +} + +// NewExperimentMeasurer creates a new ExperimentMeasurer. +func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { + return Measurer{Config: config} +} + +// SummaryKeys contains summary keys for this experiment. +// +// Note that this structure is part of the ABI contract with ooniprobe +// therefore we should be careful when changing it. +type SummaryKeys struct { + IsAnomaly bool `json:"-"` +} + +// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. +func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { + sk := SummaryKeys{IsAnomaly: false} + _, ok := measurement.TestKeys.(*TestKeys) + if !ok { + return sk, errors.New("invalid test keys type") + } + return sk, nil +} diff --git a/internal/engine/experiment/dht/dht_test.go b/internal/engine/experiment/dht/dht_test.go new file mode 100644 index 0000000..cf7eae5 --- /dev/null +++ b/internal/engine/experiment/dht/dht_test.go @@ -0,0 +1,118 @@ +package dht + +import ( + "context" + "errors" + "fmt" + "log" + "testing" + + "github.com/anacrolix/dht/v2" + "github.com/ooni/probe-cli/v3/internal/engine/mockable" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func TestMeasurer_run(t *testing.T) { + // runHelper is an helper function to run this set of tests. + runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { + measurer := NewExperimentMeasurer(Config{}) + ctx := context.Background() + measurement := &model.Measurement{ + Input: model.MeasurementTarget(input), + } + session := &mockable.Session{ + MockableLogger: model.DiscardLogger, + } + + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(model.DiscardLogger), + Measurement: measurement, + Session: session, + } + + err := measurer.Run(ctx, args) + return measurement, measurer, err + } + + t.Run("with empty input", func(t *testing.T) { + _, _, err := runHelper("") + if !errors.Is(err, errNoInputProvided) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid URL", func(t *testing.T) { + _, _, err := runHelper("\t") + if !errors.Is(err, errInputIsNotAnURL) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with invalid scheme", func(t *testing.T) { + _, _, err := runHelper("https://8.8.8.8:443/") + if !errors.Is(err, errInvalidScheme) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with missing port", func(t *testing.T) { + _, _, err := runHelper("dht://8.8.8.8") + if !errors.Is(err, errMissingPort) { + t.Fatal("unexpected error", err) + } + }) + + t.Run("with local listener", func(t *testing.T) { + conf := new(dht.ServerConfig) + conf.StartingNodes = func() (addrs []dht.Addr, err error) { + return []dht.Addr{}, nil + } + conf.Passive = false + dht, err := dht.NewServer(conf) + if err != nil { + log.Fatal(err) + } + defer dht.Close() + _, _ = dht.Bootstrap() + + println(dht.Addr().String()) + url := fmt.Sprintf("dht://%s", dht.Addr().String()) + + meas, m, err := runHelper(url) + if err != nil { + t.Fatal(err) + } + + tk := meas.TestKeys.(*TestKeys) + + if tk.Failure != "" { + t.Fatal(tk.Failure) + } + + if len(tk.Runs) != 1 { + t.Fatal("Expected one DHT run") + } + + run := tk.Runs[0] + if run.Failure != "" { + t.Fatal(run.Failure) + } + + if run.BootstrapNum != 1 { + t.Fatal("Expected only one bootstrap node") + } + + if run.PeersRespondedNum != 1 { + t.Fatal("Expected bootstrap node to respond") + } + + ask, err := m.GetSummaryKeys(meas) + if err != nil { + t.Fatal("cannot obtain summary") + } + summary := ask.(SummaryKeys) + if summary.IsAnomaly { + t.Fatal("expected no anomaly") + } + }) +} diff --git a/internal/engine/experiment/dnscheck/dnscheck.go b/internal/engine/experiment/dnscheck/dnscheck.go index 66b805d..d9912cb 100644 --- a/internal/engine/experiment/dnscheck/dnscheck.go +++ b/internal/engine/experiment/dnscheck/dnscheck.go @@ -120,10 +120,11 @@ var ( ) // Run implements model.ExperimentSession.Run -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session + // 1. fill the measurement with test keys tk := new(TestKeys) tk.Lookups = make(map[string]urlgetter.TestKeys) diff --git a/internal/engine/experiment/dnscheck/dnscheck_test.go b/internal/engine/experiment/dnscheck/dnscheck_test.go index 2bc1966..c217eb6 100644 --- a/internal/engine/experiment/dnscheck/dnscheck_test.go +++ b/internal/engine/experiment/dnscheck/dnscheck_test.go @@ -56,12 +56,12 @@ func TestExperimentNameAndVersion(t *testing.T) { func TestDNSCheckFailsWithoutInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{Domain: "example.com"}) - err := measurer.Run( - context.Background(), - newsession(), - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: new(model.Measurement), + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrInputRequired) { t.Fatal("expected no input error") } @@ -69,12 +69,12 @@ func TestDNSCheckFailsWithoutInput(t *testing.T) { func TestDNSCheckFailsWithInvalidURL(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) - err := measurer.Run( - context.Background(), - newsession(), - &model.Measurement{Input: "Not a valid URL \x7f"}, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{Input: "Not a valid URL \x7f"}, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrInvalidURL) { t.Fatal("expected invalid input error") } @@ -82,12 +82,12 @@ func TestDNSCheckFailsWithInvalidURL(t *testing.T) { func TestDNSCheckFailsWithUnsupportedProtocol(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) - err := measurer.Run( - context.Background(), - newsession(), - &model.Measurement{Input: "file://1.1.1.1"}, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{Input: "file://1.1.1.1"}, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, ErrUnsupportedURLScheme) { t.Fatal("expected unsupported scheme error") } @@ -100,12 +100,12 @@ func TestWithCancelledContext(t *testing.T) { DefaultAddrs: "1.1.1.1 1.0.0.1", }) measurement := &model.Measurement{Input: "dot://one.one.one.one"} - err := measurer.Run( - ctx, - newsession(), - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -147,12 +147,12 @@ func TestDNSCheckValid(t *testing.T) { DefaultAddrs: "1.1.1.1 1.0.0.1", }) measurement := model.Measurement{Input: "dot://one.one.one.one:853"} - err := measurer.Run( - context.Background(), - newsession(), - &measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &measurement, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } @@ -195,12 +195,12 @@ func TestDNSCheckWait(t *testing.T) { measurer := &Measurer{Endpoints: endpoints} run := func(input string) { measurement := model.Measurement{Input: model.MeasurementTarget(input)} - err := measurer.Run( - context.Background(), - newsession(), - &measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &measurement, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatalf("unexpected error: %s", err.Error()) } diff --git a/internal/engine/experiment/dnsping/dnsping.go b/internal/engine/experiment/dnsping/dnsping.go index 648b323..cef7a93 100644 --- a/internal/engine/experiment/dnsping/dnsping.go +++ b/internal/engine/experiment/dnsping/dnsping.go @@ -85,12 +85,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session if measurement.Input == "" { return errNoInputProvided } diff --git a/internal/engine/experiment/dnsping/dnsping_test.go b/internal/engine/experiment/dnsping/dnsping_test.go index 3255a31..0a9ef50 100644 --- a/internal/engine/experiment/dnsping/dnsping_test.go +++ b/internal/engine/experiment/dnsping/dnsping_test.go @@ -61,7 +61,12 @@ func TestMeasurer_run(t *testing.T) { MockableLogger: model.DiscardLogger, } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) return meas, m, err } diff --git a/internal/engine/experiment/example/example.go b/internal/engine/experiment/example/example.go index 1a7f329..c2c1c01 100644 --- a/internal/engine/experiment/example/example.go +++ b/internal/engine/experiment/example/example.go @@ -57,10 +57,10 @@ func (m Measurer) ExperimentVersion() string { var ErrFailure = errors.New("mocked error") // Run implements model.ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session var err error if m.config.ReturnError { err = ErrFailure diff --git a/internal/engine/experiment/example/example_test.go b/internal/engine/experiment/example/example_test.go index dc7e218..29c28fa 100644 --- a/internal/engine/experiment/example/example_test.go +++ b/internal/engine/experiment/example/example_test.go @@ -26,7 +26,12 @@ func TestSuccess(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(sess.Logger()) measurement := new(model.Measurement) - err := m.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := m.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -47,7 +52,12 @@ func TestFailure(t *testing.T) { ctx := context.Background() sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(sess.Logger()) - err := m.Run(ctx, sess, new(model.Measurement), callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: new(model.Measurement), + Session: sess, + } + err := m.Run(ctx, args) if !errors.Is(err, example.ErrFailure) { t.Fatal("expected an error here") } diff --git a/internal/engine/experiment/fbmessenger/fbmessenger.go b/internal/engine/experiment/fbmessenger/fbmessenger.go index 49bcbeb..3901845 100644 --- a/internal/engine/experiment/fbmessenger/fbmessenger.go +++ b/internal/engine/experiment/fbmessenger/fbmessenger.go @@ -157,10 +157,10 @@ func (m Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() urlgetter.RegisterExtensions(measurement) diff --git a/internal/engine/experiment/fbmessenger/fbmessenger_test.go b/internal/engine/experiment/fbmessenger/fbmessenger_test.go index 0545cd4..fa631c3 100644 --- a/internal/engine/experiment/fbmessenger/fbmessenger_test.go +++ b/internal/engine/experiment/fbmessenger/fbmessenger_test.go @@ -35,7 +35,12 @@ func TestSuccess(t *testing.T) { sess := newsession(t) measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -97,7 +102,12 @@ func TestWithCancelledContext(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/hhfm/hhfm.go b/internal/engine/experiment/hhfm/hhfm.go index 574cfbd..8518432 100644 --- a/internal/engine/experiment/hhfm/hhfm.go +++ b/internal/engine/experiment/hhfm/hhfm.go @@ -90,10 +90,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session ctx, cancel := context.WithTimeout(ctx, 30*time.Second) defer cancel() urlgetter.RegisterExtensions(measurement) diff --git a/internal/engine/experiment/hhfm/hhfm_test.go b/internal/engine/experiment/hhfm/hhfm_test.go index 4d95f70..157ab18 100644 --- a/internal/engine/experiment/hhfm/hhfm_test.go +++ b/internal/engine/experiment/hhfm/hhfm_test.go @@ -45,7 +45,12 @@ func TestSuccess(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -153,7 +158,12 @@ func TestCancelledContext(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -259,7 +269,12 @@ func TestNoHelpers(t *testing.T) { sess := &mockable.Session{} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) { t.Fatal("not the error we expected") } @@ -309,7 +324,12 @@ func TestNoActualHelpersInList(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hhfm.ErrNoAvailableTestHelpers) { t.Fatal("not the error we expected") } @@ -362,7 +382,12 @@ func TestWrongTestHelperType(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hhfm.ErrInvalidHelperType) { t.Fatal("not the error we expected") } @@ -415,7 +440,12 @@ func TestNewRequestFailure(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { t.Fatal("not the error we expected") } @@ -472,7 +502,12 @@ func TestInvalidJSONBody(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/hirl/hirl.go b/internal/engine/experiment/hirl/hirl.go index 6cc3517..e4931d9 100644 --- a/internal/engine/experiment/hirl/hirl.go +++ b/internal/engine/experiment/hirl/hirl.go @@ -78,10 +78,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session tk := new(TestKeys) measurement.TestKeys = tk if len(m.Methods) < 1 { diff --git a/internal/engine/experiment/hirl/hirl_test.go b/internal/engine/experiment/hirl/hirl_test.go index 1299307..40fec23 100644 --- a/internal/engine/experiment/hirl/hirl_test.go +++ b/internal/engine/experiment/hirl/hirl_test.go @@ -42,7 +42,12 @@ func TestSuccess(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -91,7 +96,12 @@ func TestCancelledContext(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -190,7 +200,12 @@ func TestWithFakeMethods(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -251,7 +266,12 @@ func TestWithNoMethods(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hirl.ErrNoMeasurementMethod) { t.Fatal("not the error we expected") } @@ -279,7 +299,12 @@ func TestNoHelpers(t *testing.T) { sess := &mockable.Session{} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) { t.Fatal("not the error we expected") } @@ -311,7 +336,12 @@ func TestNoActualHelperInList(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hirl.ErrNoAvailableTestHelpers) { t.Fatal("not the error we expected") } @@ -346,7 +376,12 @@ func TestWrongTestHelperType(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, hirl.ErrInvalidHelperType) { t.Fatal("not the error we expected") } diff --git a/internal/engine/experiment/httphostheader/httphostheader.go b/internal/engine/experiment/httphostheader/httphostheader.go index 70e4087..0a6a5b4 100644 --- a/internal/engine/experiment/httphostheader/httphostheader.go +++ b/internal/engine/experiment/httphostheader/httphostheader.go @@ -46,12 +46,10 @@ func (m *Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session if measurement.Input == "" { return errors.New("experiment requires input") } diff --git a/internal/engine/experiment/httphostheader/httphostheader_test.go b/internal/engine/experiment/httphostheader/httphostheader_test.go index efd88ce..2c9c556 100644 --- a/internal/engine/experiment/httphostheader/httphostheader_test.go +++ b/internal/engine/experiment/httphostheader/httphostheader_test.go @@ -30,12 +30,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{ TestHelperURL: "http://www.google.com", }) - err := measurer.Run( - context.Background(), - newsession(), - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{}, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if err == nil || err.Error() != "experiment requires input" { t.Fatal("not the error we expected") } @@ -44,12 +44,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) { func TestMeasurerMeasureNoTestHelper(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := &model.Measurement{Input: "x.org"} - err := measurer.Run( - context.Background(), - newsession(), - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -75,12 +75,12 @@ func TestRunnerHTTPSetHostHeader(t *testing.T) { measurement := &model.Measurement{ Input: "x.org", } - err := measurer.Run( - context.Background(), - newsession(), - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if host != "x.org" { t.Fatal("not the host we expected") } diff --git a/internal/engine/experiment/imap/.smtp.go.swp b/internal/engine/experiment/imap/.smtp.go.swp deleted file mode 100644 index c4f8bb6..0000000 Binary files a/internal/engine/experiment/imap/.smtp.go.swp and /dev/null differ diff --git a/internal/engine/experiment/imap/.smtp_test.go.swp b/internal/engine/experiment/imap/.smtp_test.go.swp deleted file mode 100644 index 5a742be..0000000 Binary files a/internal/engine/experiment/imap/.smtp_test.go.swp and /dev/null differ diff --git a/internal/engine/experiment/imap/imap.go b/internal/engine/experiment/imap/imap.go deleted file mode 100644 index 93e9386..0000000 --- a/internal/engine/experiment/imap/imap.go +++ /dev/null @@ -1,416 +0,0 @@ -package imap - -import ( - "bufio" - "context" - "crypto/tls" - "fmt" - "github.com/pkg/errors" - "net" - //"net/smtp" - "net/url" - "strings" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/tracex" -) - -var ( - // errNoInputProvided indicates you didn't provide any input - errNoInputProvided = errors.New("not input provided") - - // errInputIsNotAnURL indicates that input is not an URL - errInputIsNotAnURL = errors.New("input is not an URL") - - // errInvalidScheme indicates that the scheme is invalid - errInvalidScheme = errors.New("scheme must be smtp(s)") -) - -const ( - testName = "imap" - testVersion = "0.0.1" -) - -// Config contains the experiment config. -type Config struct{} - -type RuntimeConfig struct { - host string - port string - forced_tls bool - noop_count uint8 -} - -func config(input model.MeasurementTarget) (*RuntimeConfig, error) { - if input == "" { - // TODO: static input data (eg. gmail/riseup..) - return nil, errNoInputProvided - } - - parsed, err := url.Parse(string(input)) - if err != nil { - return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) - } - if parsed.Scheme != "imap" && parsed.Scheme != "imaps" { - return nil, errInvalidScheme - } - - port := "" - - if parsed.Port() == "" { - // Default ports for StartTLS and forced TLS respectively - if parsed.Scheme == "imap" { - port = "143" - } else { - port = "993" - } - } else { - // Valid port is checked by URL parsing - port = parsed.Port() - } - - valid_config := RuntimeConfig{ - host: parsed.Hostname(), - forced_tls: parsed.Scheme == "imaps", - port: port, - noop_count: 10, - } - - return &valid_config, nil -} - -// TestKeys contains the experiment results - -type TestKeys struct { - Queries []*model.ArchivalDNSLookupResult `json:"queries"` - Runs map[string]*IndividualTestKeys `json:"runs"` - // Used for global failure (DNS resolution) - Failure string `json:"failure"` - // Indicates global failure or individual test failure - Failed bool `json:"failed"` -} - -// IndividualTestKeys contains results for TCP/IP level stuff for each address found -// in the DNS lookup -type IndividualTestKeys struct { - NoOpCounter uint8 - TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` - TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` - // Individual failure aborting the test run for this address/port combo - Failure *string `json:"failure"` -} - -type Measurer struct { - // Config contains the experiment settings. If empty we - // will be using default settings. - Config Config - - // Getter is an optional getter to be used for testing. - Getter urlgetter.MultiGetter -} - -// ExperimentName implements ExperimentMeasurer.ExperimentName -func (m Measurer) ExperimentName() string { - return testName -} - -// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion -func (m Measurer) ExperimentVersion() string { - return testVersion -} - -// Manages sequential TCP sessions to the same hostname (over different IPs) -// don't use in parallel! -type TCPRunner struct { - trace *measurexlite.Trace - logger model.Logger - ctx context.Context - tk *TestKeys - tlsconfig *tls.Config - host string - port string - // addr is changed everytime TCPRunner.conn(addr) is called - addr string -} - -type TCPSession struct { - addr string - port string - runner *TCPRunner - tk *IndividualTestKeys - tls bool - raw_conn *net.Conn - tls_conn *net.Conn -} - -func (s *TCPSession) Close() { - if s.tls { - var conn = *s.tls_conn - conn.Close() - } else { - var conn = *s.raw_conn - conn.Close() - } -} - -func (s *TCPSession) current_conn() net.Conn { - if s.tls { - return *s.tls_conn - } else { - return *s.raw_conn - } -} - -func (r *TCPRunner) run_key() string { - return net.JoinHostPort(r.addr, r.port) -} - -func (r *TCPRunner) get_run() *IndividualTestKeys { - if r.tk.Runs == nil { - r.tk.Runs = make(map[string]*IndividualTestKeys) - } - key := r.run_key() - val, exists := r.tk.Runs[key] - if exists { - return val - } else { - r.tk.Runs[key] = &IndividualTestKeys{} - return r.tk.Runs[key] - } -} - -func (r *TCPRunner) conn(addr string, port string) (*TCPSession, bool) { - r.addr = addr - run := r.get_run() - - s := new(TCPSession) - if !s.conn(addr, port, r, run) { - return nil, false - } - return s, true -} - -func (r *TCPRunner) dial(addr string, port string) (net.Conn, error) { - dialer := r.trace.NewDialerWithoutResolver(r.logger) - conn, err := dialer.DialContext(r.ctx, "tcp", net.JoinHostPort(addr, port)) - run := r.get_run() - run.TCPConnect = append(run.TCPConnect, r.trace.TCPConnects()...) - return conn, err - -} - -func (s *TCPSession) conn(addr string, port string, runner *TCPRunner, tk *IndividualTestKeys) bool { - // Initialize addr field and corresponding errors in TestKeys - s.addr = addr - s.port = port - s.tls = false - s.runner = runner - s.tk = tk - - conn, err := runner.dial(addr, port) - if err != nil { - s.error(err) - return false - } - s.raw_conn = &conn - - return true -} - -func (s *TCPSession) error(err error) { - s.runner.tk.Failed = true - s.tk.Failure = tracex.NewFailure(err) - //s. = append(s.errors, tracex.NewFailure(err)) -} - -func (r *TCPRunner) resolve(host string) ([]string, bool) { - r.logger.Infof("Resolving DNS for %s", host) - resolver := r.trace.NewStdlibResolver(r.logger) - addrs, err := resolver.LookupHost(r.ctx, host) - r.tk.Queries = append(r.tk.Queries, r.trace.DNSLookupsFromRoundTrip()...) - if err != nil { - r.tk.Failure = *tracex.NewFailure(err) - return []string{}, false - } - r.logger.Infof("Finished DNS for %s: %v", host, addrs) - - return addrs, true -} - -func (s *TCPSession) handshake() bool { - if s.tls { - // TLS already initialized... - return true - } - s.runner.logger.Infof("Starting TLS handshake with %s:%s", s.addr, s.port) - thx := s.runner.trace.NewTLSHandshakerStdlib(s.runner.logger) - tconn, _, err := thx.Handshake(s.runner.ctx, *s.raw_conn, s.runner.tlsconfig) - s.tk.TLSHandshakes = append(s.tk.TLSHandshakes, s.runner.trace.FirstTLSHandshakeOrNil()) - if err != nil { - s.error(err) - return false - } - - s.tls = true - s.tls_conn = &tconn - s.runner.logger.Infof("Handshake succeeded") - return true -} - -func (s *TCPSession) starttls(message string) bool { - if s.tls { - // TLS already initialized... - return true - } - if message != "" { - s.runner.logger.Infof("Asking for StartTLS upgrade") - s.current_conn().Write([]byte(message)) - } - return s.handshake() -} - -func (s *TCPSession) imap(noop uint8) bool { - conn := s.current_conn() - - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - s.error(err) - return false - } - if !strings.Contains(command, "CAPABILITY") { - s.error(errors.New("Unexpected IMAP reply: " + command)) - return false - } - - - if noop > 0 { - s.runner.logger.Infof("Trying to generate no-op traffic") - s.tk.NoOpCounter = 0 - for s.tk.NoOpCounter < noop { - s.tk.NoOpCounter += 1 - s.runner.logger.Infof("NoOp Iteration %d", s.tk.NoOpCounter) - - conn.Write([]byte("A1 NOOP\n")) - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - s.error(err) - break - } - if !strings.Contains(command, "OK NOOP") { - s.error(errors.New("Unexpected IMAP reply: " + command)) - break - } - } - - if s.tk.NoOpCounter == noop { - s.runner.logger.Infof("Successfully generated no-op traffic") - return true - } else { - s.runner.logger.Infof("Failed no-op traffic at iteration %d", s.tk.NoOpCounter) - return false - } - } - - return true -} - -// Run implements ExperimentMeasurer.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { - log := sess.Logger() - trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) - - config, err := config(measurement.Input) - if err != nil { - // Invalid input data, we don't even generate report - return err - } - - tk := new(TestKeys) - measurement.TestKeys = tk - - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - tlsconfig := tls.Config{ - InsecureSkipVerify: false, - ServerName: config.host, - } - - runner := &TCPRunner{ - trace: trace, - logger: log, - ctx: ctx, - tk: tk, - tlsconfig: &tlsconfig, - host: config.host, - port: config.port, - } - - // First resolve DNS - addrs, success := runner.resolve(config.host) - if !success { - return nil - } - - for _, addr := range addrs { - tcp_session, success := runner.conn(addr, config.port) - if !success { - continue - } - defer tcp_session.Close() - - if config.forced_tls { - // Direct TLS connection - if !tcp_session.handshake() { - continue - } - - // Try EHLO + NoOps - if !tcp_session.imap(config.noop_count) { - continue - } - } else { - // StartTLS... - if !tcp_session.starttls("A1 STARTTLS\n") { - continue - } - - if !tcp_session.imap(config.noop_count) { - continue - } - } - } - - return nil -} - -// NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { - return Measurer{Config: config} -} - -// SummaryKeys contains summary keys for this experiment. -// -// Note that this structure is part of the ABI contract with ooniprobe -// therefore we should be careful when changing it. -type SummaryKeys struct { - //DNSBlocking bool `json:"facebook_dns_blocking"` - //TCPBlocking bool `json:"facebook_tcp_blocking"` - IsAnomaly bool `json:"-"` -} - -// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. -func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { - sk := SummaryKeys{IsAnomaly: false} - _, ok := measurement.TestKeys.(*TestKeys) - if !ok { - return sk, errors.New("invalid test keys type") - } - return sk, nil -} diff --git a/internal/engine/experiment/imap/imap_test.go b/internal/engine/experiment/imap/imap_test.go deleted file mode 100644 index 1fbfc15..0000000 --- a/internal/engine/experiment/imap/imap_test.go +++ /dev/null @@ -1,186 +0,0 @@ -package imap - -import ( - "bufio" - "context" - "crypto/tls" - //"encoding/json" - "errors" - "fmt" - "net" - "strings" - "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/model" -) - -func plaintextListener() net.Listener { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { - panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) - } - } - return l -} - -func tlsListener(l net.Listener) net.Listener { - return tls.NewListener(l, &tls.Config{}) -} - -func listener_addr(l net.Listener) string { - return l.Addr().String() -} - -func ValidIMAPServer(conn net.Conn) { - for { - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - return - } - - if strings.Contains(command, "NOOP") { - conn.Write([]byte("A1 OK NOOP completed.\n")) - } else if command == "STARTTLS" { - conn.Write([]byte("A1 OK Begin TLS negotiation now.\n")) - // TODO: conn.Close does not actually close connection? or does client not detect it? - conn.Close() - return - } - conn.Write([]byte("\n")) - } -} - -func TCPServer(l net.Listener) { - for { - conn, err := l.Accept() - if err != nil { - continue - } - defer conn.Close() - conn.Write([]byte("* OK [CAPABILITY IMAP4rev1 SASL-IR LOGIN-REFERRALS ID ENABLE IDLE LITERAL+ STARTTLS LOGINDISABLED] howdy, ready.\n")) - ValidIMAPServer(conn) - } -} - -func TestMeasurer_run(t *testing.T) { - // runHelper is an helper function to run this set of tests. - runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { - m := NewExperimentMeasurer(Config{}) - if m.ExperimentName() != "imap" { - t.Fatal("invalid experiment name") - } - if m.ExperimentVersion() != "0.0.1" { - t.Fatal("invalid experiment version") - } - ctx := context.Background() - meas := &model.Measurement{ - Input: model.MeasurementTarget(input), - } - sess := &mockable.Session{ - MockableLogger: model.DiscardLogger, - } - callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) - return meas, m, err - } - - t.Run("with empty input", func(t *testing.T) { - _, _, err := runHelper("") - if !errors.Is(err, errNoInputProvided) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid URL", func(t *testing.T) { - _, _, err := runHelper("\t") - if !errors.Is(err, errInputIsNotAnURL) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid scheme", func(t *testing.T) { - _, _, err := runHelper("https://8.8.8.8:443/") - if !errors.Is(err, errInvalidScheme) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with broken TLS", func(t *testing.T) { - p := plaintextListener() - defer p.Close() - - l := tlsListener(p) - defer l.Close() - addr := listener_addr(l) - go TCPServer(l) - - meas, m, err := runHelper("imaps://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { - t.Fatal("expected unrecognized_name in TLS handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) - - t.Run("with broken starttls", func(t *testing.T) { - l := plaintextListener() - defer l.Close() - addr := listener_addr(l) - - go TCPServer(l) - - meas, m, err := runHelper("imap://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - //bs, _ := json.Marshal(tk) - //fmt.Println(string(bs)) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "unknown_failure: tls: first record does not look like a TLS handshake" { - - t.Fatal("expected broken handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) -} diff --git a/internal/engine/experiment/ndt7/ndt7.go b/internal/engine/experiment/ndt7/ndt7.go index 8b403c6..7e4bb23 100644 --- a/internal/engine/experiment/ndt7/ndt7.go +++ b/internal/engine/experiment/ndt7/ndt7.go @@ -210,10 +210,10 @@ func (m *Measurer) doUpload( } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session tk := new(TestKeys) tk.Protocol = 7 measurement.TestKeys = tk diff --git a/internal/engine/experiment/ndt7/ndt7_test.go b/internal/engine/experiment/ndt7/ndt7_test.go index 5ad4f1c..7da663e 100644 --- a/internal/engine/experiment/ndt7/ndt7_test.go +++ b/internal/engine/experiment/ndt7/ndt7_test.go @@ -84,7 +84,12 @@ func TestRunWithCancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // immediately cancel meas := &model.Measurement{} - err := m.Run(ctx, sess, meas, model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) // Here we get nil because we still want to submit this measurement if !errors.Is(err, nil) { t.Fatal("not the error we expected") @@ -104,15 +109,15 @@ func TestGood(t *testing.T) { } measurement := new(model.Measurement) measurer := NewExperimentMeasurer(Config{}) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableHTTPClient: http.DefaultClient, MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -133,15 +138,15 @@ func TestFailDownload(t *testing.T) { cancel() } meas := &model.Measurement{} - err := measurer.Run( - ctx, - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: meas, + Session: &mockable.Session{ MockableHTTPClient: http.DefaultClient, MockableLogger: log.Log, }, - meas, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(ctx, args) // We expect a nil failure here because we want to submit anyway // a measurement that failed to connect to m-lab. if err != nil { @@ -164,15 +169,15 @@ func TestFailUpload(t *testing.T) { cancel() } meas := &model.Measurement{} - err := measurer.Run( - ctx, - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: meas, + Session: &mockable.Session{ MockableHTTPClient: http.DefaultClient, MockableLogger: log.Log, }, - meas, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(ctx, args) // Here we expect a nil error because we want to submit this measurement if err != nil { t.Fatal(err) @@ -197,15 +202,15 @@ func TestDownloadJSONUnmarshalFail(t *testing.T) { seenError = true return expected } - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{}, + Session: &mockable.Session{ MockableHTTPClient: http.DefaultClient, MockableLogger: log.Log, }, - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/portfiltering/measurer.go b/internal/engine/experiment/portfiltering/measurer.go index 76c9a2d..487f036 100644 --- a/internal/engine/experiment/portfiltering/measurer.go +++ b/internal/engine/experiment/portfiltering/measurer.go @@ -38,12 +38,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session // TODO(DecFox): Replace the localhost deployment with an OONI testhelper // Ensure that we only do this once we have a deployed testhelper testhelper := "http://127.0.0.1" diff --git a/internal/engine/experiment/portfiltering/measurer_test.go b/internal/engine/experiment/portfiltering/measurer_test.go index e3a3876..c5ba589 100644 --- a/internal/engine/experiment/portfiltering/measurer_test.go +++ b/internal/engine/experiment/portfiltering/measurer_test.go @@ -29,7 +29,12 @@ func TestMeasurer_run(t *testing.T) { } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) ctx := context.Background() - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/psiphon/psiphon.go b/internal/engine/experiment/psiphon/psiphon.go index 5f55aea..45ed762 100644 --- a/internal/engine/experiment/psiphon/psiphon.go +++ b/internal/engine/experiment/psiphon/psiphon.go @@ -66,10 +66,10 @@ func (m *Measurer) printprogress( } // Run runs the measurement -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session const maxruntime = 300 ctx, cancel := context.WithTimeout(ctx, maxruntime*time.Second) var ( diff --git a/internal/engine/experiment/psiphon/psiphon_test.go b/internal/engine/experiment/psiphon/psiphon_test.go index 112ccdf..4186448 100644 --- a/internal/engine/experiment/psiphon/psiphon_test.go +++ b/internal/engine/experiment/psiphon/psiphon_test.go @@ -33,8 +33,12 @@ func TestRunWithCancelledContext(t *testing.T) { ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately measurement := new(model.Measurement) - err := measurer.Run(ctx, newfakesession(), measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newfakesession(), + } + err := measurer.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit the measurement t.Fatal("expected another error here") } @@ -64,8 +68,12 @@ func TestRunWithCustomInputAndCancelledContext(t *testing.T) { } ctx, cancel := context.WithCancel(context.Background()) cancel() // fail immediately - err := measurer.Run(ctx, newfakesession(), measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newfakesession(), + } + err := measurer.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit the measurement t.Fatal("expected another error here") } @@ -84,7 +92,12 @@ func TestRunWillPrintSomethingWithCancelledContext(t *testing.T) { cancel() // fail after we've given the printer a chance to run } observer := observerCallbacks{progress: &atomicx.Int64{}} - err := measurer.Run(ctx, newfakesession(), measurement, observer) + args := &model.ExperimentArgs{ + Callbacks: observer, + Measurement: measurement, + Session: newfakesession(), + } + err := measurer.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit the measurement t.Fatal("expected another error here") } diff --git a/internal/engine/experiment/quicping/quicping.go b/internal/engine/experiment/quicping/quicping.go index 972f68e..cedf7c4 100644 --- a/internal/engine/experiment/quicping/quicping.go +++ b/internal/engine/experiment/quicping/quicping.go @@ -221,12 +221,11 @@ func (m *Measurer) receiver( } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session + host := string(measurement.Input) // allow URL input if u, err := url.ParseRequestURI(host); err == nil { diff --git a/internal/engine/experiment/quicping/quicping_test.go b/internal/engine/experiment/quicping/quicping_test.go index 79a214f..34a8023 100644 --- a/internal/engine/experiment/quicping/quicping_test.go +++ b/internal/engine/experiment/quicping/quicping_test.go @@ -33,8 +33,12 @@ func TestInvalidHost(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("a.a.a.a") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err == nil { t.Fatal("expected an error here") } @@ -53,8 +57,12 @@ func TestURLInput(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("https://google.com/") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("unexpected error") } @@ -73,8 +81,12 @@ func TestSuccess(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("google.com") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("did not expect an error here") } @@ -117,8 +129,12 @@ func TestWithCancelledContext(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} ctx, cancel := context.WithCancel(context.Background()) cancel() - err := measurer.Run(ctx, sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal("did not expect an error here") } @@ -138,8 +154,12 @@ func TestListenFails(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("google.com") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err == nil { t.Fatal("expected an error here") } @@ -182,8 +202,12 @@ func TestWriteFails(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("google.com") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("unexpected error") } @@ -239,8 +263,12 @@ func TestReadFails(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("google.com") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("unexpected error") } @@ -271,8 +299,12 @@ func TestNoResponse(t *testing.T) { measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("ooni.org") sess := &mockable.Session{MockableLogger: log.Log} - err := measurer.Run(context.Background(), sess, measurement, - model.NewPrinterCallbacks(log.Log)) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal("did not expect an error here") } diff --git a/internal/engine/experiment/riseupvpn/riseupvpn.go b/internal/engine/experiment/riseupvpn/riseupvpn.go index 62dd7e3..10aa8de 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn.go @@ -175,8 +175,11 @@ func (m Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run. -func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session + ctx, cancel := context.WithTimeout(ctx, 90*time.Second) defer cancel() testkeys := NewTestKeys() diff --git a/internal/engine/experiment/riseupvpn/riseupvpn_test.go b/internal/engine/experiment/riseupvpn/riseupvpn_test.go index 0e67541..5ae4637 100644 --- a/internal/engine/experiment/riseupvpn/riseupvpn_test.go +++ b/internal/engine/experiment/riseupvpn/riseupvpn_test.go @@ -100,7 +100,7 @@ const ( "cert": "XXXXXXXXXXXXXXXXXXXXXXXXX", "iatMode": "0" } - }, + }, { "type":"openvpn", "protocols":[ @@ -328,7 +328,12 @@ func TestInvalidCaCert(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -599,7 +604,12 @@ func TestMissingTransport(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err = measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err = measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -790,14 +800,14 @@ func runDefaultMockTest(t *testing.T, multiGetter urlgetter.MultiGetter) *model. } measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) diff --git a/internal/engine/experiment/run/dnscheck.go b/internal/engine/experiment/run/dnscheck.go index da3fd84..538e9a0 100644 --- a/internal/engine/experiment/run/dnscheck.go +++ b/internal/engine/experiment/run/dnscheck.go @@ -21,5 +21,10 @@ func (m *dnsCheckMain) do(ctx context.Context, input StructuredInput, measurement.TestName = exp.ExperimentName() measurement.TestVersion = exp.ExperimentVersion() measurement.Input = model.MeasurementTarget(input.Input) - return exp.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + return exp.Run(ctx, args) } diff --git a/internal/engine/experiment/run/run.go b/internal/engine/experiment/run/run.go index 6c38057..a25cf89 100644 --- a/internal/engine/experiment/run/run.go +++ b/internal/engine/experiment/run/run.go @@ -46,10 +46,10 @@ type StructuredInput struct { } // Run implements ExperimentMeasurer.ExperimentVersion. -func (Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session var input StructuredInput if err := json.Unmarshal([]byte(measurement.Input), &input); err != nil { return err diff --git a/internal/engine/experiment/run/run_test.go b/internal/engine/experiment/run/run_test.go index 06ae406..6fb3fde 100644 --- a/internal/engine/experiment/run/run_test.go +++ b/internal/engine/experiment/run/run_test.go @@ -31,7 +31,12 @@ func TestRunDNSCheckWithCancelledContext(t *testing.T) { cancel() // fail immediately sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) // TODO(bassosimone): here we could improve the tests by checking // whether the result makes sense for a cancelled context. if err != nil { @@ -62,7 +67,12 @@ func TestRunURLGetterWithCancelledContext(t *testing.T) { cancel() // fail immediately sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { // here we expected nil b/c we want to submit the measurement t.Fatal(err) } @@ -86,7 +96,12 @@ func TestRunWithInvalidJSON(t *testing.T) { ctx := context.Background() sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err == nil || err.Error() != "invalid character '}' looking for beginning of value" { t.Fatalf("not the error we expected: %+v", err) } @@ -100,7 +115,12 @@ func TestRunWithUnknownExperiment(t *testing.T) { ctx := context.Background() sess := &mockable.Session{MockableLogger: log.Log} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err == nil || err.Error() != "no such experiment: antani" { t.Fatalf("not the error we expected: %+v", err) } diff --git a/internal/engine/experiment/run/urlgetter.go b/internal/engine/experiment/run/urlgetter.go index 085f78f..753c051 100644 --- a/internal/engine/experiment/run/urlgetter.go +++ b/internal/engine/experiment/run/urlgetter.go @@ -18,5 +18,10 @@ func (m *urlGetterMain) do(ctx context.Context, input StructuredInput, measurement.TestName = exp.ExperimentName() measurement.TestVersion = exp.ExperimentVersion() measurement.Input = model.MeasurementTarget(input.Input) - return exp.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + return exp.Run(ctx, args) } diff --git a/internal/engine/experiment/signal/signal.go b/internal/engine/experiment/signal/signal.go index 07e01bb..df89fac 100644 --- a/internal/engine/experiment/signal/signal.go +++ b/internal/engine/experiment/signal/signal.go @@ -141,8 +141,10 @@ func (m Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run -func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() urlgetter.RegisterExtensions(measurement) diff --git a/internal/engine/experiment/signal/signal_test.go b/internal/engine/experiment/signal/signal_test.go index ecadebe..f78e859 100644 --- a/internal/engine/experiment/signal/signal_test.go +++ b/internal/engine/experiment/signal/signal_test.go @@ -25,14 +25,14 @@ func TestNewExperimentMeasurer(t *testing.T) { func TestGood(t *testing.T) { measurer := signal.NewExperimentMeasurer(signal.Config{}) measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -103,14 +103,14 @@ func TestBadSignalCA(t *testing.T) { SignalCA: "INVALIDCA", }) measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err.Error() != "AppendCertsFromPEM failed" { t.Fatal("not the error we expected") } diff --git a/internal/engine/experiment/simplequicping/simplequicping.go b/internal/engine/experiment/simplequicping/simplequicping.go index eb1ebca..44d2f7e 100644 --- a/internal/engine/experiment/simplequicping/simplequicping.go +++ b/internal/engine/experiment/simplequicping/simplequicping.go @@ -112,12 +112,11 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session + if measurement.Input == "" { return errNoInputProvided } diff --git a/internal/engine/experiment/simplequicping/simplequicping_test.go b/internal/engine/experiment/simplequicping/simplequicping_test.go index cdd4138..12bd5ce 100644 --- a/internal/engine/experiment/simplequicping/simplequicping_test.go +++ b/internal/engine/experiment/simplequicping/simplequicping_test.go @@ -65,7 +65,12 @@ func TestMeasurer_run(t *testing.T) { MockableLogger: model.DiscardLogger, } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) return meas, m, err } diff --git a/internal/engine/experiment/smtp/smtp.go b/internal/engine/experiment/smtp/smtp.go deleted file mode 100644 index 568db04..0000000 --- a/internal/engine/experiment/smtp/smtp.go +++ /dev/null @@ -1,413 +0,0 @@ -package smtp - -import ( - "context" - "crypto/tls" - "fmt" - "github.com/pkg/errors" - "net" - "net/smtp" - "net/url" - "time" - - "github.com/ooni/probe-cli/v3/internal/engine/experiment/urlgetter" - "github.com/ooni/probe-cli/v3/internal/measurexlite" - "github.com/ooni/probe-cli/v3/internal/model" - "github.com/ooni/probe-cli/v3/internal/tracex" -) - -var ( - // errNoInputProvided indicates you didn't provide any input - errNoInputProvided = errors.New("not input provided") - - // errInputIsNotAnURL indicates that input is not an URL - errInputIsNotAnURL = errors.New("input is not an URL") - - // errInvalidScheme indicates that the scheme is invalid - errInvalidScheme = errors.New("scheme must be smtp(s)") -) - -const ( - testName = "smtp" - testVersion = "0.0.1" -) - -// Config contains the experiment config. -type Config struct{} - -type RuntimeConfig struct { - host string - port string - forced_tls bool - noop_count uint8 -} - -func config(input model.MeasurementTarget) (*RuntimeConfig, error) { - if input == "" { - // TODO: static input data (eg. gmail/riseup..) - return nil, errNoInputProvided - } - - parsed, err := url.Parse(string(input)) - if err != nil { - return nil, fmt.Errorf("%w: %s", errInputIsNotAnURL, err.Error()) - } - if parsed.Scheme != "smtp" && parsed.Scheme != "smtps" { - return nil, errInvalidScheme - } - - port := "" - - if parsed.Port() == "" { - // Default ports for StartTLS and forced TLS respectively - if parsed.Scheme == "smtp" { - port = "587" - } else { - port = "465" - } - } else { - // Valid port is checked by URL parsing - port = parsed.Port() - } - - valid_config := RuntimeConfig{ - host: parsed.Hostname(), - forced_tls: parsed.Scheme == "smtps", - port: port, - noop_count: 10, - } - - return &valid_config, nil -} - -// TestKeys contains the experiment results - -type TestKeys struct { - Queries []*model.ArchivalDNSLookupResult `json:"queries"` - Runs map[string]*IndividualTestKeys `json:"runs"` - // Used for global failure (DNS resolution) - Failure string `json:"failure"` - // Indicates global failure or individual test failure - Failed bool `json:"failed"` -} - -// IndividualTestKeys contains results for TCP/IP level stuff for each address found -// in the DNS lookup -type IndividualTestKeys struct { - NoOpCounter uint8 - TCPConnect []*model.ArchivalTCPConnectResult `json:"tcp_connect"` - TLSHandshakes []*model.ArchivalTLSOrQUICHandshakeResult `json:"tls_handshakes"` - // Individual failure aborting the test run for this address/port combo - Failure *string `json:"failure"` -} - -type Measurer struct { - // Config contains the experiment settings. If empty we - // will be using default settings. - Config Config - - // Getter is an optional getter to be used for testing. - Getter urlgetter.MultiGetter -} - -// ExperimentName implements ExperimentMeasurer.ExperimentName -func (m Measurer) ExperimentName() string { - return testName -} - -// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion -func (m Measurer) ExperimentVersion() string { - return testVersion -} - -// Manages sequential TCP sessions to the same hostname (over different IPs) -// don't use in parallel! -type TCPRunner struct { - trace *measurexlite.Trace - logger model.Logger - ctx context.Context - tk *TestKeys - tlsconfig *tls.Config - host string - port string - // addr is changed everytime TCPRunner.conn(addr) is called - addr string -} - -type TCPSession struct { - addr string - port string - runner *TCPRunner - tk *IndividualTestKeys - tls bool - raw_conn *net.Conn - tls_conn *net.Conn -} - -func (s *TCPSession) Close() { - if s.tls { - var conn = *s.tls_conn - conn.Close() - } else { - var conn = *s.raw_conn - conn.Close() - } -} - -func (s *TCPSession) current_conn() net.Conn { - if s.tls { - return *s.tls_conn - } else { - return *s.raw_conn - } -} - -func (r *TCPRunner) run_key() string { - return net.JoinHostPort(r.addr, r.port) -} - -func (r *TCPRunner) get_run() *IndividualTestKeys { - if r.tk.Runs == nil { - r.tk.Runs = make(map[string]*IndividualTestKeys) - } - key := r.run_key() - val, exists := r.tk.Runs[key] - if exists { - return val - } else { - r.tk.Runs[key] = &IndividualTestKeys{} - return r.tk.Runs[key] - } -} - -func (r *TCPRunner) conn(addr string, port string) (*TCPSession, bool) { - r.addr = addr - run := r.get_run() - - s := new(TCPSession) - if !s.conn(addr, port, r, run) { - return nil, false - } - return s, true -} - -func (r *TCPRunner) dial(addr string, port string) (net.Conn, error) { - dialer := r.trace.NewDialerWithoutResolver(r.logger) - conn, err := dialer.DialContext(r.ctx, "tcp", net.JoinHostPort(addr, port)) - run := r.get_run() - run.TCPConnect = append(run.TCPConnect, r.trace.TCPConnects()...) - return conn, err - -} - -func (s *TCPSession) conn(addr string, port string, runner *TCPRunner, tk *IndividualTestKeys) bool { - // Initialize addr field and corresponding errors in TestKeys - s.addr = addr - s.port = port - s.tls = false - s.runner = runner - s.tk = tk - - conn, err := runner.dial(addr, port) - if err != nil { - s.error(err) - return false - } - s.raw_conn = &conn - - return true -} - -func (s *TCPSession) error(err error) { - s.runner.tk.Failed = true - s.tk.Failure = tracex.NewFailure(err) - //s. = append(s.errors, tracex.NewFailure(err)) -} - -func (r *TCPRunner) resolve(host string) ([]string, bool) { - r.logger.Infof("Resolving DNS for %s", host) - resolver := r.trace.NewStdlibResolver(r.logger) - addrs, err := resolver.LookupHost(r.ctx, host) - r.tk.Queries = append(r.tk.Queries, r.trace.DNSLookupsFromRoundTrip()...) - if err != nil { - r.tk.Failure = *tracex.NewFailure(err) - return []string{}, false - } - r.logger.Infof("Finished DNS for %s: %v", host, addrs) - - return addrs, true -} - -func (s *TCPSession) handshake() bool { - if s.tls { - // TLS already initialized... - return true - } - s.runner.logger.Infof("Starting TLS handshake with %s:%s", s.addr, s.port) - thx := s.runner.trace.NewTLSHandshakerStdlib(s.runner.logger) - tconn, _, err := thx.Handshake(s.runner.ctx, *s.raw_conn, s.runner.tlsconfig) - s.tk.TLSHandshakes = append(s.tk.TLSHandshakes, s.runner.trace.FirstTLSHandshakeOrNil()) - if err != nil { - s.error(err) - return false - } - - s.tls = true - s.tls_conn = &tconn - s.runner.logger.Infof("Handshake succeeded") - return true -} - -func (s *TCPSession) starttls(message string) bool { - if s.tls { - // TLS already initialized... - return true - } - if message != "" { - s.runner.logger.Infof("Asking for StartTLS upgrade") - s.current_conn().Write([]byte(message)) - } - return s.handshake() -} - -func (s *TCPSession) smtp(ehlo string, noop uint8) bool { - // Auto-choose plaintext/TCP session - client, err := smtp.NewClient(s.current_conn(), ehlo) - if err != nil { - s.error(err) - return false - } - err = client.Hello(ehlo) - if err != nil { - s.error(err) - return false - } - - if noop > 0 { - s.runner.logger.Infof("Trying to generate more no-op traffic") - // TODO: noop counter per IP address - s.tk.NoOpCounter = 0 - for s.tk.NoOpCounter < noop { - s.tk.NoOpCounter += 1 - s.runner.logger.Infof("NoOp Iteration %d", s.tk.NoOpCounter) - err = client.Noop() - if err != nil { - s.error(err) - break - } - } - - if s.tk.NoOpCounter == noop { - s.runner.logger.Infof("Successfully generated no-op traffic") - return true - } else { - s.runner.logger.Infof("Failed no-op traffic at iteration %d", s.tk.NoOpCounter) - return false - } - } - - return true -} - -// Run implements ExperimentMeasurer.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { - log := sess.Logger() - trace := measurexlite.NewTrace(0, measurement.MeasurementStartTimeSaved) - - config, err := config(measurement.Input) - if err != nil { - // Invalid input data, we don't even generate report - return err - } - - tk := new(TestKeys) - measurement.TestKeys = tk - - ctx, cancel := context.WithTimeout(ctx, 60*time.Second) - defer cancel() - - tlsconfig := tls.Config{ - InsecureSkipVerify: false, - ServerName: config.host, - } - - runner := &TCPRunner{ - trace: trace, - logger: log, - ctx: ctx, - tk: tk, - tlsconfig: &tlsconfig, - host: config.host, - port: config.port, - } - - // First resolve DNS - addrs, success := runner.resolve(config.host) - if !success { - return nil - } - - for _, addr := range addrs { - tcp_session, success := runner.conn(addr, config.port) - if !success { - continue - } - defer tcp_session.Close() - - if config.forced_tls { - // Direct TLS connection - if !tcp_session.handshake() { - continue - } - - // Try EHLO + NoOps - if !tcp_session.smtp("localhost", config.noop_count) { - continue - } - } else { - // StartTLS... first try plaintext EHLO - if !tcp_session.smtp("localhost", 0) { - continue - } - - // Upgrade via StartTLS and try EHLO + NoOps - if !tcp_session.starttls("STARTTLS\n") { - continue - } - - if !tcp_session.smtp("localhost", config.noop_count) { - continue - } - } - } - - return nil -} - -// NewExperimentMeasurer creates a new ExperimentMeasurer. -func NewExperimentMeasurer(config Config) model.ExperimentMeasurer { - return Measurer{Config: config} -} - -// SummaryKeys contains summary keys for this experiment. -// -// Note that this structure is part of the ABI contract with ooniprobe -// therefore we should be careful when changing it. -type SummaryKeys struct { - //DNSBlocking bool `json:"facebook_dns_blocking"` - //TCPBlocking bool `json:"facebook_tcp_blocking"` - IsAnomaly bool `json:"-"` -} - -// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys. -func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) { - sk := SummaryKeys{IsAnomaly: false} - _, ok := measurement.TestKeys.(*TestKeys) - if !ok { - return sk, errors.New("invalid test keys type") - } - return sk, nil -} diff --git a/internal/engine/experiment/smtp/smtp_test.go b/internal/engine/experiment/smtp/smtp_test.go deleted file mode 100644 index e49af52..0000000 --- a/internal/engine/experiment/smtp/smtp_test.go +++ /dev/null @@ -1,185 +0,0 @@ -package smtp - -import ( - "bufio" - "context" - "crypto/tls" - "errors" - "fmt" - "net" - "strings" - "testing" - - "github.com/ooni/probe-cli/v3/internal/engine/mockable" - "github.com/ooni/probe-cli/v3/internal/model" -) - -func plaintextListener() net.Listener { - l, err := net.Listen("tcp", "127.0.0.1:0") - if err != nil { - if l, err = net.Listen("tcp6", "[::1]:0"); err != nil { - panic(fmt.Sprintf("httptest: failed to listen on a port: %v", err)) - } - } - return l -} - -func tlsListener(l net.Listener) net.Listener { - return tls.NewListener(l, &tls.Config{}) -} - -func listener_addr(l net.Listener) string { - return l.Addr().String() -} - -func ValidSMTPServer(conn net.Conn) { - for { - command, err := bufio.NewReader(conn).ReadString('\n') - if err != nil { - return - } - - if command == "" { - } else if command == "NOOP" { - conn.Write([]byte("250 2.0.0 Ok\n")) - } else if command == "STARTTLS" { - conn.Write([]byte("220 2.0.0 Ready to start TLS\n")) - // TODO: conn.Close does not actually close connection? or does client not detect it? - conn.Close() - return - } else if strings.HasPrefix(command, "EHLO") { - conn.Write([]byte("250 mock.example.com\n")) - } - conn.Write([]byte("\n")) - } -} - -func TCPServer(l net.Listener) { - for { - conn, err := l.Accept() - if err != nil { - continue - } - defer conn.Close() - conn.Write([]byte("220 mock.example.com ESMTP (spam is not appreciated)\n")) - ValidSMTPServer(conn) - } -} - -func TestMeasurer_run(t *testing.T) { - // runHelper is an helper function to run this set of tests. - runHelper := func(input string) (*model.Measurement, model.ExperimentMeasurer, error) { - m := NewExperimentMeasurer(Config{}) - if m.ExperimentName() != "smtp" { - t.Fatal("invalid experiment name") - } - if m.ExperimentVersion() != "0.0.1" { - t.Fatal("invalid experiment version") - } - ctx := context.Background() - meas := &model.Measurement{ - Input: model.MeasurementTarget(input), - } - sess := &mockable.Session{ - MockableLogger: model.DiscardLogger, - } - callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) - return meas, m, err - } - - t.Run("with empty input", func(t *testing.T) { - _, _, err := runHelper("") - if !errors.Is(err, errNoInputProvided) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid URL", func(t *testing.T) { - _, _, err := runHelper("\t") - if !errors.Is(err, errInputIsNotAnURL) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with invalid scheme", func(t *testing.T) { - _, _, err := runHelper("https://8.8.8.8:443/") - if !errors.Is(err, errInvalidScheme) { - t.Fatal("unexpected error", err) - } - }) - - t.Run("with broken TLS", func(t *testing.T) { - p := plaintextListener() - defer p.Close() - - l := tlsListener(p) - defer l.Close() - addr := listener_addr(l) - go TCPServer(l) - - meas, m, err := runHelper("smtps://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "unknown_failure: remote error: tls: unrecognized name" { - t.Fatal("expected unrecognized_name in TLS handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) - - t.Run("with broken starttls", func(t *testing.T) { - l := plaintextListener() - defer l.Close() - addr := listener_addr(l) - - go TCPServer(l) - - meas, m, err := runHelper("smtp://" + addr) - if err != nil { - t.Fatal(err) - } - - tk := meas.TestKeys.(*TestKeys) - - for _, run := range tk.Runs { - for _, handshake := range run.TLSHandshakes { - if *handshake.Failure != "generic_timeout_error" { - t.Fatal("expected timeout in TLS handshake") - } - } - - if run.NoOpCounter != 0 { - t.Fatalf("expected to not have any noops, not %d noops", run.NoOpCounter) - } - } - - ask, err := m.GetSummaryKeys(meas) - if err != nil { - t.Fatal("cannot obtain summary") - } - summary := ask.(SummaryKeys) - if summary.IsAnomaly { - t.Fatal("expected no anomaly") - } - }) -} diff --git a/internal/engine/experiment/sniblocking/sniblocking.go b/internal/engine/experiment/sniblocking/sniblocking.go index 4e4df18..9bb3a40 100644 --- a/internal/engine/experiment/sniblocking/sniblocking.go +++ b/internal/engine/experiment/sniblocking/sniblocking.go @@ -233,12 +233,10 @@ func maybeURLToSNI(input model.MeasurementTarget) (model.MeasurementTarget, erro } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session m.mu.Lock() if m.cache == nil { m.cache = make(map[string]Subresult) diff --git a/internal/engine/experiment/sniblocking/sniblocking_test.go b/internal/engine/experiment/sniblocking/sniblocking_test.go index 63ec9c7..7424fbf 100644 --- a/internal/engine/experiment/sniblocking/sniblocking_test.go +++ b/internal/engine/experiment/sniblocking/sniblocking_test.go @@ -116,12 +116,12 @@ func TestMeasurerMeasureNoMeasurementInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{ ControlSNI: "example.com", }) - err := measurer.Run( - context.Background(), - newsession(), - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{}, + Session: newsession(), + } + err := measurer.Run(context.Background(), args) if err.Error() != "Experiment requires measurement.Input" { t.Fatal("not the error we expected") } @@ -136,12 +136,12 @@ func TestMeasurerMeasureWithInvalidInput(t *testing.T) { measurement := &model.Measurement{ Input: "\t", } - err := measurer.Run( - ctx, - newsession(), - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + } + err := measurer.Run(ctx, args) if err == nil { t.Fatal("expected an error here") } @@ -156,12 +156,12 @@ func TestMeasurerMeasureWithCancelledContext(t *testing.T) { measurement := &model.Measurement{ Input: "kernel.org", } - err := measurer.Run( - ctx, - newsession(), - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: newsession(), + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/stunreachability/stunreachability.go b/internal/engine/experiment/stunreachability/stunreachability.go index ba278c0..870500f 100644 --- a/internal/engine/experiment/stunreachability/stunreachability.go +++ b/internal/engine/experiment/stunreachability/stunreachability.go @@ -73,10 +73,10 @@ var errStunMissingPortInURL = errors.New("stun: missing port in URL") var errUnsupportedURLScheme = errors.New("stun: unsupported URL scheme") // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session tk := new(TestKeys) measurement.TestKeys = tk registerExtensions(measurement) diff --git a/internal/engine/experiment/stunreachability/stunreachability_test.go b/internal/engine/experiment/stunreachability/stunreachability_test.go index 0b6c2be..209fd18 100644 --- a/internal/engine/experiment/stunreachability/stunreachability_test.go +++ b/internal/engine/experiment/stunreachability/stunreachability_test.go @@ -32,12 +32,12 @@ func TestMeasurerExperimentNameVersion(t *testing.T) { func TestRunWithoutInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, errStunMissingInput) { t.Fatal("not the error we expected", err) } @@ -47,12 +47,12 @@ func TestRunWithInvalidURL(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("\t") // <- invalid URL - err := measurer.Run( - context.Background(), - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(context.Background(), args) if err == nil || !strings.HasSuffix(err.Error(), "invalid control character in URL") { t.Fatal("not the error we expected", err) } @@ -62,12 +62,12 @@ func TestRunWithNoPort(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("stun://stun.ekiga.net") - err := measurer.Run( - context.Background(), - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, errStunMissingPortInURL) { t.Fatal("not the error we expected", err) } @@ -77,12 +77,12 @@ func TestRunWithUnsupportedURLScheme(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget("https://stun.ekiga.net:3478") - err := measurer.Run( - context.Background(), - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, errUnsupportedURLScheme) { t.Fatal("not the error we expected", err) } @@ -92,14 +92,14 @@ func TestRunWithInput(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: model.DiscardLogger, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -124,14 +124,14 @@ func TestCancelledContext(t *testing.T) { measurer := NewExperimentMeasurer(Config{}) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) - err := measurer.Run( - ctx, - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: model.DiscardLogger, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit t.Fatal("not the error we expected", err) } @@ -166,14 +166,14 @@ func TestNewClientFailure(t *testing.T) { measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: model.DiscardLogger, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, nil) { // nil because we want to submit t.Fatal("not the error we expected") } @@ -202,14 +202,14 @@ func TestStartFailure(t *testing.T) { measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: model.DiscardLogger, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, nil) { // nil because we want to submit t.Fatal("not the error we expected") } @@ -242,14 +242,14 @@ func TestReadFailure(t *testing.T) { measurer := NewExperimentMeasurer(*config) measurement := new(model.Measurement) measurement.Input = model.MeasurementTarget(defaultInput) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: model.DiscardLogger, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, nil) { // nil because we want to submit t.Fatal("not the error we expected") } diff --git a/internal/engine/experiment/tcpping/tcpping.go b/internal/engine/experiment/tcpping/tcpping.go index c8cfbae..0354233 100644 --- a/internal/engine/experiment/tcpping/tcpping.go +++ b/internal/engine/experiment/tcpping/tcpping.go @@ -82,12 +82,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session if measurement.Input == "" { return errNoInputProvided } diff --git a/internal/engine/experiment/tcpping/tcpping_test.go b/internal/engine/experiment/tcpping/tcpping_test.go index faec432..330b9cb 100644 --- a/internal/engine/experiment/tcpping/tcpping_test.go +++ b/internal/engine/experiment/tcpping/tcpping_test.go @@ -51,7 +51,12 @@ func TestMeasurer_run(t *testing.T) { MockableLogger: model.DiscardLogger, } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) return meas, m, err } diff --git a/internal/engine/experiment/telegram/telegram.go b/internal/engine/experiment/telegram/telegram.go index c9a0902..d9f606e 100644 --- a/internal/engine/experiment/telegram/telegram.go +++ b/internal/engine/experiment/telegram/telegram.go @@ -101,8 +101,11 @@ func (m Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run -func (m Measurer) Run(ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() urlgetter.RegisterExtensions(measurement) diff --git a/internal/engine/experiment/telegram/telegram_test.go b/internal/engine/experiment/telegram/telegram_test.go index 4d206bf..8a1417c 100644 --- a/internal/engine/experiment/telegram/telegram_test.go +++ b/internal/engine/experiment/telegram/telegram_test.go @@ -28,14 +28,14 @@ func TestNewExperimentMeasurer(t *testing.T) { func TestGood(t *testing.T) { measurer := telegram.NewExperimentMeasurer(telegram.Config{}) measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -297,7 +297,12 @@ func TestWeConfigureWebChecksToFailOnHTTPError(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := measurer.Run(ctx, args); err != nil { t.Fatal(err) } if called.Load() < 1 { diff --git a/internal/engine/experiment/tlsmiddlebox/measurer.go b/internal/engine/experiment/tlsmiddlebox/measurer.go index d157ab0..363e7a6 100644 --- a/internal/engine/experiment/tlsmiddlebox/measurer.go +++ b/internal/engine/experiment/tlsmiddlebox/measurer.go @@ -52,12 +52,10 @@ var ( ) // // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session if measurement.Input == "" { return errNoInputProvided } diff --git a/internal/engine/experiment/tlsmiddlebox/measurer_test.go b/internal/engine/experiment/tlsmiddlebox/measurer_test.go index f5bf815..318fd5d 100644 --- a/internal/engine/experiment/tlsmiddlebox/measurer_test.go +++ b/internal/engine/experiment/tlsmiddlebox/measurer_test.go @@ -38,7 +38,12 @@ func TestMeasurer_input_failure(t *testing.T) { }, } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) return meas, m, err } diff --git a/internal/engine/experiment/tlsping/tlsping.go b/internal/engine/experiment/tlsping/tlsping.go index eda8023..de51864 100644 --- a/internal/engine/experiment/tlsping/tlsping.go +++ b/internal/engine/experiment/tlsping/tlsping.go @@ -112,12 +112,10 @@ var ( ) // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session if measurement.Input == "" { return errNoInputProvided } diff --git a/internal/engine/experiment/tlsping/tlsping_test.go b/internal/engine/experiment/tlsping/tlsping_test.go index 716549f..fad4388 100644 --- a/internal/engine/experiment/tlsping/tlsping_test.go +++ b/internal/engine/experiment/tlsping/tlsping_test.go @@ -58,7 +58,12 @@ func TestMeasurer_run(t *testing.T) { MockableLogger: model.DiscardLogger, } callbacks := model.NewPrinterCallbacks(model.DiscardLogger) - err := m.Run(ctx, sess, meas, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: meas, + Session: sess, + } + err := m.Run(ctx, args) return meas, m, err } diff --git a/internal/engine/experiment/tlstool/tlstool.go b/internal/engine/experiment/tlstool/tlstool.go index 812d081..9791b2a 100644 --- a/internal/engine/experiment/tlstool/tlstool.go +++ b/internal/engine/experiment/tlstool/tlstool.go @@ -78,12 +78,11 @@ var allMethods = []method{{ }} // Run implements ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session + // TODO(bassosimone): wondering whether this experiment should // actually be merged with sniblocking instead? tk := new(TestKeys) diff --git a/internal/engine/experiment/tlstool/tlstool_test.go b/internal/engine/experiment/tlstool/tlstool_test.go index 6f95dde..55a33bc 100644 --- a/internal/engine/experiment/tlstool/tlstool_test.go +++ b/internal/engine/experiment/tlstool/tlstool_test.go @@ -27,12 +27,12 @@ func TestRunWithExplicitSNI(t *testing.T) { }) measurement := new(model.Measurement) measurement.Input = "8.8.8.8:853" - err := measurer.Run( - ctx, - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -43,12 +43,12 @@ func TestRunWithImplicitSNI(t *testing.T) { measurer := tlstool.NewExperimentMeasurer(tlstool.Config{}) measurement := new(model.Measurement) measurement.Input = "dns.google:853" - err := measurer.Run( - ctx, - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -60,12 +60,12 @@ func TestRunWithCancelledContext(t *testing.T) { measurer := tlstool.NewExperimentMeasurer(tlstool.Config{}) measurement := new(model.Measurement) measurement.Input = "dns.google:853" - err := measurer.Run( - ctx, - &mockable.Session{}, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/tor/tor.go b/internal/engine/experiment/tor/tor.go index 19c8f50..ffd52fa 100644 --- a/internal/engine/experiment/tor/tor.go +++ b/internal/engine/experiment/tor/tor.go @@ -166,12 +166,10 @@ func (m *Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session targets, err := m.gimmeTargets(ctx, sess) if err != nil { return err // fail the measurement if we cannot get any target diff --git a/internal/engine/experiment/tor/tor_test.go b/internal/engine/experiment/tor/tor_test.go index f2eabeb..4e02549 100644 --- a/internal/engine/experiment/tor/tor_test.go +++ b/internal/engine/experiment/tor/tor_test.go @@ -36,14 +36,14 @@ func TestMeasurerMeasureFetchTorTargetsError(t *testing.T) { measurer.fetchTorTargets = func(ctx context.Context, sess model.ExperimentSession, cc string) (map[string]model.OOAPITorTarget, error) { return nil, expected } - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{}, + Session: &mockable.Session{ MockableLogger: log.Log, }, - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if !errors.Is(err, expected) { t.Fatal("not the error we expected") } @@ -55,14 +55,14 @@ func TestMeasurerMeasureFetchTorTargetsEmptyList(t *testing.T) { return nil, nil } measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{ MockableLogger: log.Log, }, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -79,14 +79,14 @@ func TestMeasurerMeasureGoodWithMockedOrchestra(t *testing.T) { measurer.fetchTorTargets = func(ctx context.Context, sess model.ExperimentSession, cc string) (map[string]model.OOAPITorTarget, error) { return nil, nil } - err := measurer.Run( - context.Background(), - &mockable.Session{ + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: &model.Measurement{}, + Session: &mockable.Session{ MockableLogger: log.Log, }, - new(model.Measurement), - model.NewPrinterCallbacks(log.Log), - ) + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -99,12 +99,12 @@ func TestMeasurerMeasureGood(t *testing.T) { measurer := NewMeasurer(Config{}) sess := newsession() measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - sess, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } @@ -142,12 +142,12 @@ func TestMeasurerMeasureSanitiseOutput(t *testing.T) { key: staticPrivateTestingTarget, } measurement := new(model.Measurement) - err := measurer.Run( - context.Background(), - sess, - measurement, - model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: sess, + } + err := measurer.Run(context.Background(), args) if err != nil { t.Fatal(err) } diff --git a/internal/engine/experiment/torsf/integration_test.go b/internal/engine/experiment/torsf/integration_test.go index 80feb82..11e846d 100644 --- a/internal/engine/experiment/torsf/integration_test.go +++ b/internal/engine/experiment/torsf/integration_test.go @@ -34,7 +34,12 @@ func TestRunWithExistingTor(t *testing.T) { MockableLogger: log.Log, MockableTempDir: tempdir, } - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { t.Fatal(err) } } diff --git a/internal/engine/experiment/torsf/torsf.go b/internal/engine/experiment/torsf/torsf.go index aef0f82..99ad54c 100644 --- a/internal/engine/experiment/torsf/torsf.go +++ b/internal/engine/experiment/torsf/torsf.go @@ -124,10 +124,10 @@ const maxRuntime = 600 * time.Second // set the relevant OONI error inside of the measurement and // return nil. This is important because the caller may not submit // the measurement if this method returns an error. -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session ptl, sfdialer, err := m.setup(ctx, sess.Logger()) if err != nil { // we cannot setup the experiment diff --git a/internal/engine/experiment/torsf/torsf_test.go b/internal/engine/experiment/torsf/torsf_test.go index e71c666..afdd81a 100644 --- a/internal/engine/experiment/torsf/torsf_test.go +++ b/internal/engine/experiment/torsf/torsf_test.go @@ -47,7 +47,12 @@ func TestFailureWithInvalidRendezvousMethod(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - err := m.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := m.Run(ctx, args) if !errors.Is(err, ptx.ErrSnowflakeNoSuchRendezvousMethod) { t.Fatal("unexpected error", err) } @@ -70,7 +75,12 @@ func TestFailureToStartPTXListener(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); !errors.Is(err, expected) { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); !errors.Is(err, expected) { t.Fatal("not the error we expected", err) } if tk := measurement.TestKeys; tk != nil { @@ -108,7 +118,12 @@ func TestSuccessWithMockedTunnelStart(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } if called.Load() != 1 { @@ -168,7 +183,12 @@ func TestWithCancelledContext(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } tk := measurement.TestKeys.(*TestKeys) @@ -231,7 +251,12 @@ func TestFailureToStartTunnel(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } tk := measurement.TestKeys.(*TestKeys) diff --git a/internal/engine/experiment/urlgetter/urlgetter.go b/internal/engine/experiment/urlgetter/urlgetter.go index 8a6593c..3414607 100644 --- a/internal/engine/experiment/urlgetter/urlgetter.go +++ b/internal/engine/experiment/urlgetter/urlgetter.go @@ -97,10 +97,10 @@ func (m Measurer) ExperimentVersion() string { } // Run implements model.ExperimentSession.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session // When using the urlgetter experiment directly, there is a nonconfigurable // default timeout that applies. When urlgetter is used as a library, it's // instead the responsibility of the user of urlgetter to set timeouts. Note diff --git a/internal/engine/experiment/urlgetter/urlgetter_test.go b/internal/engine/experiment/urlgetter/urlgetter_test.go index eaa8f7b..f9e62da 100644 --- a/internal/engine/experiment/urlgetter/urlgetter_test.go +++ b/internal/engine/experiment/urlgetter/urlgetter_test.go @@ -23,10 +23,12 @@ func TestMeasurer(t *testing.T) { } measurement := new(model.Measurement) measurement.Input = "https://www.google.com" - err := m.Run( - ctx, &mockable.Session{}, - measurement, model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := m.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit the measurement t.Fatal("not the error we expected") } @@ -60,10 +62,12 @@ func TestMeasurerDNSCache(t *testing.T) { } measurement := new(model.Measurement) measurement.Input = "https://www.google.com" - err := m.Run( - ctx, &mockable.Session{}, - measurement, model.NewPrinterCallbacks(log.Log), - ) + args := &model.ExperimentArgs{ + Callbacks: model.NewPrinterCallbacks(log.Log), + Measurement: measurement, + Session: &mockable.Session{}, + } + err := m.Run(ctx, args) if !errors.Is(err, nil) { // nil because we want to submit the measurement t.Fatal("not the error we expected") } diff --git a/internal/engine/experiment/vanillator/integration_test.go b/internal/engine/experiment/vanillator/integration_test.go index c569303..c86ae9e 100644 --- a/internal/engine/experiment/vanillator/integration_test.go +++ b/internal/engine/experiment/vanillator/integration_test.go @@ -34,7 +34,12 @@ func TestRunWithExistingTor(t *testing.T) { MockableLogger: log.Log, MockableTempDir: tempdir, } - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { t.Fatal(err) } } diff --git a/internal/engine/experiment/vanillator/vanillator.go b/internal/engine/experiment/vanillator/vanillator.go index 593f07c..1b6d687 100644 --- a/internal/engine/experiment/vanillator/vanillator.go +++ b/internal/engine/experiment/vanillator/vanillator.go @@ -106,10 +106,10 @@ const maxRuntime = 200 * time.Second // set the relevant OONI error inside of the measurement and // return nil. This is important because the caller may not submit // the measurement if this method returns an error. -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session m.registerExtensions(measurement) start := time.Now() ctx, cancel := context.WithTimeout(ctx, maxRuntime) diff --git a/internal/engine/experiment/vanillator/vanillator_test.go b/internal/engine/experiment/vanillator/vanillator_test.go index e8cbdd4..fda2f5e 100644 --- a/internal/engine/experiment/vanillator/vanillator_test.go +++ b/internal/engine/experiment/vanillator/vanillator_test.go @@ -59,7 +59,12 @@ func TestSuccessWithMockedTunnelStart(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } if called.Load() != 1 { @@ -113,7 +118,12 @@ func TestWithCancelledContext(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } tk := measurement.TestKeys.(*TestKeys) @@ -170,7 +180,12 @@ func TestFailureToStartTunnel(t *testing.T) { callbacks := &model.PrinterCallbacks{ Logger: model.DiscardLogger, } - if err := m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := m.Run(ctx, args); err != nil { t.Fatal(err) } tk := measurement.TestKeys.(*TestKeys) diff --git a/internal/engine/experiment/webconnectivity/control.go b/internal/engine/experiment/webconnectivity/control.go index ccaf870..bd953a1 100644 --- a/internal/engine/experiment/webconnectivity/control.go +++ b/internal/engine/experiment/webconnectivity/control.go @@ -4,9 +4,10 @@ import ( "context" "github.com/ooni/probe-cli/v3/internal/geoipx" - "github.com/ooni/probe-cli/v3/internal/httpx" + "github.com/ooni/probe-cli/v3/internal/httpapi" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" ) // Redirect to types defined inside the model package @@ -21,22 +22,23 @@ type ( // Control performs the control request and returns the response. func Control( ctx context.Context, sess model.ExperimentSession, - thAddr string, creq ControlRequest) (out ControlResponse, err error) { - clnt := &httpx.APIClientTemplate{ - BaseURL: thAddr, - HTTPClient: sess.DefaultHTTPClient(), - Logger: sess.Logger(), - UserAgent: sess.UserAgent(), - } + testhelpers []model.OOAPIService, creq ControlRequest) (ControlResponse, *model.OOAPIService, error) { + seqCaller := httpapi.NewSequenceCaller( + httpapi.MustNewPOSTJSONWithJSONResponseDescriptor(sess.Logger(), "/", creq).WithBodyLogging(true), + httpapi.NewEndpointList(sess.DefaultHTTPClient(), sess.UserAgent(), testhelpers...)..., + ) sess.Logger().Infof("control for %s...", creq.HTTPRequest) - // make sure error is wrapped - err = clnt.WithBodyLogging().Build().PostJSON(ctx, "/", creq, &out) - if err != nil { - err = netxlite.NewTopLevelGenericErrWrapper(err) - } + var out ControlResponse + idx, err := seqCaller.CallWithJSONResponse(ctx, &out) sess.Logger().Infof("control for %s... %+v", creq.HTTPRequest, model.ErrorToStringOrOK(err)) + if err != nil { + // make sure error is wrapped + err = netxlite.NewTopLevelGenericErrWrapper(err) + return ControlResponse{}, nil, err + } fillASNs(&out.DNS) - return + runtimex.Assert(idx >= 0 && idx < len(testhelpers), "idx out of bounds") + return out, &testhelpers[idx], nil } // fillASNs fills the ASNs array of ControlDNSResult. For each Addr inside diff --git a/internal/engine/experiment/webconnectivity/webconnectivity.go b/internal/engine/experiment/webconnectivity/webconnectivity.go index 526b9f7..6792311 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity.go @@ -15,7 +15,7 @@ import ( const ( testName = "web_connectivity" - testVersion = "0.4.1" + testVersion = "0.4.2" ) // Config contains the experiment config. @@ -121,12 +121,11 @@ const ( ) // Run implements ExperimentMeasurer.Run. -func (m Measurer) Run( - ctx context.Context, - sess model.ExperimentSession, - measurement *model.Measurement, - callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() tk := new(TestKeys) @@ -145,19 +144,9 @@ func (m Measurer) Run( } // 1. find test helper testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") - var testhelper *model.OOAPIService - for _, th := range testhelpers { - if th.Type == "https" { - testhelper = &th - break - } - } - if testhelper == nil { + if len(testhelpers) < 1 { return ErrNoAvailableTestHelpers } - measurement.TestHelpers = map[string]interface{}{ - "backend": testhelper, - } // 2. perform the DNS lookup step dnsBegin := time.Now() dnsResult := DNSLookup(ctx, DNSLookupConfig{ @@ -167,10 +156,11 @@ func (m Measurer) Run( tk.Queries = append(tk.Queries, dnsResult.TestKeys.Queries...) tk.DNSExperimentFailure = dnsResult.Failure epnts := NewEndpoints(URL, dnsResult.Addresses()) - sess.Logger().Infof("using control: %s", testhelper.Address) + sess.Logger().Infof("using control: %+v", testhelpers) // 3. perform the control measurement thBegin := time.Now() - tk.Control, err = Control(ctx, sess, testhelper.Address, ControlRequest{ + var usedTH *model.OOAPIService + tk.Control, usedTH, err = Control(ctx, sess, testhelpers, ControlRequest{ HTTPRequest: URL.String(), HTTPRequestHeaders: map[string][]string{ "Accept": {model.HTTPHeaderAccept}, @@ -179,6 +169,11 @@ func (m Measurer) Run( }, TCPConnect: epnts.Endpoints(), }) + if usedTH != nil { + measurement.TestHelpers = map[string]interface{}{ + "backend": usedTH, + } + } tk.THRuntime = time.Since(thBegin) tk.ControlFailure = tracex.NewFailure(err) // 4. analyze DNS results diff --git a/internal/engine/experiment/webconnectivity/webconnectivity_test.go b/internal/engine/experiment/webconnectivity/webconnectivity_test.go index c1fcb37..6b53693 100644 --- a/internal/engine/experiment/webconnectivity/webconnectivity_test.go +++ b/internal/engine/experiment/webconnectivity/webconnectivity_test.go @@ -21,7 +21,7 @@ func TestNewExperimentMeasurer(t *testing.T) { if measurer.ExperimentName() != "web_connectivity" { t.Fatal("unexpected name") } - if measurer.ExperimentVersion() != "0.4.1" { + if measurer.ExperimentVersion() != "0.4.2" { t.Fatal("unexpected version") } } @@ -37,7 +37,12 @@ func TestSuccess(t *testing.T) { sess := newsession(t, true) measurement := &model.Measurement{Input: "http://www.example.com"} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -65,7 +70,12 @@ func TestMeasureWithCancelledContext(t *testing.T) { sess := newsession(t, true) measurement := &model.Measurement{Input: "http://www.example.com"} callbacks := model.NewPrinterCallbacks(log.Log) - if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := measurer.Run(ctx, args); err != nil { t.Fatal(err) } tk := measurement.TestKeys.(*webconnectivity.TestKeys) @@ -99,7 +109,12 @@ func TestMeasureWithNoInput(t *testing.T) { sess := newsession(t, true) measurement := &model.Measurement{Input: ""} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, webconnectivity.ErrNoInput) { t.Fatal(err) } @@ -127,7 +142,12 @@ func TestMeasureWithInputNotBeingAnURL(t *testing.T) { sess := newsession(t, true) measurement := &model.Measurement{Input: "\t\t\t\t\t\t"} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, webconnectivity.ErrInputIsNotAnURL) { t.Fatal(err) } @@ -155,7 +175,12 @@ func TestMeasureWithUnsupportedInput(t *testing.T) { sess := newsession(t, true) measurement := &model.Measurement{Input: "dnslookup://example.com"} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, webconnectivity.ErrUnsupportedInput) { t.Fatal(err) } @@ -183,7 +208,12 @@ func TestMeasureWithNoAvailableTestHelpers(t *testing.T) { sess := newsession(t, false) measurement := &model.Measurement{Input: "https://www.example.com"} callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if !errors.Is(err, webconnectivity.ErrNoAvailableTestHelpers) { t.Fatal(err) } diff --git a/internal/engine/experiment/whatsapp/whatsapp.go b/internal/engine/experiment/whatsapp/whatsapp.go index 24874e2..1a1e3bf 100644 --- a/internal/engine/experiment/whatsapp/whatsapp.go +++ b/internal/engine/experiment/whatsapp/whatsapp.go @@ -154,10 +154,11 @@ func (m Measurer) ExperimentVersion() string { } // Run implements ExperimentMeasurer.Run -func (m Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session + ctx, cancel := context.WithTimeout(ctx, 60*time.Second) defer cancel() urlgetter.RegisterExtensions(measurement) diff --git a/internal/engine/experiment/whatsapp/whatsapp_test.go b/internal/engine/experiment/whatsapp/whatsapp_test.go index 2162086..c6bb573 100644 --- a/internal/engine/experiment/whatsapp/whatsapp_test.go +++ b/internal/engine/experiment/whatsapp/whatsapp_test.go @@ -35,7 +35,12 @@ func TestSuccess(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -70,7 +75,12 @@ func TestFailureAllEndpoints(t *testing.T) { sess := &mockable.Session{MockableLogger: log.Log} measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - err := measurer.Run(ctx, sess, measurement, callbacks) + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + err := measurer.Run(ctx, args) if err != nil { t.Fatal(err) } @@ -598,7 +608,12 @@ func TestWeConfigureWebChecksCorrectly(t *testing.T) { } measurement := new(model.Measurement) callbacks := model.NewPrinterCallbacks(log.Log) - if err := measurer.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err := measurer.Run(ctx, args); err != nil { t.Fatal(err) } if called.Load() != 263 { diff --git a/internal/engine/experiment_integration_test.go b/internal/engine/experiment_integration_test.go index 449edc6..865a452 100644 --- a/internal/engine/experiment_integration_test.go +++ b/internal/engine/experiment_integration_test.go @@ -475,10 +475,7 @@ func (am *antaniMeasurer) ExperimentVersion() string { return "0.1.1" } -func (am *antaniMeasurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (am *antaniMeasurer) Run(ctx context.Context, args *model.ExperimentArgs) error { return nil } diff --git a/internal/experiment/webconnectivity/cleartextflow.go b/internal/experiment/webconnectivity/cleartextflow.go index 351a27c..d515d4e 100644 --- a/internal/experiment/webconnectivity/cleartextflow.go +++ b/internal/experiment/webconnectivity/cleartextflow.go @@ -285,7 +285,7 @@ func (t *CleartextFlow) maybeFollowRedirects(ctx context.Context, resp *http.Res WaitGroup: t.WaitGroup, Referer: resp.Request.URL.String(), Session: nil, // no need to issue another control request - THAddr: "", // ditto + TestHelpers: nil, // ditto UDPAddress: t.UDPAddress, } resolvers.Start(ctx) diff --git a/internal/experiment/webconnectivity/control.go b/internal/experiment/webconnectivity/control.go index 3e58f42..bf7003a 100644 --- a/internal/experiment/webconnectivity/control.go +++ b/internal/experiment/webconnectivity/control.go @@ -8,10 +8,11 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/httpx" + "github.com/ooni/probe-cli/v3/internal/httpapi" "github.com/ooni/probe-cli/v3/internal/measurexlite" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" ) // EndpointMeasurementsStarter is used by Control to start extra @@ -51,8 +52,8 @@ type Control struct { // Session is the MANDATORY session to use. Session model.ExperimentSession - // THAddr is the MANDATORY TH's URL. - THAddr string + // TestHelpers is the MANDATORY list of test helpers. + TestHelpers []model.OOAPIService // URL is the MANDATORY URL we are measuring. URL *url.URL @@ -102,26 +103,20 @@ func (c *Control) Run(parentCtx context.Context) { // create logger for this operation ol := measurexlite.NewOperationLogger( c.Logger, - "control for %s using %s", + "control for %s using %+v", creq.HTTPRequest, - c.THAddr, + c.TestHelpers, ) - // create an API client - clnt := (&httpx.APIClientTemplate{ - Accept: "", - Authorization: "", - BaseURL: c.THAddr, - HTTPClient: c.Session.DefaultHTTPClient(), - Host: "", // use the one inside the URL - LogBody: true, - Logger: c.Logger, - UserAgent: c.Session.UserAgent(), - }).Build() + // create an httpapi sequence caller + seqCaller := httpapi.NewSequenceCaller( + httpapi.MustNewPOSTJSONWithJSONResponseDescriptor(c.Logger, "/", creq).WithBodyLogging(true), + httpapi.NewEndpointList(c.Session.DefaultHTTPClient(), c.Session.UserAgent(), c.TestHelpers...)..., + ) // issue the control request and wait for the response var cresp webconnectivity.ControlResponse - err := clnt.PostJSON(opCtx, "/", creq, &cresp) + idx, err := seqCaller.CallWithJSONResponse(opCtx, &cresp) if err != nil { // make sure error is wrapped err = netxlite.NewTopLevelGenericErrWrapper(err) @@ -134,6 +129,10 @@ func (c *Control) Run(parentCtx context.Context) { c.TestKeys.SetControl(&cresp) ol.Stop(nil) + // record the specific TH that worked + runtimex.Assert(idx >= 0 && idx < len(c.TestHelpers), "idx out of bounds") + c.TestKeys.setTestHelper(&c.TestHelpers[idx]) + // if the TH returned us addresses we did not previously were // aware of, make sure we also measure them c.maybeStartExtraMeasurements(parentCtx, cresp.DNS.Addrs) diff --git a/internal/experiment/webconnectivity/dnsresolvers.go b/internal/experiment/webconnectivity/dnsresolvers.go index 0c9df8d..8709635 100644 --- a/internal/experiment/webconnectivity/dnsresolvers.go +++ b/internal/experiment/webconnectivity/dnsresolvers.go @@ -67,8 +67,9 @@ type DNSResolvers struct { // always follow the redirect chain caused by the provided URL. Session model.ExperimentSession - // THAddr is the OPTIONAL test helper address. - THAddr string + // TestHelpers is the OPTIONAL list of test helpers. If the list is + // empty, we are not going to try to contact any test helper. + TestHelpers []model.OOAPIService // UDPAddress is the OPTIONAL address of the UDP resolver to use. If this // field is not set we use a default one (e.g., `8.8.8.8:53`). @@ -498,15 +499,15 @@ func (t *DNSResolvers) startSecureFlows( } } -// maybeStartControlFlow starts the control flow iff .Session and .THAddr are set. +// maybeStartControlFlow starts the control flow iff .Session and .TestHelpers are set. func (t *DNSResolvers) maybeStartControlFlow( ctx context.Context, ps *prioritySelector, addresses []DNSEntry, ) { - // note: for subsequent requests we don't set .Session and .THAddr hence + // note: for subsequent requests we don't set .Session and .TestHelpers hence // we are not going to query the test helper more than once - if t.Session != nil && t.THAddr != "" { + if t.Session != nil && len(t.TestHelpers) > 0 { var addrs []string for _, addr := range addresses { addrs = append(addrs, addr.Addr) @@ -518,7 +519,7 @@ func (t *DNSResolvers) maybeStartControlFlow( PrioSelector: ps, TestKeys: t.TestKeys, Session: t.Session, - THAddr: t.THAddr, + TestHelpers: t.TestHelpers, URL: t.URL, WaitGroup: t.WaitGroup, } diff --git a/internal/experiment/webconnectivity/measurer.go b/internal/experiment/webconnectivity/measurer.go index c6731e4..dc520cf 100644 --- a/internal/experiment/webconnectivity/measurer.go +++ b/internal/experiment/webconnectivity/measurer.go @@ -36,15 +36,17 @@ func (m *Measurer) ExperimentName() string { // ExperimentVersion implements model.ExperimentMeasurer. func (m *Measurer) ExperimentVersion() string { - return "0.5.18" + return "0.5.19" } // Run implements model.ExperimentMeasurer. -func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { // Reminder: when this function returns an error, the measurement result // WILL NOT be submitted to the OONI backend. You SHOULD only return an error // for fundamental errors (e.g., the input is invalid or missing). + _ = args.Callbacks + measurement := args.Measurement + sess := args.Session // make sure we have a cancellable context such that we can stop any // goroutine running in the background (e.g., priority.go's ones) @@ -89,17 +91,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // obtain the test helper's address testhelpers, _ := sess.GetTestHelpersByName("web-connectivity") - var thAddr string - for _, th := range testhelpers { - if th.Type == "https" { - thAddr = th.Address - measurement.TestHelpers = map[string]any{ - "backend": &th, - } - break - } - } - if thAddr == "" { + if len(testhelpers) < 1 { sess.Logger().Warnf("continuing without a valid TH address") tk.SetControlFailure(webconnectivity.ErrNoAvailableTestHelpers) } @@ -120,7 +112,7 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, CookieJar: jar, Referer: "", Session: sess, - THAddr: thAddr, + TestHelpers: testhelpers, UDPAddress: "", } resos.Start(ctx) @@ -137,6 +129,16 @@ func (m *Measurer) Run(ctx context.Context, sess model.ExperimentSession, // perform any deferred computation on the test keys tk.Finalize(sess.Logger()) + // set the test helper we used + // TODO(bassosimone): it may be more informative to know about all the + // test helpers we _tried_ to use, however the data format does not have + // support for that as far as I can tell... + if th := tk.getTestHelper(); th != nil { + measurement.TestHelpers = map[string]interface{}{ + "backend": th, + } + } + // return whether there was a fundamental failure, which would prevent // the measurement from being submitted to the OONI collector. return tk.fundamentalFailure diff --git a/internal/experiment/webconnectivity/secureflow.go b/internal/experiment/webconnectivity/secureflow.go index 68f41e1..6a47b73 100644 --- a/internal/experiment/webconnectivity/secureflow.go +++ b/internal/experiment/webconnectivity/secureflow.go @@ -337,7 +337,7 @@ func (t *SecureFlow) maybeFollowRedirects(ctx context.Context, resp *http.Respon WaitGroup: t.WaitGroup, Referer: resp.Request.URL.String(), Session: nil, // no need to issue another control request - THAddr: "", // ditto + TestHelpers: nil, // ditto UDPAddress: t.UDPAddress, } resolvers.Start(ctx) diff --git a/internal/experiment/webconnectivity/testkeys.go b/internal/experiment/webconnectivity/testkeys.go index 37dbc94..5cd917f 100644 --- a/internal/experiment/webconnectivity/testkeys.go +++ b/internal/experiment/webconnectivity/testkeys.go @@ -134,6 +134,10 @@ type TestKeys struct { // mu provides mutual exclusion for accessing the test keys. mu *sync.Mutex + + // testHelper is used to communicate the TH that worked to the main + // goroutine such that we can fill measurement.TestHelpers. + testHelper *model.OOAPIService } // ConnPriorityLogEntry is an entry in the TestKeys.ConnPriorityLog slice. @@ -302,6 +306,21 @@ func (tk *TestKeys) AppendConnPriorityLogEntry(entry *ConnPriorityLogEntry) { tk.mu.Unlock() } +// setTestHelper sets .testHelper in a thread safe way +func (tk *TestKeys) setTestHelper(th *model.OOAPIService) { + tk.mu.Lock() + tk.testHelper = th + tk.mu.Unlock() +} + +// getTestHelper gets .testHelper in a thread safe way +func (tk *TestKeys) getTestHelper() (th *model.OOAPIService) { + tk.mu.Lock() + th = tk.testHelper + tk.mu.Unlock() + return +} + // NewTestKeys creates a new instance of TestKeys. func NewTestKeys() *TestKeys { return &TestKeys{ @@ -348,6 +367,7 @@ func NewTestKeys() *TestKeys { ControlRequest: nil, fundamentalFailure: nil, mu: &sync.Mutex{}, + testHelper: nil, } } diff --git a/internal/httpapi/call.go b/internal/httpapi/call.go new file mode 100644 index 0000000..1228080 --- /dev/null +++ b/internal/httpapi/call.go @@ -0,0 +1,181 @@ +package httpapi + +// +// Calling HTTP APIs. +// + +import ( + "bytes" + "context" + "encoding/json" + "fmt" + "io" + "net/http" + "net/url" + "strings" + + "github.com/ooni/probe-cli/v3/internal/netxlite" +) + +// joinURLPath appends |resourcePath| to |urlPath|. +func joinURLPath(urlPath, resourcePath string) string { + if resourcePath == "" { + if urlPath == "" { + return "/" + } + return urlPath + } + if !strings.HasSuffix(urlPath, "/") { + urlPath += "/" + } + resourcePath = strings.TrimPrefix(resourcePath, "/") + return urlPath + resourcePath +} + +// newRequest creates a new http.Request from the given |ctx|, |endpoint|, and |desc|. +func newRequest(ctx context.Context, endpoint *Endpoint, desc *Descriptor) (*http.Request, error) { + URL, err := url.Parse(endpoint.BaseURL) + if err != nil { + return nil, err + } + // BaseURL and resource URL are joined if they have a path + URL.Path = joinURLPath(URL.Path, desc.URLPath) + if len(desc.URLQuery) > 0 { + URL.RawQuery = desc.URLQuery.Encode() + } else { + URL.RawQuery = "" // as documented we only honour desc.URLQuery + } + var reqBody io.Reader + if len(desc.RequestBody) > 0 { + reqBody = bytes.NewReader(desc.RequestBody) + desc.Logger.Debugf("httpapi: request body length: %d", len(desc.RequestBody)) + if desc.LogBody { + desc.Logger.Debugf("httpapi: request body: %s", string(desc.RequestBody)) + } + } + request, err := http.NewRequestWithContext(ctx, desc.Method, URL.String(), reqBody) + if err != nil { + return nil, err + } + request.Host = endpoint.Host // allow cloudfronting + if desc.Authorization != "" { + request.Header.Set("Authorization", desc.Authorization) + } + if desc.ContentType != "" { + request.Header.Set("Content-Type", desc.ContentType) + } + if desc.Accept != "" { + request.Header.Set("Accept", desc.Accept) + } + if endpoint.UserAgent != "" { + request.Header.Set("User-Agent", endpoint.UserAgent) + } + return request, nil +} + +// ErrHTTPRequestFailed indicates that the server returned >= 400. +type ErrHTTPRequestFailed struct { + // StatusCode is the status code that failed. + StatusCode int +} + +// Error implements error. +func (err *ErrHTTPRequestFailed) Error() string { + return fmt.Sprintf("httpapi: http request failed: %d", err.StatusCode) +} + +// errMaybeCensorship indicates that there was an error at the networking layer +// including, e.g., DNS, TCP connect, TLS. When we see this kind of error, we +// will consider retrying with another endpoint under the assumption that it +// may be that the current endpoint is censored. +type errMaybeCensorship struct { + // Err is the underlying error + Err error +} + +// Error implements error +func (err *errMaybeCensorship) Error() string { + return err.Err.Error() +} + +// Unwrap allows to get the underlying error +func (err *errMaybeCensorship) Unwrap() error { + return err.Err +} + +// docall calls the API represented by the given request |req| on the given |endpoint| +// and returns the response and its body or an error. +func docall(endpoint *Endpoint, desc *Descriptor, request *http.Request) (*http.Response, []byte, error) { + // Implementation note: remember to mark errors for which you want + // to retry with another endpoint using errMaybeCensorship. + response, err := endpoint.HTTPClient.Do(request) + if err != nil { + return nil, nil, &errMaybeCensorship{err} + } + defer response.Body.Close() + // Implementation note: always read and log the response body since + // it's quite useful to see the response JSON on API error. + r := io.LimitReader(response.Body, DefaultMaxBodySize) + data, err := netxlite.ReadAllContext(request.Context(), r) + if err != nil { + return response, nil, &errMaybeCensorship{err} + } + desc.Logger.Debugf("httpapi: response body length: %d bytes", len(data)) + if desc.LogBody { + desc.Logger.Debugf("httpapi: response body: %s", string(data)) + } + if response.StatusCode >= 400 { + return response, nil, &ErrHTTPRequestFailed{response.StatusCode} + } + return response, data, nil +} + +// call is like Call but also returns the response. +func call(ctx context.Context, desc *Descriptor, endpoint *Endpoint) (*http.Response, []byte, error) { + timeout := desc.Timeout + if timeout <= 0 { + timeout = DefaultCallTimeout // as documented + } + ctx, cancel := context.WithTimeout(ctx, timeout) + defer cancel() + request, err := newRequest(ctx, endpoint, desc) + if err != nil { + return nil, nil, err + } + return docall(endpoint, desc, request) +} + +// Call invokes the API described by |desc| on the given HTTP |endpoint| and +// returns the response body (as a slice of bytes) or an error. +// +// Note: this function returns ErrHTTPRequestFailed if the HTTP status code is +// greater or equal than 400. You could use errors.As to obtain a copy of the +// error that was returned and see for yourself the actual status code. +func Call(ctx context.Context, desc *Descriptor, endpoint *Endpoint) ([]byte, error) { + _, rawResponseBody, err := call(ctx, desc, endpoint) + return rawResponseBody, err +} + +// goodContentTypeForJSON tracks known-good content-types for JSON. If the content-type +// is not in this map, |CallWithJSONResponse| emits a warning message. +var goodContentTypeForJSON = map[string]bool{ + applicationJSON: true, +} + +// CallWithJSONResponse is like Call but also assumes that the response is a +// JSON body and attempts to parse it into the |response| field. +// +// Note: this function returns ErrHTTPRequestFailed if the HTTP status code is +// greater or equal than 400. You could use errors.As to obtain a copy of the +// error that was returned and see for yourself the actual status code. +func CallWithJSONResponse(ctx context.Context, desc *Descriptor, endpoint *Endpoint, response any) error { + httpResp, rawRespBody, err := call(ctx, desc, endpoint) + if err != nil { + return err + } + if ctype := httpResp.Header.Get("Content-Type"); !goodContentTypeForJSON[ctype] { + desc.Logger.Warnf("httpapi: unexpected content-type: %s", ctype) + // fallthrough + } + return json.Unmarshal(rawRespBody, response) +} diff --git a/internal/httpapi/call_test.go b/internal/httpapi/call_test.go new file mode 100644 index 0000000..8fd45d7 --- /dev/null +++ b/internal/httpapi/call_test.go @@ -0,0 +1,1163 @@ +package httpapi + +import ( + "context" + "errors" + "fmt" + "io" + "net/http" + "net/http/httptest" + "net/url" + "strings" + "syscall" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" + "github.com/ooni/probe-cli/v3/internal/netxlite" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +func Test_joinURLPath(t *testing.T) { + tests := []struct { + name string + urlPath string + resourcePath string + want string + }{{ + name: "whole path inside urlPath and empty resourcePath", + urlPath: "/robots.txt", + resourcePath: "", + want: "/robots.txt", + }, { + name: "empty urlPath and slash-prefixed resourcePath", + urlPath: "", + resourcePath: "/foo", + want: "/foo", + }, { + name: "slash urlPath and slash-prefixed resourcePath", + urlPath: "/", + resourcePath: "/foo", + want: "/foo", + }, { + name: "empty urlPath and empty resourcePath", + urlPath: "", + resourcePath: "", + want: "/", + }, { + name: "non-slash-terminated urlPath and slash-prefixed resourcePath", + urlPath: "/foo", + resourcePath: "/bar", + want: "/foo/bar", + }, { + name: "slash-terminated urlPath and slash-prefixed resourcePath", + urlPath: "/foo/", + resourcePath: "/bar", + want: "/foo/bar", + }, { + name: "slash-terminated urlPath and non-slash-prefixed resourcePath", + urlPath: "/foo", + resourcePath: "bar", + want: "/foo/bar", + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got := joinURLPath(tt.urlPath, tt.resourcePath) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func Test_newRequest(t *testing.T) { + type args struct { + ctx context.Context + endpoint *Endpoint + desc *Descriptor + } + tests := []struct { + name string + args args + wantFn func(*testing.T, *http.Request) + wantErr error + }{{ + name: "url.Parse fails", + args: args{ + ctx: nil, + endpoint: &Endpoint{ + BaseURL: "\t\t\t", // does not parse! + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: "", + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: nil, + wantErr: errors.New(`parse "\t\t\t": net/url: invalid control character in URL`), + }, { + name: "http.NewRequestWithContext fails", + args: args{ + ctx: nil, // causes http.NewRequestWithContext to fail + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: "", + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: nil, + wantErr: errors.New("net/http: nil Context"), + }, { + name: "successful case with GET method, no body, and no extra headers", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: http.MethodGet, + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodGet { + t.Fatal("invalid method") + } + if req.URL.String() != "https://example.com/" { + t.Fatal("invalid URL") + } + if req.Body != nil { + t.Fatal("invalid body", req.Body) + } + }, + wantErr: nil, + }, { + name: "successful case with POST method and body", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: 0, + Method: http.MethodPost, + RequestBody: []byte("deadbeef"), + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodPost { + t.Fatal("invalid method") + } + if req.URL.String() != "https://example.com/" { + t.Fatal("invalid URL") + } + data, err := netxlite.ReadAllContext(context.Background(), req.Body) + if err != nil { + t.Fatal(err) + } + if diff := cmp.Diff([]byte("deadbeef"), data); diff != "" { + t.Fatal(diff) + } + }, + wantErr: nil, + }, { + name: "with GET method and custom headers", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: nil, + Host: "antani.org", + UserAgent: "httpclient/1.0.1", + }, + desc: &Descriptor{ + Accept: "application/json", + Authorization: "deafbeef", + ContentType: "text/plain", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: http.MethodPut, + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodPut { + t.Fatal("invalid method") + } + if req.Host != "antani.org" { + t.Fatal("invalid request host") + } + if req.URL.String() != "https://example.com/" { + t.Fatal("invalid URL") + } + if req.Header.Get("Authorization") != "deafbeef" { + t.Fatal("invalid authorization") + } + if req.Header.Get("Content-Type") != "text/plain" { + t.Fatal("invalid content-type") + } + if req.Header.Get("Accept") != "application/json" { + t.Fatal("invalid accept") + } + if req.Header.Get("User-Agent") != "httpclient/1.0.1" { + t.Fatal("invalid user-agent") + } + }, + wantErr: nil, + }, { + name: "we join the urlPath with the resourcePath", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/api/v1", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: http.MethodGet, + RequestBody: nil, + Timeout: 0, + URLPath: "/test-list/urls", + URLQuery: nil, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodGet { + t.Fatal("invalid method") + } + if req.URL.String() != "https://www.example.com/api/v1/test-list/urls" { + t.Fatal("invalid URL") + } + }, + wantErr: nil, + }, { + name: "we discard any query element inside the Endpoint.BaseURL", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://example.org/api/v1/?probe_cc=IT", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: http.MethodGet, + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodGet { + t.Fatal("invalid method") + } + if req.URL.String() != "https://example.org/api/v1/" { + t.Fatal("invalid URL") + } + }, + wantErr: nil, + }, { + name: "we include query elements from Descriptor.URLQuery", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/api/v1/", + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: http.MethodGet, + RequestBody: nil, + Timeout: 0, + URLPath: "test-list/urls", + URLQuery: map[string][]string{ + "probe_cc": {"IT"}, + }, + }, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodGet { + t.Fatal("invalid method") + } + if req.URL.String() != "https://www.example.com/api/v1/test-list/urls?probe_cc=IT" { + t.Fatal("invalid URL") + } + }, + wantErr: nil, + }, { + name: "with as many implicitly-initialized fields as possible", + args: args{ + ctx: context.Background(), + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + }, + desc: &Descriptor{}, + }, + wantFn: func(t *testing.T, req *http.Request) { + if req == nil { + t.Fatal("expected non-nil request") + } + if req.Method != http.MethodGet { + t.Fatal("invalid method") + } + if req.URL.String() != "https://example.com/" { + t.Fatal("invalid URL") + } + }, + wantErr: nil, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := newRequest(tt.args.ctx, tt.args.endpoint, tt.args.desc) + switch { + case err == nil && tt.wantErr == nil: + // nothing + case err != nil && tt.wantErr == nil: + t.Fatalf("expected error but got %s", err.Error()) + case err == nil && tt.wantErr != nil: + t.Fatalf("expected %s but got ", tt.wantErr.Error()) + case err.Error() == tt.wantErr.Error(): + // nothing + default: + t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error()) + } + if tt.wantFn != nil { + tt.wantFn(t, got) + return + } + if got != nil { + t.Fatal("got response with nil tt.wantFn") + } + }) + } +} + +func TestCall(t *testing.T) { + type args struct { + ctx context.Context + desc *Descriptor + endpoint *Endpoint + } + tests := []struct { + name string + args args + want []byte + wantErr error + errfn func(t *testing.T, err error) + }{{ + name: "newRequest fails", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: nil, + MaxBodySize: 0, + Method: "", + RequestBody: nil, + Timeout: 0, + URLPath: "", + URLQuery: nil, + }, + endpoint: &Endpoint{ + BaseURL: "\t\t\t", // causes newRequest to fail + HTTPClient: nil, + Host: "", + UserAgent: "", + }, + }, + want: nil, + wantErr: errors.New(`parse "\t\t\t": net/url: invalid control character in URL`), + errfn: nil, + }, { + name: "endpoint.HTTPClient.Do fails", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + }, + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + }, + }, + want: nil, + wantErr: io.EOF, + errfn: func(t *testing.T, err error) { + var expect *errMaybeCensorship + if !errors.As(err, &expect) { + t.Fatal("unexpected error type") + } + }, + }, { + name: "reading body fails", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(&mocks.Reader{ + MockRead: func(b []byte) (int, error) { + return 0, netxlite.ECONNRESET + }, + }), + } + return resp, nil + }, + }, + }, + }, + want: nil, + wantErr: errors.New(netxlite.FailureConnectionReset), + errfn: func(t *testing.T, err error) { + var expect *errMaybeCensorship + if !errors.As(err, &expect) { + t.Fatal("unexpected error type") + } + }, + }, { + name: "status code indicates failure", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + }, + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(strings.NewReader("deadbeef")), + StatusCode: 403, + } + return resp, nil + }, + }, + }, + }, + want: nil, + wantErr: errors.New("httpapi: http request failed: 403"), + errfn: func(t *testing.T, err error) { + var expect *ErrHTTPRequestFailed + if !errors.As(err, &expect) { + t.Fatal("invalid error type") + } + }, + }, { + name: "success with log body flag", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + LogBody: true, // as documented by this test's name + Logger: model.DiscardLogger, + Method: http.MethodGet, + }, + endpoint: &Endpoint{ + BaseURL: "https://example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(strings.NewReader("deadbeef")), + StatusCode: 200, + } + return resp, nil + }, + }, + }, + }, + want: []byte("deadbeef"), + wantErr: nil, + errfn: nil, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := Call(tt.args.ctx, tt.args.desc, tt.args.endpoint) + switch { + case err == nil && tt.wantErr == nil: + // nothing + case err != nil && tt.wantErr == nil: + t.Fatalf("expected error but got %s", err.Error()) + case err == nil && tt.wantErr != nil: + t.Fatalf("expected %s but got ", tt.wantErr.Error()) + case err.Error() == tt.wantErr.Error(): + // nothing + default: + t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error()) + } + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestCallWithJSONResponse(t *testing.T) { + type response struct { + Name string + Age int64 + } + expectedResponse := response{ + Name: "sbs", + Age: 99, + } + type args struct { + ctx context.Context + desc *Descriptor + endpoint *Endpoint + } + tests := []struct { + name string + args args + wantErr error + errfn func(*testing.T, error) + }{{ + name: "call fails", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "\t\t\t\t", // causes failure + }, + }, + wantErr: errors.New(`parse "\t\t\t\t": net/url: invalid control character in URL`), + errfn: nil, + }, { + name: "with error during httpClient.Do", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/a", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + }, + }, + wantErr: io.EOF, + errfn: func(t *testing.T, err error) { + var expect *errMaybeCensorship + if !errors.As(err, &expect) { + t.Fatal("invalid error type") + } + }, + }, { + name: "with error when reading the response body", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/a", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(&mocks.Reader{ + MockRead: func(b []byte) (int, error) { + return 0, netxlite.ECONNRESET + }, + }), + StatusCode: 200, + } + return resp, nil + }, + }, + }, + }, + wantErr: errors.New(netxlite.FailureConnectionReset), + errfn: func(t *testing.T, err error) { + var expect *errMaybeCensorship + if !errors.As(err, &expect) { + t.Fatal("invalid error type") + } + }, + }, { + name: "with HTTP failure", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/a", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(strings.NewReader(`{"Name": "sbs", "Age": 99}`)), + StatusCode: 400, + } + return resp, nil + }, + }, + }, + }, + wantErr: errors.New("httpapi: http request failed: 400"), + errfn: func(t *testing.T, err error) { + var expect *ErrHTTPRequestFailed + if !errors.As(err, &expect) { + t.Fatal("invalid error type") + } + }, + }, { + name: "with good response and missing header", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/a", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Body: io.NopCloser(strings.NewReader(`{"Name": "sbs", "Age": 99}`)), + StatusCode: 200, + } + return resp, nil + }, + }, + }, + }, + wantErr: nil, + errfn: nil, + }, { + name: "with good response and good header", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + Logger: model.DiscardLogger, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/a", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Header: http.Header{ + "Content-Type": {"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{"Name": "sbs", "Age": 99}`)), + StatusCode: 200, + } + return resp, nil + }, + }, + }, + }, + wantErr: nil, + errfn: nil, + }, { + name: "response is not JSON", + args: args{ + ctx: context.Background(), + desc: &Descriptor{ + LogBody: false, + Logger: model.DiscardLogger, + Method: http.MethodGet, + }, + endpoint: &Endpoint{ + BaseURL: "https://www.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + Header: http.Header{ + "Content-Type": {"application/json"}, + }, + Body: io.NopCloser(strings.NewReader(`{`)), // invalid JSON + StatusCode: 200, + } + return resp, nil + }, + }, + }, + }, + wantErr: errors.New("unexpected end of JSON input"), + errfn: nil, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + var response response + err := CallWithJSONResponse(tt.args.ctx, tt.args.desc, tt.args.endpoint, &response) + switch { + case err == nil && tt.wantErr == nil: + if diff := cmp.Diff(expectedResponse, response); err != nil { + t.Fatal(diff) + } + case err != nil && tt.wantErr == nil: + t.Fatalf("expected error but got %s", err.Error()) + case err == nil && tt.wantErr != nil: + t.Fatalf("expected %s but got ", tt.wantErr.Error()) + case err.Error() == tt.wantErr.Error(): + // nothing + default: + t.Fatalf("expected %s but got %s", err.Error(), tt.wantErr.Error()) + } + if tt.errfn != nil { + tt.errfn(t, err) + } + }) + } +} + +func TestCallHonoursContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // should fail HTTP request immediately + desc := &Descriptor{ + LogBody: false, + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/robots.txt", + } + endpoint := &Endpoint{ + BaseURL: "https://www.example.com/", + HTTPClient: http.DefaultClient, + UserAgent: model.HTTPHeaderUserAgent, + } + body, err := Call(ctx, desc, endpoint) + if !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } + if len(body) > 0 { + t.Fatal("expected zero-length body") + } +} + +func TestCallWithJSONResponseHonoursContext(t *testing.T) { + ctx, cancel := context.WithCancel(context.Background()) + cancel() // should fail HTTP request immediately + desc := &Descriptor{ + LogBody: false, + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/robots.txt", + } + endpoint := &Endpoint{ + BaseURL: "https://www.example.com/", + HTTPClient: http.DefaultClient, + UserAgent: model.HTTPHeaderUserAgent, + } + var resp url.URL + err := CallWithJSONResponse(ctx, desc, endpoint, &resp) + if !errors.Is(err, context.Canceled) { + t.Fatal("unexpected err", err) + } +} + +func TestDescriptorLogging(t *testing.T) { + + // This test was originally written for the httpx package and we have adapted it + // by keeping the ~same implementation with a custom callx function that converts + // the previous semantics of httpx to the new semantics of httpapi. + callx := func(baseURL string, logBody bool, logger model.Logger, request, response any) error { + desc := MustNewPOSTJSONWithJSONResponseDescriptor(logger, "/", request).WithBodyLogging(logBody) + runtimex.Assert(desc.LogBody == logBody, "desc.LogBody should be equal to logBody here") + endpoint := &Endpoint{ + BaseURL: baseURL, + HTTPClient: http.DefaultClient, + } + return CallWithJSONResponse(context.Background(), desc, endpoint, response) + } + + // we also needed to create a constructor for the logger + newlogger := func(logs chan string) model.Logger { + return &mocks.Logger{ + MockDebugf: func(format string, v ...interface{}) { + logs <- fmt.Sprintf(format, v...) + }, + MockWarnf: func(format string, v ...interface{}) { + logs <- fmt.Sprintf(format, v...) + }, + } + } + + t.Run("body logging enabled, 200 Ok, and without content-type", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("[]")) + }, + )) + logs := make(chan string, 1024) + defer server.Close() + var ( + input []string + output []string + ) + logger := newlogger(logs) + err := callx(server.URL, true, logger, input, &output) + var found int + close(logs) + for entry := range logs { + if strings.HasPrefix(entry, "httpapi: request body: ") { + // we expect this because body logging is enabled + found |= 1 << 0 + continue + } + if strings.HasPrefix(entry, "httpapi: response body: ") { + // we expect this because body logging is enabled + found |= 1 << 1 + continue + } + if strings.HasPrefix(entry, "httpapi: unexpected content-type: ") { + // we would expect this because the server does not send us any content-type + found |= 1 << 2 + continue + } + if strings.HasPrefix(entry, "httpapi: request body length: ") { + // we should see this because we sent a body + found |= 1 << 3 + continue + } + if strings.HasPrefix(entry, "httpapi: response body length: ") { + // we should see this because we receive a body + found |= 1 << 4 + continue + } + } + if found != (1<<0 | 1<<1 | 1<<2 | 1<<3 | 1<<4) { + t.Fatal("did not find the expected logs") + } + if err != nil { + t.Fatal(err) + } + }) + + t.Run("body logging enabled, 200 Ok, and with content-type", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Header().Add("content-type", "application/json") + w.Write([]byte("[]")) + }, + )) + logs := make(chan string, 1024) + defer server.Close() + var ( + input []string + output []string + ) + logger := newlogger(logs) + err := callx(server.URL, true, logger, input, &output) + var found int + close(logs) + for entry := range logs { + if strings.HasPrefix(entry, "httpapi: request body: ") { + // we expect this because body logging is enabled + found |= 1 << 0 + continue + } + if strings.HasPrefix(entry, "httpapi: response body: ") { + // we expect this because body logging is enabled + found |= 1 << 1 + continue + } + if strings.HasPrefix(entry, "httpapi: unexpected content-type: ") { + // we do not expect this because the server sends us a content-type + found |= 1 << 2 + continue + } + if strings.HasPrefix(entry, "httpapi: request body length: ") { + // we should see this because we sent a body + found |= 1 << 3 + continue + } + if strings.HasPrefix(entry, "httpapi: response body length: ") { + // we should see this because we receive a body + found |= 1 << 4 + continue + } + } + if found != (1<<0 | 1<<1 | 1<<3 | 1<<4) { + t.Fatal("did not find the expected logs") + } + if err != nil { + t.Fatal(err) + } + }) + + t.Run("body logging enabled and 401 Unauthorized", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(401) + w.Write([]byte("[]")) + }, + )) + logs := make(chan string, 1024) + defer server.Close() + var ( + input []string + output []string + ) + logger := newlogger(logs) + err := callx(server.URL, true, logger, input, &output) + var found int + close(logs) + for entry := range logs { + if strings.HasPrefix(entry, "httpapi: request body: ") { + // should occur because body logging is enabled + found |= 1 << 0 + continue + } + if strings.HasPrefix(entry, "httpapi: response body: ") { + // should occur because body logging is enabled + found |= 1 << 1 + continue + } + if strings.HasPrefix(entry, "httpapi: unexpected content-type: ") { + // note: this one should not occur because the code is 401 so we're not + // actually going to parse the JSON document + found |= 1 << 2 + continue + } + if strings.HasPrefix(entry, "httpapi: request body length: ") { + // we should see this because we send a body + found |= 1 << 3 + continue + } + if strings.HasPrefix(entry, "httpapi: response body length: ") { + // we should see this because we receive a body + found |= 1 << 4 + continue + } + } + if found != (1<<0 | 1<<1 | 1<<3 | 1<<4) { + t.Fatal("did not find the expected logs") + } + var failure *ErrHTTPRequestFailed + if !errors.As(err, &failure) || failure.StatusCode != 401 { + t.Fatal("unexpected err", err) + } + }) + + t.Run("body logging NOT enabled and 200 Ok", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.Write([]byte("[]")) + }, + )) + logs := make(chan string, 1024) + defer server.Close() + var ( + input []string + output []string + ) + logger := newlogger(logs) + err := callx(server.URL, false, logger, input, &output) // no logging + var found int + close(logs) + for entry := range logs { + if strings.HasPrefix(entry, "httpapi: request body: ") { + // should not see it: body logging is disabled + found |= 1 << 0 + continue + } + if strings.HasPrefix(entry, "httpapi: response body: ") { + // should not see it: body logging is disabled + found |= 1 << 1 + continue + } + if strings.HasPrefix(entry, "httpapi: unexpected content-type: ") { + // this one should be logged ANYWAY because it's orthogonal to the + // body logging so we should see it also in this case. + found |= 1 << 2 + continue + } + if strings.HasPrefix(entry, "httpapi: request body length: ") { + // should see this because we send a body + found |= 1 << 3 + continue + } + if strings.HasPrefix(entry, "httpapi: response body length: ") { + // should see this because we're receiving a body + found |= 1 << 4 + continue + } + } + if found != (1<<2 | 1<<3 | 1<<4) { + t.Fatal("did not find the expected logs") + } + if err != nil { + t.Fatal(err) + } + }) + + t.Run("body logging NOT enabled and 401 Unauthorized", func(t *testing.T) { + server := httptest.NewServer(http.HandlerFunc( + func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(401) + w.Write([]byte("[]")) + }, + )) + logs := make(chan string, 1024) + defer server.Close() + var ( + input []string + output []string + ) + logger := newlogger(logs) + err := callx(server.URL, false, logger, input, &output) // no logging + var found int + close(logs) + for entry := range logs { + if strings.HasPrefix(entry, "httpapi: request body: ") { + // should not see it: body logging is disabled + found |= 1 << 0 + continue + } + if strings.HasPrefix(entry, "httpapi: response body: ") { + // should not see it: body logging is disabled + found |= 1 << 1 + continue + } + if strings.HasPrefix(entry, "httpapi: unexpected content-type: ") { + // should not see it because we don't parse the body on 401 errors + found |= 1 << 2 + continue + } + if strings.HasPrefix(entry, "httpapi: request body length: ") { + // we send a body so we should see it + found |= 1 << 3 + continue + } + if strings.HasPrefix(entry, "httpapi: response body length: ") { + // we receive a body so we should see it + found |= 1 << 4 + continue + } + } + if found != (1<<3 | 1<<4) { + t.Fatal("did not find the expected logs") + } + var failure *ErrHTTPRequestFailed + if !errors.As(err, &failure) || failure.StatusCode != 401 { + t.Fatal("unexpected err", err) + } + }) +} + +func Test_errMaybeCensorship_Unwrap(t *testing.T) { + t.Run("for errors.Is", func(t *testing.T) { + var err error = &errMaybeCensorship{io.EOF} + if !errors.Is(err, io.EOF) { + t.Fatal("cannot unwrap") + } + }) + + t.Run("for errors.As", func(t *testing.T) { + var err error = &errMaybeCensorship{netxlite.ECONNRESET} + var syserr syscall.Errno + if !errors.As(err, &syserr) || syserr != netxlite.ECONNRESET { + t.Fatal("cannot unwrap") + } + }) +} diff --git a/internal/httpapi/descriptor.go b/internal/httpapi/descriptor.go new file mode 100644 index 0000000..ed35e85 --- /dev/null +++ b/internal/httpapi/descriptor.go @@ -0,0 +1,155 @@ +package httpapi + +// +// HTTP API descriptor (e.g., GET /api/v1/test-list/urls) +// + +import ( + "encoding/json" + "net/http" + "net/url" + "time" + + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" +) + +// Descriptor contains the parameters for calling a given HTTP +// API (e.g., GET /api/v1/test-list/urls). +// +// The zero value of this struct is invalid. Please, fill all the +// fields marked as MANDATORY for correct initialization. +type Descriptor struct { + // Accept contains the OPTIONAL accept header. + Accept string + + // Authorization is the OPTIONAL authorization. + Authorization string + + // ContentType is the OPTIONAL content-type header. + ContentType string + + // LogBody OPTIONALLY enables logging bodies. + LogBody bool + + // Logger is the MANDATORY logger to use. + // + // For example, model.DiscardLogger. + Logger model.Logger + + // MaxBodySize is the OPTIONAL maximum response body size. If + // not set, we use the |DefaultMaxBodySize| constant. + MaxBodySize int64 + + // Method is the MANDATORY request method. + Method string + + // RequestBody is the OPTIONAL request body. + RequestBody []byte + + // Timeout is the OPTIONAL timeout for this call. If no timeout + // is specified we will use the |DefaultCallTimeout| const. + Timeout time.Duration + + // URLPath is the MANDATORY URL path. + URLPath string + + // URLQuery is the OPTIONAL query. + URLQuery url.Values +} + +// WithBodyLogging returns a SHALLOW COPY of |Descriptor| with LogBody set to |value|. You SHOULD +// only use this method when initializing the descriptor you want to use. +func (desc *Descriptor) WithBodyLogging(value bool) *Descriptor { + out := &Descriptor{} + *out = *desc + out.LogBody = value + return out +} + +// DefaultMaxBodySize is the default value for the maximum +// body size you can fetch using the httpapi package. +const DefaultMaxBodySize = 1 << 22 + +// DefaultCallTimeout is the default timeout for an httpapi call. +const DefaultCallTimeout = 60 * time.Second + +// NewGETJSONDescriptor is a convenience factory for creating a new descriptor +// that uses the GET method and expects a JSON response. +func NewGETJSONDescriptor(logger model.Logger, urlPath string) *Descriptor { + return NewGETJSONWithQueryDescriptor(logger, urlPath, url.Values{}) +} + +// applicationJSON is the content-type for JSON +const applicationJSON = "application/json" + +// NewGETJSONWithQueryDescriptor is like NewGETJSONDescriptor but it also +// allows you to provide |query| arguments. Leaving |query| nil or empty +// is equivalent to calling NewGETJSONDescriptor directly. +func NewGETJSONWithQueryDescriptor(logger model.Logger, urlPath string, query url.Values) *Descriptor { + return &Descriptor{ + Accept: applicationJSON, + Authorization: "", + ContentType: "", + LogBody: false, + Logger: logger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodGet, + RequestBody: nil, + Timeout: DefaultCallTimeout, + URLPath: urlPath, + URLQuery: query, + } +} + +// NewPOSTJSONWithJSONResponseDescriptor creates a descriptor that POSTs a JSON document +// and expects to receive back a JSON document from the API. +// +// This function ONLY fails if we cannot serialize the |request| to JSON. So, if you know +// that |request| is JSON-serializable, you can safely call MustNewPostJSONWithJSONResponseDescriptor instead. +func NewPOSTJSONWithJSONResponseDescriptor(logger model.Logger, urlPath string, request any) (*Descriptor, error) { + rawRequest, err := json.Marshal(request) + if err != nil { + return nil, err + } + desc := &Descriptor{ + Accept: applicationJSON, + Authorization: "", + ContentType: applicationJSON, + LogBody: false, + Logger: logger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodPost, + RequestBody: rawRequest, + Timeout: DefaultCallTimeout, + URLPath: urlPath, + URLQuery: nil, + } + return desc, nil +} + +// MustNewPOSTJSONWithJSONResponseDescriptor is like NewPOSTJSONWithJSONResponseDescriptor except that +// it panics in case it's not possible to JSON serialize the |request|. +func MustNewPOSTJSONWithJSONResponseDescriptor(logger model.Logger, urlPath string, request any) *Descriptor { + desc, err := NewPOSTJSONWithJSONResponseDescriptor(logger, urlPath, request) + runtimex.PanicOnError(err, "NewPOSTJSONWithJSONResponseDescriptor failed") + return desc +} + +// NewGETResourceDescriptor creates a generic descriptor for GETting a +// resource of unspecified type using the given |urlPath|. +func NewGETResourceDescriptor(logger model.Logger, urlPath string) *Descriptor { + return &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: logger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodGet, + RequestBody: nil, + Timeout: DefaultCallTimeout, + URLPath: urlPath, + URLQuery: url.Values{}, + } +} diff --git a/internal/httpapi/descriptor_test.go b/internal/httpapi/descriptor_test.go new file mode 100644 index 0000000..c3c58c0 --- /dev/null +++ b/internal/httpapi/descriptor_test.go @@ -0,0 +1,248 @@ +package httpapi + +import ( + "log" + "net/http" + "net/url" + "testing" + "time" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func TestDescriptor_WithBodyLogging(t *testing.T) { + type fields struct { + Accept string + Authorization string + ContentType string + LogBody bool + Logger model.Logger + MaxBodySize int64 + Method string + RequestBody []byte + Timeout time.Duration + URLPath string + URLQuery url.Values + } + tests := []struct { + name string + fields fields + want *Descriptor + }{{ + name: "with empty fields", + fields: fields{}, // LogBody defaults to false + want: &Descriptor{ + LogBody: true, + }, + }, { + name: "with nonempty fields", + fields: fields{ + Accept: "xx", + Authorization: "y", + ContentType: "zzz", + LogBody: false, // obviously must be false + Logger: model.DiscardLogger, + MaxBodySize: 123, + Method: "POST", + RequestBody: []byte("123"), + Timeout: 15555, + URLPath: "/", + URLQuery: map[string][]string{ + "a": {"b"}, + }, + }, + want: &Descriptor{ + Accept: "xx", + Authorization: "y", + ContentType: "zzz", + LogBody: true, + Logger: model.DiscardLogger, + MaxBodySize: 123, + Method: "POST", + RequestBody: []byte("123"), + Timeout: 15555, + URLPath: "/", + URLQuery: map[string][]string{ + "a": {"b"}, + }, + }, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + desc := &Descriptor{ + Accept: tt.fields.Accept, + Authorization: tt.fields.Authorization, + ContentType: tt.fields.ContentType, + LogBody: tt.fields.LogBody, + Logger: tt.fields.Logger, + MaxBodySize: tt.fields.MaxBodySize, + Method: tt.fields.Method, + RequestBody: tt.fields.RequestBody, + Timeout: tt.fields.Timeout, + URLPath: tt.fields.URLPath, + URLQuery: tt.fields.URLQuery, + } + got := desc.WithBodyLogging(true) + if diff := cmp.Diff(tt.want, got); diff != "" { + t.Fatal(diff) + } + }) + } +} + +func TestNewGetJSONDescriptor(t *testing.T) { + expected := &Descriptor{ + Accept: "application/json", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodGet, + RequestBody: nil, + Timeout: DefaultCallTimeout, + URLPath: "/robots.txt", + URLQuery: url.Values{}, + } + got := NewGETJSONDescriptor(model.DiscardLogger, "/robots.txt") + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } +} + +func TestNewGetJSONWithQueryDescriptor(t *testing.T) { + query := url.Values{ + "a": {"b"}, + "c": {"d"}, + } + expected := &Descriptor{ + Accept: "application/json", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodGet, + RequestBody: nil, + Timeout: DefaultCallTimeout, + URLPath: "/robots.txt", + URLQuery: query, + } + got := NewGETJSONWithQueryDescriptor(model.DiscardLogger, "/robots.txt", query) + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } +} + +func TestNewPOSTJSONWithJSONResponseDescriptor(t *testing.T) { + type request struct { + Name string + Age int64 + } + + t.Run("with failure", func(t *testing.T) { + request := make(chan int64) + got, err := NewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request) + if err == nil || err.Error() != "json: unsupported type: chan int64" { + log.Fatal("unexpected err", err) + } + if got != nil { + log.Fatal("expected to get a nil Descriptor") + } + }) + + t.Run("with success", func(t *testing.T) { + request := request{ + Name: "sbs", + Age: 99, + } + expected := &Descriptor{ + Accept: "application/json", + Authorization: "", + ContentType: "application/json", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodPost, + RequestBody: []byte(`{"Name":"sbs","Age":99}`), + Timeout: DefaultCallTimeout, + URLPath: "/robots.txt", + URLQuery: nil, + } + got, err := NewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request) + if err != nil { + log.Fatal(err) + } + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestMustNewPOSTJSONWithJSONResponseDescriptor(t *testing.T) { + type request struct { + Name string + Age int64 + } + + t.Run("with failure", func(t *testing.T) { + var panicked bool + func() { + defer func() { + if r := recover(); r != nil { + panicked = true + } + }() + request := make(chan int64) + _ = MustNewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request) + }() + if !panicked { + t.Fatal("did not panic") + } + }) + + t.Run("with success", func(t *testing.T) { + request := request{ + Name: "sbs", + Age: 99, + } + expected := &Descriptor{ + Accept: "application/json", + Authorization: "", + ContentType: "application/json", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodPost, + RequestBody: []byte(`{"Name":"sbs","Age":99}`), + Timeout: DefaultCallTimeout, + URLPath: "/robots.txt", + URLQuery: nil, + } + got := MustNewPOSTJSONWithJSONResponseDescriptor(model.DiscardLogger, "/robots.txt", request) + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } + }) +} + +func TestNewGetResourceDescriptor(t *testing.T) { + expected := &Descriptor{ + Accept: "", + Authorization: "", + ContentType: "", + LogBody: false, + Logger: model.DiscardLogger, + MaxBodySize: DefaultMaxBodySize, + Method: http.MethodGet, + RequestBody: nil, + Timeout: DefaultCallTimeout, + URLPath: "/robots.txt", + URLQuery: url.Values{}, + } + got := NewGETResourceDescriptor(model.DiscardLogger, "/robots.txt") + if diff := cmp.Diff(expected, got); diff != "" { + t.Fatal(diff) + } +} diff --git a/internal/httpapi/doc.go b/internal/httpapi/doc.go new file mode 100644 index 0000000..0c1361c --- /dev/null +++ b/internal/httpapi/doc.go @@ -0,0 +1,15 @@ +// Package httpapi contains code for calling HTTP APIs. +// +// We model HTTP APIs as follows: +// +// 1. |Endpoint| is an API endpoint (e.g., https://api.ooni.io); +// +// 2. |Descriptor| describes the specific API you want to use (e.g., +// GET /api/v1/test-list/urls with JSON response body). +// +// Generally, you use |Call| to call the API identified by a |Descriptor| +// on the specified |Endpoint|. However, there are cases where you +// need more complex calling patterns. For example, with |SequenceCaller| +// you can invoke the same API |Descriptor| with multiple equivalent +// API |Endpoint|s until one of them succeeds or all fail. +package httpapi diff --git a/internal/httpapi/endpoint.go b/internal/httpapi/endpoint.go new file mode 100644 index 0000000..acfc4d8 --- /dev/null +++ b/internal/httpapi/endpoint.go @@ -0,0 +1,76 @@ +package httpapi + +// +// HTTP API Endpoint (e.g., https://api.ooni.io) +// + +import "github.com/ooni/probe-cli/v3/internal/model" + +// Endpoint models an HTTP endpoint on which you can call +// several HTTP APIs (e.g., https://api.ooni.io) using a +// given HTTP client potentially using a circumvention tunnel +// mechanism such as psiphon or torsf. +// +// The zero value of this struct is invalid. Please, fill all the +// fields marked as MANDATORY for correct initialization. +type Endpoint struct { + // BaseURL is the MANDATORY endpoint base URL. We will honour the + // path of this URL and prepend it to the actual path specified inside + // a |Descriptor.URLPath|. However, we will always discard any query + // that may have been set inside the BaseURL. The only query string + // will be composed from the |Descriptor.URLQuery| values. + // + // For example, https://api.ooni.io. + BaseURL string + + // HTTPClient is the MANDATORY HTTP client to use. + // + // For example, http.DefaultClient. You can introduce circumvention + // here by using an HTTPClient bound to a specific tunnel. + HTTPClient model.HTTPClient + + // Host is the OPTIONAL host header to use. + // + // If this field is empty we use the BaseURL's hostname. A specific + // host header may be needed when using cloudfronting. + Host string + + // User-Agent is the OPTIONAL user-agent to use. If empty, + // we'll use the stdlib's default user-agent string. + UserAgent string +} + +// NewEndpointList constructs a list of API endpoints from |services| +// returned by the OONI backend (or known in advance). +// +// Arguments: +// +// - httpClient is the HTTP client to use for accessing the endpoints; +// +// - userAgent is the user agent you would like to use; +// +// - service is the list of services gathered from the backend. +func NewEndpointList(httpClient model.HTTPClient, + userAgent string, services ...model.OOAPIService) (out []*Endpoint) { + for _, svc := range services { + switch svc.Type { + case "https": + out = append(out, &Endpoint{ + BaseURL: svc.Address, + HTTPClient: httpClient, + Host: "", + UserAgent: userAgent, + }) + case "cloudfront": + out = append(out, &Endpoint{ + BaseURL: svc.Address, + HTTPClient: httpClient, + Host: svc.Front, + UserAgent: userAgent, + }) + default: + // nothing! + } + } + return +} diff --git a/internal/httpapi/endpoint_test.go b/internal/httpapi/endpoint_test.go new file mode 100644 index 0000000..7077e14 --- /dev/null +++ b/internal/httpapi/endpoint_test.go @@ -0,0 +1,69 @@ +package httpapi + +import ( + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" +) + +func TestNewEndpointList(t *testing.T) { + type args struct { + httpClient model.HTTPClient + userAgent string + services []model.OOAPIService + } + defaultHTTPClient := &mocks.HTTPClient{} + tests := []struct { + name string + args args + wantOut []*Endpoint + }{{ + name: "with no services", + args: args{ + httpClient: defaultHTTPClient, + userAgent: model.HTTPHeaderUserAgent, + services: nil, + }, + wantOut: nil, + }, { + name: "common cases", + args: args{ + httpClient: defaultHTTPClient, + userAgent: model.HTTPHeaderUserAgent, + services: []model.OOAPIService{{ + Address: "https://www.example.com/", + Type: "https", + Front: "", + }, { + Address: "https://www.example.org/", + Type: "cloudfront", + Front: "example.org.it", + }, { + Address: "https://nonexistent.onion/", + Type: "onion", + Front: "", + }}, + }, + wantOut: []*Endpoint{{ + BaseURL: "https://www.example.com/", + HTTPClient: defaultHTTPClient, + Host: "", + UserAgent: model.HTTPHeaderUserAgent, + }, { + BaseURL: "https://www.example.org/", + HTTPClient: defaultHTTPClient, + Host: "example.org.it", + UserAgent: model.HTTPHeaderUserAgent, + }}, + }} + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + gotOut := NewEndpointList(tt.args.httpClient, tt.args.userAgent, tt.args.services...) + if diff := cmp.Diff(tt.wantOut, gotOut); diff != "" { + t.Fatal(diff) + } + }) + } +} diff --git a/internal/httpapi/sequence.go b/internal/httpapi/sequence.go new file mode 100644 index 0000000..da1f11d --- /dev/null +++ b/internal/httpapi/sequence.go @@ -0,0 +1,92 @@ +package httpapi + +// +// Sequentially call available API endpoints until one succeed +// or all of them fail. A future implementation of this code may +// (probably should?) take into account knowledge of what is +// working and what is not working to optimize the order with +// which to try different alternatives. +// + +import ( + "context" + "errors" + + "github.com/ooni/probe-cli/v3/internal/multierror" +) + +// SequenceCaller calls the API specified by |Descriptor| once for each of +// the available |Endpoints| until one of them succeeds. +// +// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when +// the error originates in the HTTP round trip or while reading the body. +type SequenceCaller struct { + // Descriptor is the API |Descriptor|. + Descriptor *Descriptor + + // Endpoints is the list of |Endpoint| to use. + Endpoints []*Endpoint +} + +// NewSequenceCaller is a factory for creating a |SequenceCaller|. +func NewSequenceCaller(desc *Descriptor, endpoints ...*Endpoint) *SequenceCaller { + return &SequenceCaller{ + Descriptor: desc, + Endpoints: endpoints, + } +} + +// ErrAllEndpointsFailed indicates that all endpoints failed. +var ErrAllEndpointsFailed = errors.New("httpapi: all endpoints failed") + +// shouldRetry returns true when we should try with another endpoint given the +// value of |err| which could (obviously) be nil in case of success. +func (sc *SequenceCaller) shouldRetry(err error) bool { + var kind *errMaybeCensorship + belongs := errors.As(err, &kind) + return belongs +} + +// Call calls |Call| for each |Endpoint| and |Descriptor| until one endpoint succeeds. The +// return value is the response body and the selected endpoint index or the error. +// +// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when +// the error originates in the HTTP round trip or while reading the body. +func (sc *SequenceCaller) Call(ctx context.Context) ([]byte, int, error) { + var selected int + merr := multierror.New(ErrAllEndpointsFailed) + for _, epnt := range sc.Endpoints { + respBody, err := Call(ctx, sc.Descriptor, epnt) + if sc.shouldRetry(err) { + merr.Add(err) + selected++ + continue + } + // Note: some errors will lead us to return + // early as documented for this method + return respBody, selected, err + } + return nil, -1, merr +} + +// CallWithJSONResponse is like |SequenceCaller.Call| except that it invokes the +// underlying |CallWithJSONResponse| rather than invoking |Call|. +// +// CAVEAT: this code will ONLY retry API calls with subsequent endpoints when +// the error originates in the HTTP round trip or while reading the body. +func (sc *SequenceCaller) CallWithJSONResponse(ctx context.Context, response any) (int, error) { + var selected int + merr := multierror.New(ErrAllEndpointsFailed) + for _, epnt := range sc.Endpoints { + err := CallWithJSONResponse(ctx, sc.Descriptor, epnt, response) + if sc.shouldRetry(err) { + merr.Add(err) + selected++ + continue + } + // Note: some errors will lead us to return + // early as documented for this method + return selected, err + } + return -1, merr +} diff --git a/internal/httpapi/sequence_test.go b/internal/httpapi/sequence_test.go new file mode 100644 index 0000000..13cc50f --- /dev/null +++ b/internal/httpapi/sequence_test.go @@ -0,0 +1,358 @@ +package httpapi + +import ( + "context" + "errors" + "io" + "net/http" + "strings" + "testing" + + "github.com/google/go-cmp/cmp" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/model/mocks" +) + +func TestSequenceCaller(t *testing.T) { + t.Run("Call", func(t *testing.T) { + t.Run("first success", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader("deadbeef")), + } + return resp, nil + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + }, + ) + data, idx, err := sc.Call(context.Background()) + if err != nil { + t.Fatal(err) + } + if idx != 0 { + t.Fatal("invalid idx") + } + if diff := cmp.Diff([]byte("deadbeef"), data); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("first HTTP failure and we immediately stop", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 403, // should cause us to return early + Body: io.NopCloser(strings.NewReader("deadbeef")), + } + return resp, nil + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + }, + ) + data, idx, err := sc.Call(context.Background()) + var failure *ErrHTTPRequestFailed + if !errors.As(err, &failure) || failure.StatusCode != 403 { + t.Fatal("unexpected err", err) + } + if idx != 0 { + t.Fatal("invalid idx") + } + if len(data) > 0 { + t.Fatal("expected to see no response body") + } + }) + + t.Run("first network failure, second success", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to cycle to the second entry + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader("abad1dea")), + } + return resp, nil + }, + }, + }, + ) + data, idx, err := sc.Call(context.Background()) + if err != nil { + t.Fatal(err) + } + if idx != 1 { + t.Fatal("invalid idx") + } + if diff := cmp.Diff([]byte("abad1dea"), data); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("all network failure", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to cycle to the next entry + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to cycle to the next entry + }, + }, + }, + ) + data, idx, err := sc.Call(context.Background()) + if !errors.Is(err, ErrAllEndpointsFailed) { + t.Fatal("unexpected err", err) + } + if idx != -1 { + t.Fatal("invalid idx") + } + if len(data) > 0 { + t.Fatal("expected zero-length data") + } + }) + }) + + t.Run("CallWithJSONResponse", func(t *testing.T) { + type response struct { + Name string + Age int64 + } + + t.Run("first success", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(`{"Name":"sbs","Age":99}`)), + } + return resp, nil + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(`{}`)), // different + } + return resp, nil + }, + }, + }, + ) + expect := response{ + Name: "sbs", + Age: 99, + } + var got response + idx, err := sc.CallWithJSONResponse(context.Background(), &got) + if err != nil { + t.Fatal(err) + } + if idx != 0 { + t.Fatal("invalid idx") + } + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("first HTTP failure and we immediately stop", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 403, // should be enough to cause us fail immediately + Body: io.NopCloser(strings.NewReader(`{"Age": 155, "Name": "sbs"}`)), + } + return resp, nil + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF + }, + }, + }, + ) + // even though there is a JSON body we don't care about reading it + // and so we expect to see in output the zero-value struct + expect := response{ + Name: "", + Age: 0, + } + var got response + idx, err := sc.CallWithJSONResponse(context.Background(), &got) + var failure *ErrHTTPRequestFailed + if !errors.As(err, &failure) || failure.StatusCode != 403 { + t.Fatal("unexpected err", err) + } + if idx != 0 { + t.Fatal("invalid idx") + } + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("first network failure, second success", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to try the next entry + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + resp := &http.Response{ + StatusCode: 200, + Body: io.NopCloser(strings.NewReader(`{"Age":155}`)), + } + return resp, nil + }, + }, + }, + ) + expect := response{ + Name: "", + Age: 155, + } + var got response + idx, err := sc.CallWithJSONResponse(context.Background(), &got) + if err != nil { + t.Fatal(err) + } + if idx != 1 { + t.Fatal("invalid idx") + } + if diff := cmp.Diff(expect, got); diff != "" { + t.Fatal(diff) + } + }) + + t.Run("all network failure", func(t *testing.T) { + sc := NewSequenceCaller( + &Descriptor{ + Logger: model.DiscardLogger, + Method: http.MethodGet, + URLPath: "/", + }, + &Endpoint{ + BaseURL: "https://a.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to try the next entry + }, + }, + }, + &Endpoint{ + BaseURL: "https://b.example.com/", + HTTPClient: &mocks.HTTPClient{ + MockDo: func(req *http.Request) (*http.Response, error) { + return nil, io.EOF // should cause us to try the next entry + }, + }, + }, + ) + var got response + idx, err := sc.CallWithJSONResponse(context.Background(), &got) + if !errors.Is(err, ErrAllEndpointsFailed) { + t.Fatal("unexpected err", err) + } + if idx != -1 { + t.Fatal("invalid idx") + } + }) + }) +} diff --git a/internal/httpx/httpx.go b/internal/httpx/httpx.go index f330745..8083f10 100644 --- a/internal/httpx/httpx.go +++ b/internal/httpx/httpx.go @@ -1,4 +1,8 @@ // Package httpx contains http extensions. +// +// Deprecated: new code should use httpapi instead. While this package and httpapi +// are basically using the same implementation, the API exposed by httpapi allows +// us to try the same request with multiple HTTP endpoints. package httpx import ( diff --git a/internal/model/experiment.go b/internal/model/experiment.go index acaadde..1cba6c1 100644 --- a/internal/model/experiment.go +++ b/internal/model/experiment.go @@ -117,6 +117,19 @@ func (d PrinterCallbacks) OnProgress(percentage float64, message string) { d.Logger.Infof("[%5.1f%%] %s", percentage*100, message) } +// ExperimentArgs contains the arguments passed to an experiment. +type ExperimentArgs struct { + // Callbacks contains MANDATORY experiment callbacks. + Callbacks ExperimentCallbacks + + // Measurement is the MANDATORY measurement in which the experiment + // must write the results of the measurement. + Measurement *Measurement + + // Session is the MANDATORY session the experiment can use. + Session ExperimentSession +} + // ExperimentMeasurer is the interface that allows to run a // measurement for a specific experiment. type ExperimentMeasurer interface { @@ -133,10 +146,7 @@ type ExperimentMeasurer interface { // set the relevant OONI error inside of the measurement and // return nil. This is important because the caller WILL NOT submit // the measurement if this method returns an error. - Run( - ctx context.Context, sess ExperimentSession, - measurement *Measurement, callbacks ExperimentCallbacks, - ) error + Run(ctx context.Context, args *ExperimentArgs) error // GetSummaryKeys returns summary keys expected by ooni/probe-cli. GetSummaryKeys(*Measurement) (interface{}, error) diff --git a/internal/registry/bittorrent.go b/internal/registry/bittorrent.go new file mode 100644 index 0000000..7702837 --- /dev/null +++ b/internal/registry/bittorrent.go @@ -0,0 +1,22 @@ +package registry + +// +// Registers the `dnsping' experiment. +// + +import ( + "github.com/ooni/probe-cli/v3/internal/engine/experiment/bittorrent" + "github.com/ooni/probe-cli/v3/internal/model" +) + +func init() { + AllExperiments["bittorrent"] = &Factory{ + build: func(config interface{}) model.ExperimentMeasurer { + return bittorrent.NewExperimentMeasurer( + *config.(*bittorrent.Config), + ) + }, + config: &bittorrent.Config{}, + inputPolicy: model.InputOrStaticDefault, + } +} diff --git a/internal/registry/smtp.go b/internal/registry/dht.go similarity index 56% rename from internal/registry/smtp.go rename to internal/registry/dht.go index cb67e45..5332f62 100644 --- a/internal/registry/smtp.go +++ b/internal/registry/dht.go @@ -5,18 +5,18 @@ package registry // import ( - "github.com/ooni/probe-cli/v3/internal/engine/experiment/smtp" + "github.com/ooni/probe-cli/v3/internal/engine/experiment/dht" "github.com/ooni/probe-cli/v3/internal/model" ) func init() { - AllExperiments["smtp"] = &Factory{ + AllExperiments["dht"] = &Factory{ build: func(config interface{}) model.ExperimentMeasurer { - return smtp.NewExperimentMeasurer( - *config.(*smtp.Config), + return dht.NewExperimentMeasurer( + *config.(*dht.Config), ) }, - config: &smtp.Config{}, + config: &dht.Config{}, inputPolicy: model.InputOrStaticDefault, } } diff --git a/internal/tutorial/experiment/torsf/chapter01/README.md b/internal/tutorial/experiment/torsf/chapter01/README.md index d357321..eebb606 100644 --- a/internal/tutorial/experiment/torsf/chapter01/README.md +++ b/internal/tutorial/experiment/torsf/chapter01/README.md @@ -211,7 +211,12 @@ need any fancy context and we pass a `context.Background` to `Run`. ```Go ctx := context.Background() - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { log.WithError(err).Fatal("torsf experiment failed") } ``` diff --git a/internal/tutorial/experiment/torsf/chapter01/main.go b/internal/tutorial/experiment/torsf/chapter01/main.go index aef3716..d96c5f3 100644 --- a/internal/tutorial/experiment/torsf/chapter01/main.go +++ b/internal/tutorial/experiment/torsf/chapter01/main.go @@ -212,7 +212,12 @@ func main() { // // ```Go ctx := context.Background() - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { log.WithError(err).Fatal("torsf experiment failed") } // ``` diff --git a/internal/tutorial/experiment/torsf/chapter02/README.md b/internal/tutorial/experiment/torsf/chapter02/README.md index c9a2fbd..36f3323 100644 --- a/internal/tutorial/experiment/torsf/chapter02/README.md +++ b/internal/tutorial/experiment/torsf/chapter02/README.md @@ -117,10 +117,10 @@ chapters, finally, we will modify this function until it is a minimal implementation of the `torsf` experiment. ```Go -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + _ = args.Measurement + sess := args.Session ``` As you can see, this is just a stub implementation that sleeps for one second and prints a logging message. diff --git a/internal/tutorial/experiment/torsf/chapter02/main.go b/internal/tutorial/experiment/torsf/chapter02/main.go index 4464bdc..efded79 100644 --- a/internal/tutorial/experiment/torsf/chapter02/main.go +++ b/internal/tutorial/experiment/torsf/chapter02/main.go @@ -54,7 +54,12 @@ func main() { MockableLogger: log.Log, MockableTempDir: tempdir, } - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { log.WithError(err).Fatal("torsf experiment failed") } data, err := json.Marshal(measurement) diff --git a/internal/tutorial/experiment/torsf/chapter02/torsf.go b/internal/tutorial/experiment/torsf/chapter02/torsf.go index af3e0f6..2547cb4 100644 --- a/internal/tutorial/experiment/torsf/chapter02/torsf.go +++ b/internal/tutorial/experiment/torsf/chapter02/torsf.go @@ -93,10 +93,10 @@ func (m *Measurer) ExperimentVersion() string { // minimal implementation of the `torsf` experiment. // // ```Go -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + _ = args.Callbacks + _ = args.Measurement + sess := args.Session // ``` // As you can see, this is just a stub implementation that sleeps // for one second and prints a logging message. diff --git a/internal/tutorial/experiment/torsf/chapter03/README.md b/internal/tutorial/experiment/torsf/chapter03/README.md index b6b9f85..3ae6719 100644 --- a/internal/tutorial/experiment/torsf/chapter03/README.md +++ b/internal/tutorial/experiment/torsf/chapter03/README.md @@ -32,10 +32,10 @@ print periodic updates via the `callbacks`. We will defer the real work to a private function called `run`. ```Go -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session ``` Let's create an instance of `TestKeys` and let's modify diff --git a/internal/tutorial/experiment/torsf/chapter03/main.go b/internal/tutorial/experiment/torsf/chapter03/main.go index a8dad28..1e9c8ec 100644 --- a/internal/tutorial/experiment/torsf/chapter03/main.go +++ b/internal/tutorial/experiment/torsf/chapter03/main.go @@ -28,7 +28,12 @@ func main() { MockableLogger: log.Log, MockableTempDir: tempdir, } - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { log.WithError(err).Fatal("torsf experiment failed") } data, err := json.Marshal(measurement) diff --git a/internal/tutorial/experiment/torsf/chapter03/torsf.go b/internal/tutorial/experiment/torsf/chapter03/torsf.go index 97b7ae4..e48ca3a 100644 --- a/internal/tutorial/experiment/torsf/chapter03/torsf.go +++ b/internal/tutorial/experiment/torsf/chapter03/torsf.go @@ -65,10 +65,10 @@ type TestKeys struct { // real work to a private function called `run`. // // ```Go -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session // ``` // // Let's create an instance of `TestKeys` and let's modify diff --git a/internal/tutorial/experiment/torsf/chapter04/main.go b/internal/tutorial/experiment/torsf/chapter04/main.go index a8dad28..1e9c8ec 100644 --- a/internal/tutorial/experiment/torsf/chapter04/main.go +++ b/internal/tutorial/experiment/torsf/chapter04/main.go @@ -28,7 +28,12 @@ func main() { MockableLogger: log.Log, MockableTempDir: tempdir, } - if err = m.Run(ctx, sess, measurement, callbacks); err != nil { + args := &model.ExperimentArgs{ + Callbacks: callbacks, + Measurement: measurement, + Session: sess, + } + if err = m.Run(ctx, args); err != nil { log.WithError(err).Fatal("torsf experiment failed") } data, err := json.Marshal(measurement) diff --git a/internal/tutorial/experiment/torsf/chapter04/torsf.go b/internal/tutorial/experiment/torsf/chapter04/torsf.go index d6998b9..d350ee9 100644 --- a/internal/tutorial/experiment/torsf/chapter04/torsf.go +++ b/internal/tutorial/experiment/torsf/chapter04/torsf.go @@ -99,10 +99,10 @@ type TestKeys struct { } // Run implements ExperimentMeasurer.Run. -func (m *Measurer) Run( - ctx context.Context, sess model.ExperimentSession, - measurement *model.Measurement, callbacks model.ExperimentCallbacks, -) error { +func (m *Measurer) Run(ctx context.Context, args *model.ExperimentArgs) error { + callbacks := args.Callbacks + measurement := args.Measurement + sess := args.Session testkeys := &TestKeys{} measurement.TestKeys = testkeys start := time.Now()