diff --git a/.gitignore b/.gitignore index bcdda4a..fde82b1 100644 --- a/.gitignore +++ b/.gitignore @@ -1,4 +1,4 @@ -/vendor /dist +/ooni.cov *.njson .DS_Store diff --git a/.travis.yml b/.travis.yml index 1b86ea9..3f5e1b0 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,13 +1,12 @@ -# Due to MK dependencies only macOS development is supported -os: osx -language: go -go: -- 1.9.x -- 1.x -install: -- make install-dev-deps -- dep ensure -- make download-mk-libs-macos +matrix: + include: + - os: linux + dist: xenial + language: minimal + services: + - docker + - os: osx + language: minimal + osx_image: xcode10.2 script: -- make test-internal -- make build +- ./build.sh _travis-${TRAVIS_OS_NAME} diff --git a/Dockerfile b/Dockerfile new file mode 100644 index 0000000..2c26371 --- /dev/null +++ b/Dockerfile @@ -0,0 +1,3 @@ +FROM openobservatory/mk-alpine:20190509 +RUN apk add --no-progress git go +ADD . /oonibuild diff --git a/Gopkg.lock b/Gopkg.lock deleted file mode 100644 index 71eacc3..0000000 --- a/Gopkg.lock +++ /dev/null @@ -1,162 +0,0 @@ -# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. - - -[[projects]] - name = "github.com/alecthomas/kingpin" - packages = ["."] - revision = "947dcec5ba9c011838740e680966fd7087a71d0d" - version = "v2.2.6" - -[[projects]] - branch = "master" - name = "github.com/alecthomas/template" - packages = [ - ".", - "parse" - ] - revision = "a0175ee3bccc567396460bf5acd36800cb10c49c" - -[[projects]] - branch = "master" - name = "github.com/alecthomas/units" - packages = ["."] - revision = "2efee857e7cfd4f3d0138cc3cbb1b4966962b93a" - -[[projects]] - name = "github.com/apex/log" - packages = ["."] - revision = "0296d6eb16bb28f8a0c55668affcf4876dc269be" - version = "v1.0.0" - -[[projects]] - name = "github.com/certifi/gocertifi" - packages = ["."] - revision = "deb3ae2ef2610fde3330947281941c562861188b" - version = "2018.01.18" - -[[projects]] - name = "github.com/fatih/color" - packages = ["."] - revision = "5b77d2a35fb0ede96d138fc9a99f5c9b6aef11b4" - version = "v1.7.0" - -[[projects]] - branch = "master" - name = "github.com/getsentry/raven-go" - packages = ["."] - revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa" - -[[projects]] - name = "github.com/mattn/go-colorable" - packages = ["."] - revision = "167de6bfdfba052fa6b2d3664c8f5272e23c9072" - version = "v0.0.9" - -[[projects]] - name = "github.com/mattn/go-isatty" - packages = ["."] - revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39" - version = "v0.0.3" - -[[projects]] - name = "github.com/mattn/go-sqlite3" - packages = ["."] - revision = "323a32be5a2421b8c7087225079c6c900ec397cd" - version = "v1.7.0" - -[[projects]] - branch = "master" - name = "github.com/measurement-kit/go-measurement-kit" - packages = ["."] - revision = "156da129bf90ad7a1880677eb3902c4b74c18b79" - -[[projects]] - branch = "master" - name = "github.com/mgutz/ansi" - packages = ["."] - revision = "9520e82c474b0a04dd04f8a40959027271bab992" - -[[projects]] - name = "github.com/oschwald/geoip2-golang" - packages = ["."] - revision = "7118115686e16b77967cdbf55d1b944fe14ad312" - version = "v1.2.1" - -[[projects]] - name = "github.com/oschwald/maxminddb-golang" - packages = ["."] - revision = "c5bec84d1963260297932a1b7a1753c8420717a7" - version = "v1.3.0" - -[[projects]] - name = "github.com/pkg/errors" - packages = ["."] - revision = "645ef00459ed84a119197bfb8d8205042c6df63d" - version = "v0.8.0" - -[[projects]] - branch = "master" - name = "github.com/rubenv/sql-migrate" - packages = [ - ".", - "sqlparse" - ] - revision = "081fe17d19ff4e2dd9f5a0c1158e6bcf74da6906" - -[[projects]] - name = "github.com/shuLhan/go-bindata" - packages = [ - ".", - "go-bindata" - ] - revision = "73d86c43eaae43914eb3ef264bcb4d8cb604fb51" - version = "v3.3.0" - -[[projects]] - branch = "master" - name = "golang.org/x/sys" - packages = [ - "unix", - "windows" - ] - revision = "c11f84a56e43e20a78cee75a7c034031ecf57d1f" - -[[projects]] - name = "gopkg.in/AlecAivazis/survey.v1" - packages = [ - ".", - "core", - "terminal" - ] - revision = "e752db451e07e09c7d7dc8cada807a44bdb0fd47" - version = "v1.5.3" - -[[projects]] - name = "gopkg.in/gorp.v1" - packages = ["."] - revision = "c87af80f3cc5036b55b83d77171e156791085e2e" - version = "v1.7.1" - -[[projects]] - name = "upper.io/db.v3" - packages = [ - ".", - "internal/cache", - "internal/cache/hashstructure", - "internal/immutable", - "internal/sqladapter", - "internal/sqladapter/compat", - "internal/sqladapter/exql", - "lib/reflectx", - "lib/sqlbuilder", - "sqlite" - ] - revision = "199d13d76c7cfba05ea0327375056fdabc8bea80" - version = "v3.5.4" - -[solve-meta] - analyzer-name = "dep" - analyzer-version = 1 - inputs-digest = "45f2fee1cf7e2abceb2500715c0f158ad17d51b52f8200cb1bb51c369b15402f" - solver-name = "gps-cdcl" - solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml deleted file mode 100644 index b14984a..0000000 --- a/Gopkg.toml +++ /dev/null @@ -1,67 +0,0 @@ -# Gopkg.toml example -# -# Refer to https://github.com/golang/dep/blob/master/docs/Gopkg.toml.md -# for detailed Gopkg.toml documentation. -# -# required = ["github.com/user/thing/cmd/thing"] -# ignored = ["github.com/user/project/pkgX", "bitbucket.org/user/project/pkgA/pkgY"] -# -# [[constraint]] -# name = "github.com/user/project" -# version = "1.0.0" -# -# [[constraint]] -# name = "github.com/user/project2" -# branch = "dev" -# source = "github.com/myfork/project2" -# -# [[override]] -# name = "github.com/x/y" -# version = "2.4.0" -# -# [prune] -# non-go = false -# go-tests = true -# unused-packages = true - -required = ["github.com/shuLhan/go-bindata/go-bindata"] - -[[constraint]] - name = "github.com/alecthomas/kingpin" - version = "2.2.6" - -[prune] - go-tests = true - unused-packages = true - -[[constraint]] - name = "github.com/pkg/errors" - version = "0.8.0" - -[[constraint]] - name = "gopkg.in/AlecAivazis/survey.v1" - version = "1.4.1" - -[[constraint]] - name = "github.com/apex/log" - version = "1.0.0" - -[[constraint]] - branch = "master" - name = "github.com/measurement-kit/go-measurement-kit" - -[[constraint]] - name = "github.com/shuLhan/go-bindata" - version = "3.3.0" - -[[constraint]] - name = "github.com/oschwald/geoip2-golang" - version = "1.2.1" - -[[constraint]] - branch = "master" - name = "github.com/getsentry/raven-go" - -[[constraint]] - name = "upper.io/db.v3" - version = "3.5.4" diff --git a/Makefile b/Makefile index e8f1a0d..c687658 100644 --- a/Makefile +++ b/Makefile @@ -1,29 +1,28 @@ GO ?= go install-dev-deps: - @$(GO) get -u github.com/golang/dep/... @$(GO) get golang.org/x/tools/cmd/cover @$(GO) get github.com/mattn/goveralls build: @echo "Building dist/ooni" - @$(GO) build -i -o dist/ooni cmd/ooni/main.go + @$(GO) build -o dist/ooni cmd/ooni/main.go .PHONY: build build-windows: - @echo "Building dist/ooni.exe" - CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build -o dist/ooni.exe -x cmd/ooni/main.go + @echo "Building dist/windows/amd64/ooni.exe" + @./build.sh windows -build-all: build build-windows -.PHONY: build-all +build-linux: + @echo "Building dist/linux/amd64/ooni" + @./build.sh linux -download-mk-libs: - @echo "updating mk-libs" - @cd vendor/github.com/measurement-kit/go-measurement-kit && ./download-libs.sh -download-mk-libs-macos: - @echo "updating mk-libs" - @cd vendor/github.com/measurement-kit/go-measurement-kit && ./download-libs.sh macos -.PHONY: update-mk-libs-macos +build-macos: + @echo "Building dist/macos/amd64/ooni" + @./build.sh macos + +build-all: build-windows build-linux build-macos +.PHONY: build-all build-windows build-linux build-macos bindata: @$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \ diff --git a/Readme.md b/Readme.md index f352de3..4924447 100644 --- a/Readme.md +++ b/Readme.md @@ -4,61 +4,14 @@ The next generation OONI Probe Command Line Interface. ## Development setup -Be sure you have golang >= 1.8. +Be sure you have golang >= 1.12. We use Go modules. Make sure you +don't have GOPATH set. -This project uses [`dep`](https://golang.github.io/dep/) with the `vendor/` dir -in `.gitignore`. +Run -Once you have `dep` installed, run: - -``` -dep ensure +```bash +./build.sh help ``` -Next, you'll need a recent version of [Measurement Kit](http://github.com/measurement-kit). - -Building a ooni binary for windows and macOS is currently only supported on a -macOS system. - -For building a linux ooni binary, you will need a linux system and follow the -intruction in the linux section. - -### macOS - -On macOS you can build a windows and macOS ooni binary. - -This can be done by running: - -``` -make download-mk-libs -``` - -This will download the prebuilt measurement-kit binaries. - -Then you can build a macOS build by running: - -``` -make build -``` - -And a windows build by running: - -``` -make build-windows -``` - -### linux - -On linux you can only build a linux ooni binary for amd64. - -This can be done by running: - -``` -make download-mk-libs -``` - -Then you can build ooni by running: - -``` -make build -``` +to get information on the supported systems as well as to get +instructions on how to install dependencies. diff --git a/build.sh b/build.sh new file mode 100755 index 0000000..7876a90 --- /dev/null +++ b/build.sh @@ -0,0 +1,77 @@ +#!/bin/sh +set -e + +if [ "$GOPATH" != "" ]; then + echo "$0: WARNING: unsetting GOPATH as we don't need it" 1>&2 + unset GOPATH +fi + +if [ "$1" = "windows" ]; then + set -x + CC=x86_64-w64-mingw32-gcc CXX=x86_64-w64-mingw32-g++ \ + CGO_ENABLED=1 GOOS=windows GOARCH=amd64 \ + go build -o dist/windows/amd64/ooni.exe -v ./cmd/ooni + +elif [ "$1" = "linux" ]; then + set -x + docker build -t oonibuild . + docker run -v `pwd`:/oonibuild -w /oonibuild -t oonibuild \ + go build -o dist/linux/amd64/ooni -v ./cmd/ooni + +elif [ "$1" = "macos" ]; then + set -x + go build -o dist/macos/amd64/ooni -v ./cmd/ooni + +elif [ "$1" = "_travis-linux" ]; then + set -x + $0 linux + docker run -v `pwd`:/oonibuild -w /oonibuild -t oonibuild \ + go test -v -coverprofile=ooni.cov ./... + +elif [ "$1" = "_travis-osx" ]; then + set -x + brew tap measurement-kit/measurement-kit + brew update + brew upgrade + brew install measurement-kit + $0 macos + go test -v -coverprofile=ooni.cov ./... + +elif [ "$1" = "help" ]; then + echo "Usage: $0 linux | macos | windows" + echo "" + echo "Builds OONI on supported systems. The output binary will" + echo "be saved at './dist///ooni[.exe]'." + echo "" + echo "# Linux" + echo "" + echo "To compile for Linux we use a docker container with the binary" + echo "Measurement Kit dependency installed. So you need docker installed." + echo "" + echo "# macOS" + echo "" + echo "You must be on macOS. You must install Measurement Kit once using:" + echo "" + echo "- brew tap measurement-kit/measurement-kit" + echo "- brew install measurement-kit" + echo "" + echo "You should keep Measurement Kit up-to-date using:" + echo "" + echo "- brew upgrade" + echo "" + echo "# Windows" + echo "" + echo "You must be on macOS. You must install Measurement Kit once using:" + echo "" + echo "- brew tap measurement-kit/measurement-kit" + echo "- brew install mingw-w64-measurement-kit" + echo "" + echo "You should keep Measurement Kit up-to-date using:" + echo "" + echo "- brew upgrade" + echo "" + +else + echo "Invalid usage; try '$0 help' for more help." 1>&2 + exit 1 +fi diff --git a/go.mod b/go.mod new file mode 100644 index 0000000..daa4d04 --- /dev/null +++ b/go.mod @@ -0,0 +1,30 @@ +module github.com/ooni/probe-cli + +go 1.12 + +require ( + github.com/alecthomas/kingpin v2.2.6+incompatible + github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc // indirect + github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf // indirect + github.com/apex/log v1.1.0 + github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 // indirect + github.com/fatih/color v1.7.0 + github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea + github.com/go-sql-driver/mysql v1.4.1 // indirect + github.com/gobuffalo/packr v1.25.0 // indirect + github.com/lib/pq v1.1.1 // indirect + github.com/mattn/go-colorable v0.0.9 + github.com/mattn/go-isatty v0.0.7 // indirect + github.com/mattn/go-sqlite3 v1.10.0 // indirect + github.com/measurement-kit/go-measurement-kit v0.0.0-20190509222408-0d0cd3233336 + github.com/oschwald/geoip2-golang v1.2.1 + github.com/oschwald/maxminddb-golang v1.3.0 // indirect + github.com/pkg/errors v0.8.1 + github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051 + github.com/ziutek/mymysql v1.5.4 // indirect + golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 // indirect + google.golang.org/appengine v1.5.0 // indirect + gopkg.in/AlecAivazis/survey.v1 v1.8.4 + gopkg.in/gorp.v1 v1.7.2 // indirect + upper.io/db.v3 v3.5.7+incompatible +) diff --git a/go.sum b/go.sum new file mode 100644 index 0000000..b5c5cfa --- /dev/null +++ b/go.sum @@ -0,0 +1,132 @@ +github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8 h1:xzYJEypr/85nBpB11F9br+3HUrpgb+fcm5iADzXXYEw= +github.com/Netflix/go-expect v0.0.0-20180615182759-c93bf25de8e8/go.mod h1:oX5x61PbNXchhh0oikYAH+4Pcfw5LKv21+Jnpr6r6Pc= +github.com/alecthomas/kingpin v2.2.6+incompatible h1:5svnBTFgJjZvGKyYBtMB0+m5wvrbUHiqye8wRJMlnYI= +github.com/alecthomas/kingpin v2.2.6+incompatible/go.mod h1:59OFYbFVLKQKq+mqrL6Rw5bR0c3ACQaawgXx0QYndlE= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc h1:cAKDfWh5VpdgMhJosfJnn5/FoN2SRZ4p7fJNX58YPaU= +github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf h1:qet1QNfXsQxTZqLG4oE62mJzwPIB8+Tee4RNCL9ulrY= +github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= +github.com/apex/log v1.1.0 h1:J5rld6WVFi6NxA6m8GJ1LJqu3+GiTFIt3mYv27gdQWI= +github.com/apex/log v1.1.0/go.mod h1:yA770aXIDQrhVOIGurT/pVdfCpSq1GQV/auzMN5fzvY= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261 h1:6/yVvBsKeAw05IUj4AzvrxaCnDjN4nUqKjW9+w5wixg= +github.com/certifi/gocertifi v0.0.0-20180118203423-deb3ae2ef261/go.mod h1:GJKEexRPVJrBSOjoqN5VNOIKJ5Q3RViH6eu3puDRwx4= +github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= +github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/davecgh/go-spew v1.1.1 h1:vj9j/u1bqnvCEfJOwUhtlOARqs3+rkHYY13jYWTU97c= +github.com/davecgh/go-spew v1.1.1/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= +github.com/fatih/color v1.7.0 h1:DkWD4oS2D8LGGgTQ6IvwJJXSL5Vp2ffcQg58nFV38Ys= +github.com/fatih/color v1.7.0/go.mod h1:Zm6kSWBoL9eyXnKyktHP6abPY2pDugNf5KwzbycvMj4= +github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea h1:vpAHg3H71YFc5TqRSqbhMq8Wd0kdPoQMMAeQSgnUTpg= +github.com/getsentry/raven-go v0.0.0-20190419175539-919484f041ea/go.mod h1:KungGk8q33+aIAZUIVWZDr2OfAEBsO49PX4NzFV5kcQ= +github.com/go-sql-driver/mysql v1.4.1 h1:g24URVg0OFbNUTx9qqY1IRZ9D9z3iPyi5zKhQZpNwpA= +github.com/go-sql-driver/mysql v1.4.1/go.mod h1:zAC/RDZ24gD3HViQzih4MyKcchzm+sOG5ZlKdlhCg5w= +github.com/gobuffalo/attrs v0.0.0-20190224210810-a9411de4debd/go.mod h1:4duuawTqi2wkkpB4ePgWMaai6/Kc6WEz83bhFwpHzj0= +github.com/gobuffalo/depgen v0.0.0-20190329151759-d478694a28d3/go.mod h1:3STtPUQYuzV0gBVOY3vy6CfMm/ljR4pABfrTeHNLHUY= +github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/envy v1.7.0 h1:GlXgaiBkmrYMHco6t4j7SacKO4XUjvh5pwXh0f4uxXU= +github.com/gobuffalo/envy v1.7.0/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI= +github.com/gobuffalo/flect v0.1.0/go.mod h1:d2ehjJqGOH/Kjqcoz+F7jHTBbmDb38yXA598Hb50EGs= +github.com/gobuffalo/flect v0.1.1/go.mod h1:8JCgGVbRjJhVgD6399mQr4fx5rRfGKVzFjbj6RE/9UI= +github.com/gobuffalo/genny v0.0.0-20190329151137-27723ad26ef9/go.mod h1:rWs4Z12d1Zbf19rlsn0nurr75KqhYp52EAGGxTbBhNk= +github.com/gobuffalo/genny v0.0.0-20190403191548-3ca520ef0d9e/go.mod h1:80lIj3kVJWwOrXWWMRzzdhW3DsrdjILVil/SFKBzF28= +github.com/gobuffalo/gitgen v0.0.0-20190315122116-cc086187d211/go.mod h1:vEHJk/E9DmhejeLeNt7UVvlSGv3ziL+djtTr3yyzcOw= +github.com/gobuffalo/gogen v0.0.0-20190315121717-8f38393713f5/go.mod h1:V9QVDIxsgKNZs6L2IYiGR8datgMhB577vzTDqypH360= +github.com/gobuffalo/logger v0.0.0-20190315122211-86e12af44bc2/go.mod h1:QdxcLw541hSGtBnhUc4gaNIXRjiDppFGaDqzbrBd3v8= +github.com/gobuffalo/mapi v1.0.1/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/mapi v1.0.2/go.mod h1:4VAGh89y6rVOvm5A8fKFxYG+wIW6LO1FMTG9hnKStFc= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0 h1:P6naWPiHm/7R3eYx/ub3VhaW9G+1xAMJ6vzACePaGPI= +github.com/gobuffalo/packd v0.0.0-20190315124812-a385830c7fc0/go.mod h1:M2Juc+hhDXf/PnmBANFCqx4DM3wRbgDvnVWeG2RIxq4= +github.com/gobuffalo/packr v1.25.0 h1:NtPK45yOKFdTKHTvRGKL+UIKAKmJVWIVJOZBDI/qEdY= +github.com/gobuffalo/packr v1.25.0/go.mod h1:NqsGg8CSB2ZD+6RBIRs18G7aZqdYDlYNNvsSqP6T4/U= +github.com/gobuffalo/packr/v2 v2.0.9/go.mod h1:emmyGweYTm6Kdper+iywB6YK5YzuKchGtJQZ0Odn4pQ= +github.com/gobuffalo/packr/v2 v2.1.0/go.mod h1:n90ZuXIc2KN2vFAOQascnPItp9A2g9QYSvYvS3AjQEM= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754 h1:tpom+2CJmpzAWj5/VEHync2rJGi+epHNIeRSWjzGA+4= +github.com/gobuffalo/syncx v0.0.0-20190224160051-33c29581e754/go.mod h1:HhnNqWY95UYwwW3uSASeV7vtgYkT2t16hJgV3AEPUpw= +github.com/golang/protobuf v1.2.0/go.mod h1:6lQm79b+lXiMfvg/cZm0SGofjICqVBUtrP5yJMmIC1U= +github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174 h1:WlZsjVhE8Af9IcZDGgJGQpNflI3+MJSBhsgT5PCtzBQ= +github.com/hinshun/vt10x v0.0.0-20180616224451-1954e6464174/go.mod h1:DqJ97dSdRW1W22yXSB90986pcOyQ7r45iio1KN2ez1A= +github.com/inconshreveable/mousetrap v1.0.0/go.mod h1:PxqpIevigyE2G7u3NXJIT2ANytuPF1OarO4DADm73n8= +github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= +github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 h1:Z9n2FFNUXsshfwJMBgNA0RU6/i7WVaAegv3PtuIHPMs= +github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51/go.mod h1:CzGEWj7cYgsdH8dAjBGEr58BoE7ScuLd+fwFZ44+/x8= +github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= +github.com/kr/pretty v0.1.0 h1:L/CwN0zerZDmRFUapSPitk6f+Q3+0za1rQkzVuMiMFI= +github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORNo= +github.com/kr/pty v1.1.1 h1:VkoXIwSboBpnk99O/KFauAEILuNHv5DVFKZMBN/gUgw= +github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= +github.com/kr/text v0.1.0 h1:45sCR5RtlFHMR4UwH9sdQ5TC8v0qDQCHnXt+kaKSTVE= +github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= +github.com/lib/pq v1.1.1 h1:sJZmqHoEaY7f+NPP8pgLB/WxulyR3fewgCM2qaSlBb4= +github.com/lib/pq v1.1.1/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/markbates/oncer v0.0.0-20181203154359-bf2de49a0be2/go.mod h1:Ld9puTsIW75CHf65OeIOkyKbteujpZVXDpWK6YGZbxE= +github.com/markbates/safe v1.0.1/go.mod h1:nAqgmRi7cY2nqMc92/bSEeQA+R4OheNU2T1kNSCBdG0= +github.com/mattn/go-colorable v0.0.9 h1:UVL0vNpWh04HeJXV0KLcaT7r06gOH2l4OW6ddYRUIY4= +github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= +github.com/mattn/go-isatty v0.0.3/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNxMWT7Zi4= +github.com/mattn/go-isatty v0.0.7 h1:UvyT9uN+3r7yLEYSlJsbQGdsaB/a0DlgWP3pql6iwOc= +github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-sqlite3 v1.10.0 h1:jbhqpg7tQe4SupckyijYiy0mJJ/pRyHvXf7JdWK860o= +github.com/mattn/go-sqlite3 v1.10.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= +github.com/measurement-kit/go-measurement-kit v0.0.0-20190509222408-0d0cd3233336 h1:V68g+e6YpllWozgpR40rfLFWWS7iHDDy1UJj4IbiDcw= +github.com/measurement-kit/go-measurement-kit v0.0.0-20190509222408-0d0cd3233336/go.mod h1:U+1073kDD7WgZrs+F9fKgFVWXikYAB7LZAWOd7CMHyQ= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b h1:j7+1HpAFS1zy5+Q4qx1fWh90gTKwiN4QCGoY9TWyyO4= +github.com/mgutz/ansi v0.0.0-20170206155736-9520e82c474b/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= +github.com/oschwald/geoip2-golang v1.2.1 h1:3iz+jmeJc6fuCyWeKgtXSXu7+zvkxJbHFXkMT5FVebU= +github.com/oschwald/geoip2-golang v1.2.1/go.mod h1:0LTTzix/Ao1uMvOhAV4iLU0Lz7eCrP94qZWBTDKf0iE= +github.com/oschwald/maxminddb-golang v1.3.0 h1:oTh8IBSj10S5JNlUDg5WjJ1QdBMdeaZIkPEVfESSWgE= +github.com/oschwald/maxminddb-golang v1.3.0/go.mod h1:3jhIUymTJ5VREKyIhWm66LJiQt04F0UCDdodShpjWsY= +github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= +github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= +github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= +github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= +github.com/rogpeppe/go-internal v1.1.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.2.2/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rogpeppe/go-internal v1.3.0 h1:RR9dF3JtopPvtkroDZuVD7qquD0bnHlKSqaQhgwt8yk= +github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= +github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051 h1:p32bQkgLiadYiOqs294BAx/7f1Aerfva8rj+rVvzR0A= +github.com/rubenv/sql-migrate v0.0.0-20190327083759-54bad0a9b051/go.mod h1:WS0rl9eEliYI8DPnr3TOwz4439pay+qNgzJoVya/DmY= +github.com/sirupsen/logrus v1.4.0/go.mod h1:LxeOpSwHxABJmUn/MG1IvRgCAasNZTLOkJPxbbu5VWo= +github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= +github.com/spf13/cobra v0.0.3/go.mod h1:1l0Ry5zgKvJasoi3XT1TypsSe7PqH0Sj9dhYf7v3XqQ= +github.com/spf13/pflag v1.0.3/go.mod h1:DYY7MBk1bdzusC3SYhjObp+wFpr4gzcvqqNjLnInEg4= +github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= +github.com/stretchr/testify v1.2.1 h1:52QO5WkIUcHGIR7EnGagH88x1bUzqGXTC5/1bDTUQ7U= +github.com/stretchr/testify v1.2.1/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= +github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= +github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= +github.com/ziutek/mymysql v1.5.4 h1:GB0qdRGsTwQSBVYuVShFBKaXSnSnYYC2d9knnE1LHFs= +github.com/ziutek/mymysql v1.5.4/go.mod h1:LMSpPZ6DbqWFxNCHW77HeMg9I646SAhApZ/wKdgO/C0= +golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b h1:Elez2XeF2p9uyVj0yEUDqQ56NFcDtcBNkYP7yv8YbUE= +golang.org/x/crypto v0.0.0-20190123085648-057139ce5d2b/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2 h1:VklqNMn3ovrHsnt90PveolxSbWFaJdECFbxSq0Mqo2M= +golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= +golang.org/x/net v0.0.0-20180724234803-3673e40ba225/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= +golang.org/x/net v0.0.0-20190311183353-d8887717615a/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= +golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sys v0.0.0-20180606202747-9527bec2660b/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862 h1:rM0ROo5vb9AdYJi1110yjWGMej9ITfKddS89P3Fkhug= +golang.org/x/sys v0.0.0-20190509141414-a5b02f93d862/go.mod h1:h1NjWce9XRLGQEsW7wpKNCjG9DtNlClVuFLEZdDNbEs= +golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= +golang.org/x/tools v0.0.0-20190329151228-23e29df326fe/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +golang.org/x/tools v0.0.0-20190404132500-923d25813098/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= +google.golang.org/appengine v1.5.0 h1:KxkO13IPW4Lslp2bz+KHP2E3gtFlrIGNThxkZQ3g+4c= +google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4= +gopkg.in/AlecAivazis/survey.v1 v1.8.4 h1:10xXXN3wgIhPheb5NI58zFgZv32Ana7P3Tl4shW+0Qc= +gopkg.in/AlecAivazis/survey.v1 v1.8.4/go.mod h1:iBNOmqKz/NUbZx3bA+4hAGLRC7fSK7tgtVDT4tB22XA= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127 h1:qIbj1fsPNlZgppZ+VLlY7N33q108Sa+fhmuc+sWQYwY= +gopkg.in/check.v1 v1.0.0-20180628173108-788fd7840127/go.mod h1:Co6ibVJAznAaIkqp8huTwlJQCZ016jof/cbN4VW5Yz0= +gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= +gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= +gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= +upper.io/db.v3 v3.5.7+incompatible h1:3MJSnJQ+NMxBxuNwO+gOKFiugwv+f61LbyuZYSPzoi4= +upper.io/db.v3 v3.5.7+incompatible/go.mod h1:FgTdD24eBjJAbPKsQSiHUNgXjOR4Lub3u1UMHSIh82Y= diff --git a/internal/cli/geoip/geoip.go b/internal/cli/geoip/geoip.go index 0de3620..f25d2c8 100644 --- a/internal/cli/geoip/geoip.go +++ b/internal/cli/geoip/geoip.go @@ -29,7 +29,6 @@ func init() { geoipPath := utils.GeoIPDir(ctx.Home) if *shouldUpdate { utils.DownloadGeoIPDatabaseFiles(geoipPath) - utils.DownloadLegacyGeoIPDatabaseFiles(geoipPath) } loc, err := utils.GeoIPLookup(geoipPath) diff --git a/nettests/nettests.go b/nettests/nettests.go index 9713a7a..62c652f 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -87,8 +87,8 @@ func (c *Controller) Init(nt *mk.Nettest) error { testName := strcase.ToSnake(nt.Name) resultID := c.res.ID reportFilePath := c.msmtPath - geoIPCountryPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat") - geoIPASNPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat") + geoIPCountryPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoLite2-Country.mmdb") + geoIPASNPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoLite2-ASN.mmdb") caBundlePath := getCaBundlePath() msmtPath := c.msmtPath diff --git a/ooni.go b/ooni.go index 159398f..a317f0e 100644 --- a/ooni.go +++ b/ooni.go @@ -80,12 +80,6 @@ func (c *Context) MaybeDownloadDataFiles() error { return err } } - if _, err := os.Stat(path.Join(geoipDir, "GeoIP.dat")); os.IsNotExist(err) { - log.Debugf("Downloading legacy GeoIP database Files") - if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil { - return err - } - } return nil } diff --git a/utils/geoip.go b/utils/geoip.go index b02be27..be2cc84 100644 --- a/utils/geoip.go +++ b/utils/geoip.go @@ -31,11 +31,6 @@ var geoipFiles = map[string]string{ "GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz", } -var legacyGeoipFiles = map[string]string{ - "GeoIPASNum.dat": "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz", - "GeoIP.dat": "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz", -} - // Download the file to a temporary location func downloadToTemp(url string) (string, error) { out, err := ioutil.TempFile(os.TempDir(), "maxmind") @@ -56,40 +51,6 @@ func downloadToTemp(url string) (string, error) { return out.Name(), nil } -// DownloadLegacyGeoIPDatabaseFiles into the target directory -func DownloadLegacyGeoIPDatabaseFiles(dir string) error { - for filename, url := range legacyGeoipFiles { - dstPath := filepath.Join(dir, filename) - - tmpPath, err := downloadToTemp(url) - if err != nil { - return err - } - - // Extract the tar.gz file - f, err := os.Open(tmpPath) - defer f.Close() - if err != nil { - return errors.Wrap(err, "failed to read file") - } - - gzf, err := gzip.NewReader(f) - if err != nil { - return errors.Wrap(err, "failed to create gzip reader") - } - - outFile, err := os.Create(dstPath) - if err != nil { - return errors.Wrap(err, "error creating file") - } - if _, err := io.Copy(outFile, gzf); err != nil { - return errors.Wrap(err, "error reading file from gzip") - } - outFile.Close() - } - return nil -} - // DownloadGeoIPDatabaseFiles into the target directory func DownloadGeoIPDatabaseFiles(dir string) error { for filename, url := range geoipFiles {