From 322394fe63a64ca750f93c00b35d6988b553c354 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Tue, 2 Mar 2021 12:08:24 +0100 Subject: [PATCH] feat: use go1.16 and resources embedding (#235) * feat: use go1.16 embedding for resources We want to embed everything that can be easily embedded. We should, at a minimum, replace the downloading of resources and bindata. Ref: https://github.com/ooni/probe/issues/1367. * fix: get rid of bindata and use go embed instead * fix: start unbreaking some automatic tests * fix: fetch resources as part of the mobile build * fix: convert more stuff to go1.16 I still expect many breakages, but we'll fix them. * fix: make the windows CI green * fix: get resources before running QA * fix: go1.16 uses modules by default * hopefully fix all other outstanding issues * fix(QA/telegram.py): add another DC IP address * Apply suggestions from code review --- .github/workflows/alltests.yml | 3 +- .github/workflows/android.yml | 2 +- .github/workflows/bindata.yml | 22 - .github/workflows/coverage.yml | 3 +- .github/workflows/cross.yml | 4 +- .github/workflows/debian.yml | 2 +- .github/workflows/e2eminiooni.yml | 2 +- .github/workflows/generate.yml | 2 +- .github/workflows/ios.yml | 2 +- .github/workflows/jafar.yml | 2 +- .github/workflows/linux.yml | 2 +- .github/workflows/macos.yml | 2 +- .github/workflows/miniooni.yml | 2 +- .github/workflows/qafbmessenger.yml | 2 +- .github/workflows/qahhfm.yml | 2 +- .github/workflows/qahirl.yml | 2 +- .github/workflows/qatelegram.yml | 2 +- .github/workflows/qawebconnectivity.yml | 2 +- .github/workflows/qawhatsapp.yml | 2 +- .github/workflows/shorttests.yml | 3 +- .github/workflows/windows.yml | 2 +- E2E/miniooni.bash | 1 + QA/Dockerfile | 2 +- QA/pyrun.sh | 1 + QA/telegram.py | 1 + Readme.md | 22 +- build-android.bash | 3 +- build-ios.bash | 3 +- build-miniooni.sh | 3 + build.sh | 15 +- cmd/ooniprobe/internal/bindata/bindata.go | 377 ------------------ cmd/ooniprobe/internal/database/database.go | 32 +- .../migrations/1_create_msmt_results.sql | 0 .../migrations/2_single_msmt_file.sql | 0 .../internal/ooni}/default-config.json | 0 cmd/ooniprobe/internal/ooni/ooni.go | 12 +- data/README.md | 3 - go.mod | 2 +- internal/cmd/getresources/getresources.go | 51 +++ internal/cmd/jafar/README.md | 2 +- internal/engine/geolocate/mmdblookup_test.go | 14 +- .../engine/netx/archival/archival_test.go | 10 +- internal/engine/resources/README.md | 3 - internal/engine/resources/doc.go | 3 + internal/engine/resources/resources.go | 104 ----- internal/engine/resources/resources_test.go | 180 --------- .../private => resourcesmanager}/.gitignore | 1 + .../resourcesmanager/resourcesmanager.go | 149 +++++++ .../resourcesmanager/resourcesmanager_test.go | 142 +++++++ internal/engine/session.go | 8 +- updatebindata.sh | 10 - 51 files changed, 441 insertions(+), 780 deletions(-) delete mode 100644 .github/workflows/bindata.yml delete mode 100644 cmd/ooniprobe/internal/bindata/bindata.go rename {data => cmd/ooniprobe/internal/database}/migrations/1_create_msmt_results.sql (100%) rename {data => cmd/ooniprobe/internal/database}/migrations/2_single_msmt_file.sql (100%) rename {data => cmd/ooniprobe/internal/ooni}/default-config.json (100%) delete mode 100644 data/README.md create mode 100644 internal/cmd/getresources/getresources.go delete mode 100644 internal/engine/resources/README.md create mode 100644 internal/engine/resources/doc.go delete mode 100644 internal/engine/resources/resources.go delete mode 100644 internal/engine/resources/resources_test.go rename internal/engine/{resources/private => resourcesmanager}/.gitignore (75%) create mode 100644 internal/engine/resourcesmanager/resourcesmanager.go create mode 100644 internal/engine/resourcesmanager/resourcesmanager_test.go delete mode 100755 updatebindata.sh diff --git a/.github/workflows/alltests.yml b/.github/workflows/alltests.yml index 4d02b97..34ddc2e 100644 --- a/.github/workflows/alltests.yml +++ b/.github/workflows/alltests.yml @@ -11,6 +11,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 + - run: go run ./internal/cmd/getresources - run: go test -race -tags shaping ./... diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index 8551d1e..9933c84 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: brew install --cask android-sdk - run: echo y | sdkmanager --install "platforms;android-29" diff --git a/.github/workflows/bindata.yml b/.github/workflows/bindata.yml deleted file mode 100644 index c6ef933..0000000 --- a/.github/workflows/bindata.yml +++ /dev/null @@ -1,22 +0,0 @@ -# Make sure we can embed bindata -name: bindata -on: - push: - schedule: - - cron: "14 17 * * 3" -jobs: - test: - runs-on: "${{ matrix.os }}" - strategy: - fail-fast: false - matrix: - os: ["ubuntu-latest"] - go: ["1.14"] - steps: - - uses: actions/setup-go@v1 - with: - go-version: ${{ matrix.go }} - - uses: actions/checkout@v2 - - run: ./updatebindata.sh - - run: go mod tidy # revert changes caused by installing bindata - - run: git diff --exit-code # if this fails, run ./updatebindata.sh locally and push diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index c81a28f..2399cb5 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -9,12 +9,13 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ "1.14" ] + go: [ "1.16" ] steps: - uses: actions/setup-go@v1 with: go-version: "${{ matrix.go }}" - uses: actions/checkout@v2 + - run: go run ./internal/cmd/getresources - run: go test -short -race -tags shaping -coverprofile=probe-cli.cov ./... - uses: shogo82148/actions-goveralls@v1 with: diff --git a/.github/workflows/cross.yml b/.github/workflows/cross.yml index f97a938..932e20b 100644 --- a/.github/workflows/cross.yml +++ b/.github/workflows/cross.yml @@ -13,7 +13,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: sudo apt update - run: sudo apt install --yes mingw-w64 @@ -38,7 +38,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: brew install mingw-w64 - run: ./build.sh windows diff --git a/.github/workflows/debian.yml b/.github/workflows/debian.yml index 24d662f..a169495 100644 --- a/.github/workflows/debian.yml +++ b/.github/workflows/debian.yml @@ -17,7 +17,7 @@ jobs: sudo service docker restart - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux_amd64 - run: sudo apt-get update -q diff --git a/.github/workflows/e2eminiooni.yml b/.github/workflows/e2eminiooni.yml index 401c231..5b30ade 100644 --- a/.github/workflows/e2eminiooni.yml +++ b/.github/workflows/e2eminiooni.yml @@ -13,6 +13,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./E2E/miniooni.bash diff --git a/.github/workflows/generate.yml b/.github/workflows/generate.yml index a22f7c1..f763749 100644 --- a/.github/workflows/generate.yml +++ b/.github/workflows/generate.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: go generate ./... diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index ffe3e80..ea9e3e4 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./build-ios.bash - run: ./publish-ios.bash diff --git a/.github/workflows/jafar.yml b/.github/workflows/jafar.yml index c5c03ce..57f9b05 100644 --- a/.github/workflows/jafar.yml +++ b/.github/workflows/jafar.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: go build -v ./internal/cmd/jafar - run: sudo ./testjafar.bash diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index f37d9a4..8d09ae5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -14,7 +14,7 @@ jobs: sudo service docker restart - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: DOCKER_CLI_EXPERIMENTAL=enabled ./build.sh linux - run: ./smoketest.sh ./CLI/linux/amd64/ooniprobe diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 24991d7..1a50f6a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./build.sh darwin - run: ./smoketest.sh ./CLI/darwin/amd64/ooniprobe diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml index 81da688..350d3bd 100644 --- a/.github/workflows/miniooni.yml +++ b/.github/workflows/miniooni.yml @@ -11,7 +11,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./build-miniooni.sh linux diff --git a/.github/workflows/qafbmessenger.yml b/.github/workflows/qafbmessenger.yml index 39e8dd6..1081e8d 100644 --- a/.github/workflows/qafbmessenger.yml +++ b/.github/workflows/qafbmessenger.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "fbmessenger" diff --git a/.github/workflows/qahhfm.yml b/.github/workflows/qahhfm.yml index 8e01b84..02427f2 100644 --- a/.github/workflows/qahhfm.yml +++ b/.github/workflows/qahhfm.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "hhfm" diff --git a/.github/workflows/qahirl.yml b/.github/workflows/qahirl.yml index e758f72..0e18e97 100644 --- a/.github/workflows/qahirl.yml +++ b/.github/workflows/qahirl.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "hirl" diff --git a/.github/workflows/qatelegram.yml b/.github/workflows/qatelegram.yml index 9a7349d..5a58593 100644 --- a/.github/workflows/qatelegram.yml +++ b/.github/workflows/qatelegram.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "telegram" diff --git a/.github/workflows/qawebconnectivity.yml b/.github/workflows/qawebconnectivity.yml index a14c579..217f775 100644 --- a/.github/workflows/qawebconnectivity.yml +++ b/.github/workflows/qawebconnectivity.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "webconnectivity" diff --git a/.github/workflows/qawhatsapp.yml b/.github/workflows/qawhatsapp.yml index 135f577..1331649 100644 --- a/.github/workflows/qawhatsapp.yml +++ b/.github/workflows/qawhatsapp.yml @@ -11,6 +11,6 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: ./QA/rundocker.bash "whatsapp" diff --git a/.github/workflows/shorttests.yml b/.github/workflows/shorttests.yml index 322b8f6..ffcdb09 100644 --- a/.github/workflows/shorttests.yml +++ b/.github/workflows/shorttests.yml @@ -9,10 +9,11 @@ jobs: runs-on: ubuntu-latest strategy: matrix: - go: [ "1.14", "1.15" ] + go: [ "1.16" ] steps: - uses: actions/setup-go@v1 with: go-version: "${{ matrix.go }}" - uses: actions/checkout@v2 + - run: go run ./internal/cmd/getresources - run: go test -short -race -tags shaping ./... diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 9cf7bb1..11c839c 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -10,7 +10,7 @@ jobs: steps: - uses: actions/setup-go@v1 with: - go-version: "1.14" + go-version: "1.16" - uses: actions/checkout@v2 - run: bash.exe ./build.sh windows_amd64 # TODO(bassosimone): make windows_386 work - run: bash.exe ./smoketest.sh ./CLI/windows/amd64/ooniprobe.exe diff --git a/E2E/miniooni.bash b/E2E/miniooni.bash index 36b7ce5..01fb975 100755 --- a/E2E/miniooni.bash +++ b/E2E/miniooni.bash @@ -1,5 +1,6 @@ #!/bin/bash set -e +go run ./internal/cmd/getresources go build -v ./internal/cmd/miniooni probeservices=() probeservices+=( "https://ps1.ooni.io" ) diff --git a/QA/Dockerfile b/QA/Dockerfile index dcb3ab0..8767bcc 100644 --- a/QA/Dockerfile +++ b/QA/Dockerfile @@ -1,2 +1,2 @@ -FROM golang:1.14-alpine +FROM golang:1.16-alpine RUN apk add go git musl-dev iptables tmux bind-tools curl sudo python3 diff --git a/QA/pyrun.sh b/QA/pyrun.sh index 76f3f28..2c8db8a 100755 --- a/QA/pyrun.sh +++ b/QA/pyrun.sh @@ -1,6 +1,7 @@ #!/bin/sh set -ex export GOPATH=/jafar/QA/GOPATH GOCACHE=/jafar/QA/GOCACHE GO111MODULE=on +go run ./internal/cmd/getresources go build -v ./internal/cmd/miniooni go build -v ./internal/cmd/jafar sudo ./QA/$1.py ./miniooni diff --git a/QA/telegram.py b/QA/telegram.py index 717a0e1..1facdf2 100755 --- a/QA/telegram.py +++ b/QA/telegram.py @@ -26,6 +26,7 @@ ALL_POP_IPS = ( "149.154.175.100", "149.154.167.91", "149.154.171.5", + "95.161.76.100", ) diff --git a/Readme.md b/Readme.md index ee94ed1..5757837 100644 --- a/Readme.md +++ b/Readme.md @@ -23,8 +23,16 @@ Every top-level directory contains an explanatory README file. ## Development setup -Be sure you have golang >= 1.14 and a C compiler (when developing for Windows, you -need Mingw-w64 installed). The most basic build command is: +Be sure you have golang >= 1.16 and a C compiler (when developing for Windows, you +need Mingw-w64 installed). + +You need to download assets first using: + +```bash +go run ./internal/cmd/getresources +``` + +Then you can build using: ```bash go build -v ./cmd/ooniprobe @@ -32,16 +40,6 @@ go build -v ./cmd/ooniprobe This will generate a binary called `ooniprobe` in the current directory. -## Update bundled assets - -To update bundled assets use: - -```bash -./updatebindata.sh -``` - -Then commit the changes. - ## Android bindings ```bash diff --git a/build-android.bash b/build-android.bash index 4951e0a..a96d759 100755 --- a/build-android.bash +++ b/build-android.bash @@ -47,9 +47,8 @@ fi topdir=$(cd $(dirname $0) && pwd -P) set -x export PATH=$(go env GOPATH)/bin:$PATH -export GO111MODULE=off go get -u golang.org/x/mobile/cmd/gomobile gomobile init -export GO111MODULE=on output=MOBILE/android/oonimkall.aar +go run ./internal/cmd/getresources gomobile bind -target=android -o $output -ldflags="-s -w" ./pkg/oonimkall diff --git a/build-ios.bash b/build-ios.bash index c1d846b..70962a5 100755 --- a/build-ios.bash +++ b/build-ios.bash @@ -3,9 +3,8 @@ set -e topdir=$(cd $(dirname $0) && pwd -P) set -x export PATH=$(go env GOPATH)/bin:$PATH -export GO111MODULE=off go get -u golang.org/x/mobile/cmd/gomobile gomobile init -export GO111MODULE=on output=MOBILE/ios/oonimkall.framework +go run ./internal/cmd/getresources gomobile bind -target=ios -o $output -ldflags="-s -w" ./pkg/oonimkall diff --git a/build-miniooni.sh b/build-miniooni.sh index 18b2434..85d1610 100755 --- a/build-miniooni.sh +++ b/build-miniooni.sh @@ -2,10 +2,12 @@ set -e case $1 in macos|darwin) + go run ./internal/cmd/getresources export GOOS=darwin GOARCH=amd64 go build -o ./CLI/darwin/amd64 -ldflags="-s -w" ./internal/cmd/miniooni echo "Binary ready at ./CLI/darwin/amd64/miniooni";; linux) + go run ./internal/cmd/getresources export GOOS=linux GOARCH=386 go build -o ./CLI/linux/386 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni echo "Binary ready at ./CLI/linux/386/miniooni" @@ -19,6 +21,7 @@ case $1 in go build -o ./CLI/linux/arm64 -tags netgo -ldflags='-s -w -extldflags "-static"' ./internal/cmd/miniooni echo "Binary ready at ./CLI/linux/arm64/miniooni";; windows) + go run ./internal/cmd/getresources export GOOS=windows GOARCH=386 go build -o ./CLI/windows/386 -ldflags="-s -w" ./internal/cmd/miniooni echo "Binary ready at ./CLI/windows/386/miniooni.exe" diff --git a/build.sh b/build.sh index b66a69f..a8196ac 100755 --- a/build.sh +++ b/build.sh @@ -12,6 +12,7 @@ case $1 in ;; windows_amd64) + go run ./internal/cmd/getresources # Note! This assumes we've installed the mingw-w64 compiler. GOOS=windows GOARCH=amd64 CGO_ENABLED=1 CC=x86_64-w64-mingw32-gcc \ go build -ldflags='-s -w' ./cmd/ooniprobe @@ -22,6 +23,7 @@ case $1 in ;; windows_386) + go run ./internal/cmd/getresources # Note! This assumes we've installed the mingw-w64 compiler. GOOS=windows GOARCH=386 CGO_ENABLED=1 CC=i686-w64-mingw32-gcc \ go build -ldflags='-s -w' ./cmd/ooniprobe @@ -38,15 +40,17 @@ case $1 in ;; linux_amd64) - docker pull --platform linux/amd64 golang:1.14-alpine - docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine + go run ./internal/cmd/getresources + docker pull --platform linux/amd64 golang:1.16-alpine + docker run --platform linux/amd64 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine tar -cvzf ooniprobe_${v}_linux_amd64.tar.gz LICENSE.md Readme.md ooniprobe mv ooniprobe ./CLI/linux/amd64/ ;; linux_386) - docker pull --platform linux/386 golang:1.14-alpine - docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.14-alpine ./build.sh _alpine + go run ./internal/cmd/getresources + docker pull --platform linux/386 golang:1.16-alpine + docker run --platform linux/386 -v`pwd`:/ooni -w/ooni golang:1.16-alpine ./build.sh _alpine tar -cvzf ooniprobe_${v}_linux_386.tar.gz LICENSE.md Readme.md ooniprobe mv ooniprobe ./CLI/linux/386/ ;; @@ -60,6 +64,7 @@ case $1 in macos|darwin) set -x + go run ./internal/cmd/getresources # Note! The following line _assumes_ you have a working C compiler. If you # have Xcode command line tools installed, you are fine. go build -ldflags='-s -w' ./cmd/ooniprobe @@ -79,7 +84,7 @@ case $1 in set +x echo "Usage: $0 darwin|linux|macos|windows|release" echo "" - echo "You need a C compiler and Go >= 1.14. The C compiler must be a" + echo "You need a C compiler and Go >= 1.16. The C compiler must be a" echo "UNIX like compiler like GCC, Clang, Mingw-w64." echo "" echo "To build a static Linux binary, we use Docker and Alpine. We currently" diff --git a/cmd/ooniprobe/internal/bindata/bindata.go b/cmd/ooniprobe/internal/bindata/bindata.go deleted file mode 100644 index 00978ec..0000000 --- a/cmd/ooniprobe/internal/bindata/bindata.go +++ /dev/null @@ -1,377 +0,0 @@ -// Code generated by go-bindata. DO NOT EDIT. -// sources: -// data/README.md -// data/default-config.json -// data/migrations/1_create_msmt_results.sql -// data/migrations/2_single_msmt_file.sql - -package bindata - - -import ( - "bytes" - "compress/gzip" - "fmt" - "io" - "io/ioutil" - "os" - "path/filepath" - "strings" - "time" -) - -func bindataRead(data []byte, name string) ([]byte, error) { - gz, err := gzip.NewReader(bytes.NewBuffer(data)) - if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) - } - - var buf bytes.Buffer - _, err = io.Copy(&buf, gz) - clErr := gz.Close() - - if err != nil { - return nil, fmt.Errorf("Read %q: %v", name, err) - } - if clErr != nil { - return nil, err - } - - return buf.Bytes(), nil -} - - -type asset struct { - bytes []byte - info fileInfoEx -} - -type fileInfoEx interface { - os.FileInfo - MD5Checksum() string -} - -type bindataFileInfo struct { - name string - size int64 - mode os.FileMode - modTime time.Time - md5checksum string -} - -func (fi bindataFileInfo) Name() string { - return fi.name -} -func (fi bindataFileInfo) Size() int64 { - return fi.size -} -func (fi bindataFileInfo) Mode() os.FileMode { - return fi.mode -} -func (fi bindataFileInfo) ModTime() time.Time { - return fi.modTime -} -func (fi bindataFileInfo) MD5Checksum() string { - return fi.md5checksum -} -func (fi bindataFileInfo) IsDir() bool { - return false -} -func (fi bindataFileInfo) Sys() interface{} { - return nil -} - -var _bindataDataREADMEMd = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x3c\xca\xb1\x0d\x02\x31\x0c\x05\xd0\x3e\x53\x7c\x89\x9a\xcb\x12\x8c\xc0\x02\x49\x6c\x91\x2f\xdd\xd9\x28\x76\x0a\xb6\x47\x34\xd4\xef\xdd\xf0\xe0\xd2\x91\xbe\x3e\x78\x31\xe7\xee\xc7\xf0\xab\xba\x1b\xeb\x7b\x79\xd7\xfb\x38\x59\xa5\x65\x2b\xe5\x39\x19\x90\x7f\x1f\x6e\xd9\x68\x81\x9f\x22\x67\x4b\x30\xa0\x57\x57\x11\x15\xd0\xd2\xd1\x37\x4f\x89\xa3\x7c\x03\x00\x00\xff\xff\x64\xbc\xf9\x0a\x67\x00\x00\x00") - -func bindataDataREADMEMdBytes() ([]byte, error) { - return bindataRead( - _bindataDataREADMEMd, - "data/README.md", - ) -} - - - -func bindataDataREADMEMd() (*asset, error) { - bytes, err := bindataDataREADMEMdBytes() - if err != nil { - return nil, err - } - - info := bindataFileInfo{ - name: "data/README.md", - size: 0, - md5checksum: "", - mode: os.FileMode(0), - modTime: time.Unix(0, 0), - } - - a := &asset{bytes: bytes, info: info} - - return a, nil -} - -var _bindataDataDefaultconfigJson = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x54\x8d\x31\x0e\xc2\x30\x0c\x45\xf7\x9c\xc2\xca\xcc\x00\x6b\x2e\x63\x85\xe6\x97\x46\x4a\x9d\xca\x4e\xca\x80\x7a\x77\xd4\x08\x81\x58\x9f\xdf\xf3\x7f\x39\x22\xcf\x3b\xd4\x72\x15\x1f\xe8\x76\x19\x20\xcb\x5c\x75\x45\xe2\xa9\x8a\x41\x9a\x0f\x34\xc7\x62\x18\x57\x5b\xa2\x66\x79\xf8\x40\x67\x4d\xe4\xfb\x56\x6a\x4c\xac\xb0\x5e\x9a\xf9\x40\x4d\x3b\x1c\xd1\x31\x74\x41\x6b\xb0\xc1\x3f\xfe\x13\x77\xcb\x0d\xc6\x5d\x0b\x97\xbc\xe6\xf3\xff\xf5\x1b\xc4\xb4\x47\x99\x90\x7e\x81\x41\x12\x4f\x1a\x6d\x61\xc5\x56\xf5\x6f\xc4\x1d\xee\x1d\x00\x00\xff\xff\x5e\x8a\x1a\x13\xc6\x00\x00\x00") - -func bindataDataDefaultconfigJsonBytes() ([]byte, error) { - return bindataRead( - _bindataDataDefaultconfigJson, - "data/default-config.json", - ) -} - - - -func bindataDataDefaultconfigJson() (*asset, error) { - bytes, err := bindataDataDefaultconfigJsonBytes() - if err != nil { - return nil, err - } - - info := bindataFileInfo{ - name: "data/default-config.json", - size: 0, - md5checksum: "", - mode: os.FileMode(0), - modTime: time.Unix(0, 0), - } - - a := &asset{bytes: bytes, info: info} - - return a, nil -} - -var _bindataDataMigrations1createmsmtresultsSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x59\x6d\x73\xdb\x36\x12\xfe\xee\x5f\xb1\xe3\xe9\xf4\xec\x39\x49\x76\x72\x69\xe6\xce\xd7\x4e\xc7\xb5\x99\x9c\xda\x58\xca\xc8\xf2\x35\x99\x9b\x1b\x11\x22\x97\x12\x2a\x10\x60\xf0\x22\x46\xf7\xeb\x6f\x16\x00\x29\x52\x56\x1c\x67\xda\x0f\xa9\x48\x02\x8b\x7d\x7d\xf6\x59\x78\x38\x84\xbf\x96\x7c\xa5\x99\x45\xb8\x55\xb5\x3c\xe9\xbe\xb8\xb7\xcc\x62\x89\xd2\xfe\x82\x2b\x2e\x4f\x4e\x6e\x67\xd3\xf7\x30\xbf\xfe\xe5\x5d\x02\xa9\x46\xe3\x84\x35\xe9\x3f\x7b\x6f\x4b\x64\xc6\x69\xbf\xe7\xf0\x93\xd3\xe2\xf0\x95\x44\x5b\x2b\xbd\xa1\xd7\xc7\xcf\x4d\x64\xde\xff\xf2\x50\x3d\xa9\xe0\xcd\x2c\xb9\x9e\x27\xbd\x13\xe1\xec\x04\xfc\xcf\x05\xcf\x53\x18\x4f\xe6\xc9\xdb\x64\x06\xef\x67\xe3\xbb\xeb\xd9\x47\xf8\x2d\xf9\x08\xd7\x0f\xf3\xe9\x78\x72\x33\x4b\xee\x92\xc9\x7c\x10\x57\xa7\xf0\xef\xeb\xd9\xcd\xbf\xae\x67\x67\x2f\x7f\xf8\xe1\x1c\x26\xd3\x39\x4c\x1e\xde\xbd\x1b\xc0\x70\x08\x1f\x3e\x7c\x00\x6e\xc0\xae\xb9\x01\xa1\xe4\x0a\x50\x2a\xb7\x5a\xff\x4c\x5b\x33\x66\x71\xa5\xf4\x6e\x91\xa9\x1c\xf7\x42\x0e\x45\xcc\xd7\x08\x19\xb7\xfc\x7f\x28\x05\x5b\x42\xb3\x0b\x68\x17\x14\x4a\x83\x5d\xe3\x09\x3c\xef\xbf\xe1\x10\x0c\xb7\x38\x82\xdf\x11\x9c\x41\xda\x0a\xc6\x6a\x2e\x57\x30\x99\x4e\x12\xb0\x0a\x72\x94\xca\x7e\x8b\x40\xa9\x60\x23\x55\x2d\xfb\x9a\x8d\x4e\x1a\x5f\x66\xca\x49\xfb\xc8\xca\x97\x7b\x2b\x1b\x23\x6d\xad\x40\xa0\xb5\xa8\x21\xee\x09\x36\xd6\x6b\x9e\xad\xbd\x0b\x9f\xa7\xd5\x70\x08\x0f\xb3\x77\xb0\x44\x72\xb8\x01\xab\x4e\xce\x43\xd2\xfc\x8e\x90\x69\xa4\x64\x60\x60\xb0\x62\x3e\x2f\x2c\x5b\x8a\xe0\xc7\x26\xc5\xfc\xc3\x4b\xd0\xc8\x8c\x92\xe6\x8a\x76\xbe\x18\xc1\x1b\xa5\xc1\xa8\x12\x41\x15\xde\x6d\x5b\x8e\xb5\x81\x7a\x8d\x1a\x41\x22\xe6\xfe\xa5\x55\x96\x09\x90\xae\x5c\xa2\xa6\x85\x31\xc7\xf3\x56\xf6\x80\xa4\x71\xfb\x17\x03\x2b\x45\x5e\xb7\x0a\x96\x08\xa5\xcb\xd6\x50\x2a\x8d\x80\x45\xc1\x33\x8e\xd2\xd2\x97\x3f\x9c\xb1\x20\x94\xda\xb8\xca\x4b\xf7\x5e\x21\xb1\x5a\xd5\x06\xb8\x0c\x3e\x19\x0e\x83\x0d\x23\xfa\xf5\x72\x04\x67\xa5\x32\x16\x78\x59\x29\x6d\x99\xb4\xe7\x64\x76\xcd\x82\x44\xb6\x55\x3c\x87\xdc\x55\x82\x67\xcc\x92\x02\x0c\x96\x4e\x66\x6b\x92\xca\x65\xa1\x74\xc9\x2c\x57\x24\x99\x59\xaf\x6a\x5f\xd1\x4c\x95\x25\x7d\x55\x60\x70\x8b\x9a\x6c\x6d\x9c\x46\x0a\x3a\x83\x9a\xb6\x28\xe9\x95\x49\x3e\xb3\xb2\x12\x78\x15\x7d\x5f\xb2\x1d\xd4\xdc\xac\xbd\x22\x79\x4e\xff\xf3\x75\x11\x22\x40\xfb\x85\xca\xc2\xf1\x85\x56\x65\xe3\xe8\x4a\xab\x25\x86\x37\xf4\xf8\xf6\xfd\x3d\xc9\x53\xda\xcb\x30\xae\x22\x3b\x7d\xc8\x98\x10\xaa\xf6\xba\x36\xaa\x58\x05\xa7\x99\xd2\x1a\x33\x7b\x0a\x0c\x4a\x6e\x32\xc1\x8c\xe1\x05\xc7\x1c\x3a\xf8\x13\x05\xe6\xdc\x90\x4f\x1c\x37\x6b\x12\xb3\x44\x5b\x23\x4a\xa8\x79\xc1\x81\xc9\x1c\x4a\xb5\xe4\xe4\xe7\x3e\x74\xb4\xc8\x14\xe0\x23\x3e\x7e\x03\x84\x34\x3b\x24\x2b\xf1\x29\x2c\xb9\x0f\x85\x4a\xcb\x40\x63\xa5\xd1\xa0\xb4\x8d\xbd\x5d\x21\xb1\x62\x96\x3b\xc8\xb1\x60\x4e\x58\x8a\x49\xa5\x2a\x27\x98\xc5\x1c\x96\xcc\x60\xfe\xb5\x52\x22\x8f\x48\x2f\xf9\xfa\x7e\x32\x7a\xc6\xea\x88\x28\x9d\xca\xda\xe0\x8e\x22\xa0\xb1\x40\x8d\x32\x0b\x21\x8e\xa9\xfb\x0c\x81\xfb\xdc\x30\x03\x58\x62\xc6\x48\x7c\xdd\x4f\xa3\x53\x94\x9a\x67\xeb\xd3\xe7\x8a\xab\xb9\x8d\x85\x96\x33\xcb\x42\x09\x21\x14\xce\x3a\x8d\xa3\x6e\x2c\xec\xae\xea\xc4\xe2\xc5\xeb\x83\x50\x4c\xa5\xc7\x01\xca\x8c\x41\x4c\x0b\x8f\x77\xbc\xda\x6f\x7a\x75\xd9\xdd\x14\x02\xa8\x34\x1a\x72\x51\x88\x64\x1b\xc4\x90\xf4\xaa\x00\x26\x81\x57\xdb\x57\x94\x8c\xbc\xda\xbe\xa6\x14\xd7\x68\xcc\x73\xfc\x3f\xf7\xf5\x23\x57\x48\xc5\x5f\x51\xc4\x83\xb0\x56\x08\x08\xbe\xc1\xab\x67\x48\xba\xbc\xbc\xbc\xbc\xfa\xfa\x3f\x83\x67\x88\x0a\x89\xc8\x0d\xfc\xed\x1f\x90\xad\x99\xf6\x96\xa4\xcc\x48\x5f\x1b\x67\xaf\x3a\x1e\xea\x7a\xff\xcf\xb6\x0c\x0f\xf8\xfd\x2a\x6d\x38\x88\x2f\x52\x68\x9e\x9f\x5f\xa5\xd1\xc7\xdc\x40\xc6\x24\x41\xa1\x0a\x29\x70\x5a\xe3\x92\x5a\xaa\x39\x1d\xc0\x29\x2f\xe9\xdf\x0a\xb5\x07\x52\x99\x21\x3d\x96\x3c\xcf\x05\x2e\xd5\xe7\xd3\x10\xc6\xd4\xa2\xb1\x8b\x95\x56\xae\x3a\x28\xf9\x5e\x9a\x35\x67\xb6\x75\x95\xf3\xc2\x17\x92\x05\x63\x99\xb6\x0b\xcb\x4b\xf4\xb0\xa4\x9d\xa4\xdf\xbd\x22\x69\x01\x5f\x18\x05\x6b\xb6\xc5\x46\x9c\xcf\x7b\xab\x1a\xf4\xf3\xf9\xaf\xb6\xa8\xd7\xc8\x72\xb2\xc7\x37\xc8\xd0\x18\x34\x7a\x68\xa5\x23\x94\x5d\xa3\x86\x82\x65\x56\x69\x13\x9a\x43\x94\xb7\x52\xc0\xa5\x47\x72\x04\x32\x6c\xb4\xf7\x15\xf3\xb8\x43\xbd\x82\xed\xae\x20\xbd\x7f\xb8\x3b\x8b\xaa\x9e\xc3\x9b\xd9\xf4\x0e\x7a\x0c\x10\x6a\x2e\x04\x30\x51\xb3\x9d\x21\xff\xfe\xf8\x53\x23\x29\x8d\xbb\xc2\xa6\x7d\x20\x7d\x9f\xa3\x0f\x06\x7e\x3c\xef\x45\x75\xef\xa0\x14\x6e\xaf\xe7\xc9\x7c\x7c\x97\x1c\x78\xb6\x59\x1a\x65\xa7\x30\x4b\xae\xdf\x0d\x4e\x9a\x33\x1f\x0c\xfa\x06\xc5\x65\x4e\x9d\x12\x81\x17\xfb\xb6\xb2\x66\x06\x0c\x75\x06\x8f\x29\x41\x50\x3f\xab\xcc\x82\xe8\x01\xe6\x29\xcc\xc7\x93\x8f\x94\xea\x2f\xba\xa1\xed\xe5\x13\x55\x2a\x14\x82\xad\x48\xf8\xd1\x43\x83\x54\x5a\x98\xfb\xac\xf3\x3d\x36\x73\x9a\x92\x41\xec\x28\xfe\x92\xcb\xd5\xe8\x50\x05\x5a\xfc\x05\x05\xba\x2b\x29\x23\x16\xce\xb0\x15\x2e\x5c\x15\xfc\xf0\xf5\x95\xb9\xaa\xe5\xd1\xb5\xc3\x21\x8c\x89\xde\x50\xd7\x66\x4b\xd2\xce\xd3\xa8\xd0\xe2\x89\x36\x58\x6f\x52\xc9\x3e\xf3\xd2\x95\x20\x50\xae\xac\x87\xf2\x97\xaf\x2f\x81\x45\xa6\xec\x19\x73\x9b\xb2\x07\x6b\x55\x01\x05\x17\x08\x15\xb3\x6b\xa2\x1a\x50\x73\x99\xab\x3a\x82\x64\x77\xac\x58\xe4\x5c\x77\xe0\xe3\xf5\xe5\xa3\x18\x1c\xed\xd6\x7d\x83\x6e\xa6\x93\xfb\xf9\xec\x7a\x3c\x99\x43\x5a\x6c\x16\x9d\x0d\x11\xff\xde\x4c\x67\xc9\xf8\xed\x84\x60\xe3\xac\x2b\xef\x3c\x7e\x9f\x25\x6f\x92\x59\x32\xb9\x49\xee\x3b\x5c\xe1\x60\xe5\x63\xbc\xea\xd7\xc6\xd9\x63\xdb\xfe\x2c\x72\x5d\x35\x9f\x0a\x96\xe1\x52\xa9\xcd\xa2\x44\x63\x50\xae\x50\x37\x5f\x2c\x0a\x5c\x69\x56\x9e\xb4\x68\xce\xac\x61\x55\xd5\x3c\xaf\xad\xad\x16\x04\x1c\xa8\x17\x05\x47\x91\x2f\x4a\x26\xb9\xa7\x19\x5c\xc9\xde\x2a\x2e\xb7\x4c\xf0\x7c\xa1\xf1\x93\x23\xf8\x13\x5c\x76\x20\xc9\xac\x9b\xdf\x32\xb7\x1d\x90\xec\xc3\xe3\xeb\x57\x8f\x52\xb8\xeb\x90\xe7\x14\x7d\x77\x7d\xaf\xf2\x8f\x14\xe7\x44\xd9\x30\x17\xac\x94\x60\x72\x75\x45\xb0\xda\x54\x28\x21\x2a\xc1\xb0\xc5\x4e\x2b\x48\x43\xc1\x11\x5e\xa6\x2c\xb3\x7c\x8b\xe9\x00\x8c\x3a\xe9\x12\x10\x6e\x00\x3f\x39\xbe\x65\x22\x72\x7c\x5f\xd1\x4b\xf4\x34\x4e\x3b\x5f\xdc\x05\x13\x06\x5b\x1c\x4d\xfd\x31\x29\xcc\x93\x0f\xf3\x23\x56\x7c\xbd\xce\x63\xab\x0c\x75\xd8\x2a\xcf\x20\xc7\x80\x32\x39\x70\xb3\x70\x95\x50\x2c\xc7\xdc\x03\xd1\x00\xb8\x34\x36\x36\x04\x3f\x84\x38\xc3\xe5\xaa\x91\xd6\x2e\x5f\x14\x8c\x0b\xcc\x07\xa1\x5e\x99\x6d\xd8\x99\x54\x36\x1c\xd2\x4a\xf5\x25\xbf\xd7\x1a\x72\xd7\x46\x9f\x9a\x14\xc1\x82\xdd\x43\xd8\x81\x7d\x8d\x94\x67\x82\xe9\xe1\x59\x41\x49\xcf\x44\x9d\xf4\xd1\x69\x81\xdc\xac\x95\x13\xb9\x0f\x21\xf5\x56\xed\x97\x35\xf2\x34\x0e\x69\x03\xb7\xc7\xb5\x0a\x62\x9f\xc2\xd7\xee\x06\x5a\xed\x34\x2e\x4a\xb3\xea\x53\xfc\x06\x88\x8e\xda\xfc\x8d\x87\x74\x36\x3d\x75\x16\x41\xb4\x79\xdc\x6c\x7c\x04\x7d\x92\x56\x4c\x5b\x9e\x39\xc1\x74\xcf\x91\xd4\xf6\x96\xd4\xf6\xa2\x67\x98\xcc\xf7\xb9\x8d\x1a\x0b\x15\xf9\xc4\xc3\xd8\x43\x8d\x65\x1b\x8c\x59\x4f\x0c\x81\x65\x61\x7e\xb5\x0a\x90\x7b\x3e\xb1\xe6\x39\x02\xb7\xed\x6c\xb7\xf7\xbc\xef\x77\xd4\x42\xfd\x9c\x17\x5a\xc6\x16\xf5\x0e\x04\x32\x63\x69\x50\x6b\x67\x46\xb6\xe4\x82\xdb\x38\x69\xf4\x22\x16\xaf\x5f\x72\x45\x79\xe9\x89\x50\xc3\x8a\x62\x05\x74\x26\x13\x15\x1b\xad\x17\xd0\x31\xfa\xe7\xa3\xd1\xd1\xa8\x9d\xfc\x86\x74\x34\xa8\xb7\xa8\x87\x86\xec\x0d\xac\x6a\xc1\x73\xd0\x68\x9d\x96\x34\x90\xed\xe2\x78\x2f\x04\x12\xc3\x1a\xc1\x2f\xbb\x7e\xc9\xed\x37\x7d\x0f\x5c\x56\xce\x0e\x60\xa7\x9c\xf7\xf2\x27\x47\x7e\xf1\x9e\xa8\x38\x19\x52\xa0\x8d\xd7\x25\x5d\x43\x5a\x97\x24\x9f\xdb\x9f\x6f\x93\xb9\x47\x67\x73\x75\x71\xc1\x2a\x3e\x52\x4a\xf2\x11\x57\xf4\xfb\x62\xfb\xe2\xa2\xdb\x82\x7e\xf6\xa7\xfe\xf4\xdd\x78\xf2\xfe\x61\xfe\x7d\xab\xce\x4f\xdf\xcd\x92\xf7\xd3\xd9\x7c\x31\xbe\xdd\xcb\xb7\x9a\x65\x21\x64\x05\xd7\x34\x8d\x58\x2c\xf7\xf3\x7b\x24\x13\xff\xf9\x6f\x0a\x82\x1b\xdb\x14\xa4\x0c\x7a\xb7\x5d\xa9\x9f\xd8\x5a\xa4\x64\xda\x2a\xb2\x87\x5f\xef\xa7\x93\x70\x3d\xd0\x37\x92\xa6\xcb\x0e\x01\x45\x13\x26\x84\x2d\x13\x0e\x0d\x9c\xa5\xad\xde\xe9\x00\x52\x6f\x51\x7a\x0e\x4c\x7b\x34\x28\x9c\xd8\x7b\x8f\xb5\xdc\xa3\x23\xdc\x17\x08\x15\x01\x13\x1a\x59\xbe\x0b\xc5\x50\x69\x95\x51\xe3\x6c\xc3\x58\xf1\x0a\xa9\xbd\x0d\x3a\x58\xc2\xcb\x4a\x04\x21\x99\x40\x26\x5d\xe5\x87\xbd\x28\xa6\x45\xc9\xae\xc3\x5b\x36\xd7\x68\xdc\xaf\xe4\xc3\x9e\xee\x87\xa2\x9a\xdc\x28\x55\x43\xdc\x3d\x4b\x6a\x8a\xf6\x2b\xc3\xda\x70\x18\xaf\xc6\xf2\x51\x04\xa4\x83\x6b\xd0\xc7\x89\x4d\x28\xbf\x43\x4b\xe4\x15\x19\x0d\xd0\xcd\x85\x4d\x9b\xc7\x03\x58\x3a\xdf\x14\xc8\xc5\x95\x60\x9e\xa6\xc6\xdb\x9f\x5e\x57\x64\x36\x5c\xad\x55\x8a\x4b\xdb\x4c\xe5\x12\x99\xee\x8c\xe6\x61\x82\x46\xbc\x6a\x53\x76\xc5\xed\xda\x2d\x47\x99\x2a\x2f\x28\x73\x2f\x1a\xc7\x5f\x2c\x85\x5a\x5e\x94\xcc\x58\xd4\x17\xb9\xca\x8c\xff\x3c\x74\x8e\xe7\xa3\x32\x87\xef\xbb\xc4\xe4\x49\x39\xdc\x18\x87\xe6\xe2\xd5\xdf\x83\x47\x5a\xbb\x16\x47\x78\x18\xb1\x93\x43\x1f\x45\x64\x35\x8d\x45\x19\x33\xde\x49\x0c\x9a\xa1\xd1\x8f\x4c\x83\x90\x59\xcc\x5f\xcd\x92\x67\x69\x50\x17\xbb\x46\xd6\x52\xa8\x6c\x43\x5d\x96\xa8\x01\xc1\xa1\x84\xf1\x9d\xdf\xd8\xcc\x07\xf1\xd1\xd0\xa0\x65\x22\x14\x54\x4f\x0b\xe2\x85\xbf\x0f\x8b\x93\x29\xd4\xcc\x40\x8e\x16\x33\x9f\x00\x71\xfd\xc7\x88\x30\xe9\xaf\xd3\xf1\x24\x05\x06\xe9\xcd\xf4\x61\x32\x3f\x3b\x4f\xdb\xda\xf3\x95\xd5\x98\x17\x27\xb3\x80\xdb\xb1\x5a\x59\x7b\x69\x79\xa0\x05\x04\xfb\x95\x6e\x5f\x8c\xef\x48\xed\x70\xc7\x9b\x72\xb3\x60\x52\x95\x4c\xec\xba\x30\x7b\x64\x72\x92\xa0\x2a\xf6\xc9\x45\x4c\x30\x56\xbb\x8c\x32\x66\x10\x6f\x66\x6b\xa2\x69\xd4\x97\xba\x57\xb7\x9e\x5b\x6e\x70\x67\x5a\x62\x1b\xaf\x70\xe3\x6d\x7a\x9f\xaa\xa0\x65\x5c\x98\x78\xdf\x4b\x68\xe5\x45\x75\x7a\x94\x81\x33\xfc\x3c\xea\x36\xb0\x50\xd1\x17\x34\x24\xd1\x0f\x30\x15\x49\x57\x05\x4c\x6e\xe7\x83\xe8\x2b\xcf\xc6\x8a\xc6\x7e\x2a\x0c\x9f\x19\xe4\x96\x96\xb7\xa1\xcd\x46\xe7\x1d\x3e\x4c\x3a\xa7\xc1\xd2\x63\x0d\x07\x21\xd3\xca\x34\x17\xaa\xbd\xee\x46\x21\x0c\xb6\xd7\x2a\x5e\xaa\x81\x55\x2b\xa4\x3e\x3c\xfa\xe2\x8d\x48\xe7\x90\xc7\x23\xeb\x96\x69\xee\x0f\xf2\x4c\x82\x4b\x8b\x5a\x32\x21\x7c\x27\xa6\x16\xb0\x09\x68\xc8\xc2\x78\xe7\xef\x19\xe4\x30\xe7\x66\x73\x04\x5b\xcd\xe8\x0f\xa3\xe4\x08\xc6\xd6\x13\xc8\x92\x98\x83\x41\x69\xbc\xee\xb5\xa6\xba\x20\x9e\x1c\xe6\x3e\xd4\x80\xfe\xaa\x67\xd9\xa6\xf6\x5a\x29\xef\xc2\xbb\xdf\x7c\x84\x2a\x8d\xdb\x78\x37\xda\xd0\x0b\x12\xd2\xa0\x4f\x90\xa3\x24\xf1\x88\x4d\xbc\xa3\x2a\xd9\x5e\x18\xb1\x83\x92\xc9\x5d\x4f\x43\x7f\x6e\xe1\xef\x7f\xbb\xc8\x4c\x6f\x16\x64\xe4\xd3\x83\xe6\xc1\x14\xb9\xf7\xf5\xe3\x21\xd2\xf7\xa9\xe6\xf3\xb1\x21\xb2\xb9\x01\x39\xb6\x6e\x3a\x81\xdb\xe4\x5d\x32\x4f\xe0\xe6\xfa\xfe\xe6\xfa\x36\xf1\x9d\x62\x5c\x50\x8a\xe7\x28\xd0\x06\xde\xe3\x73\xb7\xcb\x8a\xbe\xdc\x1e\x86\x43\x60\x42\x1c\x96\x85\x89\x7f\x10\x08\x32\x73\x9a\xdc\x6b\x14\x22\xf8\xa6\x6f\x4c\x6c\x24\xe7\x3d\x1b\xfc\xdf\xdb\xf6\xdf\x68\x00\xfe\xe2\x5f\xf5\xfe\x1f\x00\x00\xff\xff\x38\xc6\x64\x22\x78\x1c\x00\x00") - -func bindataDataMigrations1createmsmtresultsSqlBytes() ([]byte, error) { - return bindataRead( - _bindataDataMigrations1createmsmtresultsSql, - "data/migrations/1_create_msmt_results.sql", - ) -} - - - -func bindataDataMigrations1createmsmtresultsSql() (*asset, error) { - bytes, err := bindataDataMigrations1createmsmtresultsSqlBytes() - if err != nil { - return nil, err - } - - info := bindataFileInfo{ - name: "data/migrations/1_create_msmt_results.sql", - size: 0, - md5checksum: "", - mode: os.FileMode(0), - modTime: time.Unix(0, 0), - } - - a := &asset{bytes: bytes, info: info} - - return a, nil -} - -var _bindataDataMigrations2singlemsmtfileSql = []byte("\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xec\x56\x4d\x6f\xdb\x38\x10\xbd\xf3\x57\xcc\xd1\xc6\x2a\x8b\xdd\xb6\xc9\xc5\xe8\x81\x91\x99\x54\xad\x4c\xa5\x14\x53\x34\x27\x89\xb5\x68\x47\x88\x4c\x0a\x24\xd5\x20\xff\xbe\x90\x3f\x6a\xda\x96\x8d\x3a\x28\x8a\x1e\x74\x9d\x8f\xc7\x21\xe7\x0d\xe7\x5d\x5c\xc0\x3f\x8b\x72\x6e\x84\x93\x30\xd6\xcf\x0a\xf9\x86\xd4\x09\x27\x17\x52\xb9\x6b\x39\x2f\x15\x42\x77\x0c\xdf\x4e\x30\xcc\xb4\x91\xe5\x5c\x65\x4f\xf2\xc5\xbe\xd7\xb3\xd9\x08\xe1\x98\x13\x06\x1c\x5f\xc7\x04\xf2\x85\x14\xb6\x31\xcb\x3c\x9b\x03\x23\x14\x4f\x08\xf0\x04\xf2\xcc\xf7\x64\x4a\x3e\xe7\x23\x84\x42\x46\x30\x27\xdd\xb9\x03\x04\x00\x3b\xc6\xac\x2c\x72\x88\x28\x27\xb7\x84\xc1\x1d\x8b\x26\x98\x3d\xc0\x27\xf2\x00\xf8\x9e\x27\x11\x0d\x19\x99\x10\xca\x83\x55\x9e\x93\xd6\x65\x4a\x2c\x64\x0e\x5f\x30\x0b\x3f\x60\x36\xb8\x7a\x37\x04\x9a\x70\xa0\xf7\x71\x1c\x1c\xa2\x5b\x27\x8c\xcb\x5c\xd9\xa6\x8c\x31\x27\x3c\x9a\x90\x53\xf1\xa6\x51\xab\x60\x46\x70\xec\x05\x76\xd4\x6d\xb3\x42\x2b\x99\x03\x8f\xe8\x43\x44\xf9\xe0\xff\x93\x85\x94\x36\x6b\xea\x4a\x8b\x42\x16\x67\xa4\xcc\x44\x59\xfd\x7a\x42\x1b\xdd\x18\x99\x2d\xec\x7c\xfb\x40\x6f\x2e\x2f\x87\xa7\xea\x39\xf3\x0c\x2f\xe9\x8c\xa3\x8c\x34\x8d\x3a\x75\x84\x91\xb5\x36\x2b\x32\x74\xa0\x35\xa6\xf2\x79\xb2\xb6\x4e\x75\x55\xc9\xa9\xd3\x26\xeb\xe0\x53\x4b\x8c\x75\x5c\x69\x33\xa1\xf4\x42\x54\x2f\x7e\x05\x3e\xa5\x5a\xde\xe7\xf0\x31\x4d\xe8\x61\x5d\xb6\xa9\x76\x49\xda\x5d\xf9\xac\xac\x64\x56\x0b\xf7\xe8\x5d\xe0\xea\xbf\xfd\x7b\x86\x09\x4d\x39\xc3\x11\xe5\x90\xcf\x9e\xb2\x2d\xfa\xd2\x0b\x70\x93\x30\x12\xdd\xd2\xe5\x00\x0c\xbc\xc3\x87\x6b\x3f\x23\x37\x84\x11\x1a\x92\x74\x53\x9a\xcd\xbb\xe2\x12\x0a\x63\x12\x13\x4e\x20\xc4\x69\x88\xc7\x64\x75\xfc\x2e\xfc\xfa\x55\x87\x3b\xa8\x8d\xa9\x5a\xc8\x8d\x0f\x0d\x47\x08\x45\x34\x25\x8c\xb7\x0f\x90\x80\x3f\xcf\x30\x40\xfb\xa3\x1c\x20\x6f\x48\x03\x74\x6c\x16\xf7\x3c\x9b\xa9\xdb\x33\x6f\x46\xec\xd0\xfc\x73\x94\x0e\x5d\x6b\x3a\xef\x39\x7c\xbe\x1e\x83\x3b\x92\xda\xc1\xf8\x43\x84\x15\xc1\x03\xe4\x11\x39\x40\x9b\x57\x0c\xd0\x71\xae\x06\xc8\xe7\xe7\xe6\xf9\x96\x84\x5c\xa2\x6d\x3a\xbb\x85\xde\x32\x0d\xb5\xed\x4e\x49\x4c\x42\x7e\xf0\xa7\xf6\x8d\xf8\x93\x8d\x00\xb8\x61\xc9\x04\x0e\xf6\xe1\x08\xa1\x31\x4b\xee\xd6\xcb\xb0\xcb\xdd\xb9\x81\xd5\x08\x75\xaf\x6d\xa2\x8a\x5d\xcf\x7d\xfd\xba\xfd\xde\x66\xa5\x9f\xe3\xd2\xc9\xb7\x50\x68\x69\x41\x69\x07\xb6\xa9\xdb\x8b\x81\x28\x8a\x52\xcd\x61\xaa\xab\x66\xa1\x2c\x68\x03\x85\xd1\x75\xbd\xb2\x29\xeb\x8c\x28\x95\xb3\x01\x58\x0d\xcf\x12\x94\x94\x45\x0b\xe7\x34\x18\x79\x31\x35\xb2\x2d\xc4\x3d\x4a\x70\xe2\x5b\x25\x41\xa8\x02\xa6\xba\x7e\x59\x9a\x0a\xe1\x04\xe8\xef\xd2\xfc\x8b\x5e\x25\x31\x74\x55\xf4\x12\xa3\x97\x18\xbd\xc4\xf0\x24\x46\x57\x97\xb6\x91\x9c\x7c\xe5\xbd\xf6\xe8\x57\xde\xef\x5e\x79\xfb\x77\xec\x45\xc9\xdf\xd7\xa1\xf6\x0b\xe9\x96\x26\xba\x2a\x4e\x49\x93\x95\xfb\x7c\x69\xf2\x23\x00\x00\xff\xff\xca\xeb\xb6\x24\x7c\x10\x00\x00") - -func bindataDataMigrations2singlemsmtfileSqlBytes() ([]byte, error) { - return bindataRead( - _bindataDataMigrations2singlemsmtfileSql, - "data/migrations/2_single_msmt_file.sql", - ) -} - - - -func bindataDataMigrations2singlemsmtfileSql() (*asset, error) { - bytes, err := bindataDataMigrations2singlemsmtfileSqlBytes() - if err != nil { - return nil, err - } - - info := bindataFileInfo{ - name: "data/migrations/2_single_msmt_file.sql", - size: 0, - md5checksum: "", - mode: os.FileMode(0), - modTime: time.Unix(0, 0), - } - - a := &asset{bytes: bytes, info: info} - - return a, nil -} - - -// -// Asset loads and returns the asset for the given name. -// It returns an error if the asset could not be found or -// could not be loaded. -// -func Asset(name string) ([]byte, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { - a, err := f() - if err != nil { - return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err) - } - return a.bytes, nil - } - return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} -} - -// -// MustAsset is like Asset but panics when Asset would return an error. -// It simplifies safe initialization of global variables. -// nolint: deadcode -// -func MustAsset(name string) []byte { - a, err := Asset(name) - if err != nil { - panic("asset: Asset(" + name + "): " + err.Error()) - } - - return a -} - -// -// AssetInfo loads and returns the asset info for the given name. -// It returns an error if the asset could not be found or could not be loaded. -// -func AssetInfo(name string) (os.FileInfo, error) { - cannonicalName := strings.Replace(name, "\\", "/", -1) - if f, ok := _bindata[cannonicalName]; ok { - a, err := f() - if err != nil { - return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err) - } - return a.info, nil - } - return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist} -} - -// -// AssetNames returns the names of the assets. -// nolint: deadcode -// -func AssetNames() []string { - names := make([]string, 0, len(_bindata)) - for name := range _bindata { - names = append(names, name) - } - return names -} - -// -// _bindata is a table, holding each asset generator, mapped to its name. -// -var _bindata = map[string]func() (*asset, error){ - "data/README.md": bindataDataREADMEMd, - "data/default-config.json": bindataDataDefaultconfigJson, - "data/migrations/1_create_msmt_results.sql": bindataDataMigrations1createmsmtresultsSql, - "data/migrations/2_single_msmt_file.sql": bindataDataMigrations2singlemsmtfileSql, -} - -// -// AssetDir returns the file names below a certain -// directory embedded in the file by go-bindata. -// For example if you run go-bindata on data/... and data contains the -// following hierarchy: -// data/ -// foo.txt -// img/ -// a.png -// b.png -// then AssetDir("data") would return []string{"foo.txt", "img"} -// AssetDir("data/img") would return []string{"a.png", "b.png"} -// AssetDir("foo.txt") and AssetDir("notexist") would return an error -// AssetDir("") will return []string{"data"}. -// -func AssetDir(name string) ([]string, error) { - node := _bintree - if len(name) != 0 { - cannonicalName := strings.Replace(name, "\\", "/", -1) - pathList := strings.Split(cannonicalName, "/") - for _, p := range pathList { - node = node.Children[p] - if node == nil { - return nil, &os.PathError{ - Op: "open", - Path: name, - Err: os.ErrNotExist, - } - } - } - } - if node.Func != nil { - return nil, &os.PathError{ - Op: "open", - Path: name, - Err: os.ErrNotExist, - } - } - rv := make([]string, 0, len(node.Children)) - for childName := range node.Children { - rv = append(rv, childName) - } - return rv, nil -} - - -type bintree struct { - Func func() (*asset, error) - Children map[string]*bintree -} - -var _bintree = &bintree{Func: nil, Children: map[string]*bintree{ - "data": {Func: nil, Children: map[string]*bintree{ - "README.md": {Func: bindataDataREADMEMd, Children: map[string]*bintree{}}, - "default-config.json": {Func: bindataDataDefaultconfigJson, Children: map[string]*bintree{}}, - "migrations": {Func: nil, Children: map[string]*bintree{ - "1_create_msmt_results.sql": {Func: bindataDataMigrations1createmsmtresultsSql, Children: map[string]*bintree{}}, - "2_single_msmt_file.sql": {Func: bindataDataMigrations2singlemsmtfileSql, Children: map[string]*bintree{}}, - }}, - }}, -}} - -// RestoreAsset restores an asset under the given directory -func RestoreAsset(dir, name string) error { - data, err := Asset(name) - if err != nil { - return err - } - info, err := AssetInfo(name) - if err != nil { - return err - } - err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755)) - if err != nil { - return err - } - err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode()) - if err != nil { - return err - } - return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime()) -} - -// RestoreAssets restores an asset under the given directory recursively -func RestoreAssets(dir, name string) error { - children, err := AssetDir(name) - // File - if err != nil { - return RestoreAsset(dir, name) - } - // Dir - for _, child := range children { - err = RestoreAssets(dir, filepath.Join(name, child)) - if err != nil { - return err - } - } - return nil -} - -func _filePath(dir, name string) string { - cannonicalName := strings.Replace(name, "\\", "/", -1) - return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...) -} diff --git a/cmd/ooniprobe/internal/database/database.go b/cmd/ooniprobe/internal/database/database.go index b5e349d..9b59047 100644 --- a/cmd/ooniprobe/internal/database/database.go +++ b/cmd/ooniprobe/internal/database/database.go @@ -2,21 +2,45 @@ package database import ( "database/sql" + "embed" + "io/ioutil" "github.com/apex/log" - "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata" migrate "github.com/rubenv/sql-migrate" "upper.io/db.v3/lib/sqlbuilder" "upper.io/db.v3/sqlite" ) +//go:embed migrations/*.sql +var efs embed.FS + +func readAsset(path string) ([]byte, error) { + filep, err := efs.Open(path) + if err != nil { + return nil, err + } + return ioutil.ReadAll(filep) +} + +func readAssetDir(path string) ([]string, error) { + var out []string + lst, err := efs.ReadDir(path) + if err != nil { + return nil, err + } + for _, e := range lst { + out = append(out, e.Name()) + } + return out, nil +} + // RunMigrations runs the database migrations func RunMigrations(db *sql.DB) error { log.Debugf("running migrations") migrations := &migrate.AssetMigrationSource{ - Asset: bindata.Asset, - AssetDir: bindata.AssetDir, - Dir: "data/migrations", + Asset: readAsset, + AssetDir: readAssetDir, + Dir: "migrations", } n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up) if err != nil { diff --git a/data/migrations/1_create_msmt_results.sql b/cmd/ooniprobe/internal/database/migrations/1_create_msmt_results.sql similarity index 100% rename from data/migrations/1_create_msmt_results.sql rename to cmd/ooniprobe/internal/database/migrations/1_create_msmt_results.sql diff --git a/data/migrations/2_single_msmt_file.sql b/cmd/ooniprobe/internal/database/migrations/2_single_msmt_file.sql similarity index 100% rename from data/migrations/2_single_msmt_file.sql rename to cmd/ooniprobe/internal/database/migrations/2_single_msmt_file.sql diff --git a/data/default-config.json b/cmd/ooniprobe/internal/ooni/default-config.json similarity index 100% rename from data/default-config.json rename to cmd/ooniprobe/internal/ooni/default-config.json diff --git a/cmd/ooniprobe/internal/ooni/ooni.go b/cmd/ooniprobe/internal/ooni/ooni.go index 676ccef..ab9c065 100644 --- a/cmd/ooniprobe/internal/ooni/ooni.go +++ b/cmd/ooniprobe/internal/ooni/ooni.go @@ -1,6 +1,7 @@ package ooni import ( + _ "embed" // because we embed a file "io/ioutil" "os" "os/signal" @@ -8,7 +9,6 @@ import ( "syscall" "github.com/apex/log" - "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/bindata" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/config" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database" "github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/enginex" @@ -239,6 +239,9 @@ func MaybeInitializeHome(home string) error { return nil } +//go:embed default-config.json +var defaultConfig []byte + // InitDefaultConfig reads the config from common locations or creates it if // missing. func InitDefaultConfig(home string) (*config.Config, error) { @@ -252,12 +255,7 @@ func InitDefaultConfig(home string) (*config.Config, error) { if err != nil { if os.IsNotExist(err) { log.Debugf("writing default config to %s", configPath) - var data []byte - data, err = bindata.Asset("data/default-config.json") - if err != nil { - return nil, err - } - if err = ioutil.WriteFile(configPath, data, 0644); err != nil { + if err = ioutil.WriteFile(configPath, defaultConfig, 0644); err != nil { return nil, err } // If the user did the informed consent procedure in diff --git a/data/README.md b/data/README.md deleted file mode 100644 index bf95010..0000000 --- a/data/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Directory github.com/ooni/probe-cli/data - -This directory contains data that is embedded into builds. diff --git a/go.mod b/go.mod index 779d196..8d5b5cc 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/ooni/probe-cli/v3 -go 1.14 +go 1.16 require ( git.torproject.org/pluggable-transports/goptlib.git v1.1.0 diff --git a/internal/cmd/getresources/getresources.go b/internal/cmd/getresources/getresources.go new file mode 100644 index 0000000..3f6ba7d --- /dev/null +++ b/internal/cmd/getresources/getresources.go @@ -0,0 +1,51 @@ +// Command getresources downloads the resources +package main + +import ( + "crypto/sha256" + "errors" + "fmt" + "io/ioutil" + "log" + "net/http" + "net/url" + "path/filepath" + + "github.com/ooni/probe-cli/v3/internal/engine/resources" +) + +func main() { + for name, ri := range resources.All { + if err := getit(name, &ri); err != nil { + log.Fatal(err) + } + } +} + +func getit(name string, ri *resources.ResourceInfo) error { + workDir := filepath.Join("internal", "engine", "resourcesmanager") + URL, err := url.Parse(resources.BaseURL) + if err != nil { + return err + } + URL.Path = ri.URLPath + log.Println("fetching", URL.String()) + resp, err := http.Get(URL.String()) + if err != nil { + return err + } + defer resp.Body.Close() + if resp.StatusCode != 200 { + return errors.New("http request failed") + } + data, err := ioutil.ReadAll(resp.Body) + if err != nil { + return err + } + checksum := fmt.Sprintf("%x", sha256.Sum256(data)) + if checksum != ri.GzSHA256 { + return errors.New("sha256 mismatch") + } + fullpath := filepath.Join(workDir, name+".gz") + return ioutil.WriteFile(fullpath, data, 0644) +} diff --git a/internal/cmd/jafar/README.md b/internal/cmd/jafar/README.md index 4cda5a8..8f1f6da 100644 --- a/internal/cmd/jafar/README.md +++ b/internal/cmd/jafar/README.md @@ -8,7 +8,7 @@ any system but it really on works on Linux. ## Building -We use Go >= 1.14. Jafar also needs the C library headers, +We use Go >= 1.16. Jafar also needs the C library headers, iptables installed, and root permissions. With Linux Alpine edge, you can compile Jafar with: diff --git a/internal/engine/geolocate/mmdblookup_test.go b/internal/engine/geolocate/mmdblookup_test.go index 07153b5..e66fbe8 100644 --- a/internal/engine/geolocate/mmdblookup_test.go +++ b/internal/engine/geolocate/mmdblookup_test.go @@ -1,12 +1,9 @@ package geolocate import ( - "context" - "net/http" "testing" - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/resources" + "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" ) const ( @@ -16,13 +13,8 @@ const ( ) func maybeFetchResources(t *testing.T) { - c := &resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: "../testdata/", - } - if err := c.Ensure(context.Background()); err != nil { + c := &resourcesmanager.CopyWorker{DestDir: "../testdata/"} + if err := c.Ensure(); err != nil { t.Fatal(err) } } diff --git a/internal/engine/netx/archival/archival_test.go b/internal/engine/netx/archival/archival_test.go index 667ecff..df38a3e 100644 --- a/internal/engine/netx/archival/archival_test.go +++ b/internal/engine/netx/archival/archival_test.go @@ -10,14 +10,13 @@ import ( "testing" "time" - "github.com/apex/log" "github.com/google/go-cmp/cmp" "github.com/gorilla/websocket" "github.com/ooni/probe-cli/v3/internal/engine/model" "github.com/ooni/probe-cli/v3/internal/engine/netx/archival" "github.com/ooni/probe-cli/v3/internal/engine/netx/errorx" "github.com/ooni/probe-cli/v3/internal/engine/netx/trace" - "github.com/ooni/probe-cli/v3/internal/engine/resources" + "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" ) func TestNewTCPConnectList(t *testing.T) { @@ -286,12 +285,7 @@ func TestNewRequestList(t *testing.T) { } func TestNewDNSQueriesList(t *testing.T) { - err := (&resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "miniooni/0.1.0-dev", - WorkDir: "../../testdata", - }).Ensure(context.Background()) + err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure() if err != nil { t.Fatal(err) } diff --git a/internal/engine/resources/README.md b/internal/engine/resources/README.md deleted file mode 100644 index f1e9428..0000000 --- a/internal/engine/resources/README.md +++ /dev/null @@ -1,3 +0,0 @@ -# Package github.com/ooni/probe-engine/resources - -This package contains code to download OONI resources. diff --git a/internal/engine/resources/doc.go b/internal/engine/resources/doc.go new file mode 100644 index 0000000..a694698 --- /dev/null +++ b/internal/engine/resources/doc.go @@ -0,0 +1,3 @@ +// Package resources contains info on resources. See also +// the resourcesmanager package. +package resources diff --git a/internal/engine/resources/resources.go b/internal/engine/resources/resources.go deleted file mode 100644 index d4afbaa..0000000 --- a/internal/engine/resources/resources.go +++ /dev/null @@ -1,104 +0,0 @@ -// Package resources contains code to download resources. -package resources - -import ( - "bytes" - "compress/gzip" - "context" - "crypto/sha256" - "fmt" - "io" - "io/ioutil" - "net/http" - "os" - "path/filepath" - - "github.com/ooni/probe-cli/v3/internal/engine/httpx" - "github.com/ooni/probe-cli/v3/internal/engine/model" -) - -// Client is a client for fetching resources. -type Client struct { - // HTTPClient is the HTTP client to use. - HTTPClient *http.Client - - // Logger is the logger to use. - Logger model.Logger - - // OSMkdirAll allows testing os.MkdirAll failures. - OSMkdirAll func(path string, perm os.FileMode) error - - // UserAgent is the user agent to use. - UserAgent string - - // WorkDir is the directory where to save resources. - WorkDir string -} - -// Ensure ensures that resources are downloaded and current. -func (c *Client) Ensure(ctx context.Context) error { - mkdirall := c.OSMkdirAll - if mkdirall == nil { - mkdirall = os.MkdirAll - } - if err := mkdirall(c.WorkDir, 0700); err != nil { - return err - } - for name, resource := range All { - if err := c.EnsureForSingleResource( - ctx, name, resource, func(real, expected string) bool { - return real == expected - }, - gzip.NewReader, ioutil.ReadAll, - ); err != nil { - return err - } - } - return nil -} - -// EnsureForSingleResource ensures that a single resource -// is downloaded and is current. -func (c *Client) EnsureForSingleResource( - ctx context.Context, name string, resource ResourceInfo, - equal func(real, expected string) bool, - gzipNewReader func(r io.Reader) (*gzip.Reader, error), - ioutilReadAll func(r io.Reader) ([]byte, error), -) error { - fullpath := filepath.Join(c.WorkDir, name) - data, err := ioutil.ReadFile(fullpath) - if err == nil { - sha256sum := fmt.Sprintf("%x", sha256.Sum256(data)) - if equal(sha256sum, resource.SHA256) { - return nil - } - c.Logger.Debugf("resources: %s is outdated", fullpath) - } else { - c.Logger.Debugf("resources: can't read %s: %s", fullpath, err.Error()) - } - data, err = (httpx.Client{ - BaseURL: BaseURL, - HTTPClient: c.HTTPClient, - Logger: c.Logger, - UserAgent: c.UserAgent, - }).FetchResourceAndVerify(ctx, resource.URLPath, resource.GzSHA256) - if err != nil { - return err - } - c.Logger.Debugf("resources: uncompress %s", fullpath) - gzreader, err := gzipNewReader(bytes.NewReader(data)) - if err != nil { - return err - } - defer gzreader.Close() // we already have a sha256 for it - data, err = ioutilReadAll(gzreader) // small file - if err != nil { - return err - } - sha256sum := fmt.Sprintf("%x", sha256.Sum256(data)) - if equal(sha256sum, resource.SHA256) == false { - return fmt.Errorf("resources: %s sha256 mismatch", fullpath) - } - c.Logger.Debugf("resources: overwrite %s", fullpath) - return ioutil.WriteFile(fullpath, data, 0600) -} diff --git a/internal/engine/resources/resources_test.go b/internal/engine/resources/resources_test.go deleted file mode 100644 index 025df30..0000000 --- a/internal/engine/resources/resources_test.go +++ /dev/null @@ -1,180 +0,0 @@ -package resources_test - -import ( - "compress/gzip" - "context" - "errors" - "io" - "io/ioutil" - "net/http" - "os" - "strings" - "testing" - - "github.com/apex/log" - "github.com/ooni/probe-cli/v3/internal/engine/resources" -) - -func TestEnsureMkdirAllFailure(t *testing.T) { - log.SetLevel(log.DebugLevel) - expected := errors.New("mocked error") - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - OSMkdirAll: func(string, os.FileMode) error { - return expected - }, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: "/foobar", - } - err := client.Ensure(context.Background()) - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } -} - -func TestEnsure(t *testing.T) { - tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test") - if err != nil { - t.Fatal(err) - } - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: tempdir, - } - err = client.Ensure(context.Background()) - if err != nil { - t.Fatal(err) - } - // the second round should be idempotent - err = client.Ensure(context.Background()) - if err != nil { - t.Fatal(err) - } -} - -func TestEnsureFailure(t *testing.T) { - log.SetLevel(log.DebugLevel) - tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test") - if err != nil { - t.Fatal(err) - } - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: tempdir, - } - ctx, cancel := context.WithCancel(context.Background()) - cancel() - err = client.Ensure(ctx) - if !errors.Is(err, context.Canceled) { - t.Fatal("not the error we expected") - } -} - -func TestEnsureFailAllComparisons(t *testing.T) { - log.SetLevel(log.DebugLevel) - tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test") - if err != nil { - t.Fatal(err) - } - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: tempdir, - } - // run once to download the resource once - err = client.EnsureForSingleResource( - context.Background(), "ca-bundle.pem", resources.ResourceInfo{ - URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz", - GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab", - SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a", - }, func(left, right string) bool { - return left == right - }, - gzip.NewReader, ioutil.ReadAll, - ) - if err != nil { - t.Fatal(err) - } - // re-run with broken comparison operator so that we should - // first redownload and then fail for invalid SHA256. - err = client.EnsureForSingleResource( - context.Background(), "ca-bundle.pem", resources.ResourceInfo{ - URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz", - GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab", - SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a", - }, func(left, right string) bool { - return false // comparison for equality always fails - }, - gzip.NewReader, ioutil.ReadAll, - ) - if err == nil || !strings.HasSuffix(err.Error(), "sha256 mismatch") { - t.Fatal("not the error we expected") - } -} - -func TestEnsureFailGzipNewReader(t *testing.T) { - log.SetLevel(log.DebugLevel) - tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test") - if err != nil { - t.Fatal(err) - } - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: tempdir, - } - expected := errors.New("mocked error") - err = client.EnsureForSingleResource( - context.Background(), "ca-bundle.pem", resources.ResourceInfo{ - URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz", - GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab", - SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a", - }, func(left, right string) bool { - return left == right - }, - func(r io.Reader) (*gzip.Reader, error) { - return nil, expected - }, - ioutil.ReadAll, - ) - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } -} - -func TestEnsureFailIoUtilReadAll(t *testing.T) { - log.SetLevel(log.DebugLevel) - tempdir, err := ioutil.TempDir("", "ooniprobe-engine-resources-test") - if err != nil { - t.Fatal(err) - } - client := resources.Client{ - HTTPClient: http.DefaultClient, - Logger: log.Log, - UserAgent: "ooniprobe-engine/0.1.0", - WorkDir: tempdir, - } - expected := errors.New("mocked error") - err = client.EnsureForSingleResource( - context.Background(), "ca-bundle.pem", resources.ResourceInfo{ - URLPath: "/ooni/probe-assets/releases/download/20190822135402/ca-bundle.pem.gz", - GzSHA256: "d5a6aa2290ee18b09cc4fb479e2577ed5ae66c253870ba09776803a5396ea3ab", - SHA256: "cb2eca3fbfa232c9e3874e3852d43b33589f27face98eef10242a853d83a437a", - }, func(left, right string) bool { - return left == right - }, - gzip.NewReader, func(r io.Reader) ([]byte, error) { - return nil, expected - }, - ) - if !errors.Is(err, expected) { - t.Fatal("not the error we expected") - } -} diff --git a/internal/engine/resources/private/.gitignore b/internal/engine/resourcesmanager/.gitignore similarity index 75% rename from internal/engine/resources/private/.gitignore rename to internal/engine/resourcesmanager/.gitignore index 9f7c7ac..de9444a 100644 --- a/internal/engine/resources/private/.gitignore +++ b/internal/engine/resourcesmanager/.gitignore @@ -1,2 +1,3 @@ /asn.mmdb.gz /country.mmdb.gz +/testdata diff --git a/internal/engine/resourcesmanager/resourcesmanager.go b/internal/engine/resourcesmanager/resourcesmanager.go new file mode 100644 index 0000000..d56895b --- /dev/null +++ b/internal/engine/resourcesmanager/resourcesmanager.go @@ -0,0 +1,149 @@ +// Package resourcesmanager contains the resources manager. +package resourcesmanager + +import ( + "compress/gzip" + "crypto/sha256" + "embed" + "errors" + "fmt" + "io" + "io/fs" + "io/ioutil" + "os" + "path/filepath" + + "github.com/ooni/probe-cli/v3/internal/engine/resources" +) + +// Errors returned by this package. +var ( + ErrDestDirEmpty = errors.New("resources: DestDir is empty") + ErrSHA256Mismatch = errors.New("resources: sha256 mismatch") +) + +// CopyWorker ensures that resources are current. You always need to set +// the DestDir attribute. All the rest is optional. +type CopyWorker struct { + DestDir string // mandatory + Different func(left, right string) bool // optional + Equal func(left, right string) bool // optional + MkdirAll func(path string, perm os.FileMode) error // optional + NewReader func(r io.Reader) (io.ReadCloser, error) // optional + Open func(path string) (fs.File, error) // optional + ReadAll func(r io.Reader) ([]byte, error) // optional + ReadFile func(filename string) ([]byte, error) // optional + WriteFile func(filename string, data []byte, perm fs.FileMode) error // optional +} + +//go:embed *.mmdb.gz +var efs embed.FS + +func (cw *CopyWorker) mkdirAll(path string, perm os.FileMode) error { + if cw.MkdirAll != nil { + return cw.MkdirAll(path, perm) + } + return os.MkdirAll(path, perm) +} + +// Ensure ensures that the resources on disk are current. +func (cw *CopyWorker) Ensure() error { + if cw.DestDir == "" { + return ErrDestDirEmpty + } + if err := cw.mkdirAll(cw.DestDir, 0700); err != nil { + return err + } + for name, resource := range resources.All { + if err := cw.ensureFor(name, &resource); err != nil { + return err + } + } + return nil +} + +func (cw *CopyWorker) readFile(path string) ([]byte, error) { + if cw.ReadFile != nil { + return cw.ReadFile(path) + } + return ioutil.ReadFile(path) +} + +func (cw *CopyWorker) equal(left, right string) bool { + if cw.Equal != nil { + return cw.Equal(left, right) + } + return left == right +} + +func (cw *CopyWorker) different(left, right string) bool { + if cw.Different != nil { + return cw.Different(left, right) + } + return left != right +} + +func (cw *CopyWorker) open(path string) (fs.File, error) { + if cw.Open != nil { + return cw.Open(path) + } + return efs.Open(path) +} + +func (cw *CopyWorker) newReader(r io.Reader) (io.ReadCloser, error) { + if cw.NewReader != nil { + return cw.NewReader(r) + } + return gzip.NewReader(r) +} + +func (cw *CopyWorker) readAll(r io.Reader) ([]byte, error) { + if cw.ReadAll != nil { + return cw.ReadAll(r) + } + return ioutil.ReadAll(r) +} + +func (cw *CopyWorker) writeFile(filename string, data []byte, perm fs.FileMode) error { + if cw.WriteFile != nil { + return cw.WriteFile(filename, data, perm) + } + return ioutil.WriteFile(filename, data, perm) +} + +func (cw *CopyWorker) sha256sum(data []byte) string { + return fmt.Sprintf("%x", sha256.Sum256(data)) +} + +func (cw *CopyWorker) allGood(rpath string, resource *resources.ResourceInfo) bool { + data, err := cw.readFile(rpath) + if err != nil { + return false + } + return cw.equal(cw.sha256sum(data), resource.SHA256) +} + +func (cw *CopyWorker) ensureFor(name string, resource *resources.ResourceInfo) error { + rpath := filepath.Join(cw.DestDir, name) + if cw.allGood(rpath, resource) { + return nil + } + filep, err := cw.open(name + ".gz") + if err != nil { + return err + } + defer filep.Close() + gzfilep, err := cw.newReader(filep) + if err != nil { + return err + } + defer gzfilep.Close() + data, err := cw.readAll(gzfilep) + if err != nil { + return err + } + if cw.different(cw.sha256sum(data), resource.SHA256) { + return ErrSHA256Mismatch + } + return cw.writeFile(rpath, data, 0600) +} diff --git a/internal/engine/resourcesmanager/resourcesmanager_test.go b/internal/engine/resourcesmanager/resourcesmanager_test.go new file mode 100644 index 0000000..5101e5f --- /dev/null +++ b/internal/engine/resourcesmanager/resourcesmanager_test.go @@ -0,0 +1,142 @@ +package resourcesmanager + +import ( + "errors" + "io" + "io/fs" + "os" + "testing" +) + +func TestAllGood(t *testing.T) { + // make sure we start from scratch + if err := os.RemoveAll("testdata"); err != nil { + t.Fatal(err) + } + // first iteration should copy the resources + cw := &CopyWorker{DestDir: "testdata"} + if err := cw.Ensure(); err != nil { + t.Fatal(err) + } + // second iteration should just ensure they're there + if err := cw.Ensure(); err != nil { + t.Fatal(err) + } +} + +func TestEmptyDestDir(t *testing.T) { + cw := &CopyWorker{DestDir: ""} + if err := cw.Ensure(); !errors.Is(err, ErrDestDirEmpty) { + t.Fatal("not the error we expected", err) + } +} + +func TestMkdirAllFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return errMocked + }, + } + if err := cw.Ensure(); !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} + +func TestOpenFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return nil + }, + ReadFile: func(path string) ([]byte, error) { + return []byte(`fake`), nil + }, + Equal: func(left, right string) bool { + return false + }, + Open: func(path string) (fs.File, error) { + return nil, errMocked + }, + } + if err := cw.Ensure(); !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} + +func TestNewReaderFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return nil + }, + Equal: func(left, right string) bool { + return false + }, + NewReader: func(r io.Reader) (io.ReadCloser, error) { + return nil, errMocked + }, + } + if err := cw.Ensure(); !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} + +func TestReadAllFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return nil + }, + Equal: func(left, right string) bool { + return false + }, + ReadAll: func(r io.Reader) ([]byte, error) { + return nil, errMocked + }, + } + if err := cw.Ensure(); !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} + +func TestSHA256Mismatch(t *testing.T) { + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return nil + }, + Equal: func(left, right string) bool { + return false + }, + Different: func(left, right string) bool { + return true + }, + } + if err := cw.Ensure(); !errors.Is(err, ErrSHA256Mismatch) { + t.Fatal("not the error we expected", err) + } +} + +func TestWriteFileFailure(t *testing.T) { + errMocked := errors.New("mocked error") + cw := &CopyWorker{ + DestDir: "testdata", + MkdirAll: func(path string, perm os.FileMode) error { + return nil + }, + Equal: func(left, right string) bool { + return false + }, + WriteFile: func(filename string, data []byte, perm fs.FileMode) error { + return errMocked + }, + } + if err := cw.Ensure(); !errors.Is(err, errMocked) { + t.Fatal("not the error we expected", err) + } +} diff --git a/internal/engine/session.go b/internal/engine/session.go index 980e7e1..b4ad9f8 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -22,6 +22,7 @@ import ( "github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/engine/resources" + "github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager" "github.com/ooni/probe-cli/v3/internal/version" ) @@ -414,12 +415,7 @@ func (s *Session) UserAgent() (useragent string) { // MaybeUpdateResources updates the resources if needed. func (s *Session) MaybeUpdateResources(ctx context.Context) error { - return (&resources.Client{ - HTTPClient: s.DefaultHTTPClient(), - Logger: s.logger, - UserAgent: s.UserAgent(), - WorkDir: s.assetsDir, - }).Ensure(ctx) + return (&resourcesmanager.CopyWorker{DestDir: s.assetsDir}).Ensure() } func (s *Session) getAvailableProbeServices() []model.Service { diff --git a/updatebindata.sh b/updatebindata.sh deleted file mode 100755 index 7bce7ab..0000000 --- a/updatebindata.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/sh -set -ex -go get -u github.com/shuLhan/go-bindata/... -gobindata=`go env GOPATH`/bin/go-bindata -version=`$gobindata -version | grep ^go-bindata | cut -d ' ' -f2` -if [ "$version" != "4.0.0" ]; then - echo "FATAL: unexpected go-bindata version" 1>&2 - exit 1 -fi -$gobindata -nometadata -o cmd/ooniprobe/internal/bindata/bindata.go -pkg bindata data/...