From 264e30f016cc0ac276dfb41fe96a95c127b837a3 Mon Sep 17 00:00:00 2001 From: Simone Basso Date: Fri, 19 Nov 2021 12:40:10 +0100 Subject: [PATCH] [forwardport] fix(mk): do nothing if we already have psiphon config (#605) (#606) This diff forward ports ea44e99451f345474738b9010ff791759a1f1367. Original commit message: - - - This change allows for producing cloud builds using the psiphon config files. We will add those files as build secrets. Only people in the organization and collaborators with at least "write" access could trigger builds containing such secrets. Before this change, `./mk` unconditionally attempted to clone github.com/ooni/probe-private. Now, it only checks whether we need to clone _if_ files are not already there. This allows us to use GitHub actions and secrets to copy the files in there _without_ needing to clone a private repo. Cloning a private repo would require us to include as repository secret an access token with full `repo` scope, which is a very broad scope. Instead, by using secrets to include psiphon config, we are narrowing down the secrets required to make a release build. See https://github.com/ooni/probe/issues/1878 This diff WILL require forward porting to the master branch. --- .github/workflows/android.yml | 12 ++++++- .github/workflows/ios.yml | 12 ++++++- .github/workflows/linux.yml | 40 ++++++++++++++++++++--- .github/workflows/macos.yml | 10 +++++- .github/workflows/miniooni.yml | 12 ++++++- .github/workflows/windows.yml | 10 +++++- internal/cmd/miniooni/libminiooni.go | 1 + internal/engine/session_nopsiphon.go | 5 +++ internal/engine/session_nopsiphon_test.go | 6 ++++ internal/engine/session_psiphon.go | 7 ++++ internal/engine/session_psiphon_test.go | 6 ++++ mk | 28 +++++++++++++--- 12 files changed, 135 insertions(+), 14 deletions(-) diff --git a/.github/workflows/android.yml b/.github/workflows/android.yml index dfa2fc3..32161b1 100644 --- a/.github/workflows/android.yml +++ b/.github/workflows/android.yml @@ -13,4 +13,14 @@ jobs: with: go-version: "1.17.3" - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" ./MOBILE/android/oonimkall.aar + with: + fetch-depth: 0 + + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - run: ./mk ./MOBILE/android/oonimkall.aar diff --git a/.github/workflows/ios.yml b/.github/workflows/ios.yml index 438dc2e..d8d92d9 100644 --- a/.github/workflows/ios.yml +++ b/.github/workflows/ios.yml @@ -13,4 +13,14 @@ jobs: with: go-version: "1.17.3" - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" XCODE_VERSION=12.4 ./MOBILE/ios/oonimkall.xcframework.zip + with: + fetch-depth: 0 + + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - run: ./mk XCODE_VERSION=12.4 ./MOBILE/ios/oonimkall.xcframework.zip diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 35f76e7..2cd8473 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -11,7 +11,15 @@ jobs: runs-on: "ubuntu-20.04" steps: - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386 + with: + fetch-depth: 0 + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/386 - run: ./E2E/ooniprobe.sh ./CLI/linux/386/ooniprobe - run: ./CLI/linux/pubdebian if: github.ref == 'refs/heads/ooniprobe-staging' @@ -24,7 +32,15 @@ jobs: runs-on: "ubuntu-20.04" steps: - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64 + with: + fetch-depth: 0 + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/amd64 - run: ./E2E/ooniprobe.sh ./CLI/linux/amd64/ooniprobe - run: ./CLI/linux/pubdebian if: github.ref == 'refs/heads/ooniprobe-staging' @@ -37,9 +53,17 @@ jobs: runs-on: "ubuntu-20.04" steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - run: sudo apt-get update -q - run: sudo apt-get install -y qemu-user-static - - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm - run: ./E2E/ooniprobe.sh ./CLI/linux/arm/ooniprobe - run: ./CLI/linux/pubdebian if: github.ref == 'refs/heads/ooniprobe-staging' @@ -52,9 +76,17 @@ jobs: runs-on: "ubuntu-20.04" steps: - uses: actions/checkout@v2 + with: + fetch-depth: 0 - run: sudo apt-get update -q - run: sudo apt-get install -y qemu-user-static - - run: ./mk OONI_PSIPHON_TAGS="" DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64 + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk DEBIAN_TILDE_VERSION=$GITHUB_RUN_NUMBER ./debian/arm64 - run: ./E2E/ooniprobe.sh ./CLI/linux/arm64/ooniprobe - run: ./CLI/linux/pubdebian if: github.ref == 'refs/heads/ooniprobe-staging' diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index 781c305..604d88a 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -12,5 +12,13 @@ jobs: with: go-version: "1.17.3" - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/darwin/amd64/ooniprobe + with: + fetch-depth: 0 + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk ./CLI/darwin/amd64/ooniprobe - run: ./E2E/ooniprobe.sh ./CLI/darwin/amd64/ooniprobe diff --git a/.github/workflows/miniooni.yml b/.github/workflows/miniooni.yml index d5dac7b..2b50d18 100644 --- a/.github/workflows/miniooni.yml +++ b/.github/workflows/miniooni.yml @@ -15,7 +15,17 @@ jobs: with: go-version: "1.17.3" - uses: actions/checkout@v2 - - run: ./mk OONI_PSIPHON_TAGS="" ./CLI/miniooni + with: + fetch-depth: 0 + + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + + - run: ./mk ./CLI/miniooni - run: ./E2E/miniooni.bash ./CLI/linux/amd64/miniooni diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index da8bd34..7e9baa6 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -12,8 +12,16 @@ jobs: with: go-version: "1.17.3" - uses: actions/checkout@v2 + with: + fetch-depth: 0 - run: sudo apt install mingw-w64 - - run: ./mk OONI_PSIPHON_TAGS="" MINGW_W64_VERSION="9.3-win32" ./CLI/windows/amd64/ooniprobe.exe + - run: | + echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key + echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age + env: + PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }} + PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }} + - run: ./mk MINGW_W64_VERSION="9.3-win32" ./CLI/windows/amd64/ooniprobe.exe - uses: actions/upload-artifact@v2 with: name: ooniprobe.exe diff --git a/internal/cmd/miniooni/libminiooni.go b/internal/cmd/miniooni/libminiooni.go index 0be5f65..e7cba04 100644 --- a/internal/cmd/miniooni/libminiooni.go +++ b/internal/cmd/miniooni/libminiooni.go @@ -159,6 +159,7 @@ func Main() { os.Exit(0) } fatalIfFalse(len(getopt.Args()) == 1, "Missing experiment name") + fatalOnError(engine.CheckEmbeddedPsiphonConfig(), "Invalid embedded psiphon config") MainWithConfiguration(getopt.Arg(0), globalOptions) } diff --git a/internal/engine/session_nopsiphon.go b/internal/engine/session_nopsiphon.go index e543c9b..f968f8b 100644 --- a/internal/engine/session_nopsiphon.go +++ b/internal/engine/session_nopsiphon.go @@ -29,3 +29,8 @@ var errPsiphonNoEmbeddedConfig = errors.New("no embedded configuration file") func (s *sessionTunnelEarlySession) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { return nil, errPsiphonNoEmbeddedConfig } + +// CheckEmbeddedPsiphonConfig checks whether we can load psiphon's config +func CheckEmbeddedPsiphonConfig() error { + return nil +} diff --git a/internal/engine/session_nopsiphon_test.go b/internal/engine/session_nopsiphon_test.go index 4564f5b..a523ad4 100644 --- a/internal/engine/session_nopsiphon_test.go +++ b/internal/engine/session_nopsiphon_test.go @@ -19,3 +19,9 @@ func TestEarlySessionNoPsiphonFetchPsiphonConfig(t *testing.T) { t.Fatal("expected nil here") } } + +func TestCheckEmbeddedPsiphonConfig(t *testing.T) { + if err := CheckEmbeddedPsiphonConfig(); err != nil { + t.Fatal(err) + } +} diff --git a/internal/engine/session_psiphon.go b/internal/engine/session_psiphon.go index bb2a0d4..f5b8a69 100644 --- a/internal/engine/session_psiphon.go +++ b/internal/engine/session_psiphon.go @@ -44,3 +44,10 @@ func (s *Session) FetchPsiphonConfig(ctx context.Context) ([]byte, error) { child := &sessionTunnelEarlySession{} return child.FetchPsiphonConfig(ctx) } + +// CheckEmbeddedPsiphonConfig checks whether we can load psiphon's config +func CheckEmbeddedPsiphonConfig() error { + child := &sessionTunnelEarlySession{} + _, err := child.FetchPsiphonConfig(context.Background()) + return err +} diff --git a/internal/engine/session_psiphon_test.go b/internal/engine/session_psiphon_test.go index 3ff4186..d65f445 100644 --- a/internal/engine/session_psiphon_test.go +++ b/internal/engine/session_psiphon_test.go @@ -18,3 +18,9 @@ func TestSessionEmbeddedPsiphonConfig(t *testing.T) { t.Fatal("expected non-nil data here") } } + +func TestCheckEmbeddedPsiphonConfig(t *testing.T) { + if err := CheckEmbeddedPsiphonConfig(); err != nil { + t.Fatal(err) + } +} diff --git a/mk b/mk index 6cc5835..75412d1 100755 --- a/mk +++ b/mk @@ -637,13 +637,31 @@ search/for/zip: @command -v zip || { echo "not found"; exit 1; } #help: -#help: The `./mk maybe/copypsiphon` command copies the private psiphon config -#help: file into the current tree unless `$(OONI_PSIPHON_TAGS)` is empty. +#help: The `./mk maybe/copypsiphon` command checks whether we want +#help: to embed the Psiphon config file into the build. To this end, +#help: this command checks whether OONI_PSIPHON_TAGS is set. In +#help: such a case, this command checks whether the required files +#help: are already in place. If not, this command fetches them +#help: by cloning the github.com/ooni/probe-private repo. +# +# Note: we check for files being already there before attempting +# to clone _because_ we put files in there using secrets when +# running cloud builds. This saves us from including a token with +# `repo` scope as a build secret, which is a very broad scope. +# +# Cloning the private repository, instead, is the way in which +# local builds get access to the psiphon config files. .PHONY: maybe/copypsiphon maybe/copypsiphon: search/for/git - test -z "$(OONI_PSIPHON_TAGS)" || $(MAKE) -f mk $(OONIPRIVATE) - test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.key ./internal/engine - test -z "$(OONI_PSIPHON_TAGS)" || cp $(OONIPRIVATE)/psiphon-config.json.age ./internal/engine + @if test "$(OONI_PSIPHON_TAGS)" = "ooni_psiphon_config"; then \ + if test ! -f ./internal/engine/psiphon-config.json.age -a \ + ! -f ./internal/engine/psiphon-config.key; then \ + echo "copying psiphon configuration file into ./internal/engine"; \ + $(MAKE) -f mk $(OONIPRIVATE) || exit 1; \ + cp $(OONIPRIVATE)/psiphon-config.key ./internal/engine || exit 1; \ + cp $(OONIPRIVATE)/psiphon-config.json.age ./internal/engine || exit 1; \ + fi; \ + fi # OONIPRIVATE is the directory where we clone the private repository. OONIPRIVATE = $(GIT_CLONE_DIR)/github.com/ooni/probe-private