c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

feat: autogen GH workflows and split build, test, and publish (#971)

Browse source 
Closes https://github.com/ooni/probe/issues/2337.
This commit is contained in:
Simone Basso 2022-10-03 14:20:54 +02:00 • committed by GitHub
commit 5466f30526
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 990 additions and 142 deletions
Download patch file Download diff file Expand all files Collapse all files

61
GHGEN/android.go Normal file
View file

@ -0,0 +1,61 @@
package main
//
// Generates Android workflow.
//
import (
"fmt"
"io"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func buildAndPublishMobileAndroid(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) <= 0, "expected no architecture matrix")
buildJob := "build_android_mobile"
artifacts := []string{
"./MOBILE/android/oonimkall.aar",
"./MOBILE/android/oonimkall-sources.jar",
"./MOBILE/android/oonimkall.pom",
}
publishJob := "publish_android_mobile"
newJob(w, buildJob, runsOnUbuntu, noDependencies, noPermissions)
newStepCheckout(w)
newStepSetupGo(w, "android-oonimkall")
newStepSetupPsiphon(w)
newStepMake(w, "MOBILE/android")
newStepUploadArtifacts(w, artifacts)
newJob(w, publishJob, runsOnUbuntu, buildJob, contentsWritePermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}
func buildAndPublishCLIAndroid(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) > 0, "expected architecture matrix")
for _, arch := range job.ArchsMatrix {
buildJob := fmt.Sprintf("build_android_cli_%s", arch)
artifacts := []string{
fmt.Sprintf("./CLI/miniooni-android-%s", arch),
fmt.Sprintf("./CLI/ooniprobe-android-%s", arch),
}
publishJob := fmt.Sprintf("publish_android_cli_%s", arch)
newJob(w, buildJob, runsOnUbuntu, noDependencies, noPermissions)
newStepCheckout(w)
newStepSetupGo(w, fmt.Sprintf("android-cli-%s", arch))
newStepSetupPsiphon(w)
newStepMake(w, fmt.Sprintf("CLI/android-%s", arch))
newStepUploadArtifacts(w, artifacts)
newJob(w, publishJob, runsOnUbuntu, buildJob, contentsWritePermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}
}

76
GHGEN/config.go Normal file
View file

@ -0,0 +1,76 @@
package main
//
// Configuration with which we will run
//
import "io"
// Job is a job to run.
type Job struct {
// Action is the job name
Action func(w io.Writer, job *Job)
// ArchsMatrix contains the architectures to iterate over
ArchsMatrix []string
}
// Config contains the configuration.
var Config = map[string][]Job{
"android": {{
Action: buildAndPublishMobileAndroid,
ArchsMatrix: []string{},
}, {
Action: buildAndPublishCLIAndroid,
ArchsMatrix: []string{
"386",
"amd64",
"arm",
"arm64",
},
}},
"ios": {{
Action: buildAndPublishMobileIOS,
ArchsMatrix: []string{},
}},
"linux": {{
Action: buildAndPublishCLILinux,
ArchsMatrix: []string{
"386",
"amd64",
"armv6",
"armv7",
"arm64",
},
}},
"macos": {{
Action: buildAndPublishCLIMacOS,
ArchsMatrix: []string{},
}},
"windows": {{
Action: buildAndPublishCLIWindows,
ArchsMatrix: []string{},
}},
}
const (
// runOnUbuntu is the Ubuntu system where to run.
runsOnUbuntu = "ubuntu-20.04"
// runsOnMacOS is the macOS system where to run.
runsOnMacOS = "macos-11"
// runsOnWindows is the windows system where to run.
runsOnWindows = "windows-2019"
)
// noPermission indicates a job does not require permissions.
var noPermissions map[string]string
// contentsWritePermissions indicates the job needs the `contents: write` permission.
var contentsWritePermissions = map[string]string{
"contents": "write",
}
// noDependencies indicates a job does not require dependencies.
var noDependencies string

34
GHGEN/ios.go Normal file
View file

@ -0,0 +1,34 @@
package main
//
// Generates iOS workflow.
//
import (
"io"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func buildAndPublishMobileIOS(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) <= 0, "expected no architecture matrix")
buildJob := "build_ios_mobile"
artifacts := []string{
"./MOBILE/ios/oonimkall.xcframework.zip",
"./MOBILE/ios/oonimkall.podspec",
}
publishJob := "publish_ios_mobile"
newJob(w, buildJob, runsOnMacOS, noDependencies, noPermissions)
newStepCheckout(w)
newStepSetupGo(w, "ios")
newStepSetupPsiphon(w)
newStepMake(w, "EXPECTED_XCODE_VERSION=13.2.1 MOBILE/ios")
newStepUploadArtifacts(w, artifacts)
newJob(w, publishJob, runsOnUbuntu, buildJob, contentsWritePermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}

58
GHGEN/linux.go Normal file
View file

@ -0,0 +1,58 @@
package main
//
// Generates Linux workflow.
//
import (
"fmt"
"io"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func buildAndPublishCLILinux(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) > 0, "expected architecture matrix")
for _, arch := range job.ArchsMatrix {
buildJob := fmt.Sprintf("build_linux_cli_%s", arch)
artifacts := []string{
fmt.Sprintf("./CLI/ooniprobe-linux-%s", arch),
fmt.Sprintf("./CLI/miniooni-linux-%s", arch),
}
testJob := fmt.Sprintf("test_linux_cli_%s", arch)
publishJob := fmt.Sprintf("publish_linux_cli_%s", arch)
newJob(w, buildJob, runsOnUbuntu, noDependencies, noPermissions)
newStepCheckout(w)
switch arch {
case "386", "amd64":
// nothing
default:
newSetupInstallQemuUserStatic(w)
}
newStepSetupPsiphon(w)
newStepSetupLinuxDockerGoCache(w, arch)
newStepMake(w, fmt.Sprintf("CLI/linux-static-%s", arch))
newStepUploadArtifacts(w, artifacts)
// We only run integration tests for amd64
switch arch {
case "amd64":
newJob(w, testJob, runsOnUbuntu, buildJob, noPermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepSetupGo(w, fmt.Sprintf("linux-%s", arch))
newStepInstallTor(w)
newStepRunOONIProbeIntegrationTests(w, "linux", arch, "")
newStepRunMiniooniIntegrationTests(w, "linux", arch, "")
newJob(w, publishJob, runsOnUbuntu, testJob, contentsWritePermissions)
default:
newJob(w, publishJob, runsOnUbuntu, buildJob, contentsWritePermissions)
}
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}
}

42
GHGEN/macos.go Normal file
View file

@ -0,0 +1,42 @@
package main
//
// Generates the macOS workflow.
//
import (
"io"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func buildAndPublishCLIMacOS(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) <= 0, "expected no architecture matrix")
buildJob := "build_darwin_cli"
artifacts := []string{
"./CLI/ooniprobe-darwin-amd64",
"./CLI/ooniprobe-darwin-arm64",
"./CLI/miniooni-darwin-amd64",
"./CLI/miniooni-darwin-arm64",
}
testJob := "test_darwin_cli"
publishJob := "publish_darwin_cli"
newJob(w, buildJob, runsOnMacOS, noDependencies, noPermissions)
newStepCheckout(w)
newStepSetupGo(w, "macos")
newStepSetupPsiphon(w)
newStepMake(w, "CLI/darwin")
newStepUploadArtifacts(w, artifacts)
newJob(w, testJob, runsOnMacOS, buildJob, noPermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, []string{"ooniprobe-darwin-amd64"})
newStepRunOONIProbeIntegrationTests(w, "darwin", "amd64", "")
newJob(w, publishJob, runsOnMacOS, testJob, contentsWritePermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}

12
GHGEN/main.go Normal file
View file

@ -0,0 +1,12 @@
// Command GHGEN regenerates selected GitHub actions.
package main
import (
_ "embed"
)
func main() {
for name, jobs := range Config {
generateWorkflowFile(name, jobs)
}
}

177
GHGEN/utils.go Normal file
View file

@ -0,0 +1,177 @@
package main
//
// Utility functions.
//
import (
"fmt"
"io"
"os"
"path/filepath"
"strings"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func newJob(w io.Writer, name, runsOn, needs string, permissions map[string]string) {
mustFprintf(w, " %s:\n", name)
mustFprintf(w, " runs-on: %s\n", runsOn)
if needs != "" {
mustFprintf(w, " needs: %s\n", needs)
}
if len(permissions) > 0 {
mustFprintf(w, " permissions:\n")
for key, value := range permissions {
mustFprintf(w, " %s: %s\n", key, value)
}
}
mustFprintf(w, " steps:\n")
}
func newStepCheckout(w io.Writer) {
mustFprintf(w, " - uses: actions/checkout@v2\n")
mustFprintf(w, " with:\n")
mustFprintf(w, " fetch-depth: 0\n")
mustFprintf(w, "\n")
}
func newStepSetupGo(w io.Writer, cacheName string) {
mustFprintf(w, " - name: Get GOVERSION content\n")
mustFprintf(w, " id: goversion\n")
mustFprintf(w, " run: echo ::set-output name=version::$(cat GOVERSION)\n")
mustFprintf(w, " - uses: magnetikonline/action-golang-cache@v2\n")
mustFprintf(w, " with:\n")
mustFprintf(w, " go-version: \"${{ steps.goversion.outputs.version }}\"\n")
mustFprintf(w, " cache-key-suffix: \"-%s-${{ steps.goversion.outputs.version }}\"\n", cacheName)
mustFprintf(w, "\n")
}
func newStepSetupPsiphon(w io.Writer) {
mustFprintf(w, " - run: |\n")
mustFprintf(w, " echo -n $PSIPHON_CONFIG_KEY > ./internal/engine/psiphon-config.key\n")
mustFprintf(w, " echo $PSIPHON_CONFIG_JSON_AGE_BASE64 | base64 -d > ./internal/engine/psiphon-config.json.age\n")
mustFprintf(w, " env:\n")
mustFprintf(w, " PSIPHON_CONFIG_KEY: ${{ secrets.PSIPHON_CONFIG_KEY }}\n")
mustFprintf(w, " PSIPHON_CONFIG_JSON_AGE_BASE64: ${{ secrets.PSIPHON_CONFIG_JSON_AGE_BASE64 }}\n")
mustFprintf(w, "\n")
}
func newStepMake(w io.Writer, target string) {
mustFprintf(w, " - run: make %s\n", target)
mustFprintf(w, "\n")
}
func newStepUploadArtifacts(w io.Writer, artifacts []string) {
for _, arti := range artifacts {
mustFprintf(w, " - uses: actions/upload-artifact@v2\n")
mustFprintf(w, " with:\n")
mustFprintf(w, " name: %s\n", filepath.Base(arti))
mustFprintf(w, " path: %s\n", arti)
mustFprintf(w, "\n")
}
}
func newStepDownloadArtifacts(w io.Writer, artifacts []string) {
for _, arti := range artifacts {
mustFprintf(w, " - uses: actions/download-artifact@v2\n")
mustFprintf(w, " with:\n")
mustFprintf(w, " name: %s\n", filepath.Base(arti))
mustFprintf(w, "\n")
}
}
func newStepGHPublish(w io.Writer, artifacts []string) {
runtimex.Assert(len(artifacts) > 0, "expected at least one artifact")
artifactsNames := []string{}
for _, arti := range artifacts {
artifactsNames = append(artifactsNames, filepath.Base(arti))
}
mustFprintf(w, " - run: ./script/ghpublish.bash %s\n", strings.Join(artifactsNames, " "))
mustFprintf(w, " env:\n")
mustFprintf(w, " GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}\n")
mustFprintf(w, "\n")
}
func newStepSetupLinuxDockerGoCache(w io.Writer, name string) {
mustFprintf(w, " - uses: actions/cache@v3\n")
mustFprintf(w, " with:\n")
mustFprintf(w, " path: GOCACHE\n")
mustFprintf(w, " key: linux-build-cache-%s\n", name)
mustFprintf(w, "\n")
}
func newSetupInstallQemuUserStatic(w io.Writer) {
mustFprintf(w, " - run: sudo apt-get update -q\n")
mustFprintf(w, " - run: sudo apt-get install -y qemu-user-static\n")
mustFprintf(w, "\n")
}
func newStepInstallTor(w io.Writer) {
mustFprintf(w, " - run: sudo apt-get update -q\n")
mustFprintf(w, " - run: sudo apt-get install -y tor\n")
mustFprintf(w, "\n")
}
func newStepRunOONIProbeIntegrationTests(w io.Writer, os, arch, ext string) {
executable := fmt.Sprintf("ooniprobe-%s-%s%s", os, arch, ext)
if os != "windows" {
mustFprintf(w, " - run: chmod +x %s\n", executable)
}
mustFprintf(w, " - run: ./E2E/ooniprobe.bash ./%s\n", executable)
mustFprintf(w, " shell: bash\n")
mustFprintf(w, "\n")
}
func newStepRunMiniooniIntegrationTests(w io.Writer, os, arch, ext string) {
executable := fmt.Sprintf("miniooni-%s-%s%s", os, arch, ext)
if os != "windows" {
mustFprintf(w, " - run: chmod +x %s\n", executable)
}
mustFprintf(w, " - run: ./E2E/miniooni.bash ./%s\n", executable)
mustFprintf(w, " shell: bash\n")
mustFprintf(w, "\n")
}
func newStepInstallMingwW64(w io.Writer) {
mustFprintf(w, " - run: sudo apt-get update -q\n")
mustFprintf(w, " - run: sudo apt-get install -y mingw-w64\n")
mustFprintf(w, "\n")
}
func mustFprintf(w io.Writer, format string, v ...any) {
_, err := fmt.Fprintf(w, format, v...)
runtimex.PanicOnError(err, "fmt.Fprintf failed")
}
func mustClose(c io.Closer) {
err := c.Close()
runtimex.PanicOnError(err, "c.Close failed")
}
func generateWorkflowFile(name string, jobs []Job) {
filename := filepath.Join(".github", "workflows", name+".yml")
fp, err := os.Create(filename)
runtimex.PanicOnError(err, "os.Create failed")
defer mustClose(fp)
mustFprintf(fp, "# File generated by `go run ./GHGEN`; DO NOT EDIT.\n")
mustFprintf(fp, "\n")
mustFprintf(fp, "name: %s\n", name)
mustFprintf(fp, "on:\n")
mustFprintf(fp, " push:\n")
mustFprintf(fp, " branches:\n")
mustFprintf(fp, " - \"release/**\"\n")
mustFprintf(fp, " - \"fullbuild\"\n")
mustFprintf(fp, " - \"%sbuild\"\n", name)
mustFprintf(fp, " tags:\n")
mustFprintf(fp, " - \"v*\"\n")
mustFprintf(fp, " schedule:\n")
mustFprintf(fp, " - cron: \"17 1 * * *\"\n")
mustFprintf(fp, "\n")
mustFprintf(fp, "jobs:\n")
for _, job := range jobs {
job.Action(fp, &job)
}
mustFprintf(fp, "# End of autogenerated file\n")
}

44
GHGEN/windows.go Normal file
View file

@ -0,0 +1,44 @@
package main
//
// Generates the Windows workflow.
//
import (
"io"
"github.com/ooni/probe-cli/v3/internal/runtimex"
)
func buildAndPublishCLIWindows(w io.Writer, job *Job) {
runtimex.Assert(len(job.ArchsMatrix) <= 0, "expected no architecture matrix")
buildJob := "build_windows_cli"
artifacts := []string{
"./CLI/ooniprobe-windows-386.exe",
"./CLI/ooniprobe-windows-amd64.exe",
"./CLI/miniooni-windows-386.exe",
"./CLI/miniooni-windows-amd64.exe",
}
testJob := "test_windows_cli"
publishJob := "publish_windows_cli"
newJob(w, buildJob, runsOnUbuntu, noDependencies, noPermissions)
newStepCheckout(w)
newStepSetupGo(w, "windows")
newStepInstallMingwW64(w)
newStepSetupPsiphon(w)
newStepMake(w, "EXPECTED_MINGW_W64_VERSION=\"9.3-win32\" CLI/windows")
newStepUploadArtifacts(w, artifacts)
newJob(w, testJob, runsOnWindows, buildJob, noPermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, []string{"ooniprobe-windows-amd64.exe"})
newStepRunOONIProbeIntegrationTests(w, "windows", "amd64", ".exe")
newJob(w, publishJob, runsOnUbuntu, testJob, contentsWritePermissions)
newStepCheckout(w)
newStepDownloadArtifacts(w, artifacts)
newStepGHPublish(w, artifacts)
}