refactor: start building an Android package (#205)

* refactor: start building an Android package

Part of https://github.com/ooni/probe/issues/1335.

This seems also a good moment to move some packages out of the
engine, e.g., oonimkall. This package, for example, is a consumer
of the engine, so it makes sense it's not _inside_ it.

* fix: committed some stuff I didn't need to commit

* fix: oonimkall needs to be public to build

The side effect is that we will probably need to bump the major
version number every time we change one of these APIs.

(We can also of course choose to violate the basic guidelines of Go
software, but I believe this is bad form.)

I have no problem in bumping the major quite frequently and in
any case this monorepo solution is convinving me more than continuing
to keep a split between engine and cli. The need to embed assets to
make the probe more reliable trumps the negative effects of having to
~frequently bump major because we expose a public API.

* fix: let's not forget about libooniffi

Honestly, I don't know what to do with this library. I added it
to provide a drop in replacement for MK but I have no idea whether
it's used and useful. I would not feel comfortable exposing it,
unlike oonimkall, since we're not using it.

It may be that the right thing to do here is just to delete the
package and reduce the amount of code we're maintaining?

* woops, we're still missing the publish android script

* fix(publish-android.bash): add proper API key

* ouch fix another place where the name changed

internal/libooniffi/.gitignore

@@ -0,0 +1,8 @@
+/asn.mmdb
+/ca-bundle.pem
+/country.mmdb
+/ffirun
+/ffirun.exe
+/libooniffi.dll
+/libooniffi.h
+/libooniffi.so

internal/libooniffi/README.md

@@ -0,0 +1,16 @@
+# Directory github.com/ooni/probe-engine/libooniffi
+
+This directory contains code to generate shared/static libraries with
+a Measurement Kit compatible ABI. To this end, we wrap the [oonimkall](
+../oonimkall) API with a simple C API.
+
+The generated libraries have a Measurement Kit compatible ABI. You can
+also instruct your compiler so that [ooniffi.h](ooniffi.h) defines macros
+that make code written for Measurement Kit compile and work. To this
+end, please see comments insider [ooniffi.h](ooniffi.h).
+
+To see how we compile this library for several systems, please take a
+look at [libooniffi.yml](../.github/workflows/libooniffi.yml).
+
+This is not used in any OONI product. We may break something
+in ooniffi without noticing it. Please, be aware of that.

internal/libooniffi/buildtest.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+cd $(dirname $0) 
+case $1 in
+  darwin)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.so .
+    clang++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun -looniffi ./testdata/ffirun.cpp
+    ./ffirun testdata/webconnectivity.json
+    ;;
+  linux)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.so .
+    g++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun ./testdata/ffirun.cpp -looniffi
+    LD_LIBRARY_PATH=. ./ffirun testdata/webconnectivity.json
+    ;;
+  windows)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.dll .
+    x86_64-w64-mingw32-g++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun.exe -looniffi ./testdata/ffirun.cpp
+    ./ffirun.exe testdata/webconnectivity.json
+    ;;
+  *)
+    echo "usage: $0 darwin|linux|windows" 1>&2
+    exit 1
+esac

internal/libooniffi/ooniffi.c

@@ -0,0 +1,17 @@
+#include "ooniffi.h"
+
+#include "_cgo_export.h"
+
+ooniffi_task_t *ooniffi_task_start(const char *settings) {
+    /* Implementation note: Go does not have the concept of const but
+       we know that the code is just making a copy of settings. */
+    return ooniffi_task_start_((char *)settings);
+}
+
+const char *ooniffi_event_serialization(ooniffi_event_t *event) {
+    /* Implementation note: Go does not have the concept of const but
+       we want to return const to very clearly communicate that the
+       returned string is owned by the event. This is what tools like
+       python's ctypes and SWIG expect from us. */
+    return (const char *)ooniffi_event_serialization_(event);
+}

internal/libooniffi/ooniffi.def

@@ -0,0 +1,7 @@
+EXPORTS
+ooniffi_task_start
+ooniffi_task_is_done
+ooniffi_task_wait_for_next_event
+ooniffi_event_serialization
+ooniffi_event_destroy
+ooniffi_task_destroy

internal/libooniffi/ooniffi.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	//#include "ooniffi.h"
+	//
+	//#include <stdint.h>
+	//#include <stdlib.h>
+	//
+	//struct ooniffi_task_ {
+	//    int64_t Handle;
+	//};
+	//
+	// struct ooniffi_event_ {
+	//    char *String;
+	//};
+	"C"
+	"sync"
+	"unsafe"
+
+	"github.com/ooni/probe-cli/v3/pkg/oonimkall"
+)
+
+var (
+	idx C.int64_t
+	m   = make(map[C.int64_t]*oonimkall.Task)
+	mu  sync.Mutex
+)
+
+func cstring(s string) *C.char {
+	return C.CString(s)
+}
+
+func freestring(s *C.char) {
+	C.free(unsafe.Pointer(s))
+}
+
+func gostring(s *C.char) string {
+	return C.GoString(s)
+}
+
+const maxIdx = C.INT64_MAX
+
+//export ooniffi_task_start_
+func ooniffi_task_start_(settings *C.char) *C.ooniffi_task_t {
+	if settings == nil {
+		return nil
+	}
+	tp, err := oonimkall.StartTask(gostring(settings))
+	if err != nil {
+		return nil
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	// TODO(bassosimone): the following if is basic protection against
+	// undefined behaviour, i.e., the counter wrapping around. A much
+	// better strategy would probably be to restart from 0. However it's
+	// also unclear if any device could run that many tests, so...
+	if idx >= maxIdx {
+		return nil
+	}
+	handle := idx
+	idx++
+	m[handle] = tp
+	task := (*C.ooniffi_task_t)(C.malloc(C.sizeof_ooniffi_task_t))
+	task.Handle = handle
+	return task
+}
+
+func setmaxidx() C.int64_t {
+	o := idx
+	idx = maxIdx
+	return o
+}
+
+func restoreidx(v C.int64_t) {
+	idx = v
+}
+
+//export ooniffi_task_wait_for_next_event
+func ooniffi_task_wait_for_next_event(task *C.ooniffi_task_t) (event *C.ooniffi_event_t) {
+	if task != nil {
+		mu.Lock()
+		tp := m[task.Handle]
+		mu.Unlock()
+		if tp != nil {
+			event = (*C.ooniffi_event_t)(C.malloc(C.sizeof_ooniffi_event_t))
+			event.String = cstring(tp.WaitForNextEvent())
+		}
+	}
+	return
+}
+
+//export ooniffi_task_is_done
+func ooniffi_task_is_done(task *C.ooniffi_task_t) C.int {
+	var isdone C.int = 1
+	if task != nil {
+		mu.Lock()
+		if tp := m[task.Handle]; tp != nil && !tp.IsDone() {
+			isdone = 0
+		}
+		mu.Unlock()
+	}
+	return isdone
+}
+
+//export ooniffi_task_interrupt
+func ooniffi_task_interrupt(task *C.ooniffi_task_t) {
+	if task != nil {
+		mu.Lock()
+		if tp := m[task.Handle]; tp != nil {
+			tp.Interrupt()
+		}
+		mu.Unlock()
+	}
+}
+
+//export ooniffi_event_serialization_
+func ooniffi_event_serialization_(event *C.ooniffi_event_t) (s *C.char) {
+	if event != nil {
+		s = event.String
+	}
+	return
+}
+
+//export ooniffi_event_destroy
+func ooniffi_event_destroy(event *C.ooniffi_event_t) {
+	if event != nil {
+		C.free(unsafe.Pointer(event.String))
+		C.free(unsafe.Pointer(event))
+	}
+}
+
+//export ooniffi_task_destroy
+func ooniffi_task_destroy(task *C.ooniffi_task_t) {
+	if task != nil {
+		mu.Lock()
+		tp := m[task.Handle]
+		delete(m, task.Handle)
+		mu.Unlock()
+		C.free(unsafe.Pointer(task))
+		if tp != nil { // drain task if needed
+			tp.Interrupt()
+			go func() {
+				for !tp.IsDone() {
+					tp.WaitForNextEvent()
+				}
+			}()
+		}
+	}
+}
+
+func main() {}

internal/libooniffi/ooniffi.h

@@ -0,0 +1,51 @@
+#ifndef INCLUDE_OONIFFI_H_
+#define INCLUDE_OONIFFI_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+/*
+ * ABI compatible with Measurement Kit v0.10.11 [1].
+ *
+ * Just replace `mk_` with `ooniffi_` and recompile.
+ *
+ * .. [1] https://github.com/measurement-kit/measurement-kit/tree/v0.10.11/
+ * 
+ * This is not used in any OONI product. We may break something
+ * in ooniffi without noticing it. Please, be aware of that.
+ */
+
+typedef struct ooniffi_task_ ooniffi_task_t;
+typedef struct ooniffi_event_ ooniffi_event_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern ooniffi_task_t *ooniffi_task_start(const char *settings);
+extern ooniffi_event_t *ooniffi_task_wait_for_next_event(ooniffi_task_t *task);
+extern int ooniffi_task_is_done(ooniffi_task_t *task);
+extern void ooniffi_task_interrupt(ooniffi_task_t *task);
+extern const char *ooniffi_event_serialization(ooniffi_event_t *str);
+extern void ooniffi_event_destroy(ooniffi_event_t *str);
+extern void ooniffi_task_destroy(ooniffi_task_t *task);
+
+#ifdef __cplusplus
+}
+#endif
+
+/*
+ * Define OONIFFI_EMULATE_MK_API to provide a MK-compatible API at
+ * compile time that will map to ooniffi's own API.
+ */
+#ifdef OONIFFI_EMULATE_MK_API
+#define mk_task_start ooniffi_task_start
+#define mk_task_wait_for_next_event ooniffi_task_wait_for_next_event
+#define mk_task_is_done ooniffi_task_is_done
+#define mk_task_interrupt ooniffi_task_interrupt
+#define mk_event_serialization ooniffi_event_serialization
+#define mk_event_destroy ooniffi_event_destroy
+#define mk_task_destroy ooniffi_task_destroy
+#endif
+
+#endif /* INCLUDE_OONIFFI_H_ */

internal/libooniffi/ooniffi_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestTaskStartNullPointer(t *testing.T) {
+	if ooniffi_task_start_(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskStartInvalidJSON(t *testing.T) {
+	settings := cstring("{")
+	defer freestring(settings)
+	if ooniffi_task_start_(settings) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskStartIdxWrapping(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	o := setmaxidx()
+	// do twice and see if it's idempotent
+	if task := ooniffi_task_start_(settings); task != nil {
+		t.Fatal("expected nil task here")
+	}
+	if task := ooniffi_task_start_(settings); task != nil {
+		t.Fatal("expected nil task here")
+	}
+	restoreidx(o)
+}
+
+func TestTaskWaitForNextEventNullPointer(t *testing.T) {
+	if ooniffi_task_wait_for_next_event(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskIsDoneNullPointer(t *testing.T) {
+	if ooniffi_task_is_done(nil) == 0 {
+		t.Fatal("expected true-ish result here")
+	}
+}
+
+func TestTaskInterruptNullPointer(t *testing.T) {
+	ooniffi_task_interrupt(nil) // mainly: we don't crash :^)
+}
+
+func TestEventSerializationNullPointer(t *testing.T) {
+	if ooniffi_event_serialization_(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestEventDestroyNullPointer(t *testing.T) {
+	ooniffi_event_destroy(nil) // mainly: we don't crash
+}
+
+func TestTaskDestroyNullPointer(t *testing.T) {
+	ooniffi_task_destroy(nil) // mainly: we don't crash
+}
+
+func TestExampleNormalUsage(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	for ooniffi_task_is_done(task) == 0 {
+		event := ooniffi_task_wait_for_next_event(task)
+		t.Logf("%s", gostring(ooniffi_event_serialization_(event)))
+		ooniffi_event_destroy(event)
+	}
+	ooniffi_task_destroy(task)
+}
+
+func TestExampleInterruptAndDestroy(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	ooniffi_task_interrupt(task)
+	ooniffi_task_destroy(task)
+}
+
+func TestExampleDestroyImmediately(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	ooniffi_task_destroy(task)
+}

internal/libooniffi/testdata/ffirun.cpp

@@ -0,0 +1,43 @@
+#include <stdlib.h>
+
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <string>
+
+#define OONIFFI_EMULATE_MK_API
+#include "ooniffi.h"
+
+int main(int argc, char **argv) {
+  if (argc != 2) {
+    std::clog << "usage: ffirun /path/to/json/settings" << std::endl;
+    exit(1);
+  }
+  std::ifstream filep(argv[1]);
+  if (!filep.good()) {
+    std::clog << "fatal: cannot open settings file" << std::endl;
+    exit(1);
+  }
+  std::string settings((std::istreambuf_iterator<char>(filep)),
+      std::istreambuf_iterator<char>());
+  auto taskp = mk_task_start(settings.c_str());
+  if (taskp == nullptr) {
+    std::clog << "fatal: cannot start task" << std::endl;
+    exit(1);
+  }
+  while (!mk_task_is_done(taskp)) {
+    auto evp = mk_task_wait_for_next_event(taskp);
+    if (evp == nullptr) {
+      std::clog << "warning: cannot wait for next event" << std::endl;
+      break;
+    }
+    auto evstr = mk_event_serialization(evp);
+    if (evstr != nullptr) {
+      std::cout << evstr << std::endl;
+    } else {
+      std::clog << "warning: cannot get event serialization" << std::endl;
+    }
+    mk_event_destroy(evp);
+  }
+  mk_task_destroy(taskp);
+}

internal/libooniffi/testdata/webconnectivity.json

@@ -0,0 +1,17 @@
+{
+  "assets_dir": ".",
+  "inputs": [
+    "https://www.example.com",
+    "https://www.example.org"
+  ],
+  "name": "WebConnectivity",
+  "log_level": "INFO",
+  "options": {
+    "no_collector": true,
+    "software_name": "ooniffi",
+    "software_version": "0.1.0-dev"
+  },
+  "state_dir": ".",
+  "temp_dir": ".",
+  "version": 1
+}