chore: merge probe-engine into probe-cli (#201)

This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335

internal/engine/libooniffi/.gitignore

@@ -0,0 +1,8 @@
+/asn.mmdb
+/ca-bundle.pem
+/country.mmdb
+/ffirun
+/ffirun.exe
+/libooniffi.dll
+/libooniffi.h
+/libooniffi.so

internal/engine/libooniffi/README.md

@@ -0,0 +1,16 @@
+# Directory github.com/ooni/probe-engine/libooniffi
+
+This directory contains code to generate shared/static libraries with
+a Measurement Kit compatible ABI. To this end, we wrap the [oonimkall](
+../oonimkall) API with a simple C API.
+
+The generated libraries have a Measurement Kit compatible ABI. You can
+also instruct your compiler so that [ooniffi.h](ooniffi.h) defines macros
+that make code written for Measurement Kit compile and work. To this
+end, please see comments insider [ooniffi.h](ooniffi.h).
+
+To see how we compile this library for several systems, please take a
+look at [libooniffi.yml](../.github/workflows/libooniffi.yml).
+
+This is not used in any OONI product. We may break something
+in ooniffi without noticing it. Please, be aware of that.

internal/engine/libooniffi/buildtest.bash

@@ -0,0 +1,26 @@
+#!/bin/bash
+set -e
+cd $(dirname $0) 
+case $1 in
+  darwin)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.so .
+    clang++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun -looniffi ./testdata/ffirun.cpp
+    ./ffirun testdata/webconnectivity.json
+    ;;
+  linux)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.so .
+    g++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun ./testdata/ffirun.cpp -looniffi
+    LD_LIBRARY_PATH=. ./ffirun testdata/webconnectivity.json
+    ;;
+  windows)
+    set -x
+    go build -ldflags '-s -w' -buildmode=c-shared -o libooniffi.dll .
+    x86_64-w64-mingw32-g++ -std=c++11 -Wall -Wextra -I. -L. -o ffirun.exe -looniffi ./testdata/ffirun.cpp
+    ./ffirun.exe testdata/webconnectivity.json
+    ;;
+  *)
+    echo "usage: $0 darwin|linux|windows" 1>&2
+    exit 1
+esac

internal/engine/libooniffi/ooniffi.c

@@ -0,0 +1,17 @@
+#include "ooniffi.h"
+
+#include "_cgo_export.h"
+
+ooniffi_task_t *ooniffi_task_start(const char *settings) {
+    /* Implementation note: Go does not have the concept of const but
+       we know that the code is just making a copy of settings. */
+    return ooniffi_task_start_((char *)settings);
+}
+
+const char *ooniffi_event_serialization(ooniffi_event_t *event) {
+    /* Implementation note: Go does not have the concept of const but
+       we want to return const to very clearly communicate that the
+       returned string is owned by the event. This is what tools like
+       python's ctypes and SWIG expect from us. */
+    return (const char *)ooniffi_event_serialization_(event);
+}

internal/engine/libooniffi/ooniffi.def

@@ -0,0 +1,7 @@
+EXPORTS
+ooniffi_task_start
+ooniffi_task_is_done
+ooniffi_task_wait_for_next_event
+ooniffi_event_serialization
+ooniffi_event_destroy
+ooniffi_task_destroy

internal/engine/libooniffi/ooniffi.go

@@ -0,0 +1,152 @@
+package main
+
+import (
+	//#include "ooniffi.h"
+	//
+	//#include <stdint.h>
+	//#include <stdlib.h>
+	//
+	//struct ooniffi_task_ {
+	//    int64_t Handle;
+	//};
+	//
+	// struct ooniffi_event_ {
+	//    char *String;
+	//};
+	"C"
+	"sync"
+	"unsafe"
+
+	"github.com/ooni/probe-cli/v3/internal/engine/oonimkall"
+)
+
+var (
+	idx C.int64_t
+	m   = make(map[C.int64_t]*oonimkall.Task)
+	mu  sync.Mutex
+)
+
+func cstring(s string) *C.char {
+	return C.CString(s)
+}
+
+func freestring(s *C.char) {
+	C.free(unsafe.Pointer(s))
+}
+
+func gostring(s *C.char) string {
+	return C.GoString(s)
+}
+
+const maxIdx = C.INT64_MAX
+
+//export ooniffi_task_start_
+func ooniffi_task_start_(settings *C.char) *C.ooniffi_task_t {
+	if settings == nil {
+		return nil
+	}
+	tp, err := oonimkall.StartTask(gostring(settings))
+	if err != nil {
+		return nil
+	}
+	mu.Lock()
+	defer mu.Unlock()
+	// TODO(bassosimone): the following if is basic protection against
+	// undefined behaviour, i.e., the counter wrapping around. A much
+	// better strategy would probably be to restart from 0. However it's
+	// also unclear if any device could run that many tests, so...
+	if idx >= maxIdx {
+		return nil
+	}
+	handle := idx
+	idx++
+	m[handle] = tp
+	task := (*C.ooniffi_task_t)(C.malloc(C.sizeof_ooniffi_task_t))
+	task.Handle = handle
+	return task
+}
+
+func setmaxidx() C.int64_t {
+	o := idx
+	idx = maxIdx
+	return o
+}
+
+func restoreidx(v C.int64_t) {
+	idx = v
+}
+
+//export ooniffi_task_wait_for_next_event
+func ooniffi_task_wait_for_next_event(task *C.ooniffi_task_t) (event *C.ooniffi_event_t) {
+	if task != nil {
+		mu.Lock()
+		tp := m[task.Handle]
+		mu.Unlock()
+		if tp != nil {
+			event = (*C.ooniffi_event_t)(C.malloc(C.sizeof_ooniffi_event_t))
+			event.String = cstring(tp.WaitForNextEvent())
+		}
+	}
+	return
+}
+
+//export ooniffi_task_is_done
+func ooniffi_task_is_done(task *C.ooniffi_task_t) C.int {
+	var isdone C.int = 1
+	if task != nil {
+		mu.Lock()
+		if tp := m[task.Handle]; tp != nil && !tp.IsDone() {
+			isdone = 0
+		}
+		mu.Unlock()
+	}
+	return isdone
+}
+
+//export ooniffi_task_interrupt
+func ooniffi_task_interrupt(task *C.ooniffi_task_t) {
+	if task != nil {
+		mu.Lock()
+		if tp := m[task.Handle]; tp != nil {
+			tp.Interrupt()
+		}
+		mu.Unlock()
+	}
+}
+
+//export ooniffi_event_serialization_
+func ooniffi_event_serialization_(event *C.ooniffi_event_t) (s *C.char) {
+	if event != nil {
+		s = event.String
+	}
+	return
+}
+
+//export ooniffi_event_destroy
+func ooniffi_event_destroy(event *C.ooniffi_event_t) {
+	if event != nil {
+		C.free(unsafe.Pointer(event.String))
+		C.free(unsafe.Pointer(event))
+	}
+}
+
+//export ooniffi_task_destroy
+func ooniffi_task_destroy(task *C.ooniffi_task_t) {
+	if task != nil {
+		mu.Lock()
+		tp := m[task.Handle]
+		delete(m, task.Handle)
+		mu.Unlock()
+		C.free(unsafe.Pointer(task))
+		if tp != nil { // drain task if needed
+			tp.Interrupt()
+			go func() {
+				for !tp.IsDone() {
+					tp.WaitForNextEvent()
+				}
+			}()
+		}
+	}
+}
+
+func main() {}

internal/engine/libooniffi/ooniffi.h

@@ -0,0 +1,51 @@
+#ifndef INCLUDE_OONIFFI_H_
+#define INCLUDE_OONIFFI_H_
+
+#include <stdint.h>
+#include <stdlib.h>
+
+/*
+ * ABI compatible with Measurement Kit v0.10.11 [1].
+ *
+ * Just replace `mk_` with `ooniffi_` and recompile.
+ *
+ * .. [1] https://github.com/measurement-kit/measurement-kit/tree/v0.10.11/
+ * 
+ * This is not used in any OONI product. We may break something
+ * in ooniffi without noticing it. Please, be aware of that.
+ */
+
+typedef struct ooniffi_task_ ooniffi_task_t;
+typedef struct ooniffi_event_ ooniffi_event_t;
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+extern ooniffi_task_t *ooniffi_task_start(const char *settings);
+extern ooniffi_event_t *ooniffi_task_wait_for_next_event(ooniffi_task_t *task);
+extern int ooniffi_task_is_done(ooniffi_task_t *task);
+extern void ooniffi_task_interrupt(ooniffi_task_t *task);
+extern const char *ooniffi_event_serialization(ooniffi_event_t *str);
+extern void ooniffi_event_destroy(ooniffi_event_t *str);
+extern void ooniffi_task_destroy(ooniffi_task_t *task);
+
+#ifdef __cplusplus
+}
+#endif
+
+/*
+ * Define OONIFFI_EMULATE_MK_API to provide a MK-compatible API at
+ * compile time that will map to ooniffi's own API.
+ */
+#ifdef OONIFFI_EMULATE_MK_API
+#define mk_task_start ooniffi_task_start
+#define mk_task_wait_for_next_event ooniffi_task_wait_for_next_event
+#define mk_task_is_done ooniffi_task_is_done
+#define mk_task_interrupt ooniffi_task_interrupt
+#define mk_event_serialization ooniffi_event_serialization
+#define mk_event_destroy ooniffi_event_destroy
+#define mk_task_destroy ooniffi_task_destroy
+#endif
+
+#endif /* INCLUDE_OONIFFI_H_ */

internal/engine/libooniffi/ooniffi_test.go

@@ -0,0 +1,139 @@
+package main
+
+import (
+	"testing"
+)
+
+func TestTaskStartNullPointer(t *testing.T) {
+	if ooniffi_task_start_(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskStartInvalidJSON(t *testing.T) {
+	settings := cstring("{")
+	defer freestring(settings)
+	if ooniffi_task_start_(settings) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskStartIdxWrapping(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	o := setmaxidx()
+	// do twice and see if it's idempotent
+	if task := ooniffi_task_start_(settings); task != nil {
+		t.Fatal("expected nil task here")
+	}
+	if task := ooniffi_task_start_(settings); task != nil {
+		t.Fatal("expected nil task here")
+	}
+	restoreidx(o)
+}
+
+func TestTaskWaitForNextEventNullPointer(t *testing.T) {
+	if ooniffi_task_wait_for_next_event(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestTaskIsDoneNullPointer(t *testing.T) {
+	if ooniffi_task_is_done(nil) == 0 {
+		t.Fatal("expected true-ish result here")
+	}
+}
+
+func TestTaskInterruptNullPointer(t *testing.T) {
+	ooniffi_task_interrupt(nil) // mainly: we don't crash :^)
+}
+
+func TestEventSerializationNullPointer(t *testing.T) {
+	if ooniffi_event_serialization_(nil) != nil {
+		t.Fatal("expected nil result here")
+	}
+}
+
+func TestEventDestroyNullPointer(t *testing.T) {
+	ooniffi_event_destroy(nil) // mainly: we don't crash
+}
+
+func TestTaskDestroyNullPointer(t *testing.T) {
+	ooniffi_task_destroy(nil) // mainly: we don't crash
+}
+
+func TestExampleNormalUsage(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	for ooniffi_task_is_done(task) == 0 {
+		event := ooniffi_task_wait_for_next_event(task)
+		t.Logf("%s", gostring(ooniffi_event_serialization_(event)))
+		ooniffi_event_destroy(event)
+	}
+	ooniffi_task_destroy(task)
+}
+
+func TestExampleInterruptAndDestroy(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	ooniffi_task_interrupt(task)
+	ooniffi_task_destroy(task)
+}
+
+func TestExampleDestroyImmediately(t *testing.T) {
+	settings := cstring(`{
+		"assets_dir": "../testdata/oonimkall/assets",
+		"log_level": "DEBUG",
+		"name": "Example",
+		"options": {
+			"software_name": "oonimkall-test",
+			"software_version": "0.1.0"
+		},
+		"state_dir": "../testdata/oonimkall/state",
+		"temp_dir": "../testdata/oonimkall/tmp"
+	}`)
+	defer freestring(settings)
+	task := ooniffi_task_start_(settings)
+	if task == nil {
+		t.Fatal("expected non-nil task here")
+	}
+	ooniffi_task_destroy(task)
+}

internal/engine/libooniffi/testdata/ffirun.cpp

@@ -0,0 +1,43 @@
+#include <stdlib.h>
+
+#include <fstream>
+#include <iostream>
+#include <iterator>
+#include <string>
+
+#define OONIFFI_EMULATE_MK_API
+#include "ooniffi.h"
+
+int main(int argc, char **argv) {
+  if (argc != 2) {
+    std::clog << "usage: ffirun /path/to/json/settings" << std::endl;
+    exit(1);
+  }
+  std::ifstream filep(argv[1]);
+  if (!filep.good()) {
+    std::clog << "fatal: cannot open settings file" << std::endl;
+    exit(1);
+  }
+  std::string settings((std::istreambuf_iterator<char>(filep)),
+      std::istreambuf_iterator<char>());
+  auto taskp = mk_task_start(settings.c_str());
+  if (taskp == nullptr) {
+    std::clog << "fatal: cannot start task" << std::endl;
+    exit(1);
+  }
+  while (!mk_task_is_done(taskp)) {
+    auto evp = mk_task_wait_for_next_event(taskp);
+    if (evp == nullptr) {
+      std::clog << "warning: cannot wait for next event" << std::endl;
+      break;
+    }
+    auto evstr = mk_event_serialization(evp);
+    if (evstr != nullptr) {
+      std::cout << evstr << std::endl;
+    } else {
+      std::clog << "warning: cannot get event serialization" << std::endl;
+    }
+    mk_event_destroy(evp);
+  }
+  mk_task_destroy(taskp);
+}

internal/engine/libooniffi/testdata/webconnectivity.json

@@ -0,0 +1,17 @@
+{
+  "assets_dir": ".",
+  "inputs": [
+    "https://www.example.com",
+    "https://www.example.org"
+  ],
+  "name": "WebConnectivity",
+  "log_level": "INFO",
+  "options": {
+    "no_collector": true,
+    "software_name": "ooniffi",
+    "software_version": "0.1.0-dev"
+  },
+  "state_dir": ".",
+  "temp_dir": ".",
+  "version": 1
+}