b7cc309901
This diff re-implements the vanilla_tor experiment. This experiment was part of the ooni/probe-legacy implementation. The reference issue is https://github.com/ooni/probe/issues/803. We didn't consider the possible improvements mentioned by the https://github.com/ooni/probe/issues/803#issuecomment-598715694 comment, which means we'll need to create a follow-up issue for them. We will then decide whether, when, and how to implement those follow-up measurements either into `vanilla_tor` or into the existing `tor` experiment. This novel `vanilla_tor` implementation emits test_keys that are mostly compatible with the original implementation, however: 1. the `timeout` is a `float64` rather than integer (but the default timeout is an integer, so there are no JSON-visible changes); 2. the `tor_log` string is gone and replaced by the `tor_logs` list of strings, which contains the same information; 3. the definition of `error` has been augmented to include the case in which there is an unknown error; 4. the implementation of vanilla_tor mirrors closely the one of torsf and we have taken steps to make the two implementations as comparable as possible in terms of the generated JSON measurement. The main reason why we replaced `tor_log` with `tor_logs` are: 1. that `torsf` already used that; 2. that reading the JSON is easier with this implementation compared to an implementation where all logs are into the same string. If one is processing the new data format using Python, then it will not be difficult convert `tor_log` to `tor_logs`. In any case, because we extract the most interesting fields (e.g., the percentage of the bootstrap where tor fails), it seems that logs are probably more useful as something you want to read in edge cases (I guess). Also, because we want `torsf` and `vanilla_tor` to have similar JSONs, we renamed `torsf`'s `default_timeout` to `timeout`. This change has little to none real-world impact, because no stable version of OONI Probe has ever shipped a `torsf` producing the `default_timeout` field. Regarding the structure of this diff, we have: 1. factored code to parse tor logs into a separate package; 2. implemented `vanilla_tor` as a stripped down `torsf` and added further changes to ensure compatibility with the previous `vanilla_tor`'s data format; 3. improved `torsf` to merge back the changes in `vanilla_tor`, so the two data formats of the two experiments are as similar as possible. We believe producing as similar as possible data formats helps anyone who's reading measurements generated by both experiments. We have retained/introduced `vanilla_tor`'s `error` field, which is not very useful when one has a more precise failure but is still what `vanilla_tor` used to emit, so it makes sense to also have this field. In addition to changing the implementation, we also updated the specs. As part of our future work, we may want to consider factoring the common code of these two experiments into the same underlying support library.
106 lines
3.2 KiB
Go
106 lines
3.2 KiB
Go
// Package torlogs contains code to read tor logs.
|
|
package torlogs
|
|
|
|
import (
|
|
"bytes"
|
|
"errors"
|
|
"fmt"
|
|
"os"
|
|
"regexp"
|
|
"strconv"
|
|
|
|
"github.com/ooni/probe-cli/v3/internal/model"
|
|
)
|
|
|
|
var (
|
|
// ErrEmptyLogFilePath indicates that the log file path is empty.
|
|
ErrEmptyLogFilePath = errors.New("torlogs: empty log file path")
|
|
|
|
// ErrCannotReadLogFile indicates we cannot read the log file.
|
|
ErrCannotReadLogFile = errors.New("torlogs: cannot read the log file")
|
|
|
|
// ErrNoBootstrapLogs indicates we could not find any bootstrap log in the log file.
|
|
ErrNoBootstrapLogs = errors.New("torlogs: no bootstrap logs")
|
|
|
|
// ErrCannotFindSubmatches indicates we cannot find submatches.
|
|
ErrCannotFindSubmatches = errors.New("torlogs: cannot find submatches")
|
|
)
|
|
|
|
// torBootstrapRegexp helps to extract progress info from logs.
|
|
//
|
|
// See https://regex101.com/r/Do07qd/1.
|
|
var torBootstrapRegexp = regexp.MustCompile(
|
|
`^[A-Za-z0-9.: ]+ \[notice\] Bootstrapped ([0-9]+)% \(([A-Za-z_]+)\): ([A-Za-z0-9 ]+)$`)
|
|
|
|
// ReadBootstrapLogs reads tor logs from the given file and
|
|
// returns a list of bootstrap-related logs.
|
|
func ReadBootstrapLogs(logFilePath string) ([]string, error) {
|
|
// Implementation note:
|
|
//
|
|
// Tor is know to be good software that does not break its output
|
|
// unnecessarily and that does not include PII into its logs unless
|
|
// explicitly asked to. This fact gives me confidence that we can
|
|
// safely include this subset of the logs into the results.
|
|
//
|
|
// On this note, I think it's safe to include timestamps from the
|
|
// logs into the output, since we have a timestamp for the whole
|
|
// experiment already, so we don't leak much more by also including
|
|
// the Tor proper timestamps into the results.
|
|
if logFilePath == "" {
|
|
return nil, ErrEmptyLogFilePath
|
|
}
|
|
data, err := os.ReadFile(logFilePath)
|
|
if err != nil {
|
|
return nil, fmt.Errorf("%w: %s", ErrCannotReadLogFile, err.Error())
|
|
}
|
|
var out []string
|
|
for _, bline := range bytes.Split(data, []byte("\n")) {
|
|
if torBootstrapRegexp.Match(bline) {
|
|
out = append(out, string(bline))
|
|
}
|
|
}
|
|
if len(out) <= 0 {
|
|
return nil, ErrNoBootstrapLogs
|
|
}
|
|
return out, nil
|
|
}
|
|
|
|
// ReadBootstrapLogsOrWarn is like ReadBootstrapLogs except that it does
|
|
// not return an error on failure, rather it emits a warning.
|
|
func ReadBootstrapLogsOrWarn(logger model.Logger, logFilePath string) []string {
|
|
logs, err := ReadBootstrapLogs(logFilePath)
|
|
if err != nil {
|
|
logger.Warnf("%s", err.Error())
|
|
return nil
|
|
}
|
|
return logs
|
|
}
|
|
|
|
// BootstrapInfo contains info extracted from a bootstrap log line.
|
|
type BootstrapInfo struct {
|
|
// Progress is the progress (between 0 and 100)
|
|
Progress int64
|
|
|
|
// Tag is the machine readable description of the bootstrap state.
|
|
Tag string
|
|
|
|
// Summary is the human readable summary.
|
|
Summary string
|
|
}
|
|
|
|
// ParseBootstrapLogLine takes in input a bootstrap log line and returns
|
|
// in output the components of such a log line.
|
|
func ParseBootstrapLogLine(logLine string) (*BootstrapInfo, error) {
|
|
values := torBootstrapRegexp.FindStringSubmatch(logLine)
|
|
if len(values) != 4 {
|
|
return nil, ErrCannotFindSubmatches
|
|
}
|
|
progress, _ := strconv.ParseInt(values[1], 10, 64)
|
|
bi := &BootstrapInfo{
|
|
Progress: progress,
|
|
Tag: values[2],
|
|
Summary: values[3],
|
|
}
|
|
return bi, nil
|
|
}
|