fix(oonimkall): run tests with InputOrStaticDefault policy (#634)

Previous work to make https://github.com/ooni/probe/issues/1814
possible has broken running stunreachability on mobile.

This diff repairs the blunder and allows to run any experiment
using InputOrStaticDefault with oonimkall.

Diff extracted from https://github.com/ooni/probe-cli/pull/539.
This commit is contained in:
Simone Basso 2021-12-03 17:43:09 +01:00 committed by GitHub
parent 1896d2172a
commit dc9fbe9c64
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 299 additions and 236 deletions

View File

@ -183,10 +183,10 @@ var dnsCheckDefaultInput = []string{
var stunReachabilityDefaultInput = stuninput.AsnStunReachabilityInput()
// staticBareInputForExperiment returns the list of strings an
// StaticBareInputForExperiment returns the list of strings an
// experiment should use as static input. In case there is no
// static input for this experiment, we return an error.
func staticBareInputForExperiment(name string) ([]string, error) {
func StaticBareInputForExperiment(name string) ([]string, error) {
// Implementation note: we may be called from pkg/oonimkall
// with a non-canonical experiment name, so we need to convert
// the experiment name to be canonical before proceeding.
@ -203,7 +203,7 @@ func staticBareInputForExperiment(name string) ([]string, error) {
// staticInputForExperiment returns the static input for the given experiment
// or an error if there's no static input for the experiment.
func staticInputForExperiment(name string) ([]model.URLInfo, error) {
return stringListToModelURLInfo(staticBareInputForExperiment(name))
return stringListToModelURLInfo(StaticBareInputForExperiment(name))
}
// loadOrStaticDefault implements the InputOrStaticDefault policy.

View File

@ -154,12 +154,46 @@ func (r *runnerForTask) Run(ctx context.Context) {
})
builder.SetCallbacks(&runnerCallbacks{emitter: r.emitter})
if len(r.settings.Inputs) <= 0 {
switch builder.InputPolicy() {
case engine.InputOrQueryBackend, engine.InputStrictlyRequired:
// TODO(bassosimone): replace the following code with an
// invocation of the InputLoader. Since I am making these
// changes before a release and I've already changed the
// code a lot, I'd rather avoid changing it even more,
// for the following reason:
//
// If we add an call InputLoader here, this code will
// magically invoke check-in for InputOrQueryBackend,
// which we need to make sure the app can handle. This is
// the main reason why now I don't fill like properly
// fixing this code and use InputLoader: too much work
// in too little time, so mistakes more likely.
//
// In fact, our current app assumes that it's its
// responsibility to load the inputs, not oonimkall's.
switch builder.InputPolicy() {
case engine.InputOrQueryBackend, engine.InputStrictlyRequired:
if len(r.settings.Inputs) <= 0 {
r.emitter.EmitFailureStartup("no input provided")
return
}
case engine.InputOrStaticDefault:
if len(r.settings.Inputs) <= 0 {
inputs, err := engine.StaticBareInputForExperiment(r.settings.Name)
if err != nil {
r.emitter.EmitFailureStartup("no default static input for this experiment")
return
}
r.settings.Inputs = inputs
}
case engine.InputOptional:
if len(r.settings.Inputs) <= 0 {
r.settings.Inputs = append(r.settings.Inputs, "")
}
default: // treat this case as engine.InputNone.
if len(r.settings.Inputs) > 0 {
r.emitter.EmitFailureStartup("experiment does not accept input")
return
}
r.settings.Inputs = append(r.settings.Inputs, "")
}
experiment := builder.NewExperimentInstance()

View File

@ -255,19 +255,12 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeFailureStartup,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
@ -280,19 +273,12 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeFailureStartup,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
@ -305,35 +291,20 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 1,
}, {
Key: eventTypeFailureIPLookup,
Count: 1,
}, {
Key: eventTypeFailureASNLookup,
Count: 1,
}, {
Key: eventTypeFailureCCLookup,
Count: 1,
}, {
Key: eventTypeFailureResolverLookup,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeFailureIPLookup, Count: 1},
{Key: eventTypeFailureASNLookup, Count: 1},
{Key: eventTypeFailureCCLookup, Count: 1},
{Key: eventTypeFailureResolverLookup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with missing input and input or query backend", func(t *testing.T) {
t.Run("with missing input and InputOrQueryBackend policy", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
@ -342,32 +313,19 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeFailureStartup,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with missing input and input strictly required", func(t *testing.T) {
t.Run("with missing input and InputStrictlyRequired policy", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
@ -376,28 +334,61 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeFailureStartup,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run(
"with InputOrStaticDefault policy and experiment with no static input",
func(t *testing.T) {
runner, emitter := newRunnerForTesting()
runner.settings.Name = "Antani" // no input for this experiment
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputOrStaticDefault
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with InputNone policy and provided input", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
runner.settings.Inputs = append(runner.settings.Inputs, "https://x.org/")
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputNone
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeFailureStartup, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
@ -410,121 +401,72 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeFailureReportCreate,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeFailureReportCreate, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with success and no input", func(t *testing.T) {
t.Run("with success and InputNone policy", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputNone
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 1,
}, {
Key: eventTypeStatusReportCreate,
Count: 1,
}, {
Key: eventTypeStatusMeasurementStart,
Count: 1,
}, {
Key: eventTypeMeasurement,
Count: 1,
}, {
Key: eventTypeStatusMeasurementSubmission,
Count: 1,
}, {
Key: eventTypeStatusMeasurementDone,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with measurement failure and no input", func(t *testing.T) {
t.Run("with measurement failure and InputNone policy", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputNone
}
fake.MockableMeasureWithContext = func(ctx context.Context, input string) (measurement *model.Measurement, err error) {
return nil, errors.New("preconditions error")
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 1,
}, {
Key: eventTypeStatusReportCreate,
Count: 1,
}, {
Key: eventTypeStatusMeasurementStart,
Count: 1,
}, {
Key: eventTypeFailureMeasurement,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeFailureMeasurement, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with success and input", func(t *testing.T) {
t.Run("with success and InputStrictlyRequired", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
runner.settings.Inputs = []string{"a", "b", "c", "d"}
fake := fakeSuccessfulRun()
@ -572,6 +514,118 @@ func TestTaskRunnerRun(t *testing.T) {
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with success and InputOptional and input", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
runner.settings.Inputs = []string{"a", "b", "c", "d"}
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputOptional
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
//
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
//
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
//
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
//
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
//
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with success and InputOptional and no input", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputOptional
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
//
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
//
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with success and InputOrStaticDefault", func(t *testing.T) {
experimentName := "DNSCheck"
runner, emitter := newRunnerForTesting()
runner.settings.Name = experimentName
fake := fakeSuccessfulRun()
fake.MockableInputPolicy = func() engine.InputPolicy {
return engine.InputOrStaticDefault
}
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
}
allEntries, err := engine.StaticBareInputForExperiment(experimentName)
if err != nil {
t.Fatal(err)
}
// write the correct entries for each expected measurement.
for idx := 0; idx < len(allEntries); idx++ {
expect = append(expect, eventKeyCount{Key: eventTypeStatusMeasurementStart, Count: 1})
expect = append(expect, eventKeyCount{Key: eventTypeStatusProgress, Count: 1})
expect = append(expect, eventKeyCount{Key: eventTypeMeasurement, Count: 1})
expect = append(expect, eventKeyCount{Key: eventTypeStatusMeasurementSubmission, Count: 1})
expect = append(expect, eventKeyCount{Key: eventTypeStatusMeasurementDone, Count: 1})
}
expect = append(expect, eventKeyCount{Key: eventTypeStatusEnd, Count: 1})
assertReducedEventsLike(t, expect, reduced)
})
t.Run("with succes and max runtime", func(t *testing.T) {
runner, emitter := newRunnerForTesting()
runner.settings.Inputs = []string{"a", "b", "c", "d"}
@ -696,46 +750,21 @@ func TestTaskRunnerRun(t *testing.T) {
runner.sessionBuilder = fake
events := runAndCollect(runner, emitter)
reduced := reduceEventsKeysIgnoreLog(events)
expect := []eventKeyCount{{
Key: eventTypeStatusQueued,
Count: 1,
}, {
Key: eventTypeStatusStarted,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 3,
}, {
Key: eventTypeStatusGeoIPLookup,
Count: 1,
}, {
Key: eventTypeStatusResolverLookup,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 1,
}, {
Key: eventTypeStatusReportCreate,
Count: 1,
}, {
Key: eventTypeStatusMeasurementStart,
Count: 1,
}, {
Key: eventTypeStatusProgress,
Count: 1,
}, {
Key: eventTypeMeasurement,
Count: 1,
}, {
Key: eventTypeStatusMeasurementSubmission,
Count: 1,
}, {
Key: eventTypeStatusMeasurementDone,
Count: 1,
}, {
Key: eventTypeStatusEnd,
Count: 1,
}}
expect := []eventKeyCount{
{Key: eventTypeStatusQueued, Count: 1},
{Key: eventTypeStatusStarted, Count: 1},
{Key: eventTypeStatusProgress, Count: 3},
{Key: eventTypeStatusGeoIPLookup, Count: 1},
{Key: eventTypeStatusResolverLookup, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeStatusReportCreate, Count: 1},
{Key: eventTypeStatusMeasurementStart, Count: 1},
{Key: eventTypeStatusProgress, Count: 1},
{Key: eventTypeMeasurement, Count: 1},
{Key: eventTypeStatusMeasurementSubmission, Count: 1},
{Key: eventTypeStatusMeasurementDone, Count: 1},
{Key: eventTypeStatusEnd, Count: 1},
}
assertReducedEventsLike(t, expect, reduced)
})
}