Refactor the list measurements function to make use of nested queries (#662)

* Refactor the list measurements function to make use of nested queries

With a dataset of 1489 test, the ooniprobe list command went from
taking 17.27s to run, to requiring 0.17s or a 100x speed boost

See https://github.com/ooni/probe/issues/1966

* Remove dead code from actions

* Improve the tests for the ListResults function

* Add test for AnomalyCount

* Add more documentation about the merging of the test_keys
This commit is contained in:
Arturo Filastò
2022-01-14 11:24:43 +01:00
committed by GitHub
parent b5da8be183
commit a884481b12
4 changed files with 86 additions and 80 deletions
+20 -9
View File
@@ -1,6 +1,8 @@
package list
import (
"strings"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
@@ -92,14 +94,23 @@ func init() {
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(probeCLI.DB(), result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
testKeys, err := database.GetResultTestKeys(probeCLI.DB(), result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to get testKeys")
testKeys := "{}"
// We only care to expose in the testKeys the value of the ndt test result
if result.TestGroupName == "performance" {
// The test_keys column are concanetated with the "|" character as a separator.
// We consider this to be safe since we only really care about values of the
// performance test_keys where the values are all numbers and none of the keys
// contain the "|" character.
for _, e := range strings.Split(result.TestKeys, "|") {
// We use the presence of the "download" key to indicate we have found the
// ndt test_keys, since the dash result does not contain it.
if strings.Contains(e, "download") {
testKeys = e
}
}
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
@@ -110,8 +121,8 @@ func init() {
Country: result.Network.CountryCode,
ASN: result.Network.ASN,
TestKeys: testKeys,
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
MeasurementCount: result.TotalCount,
MeasurementAnomalyCount: result.AnomalyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,