ooni-probe-cli/cmd/ooniprobe/internal/cli/list/list.go
Arturo Filastò a884481b12
Refactor the list measurements function to make use of nested queries (#662)
* Refactor the list measurements function to make use of nested queries

With a dataset of 1489 test, the ooniprobe list command went from
taking 17.27s to run, to requiring 0.17s or a 100x speed boost

See https://github.com/ooni/probe/issues/1966

* Remove dead code from actions

* Improve the tests for the ListResults function

* Add test for AnomalyCount

* Add more documentation about the merging of the test_keys
2022-01-14 11:24:43 +01:00

141 lines
4.9 KiB
Go

package list
import (
"strings"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/cli/root"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/database"
"github.com/ooni/probe-cli/v3/cmd/ooniprobe/internal/output"
)
func init() {
cmd := root.Command("list", "List results")
resultID := cmd.Arg("id", "the id of the result to list measurements for").Int64()
cmd.Action(func(_ *kingpin.ParseContext) error {
probeCLI, err := root.Init()
if err != nil {
log.WithError(err).Error("failed to initialize root context")
return err
}
if *resultID > 0 {
measurements, err := database.ListMeasurements(probeCLI.DB(), *resultID)
if err != nil {
log.WithError(err).Error("failed to list measurements")
return err
}
msmtSummary := output.MeasurementSummaryData{
TotalCount: 0,
AnomalyCount: 0,
DataUsageUp: 0.0,
DataUsageDown: 0.0,
TotalRuntime: 0,
ASN: 0,
NetworkName: "",
NetworkCountryCode: "ZZ",
}
isFirst := true
isLast := false
for idx, msmt := range measurements {
if idx > 0 {
isFirst = false
}
if idx == len(measurements)-1 {
isLast = true
}
// We assume that since these are summary level information the first
// item will contain the information necessary.
if isFirst {
msmtSummary.TotalRuntime = msmt.Result.Runtime
msmtSummary.DataUsageUp = msmt.DataUsageUp
msmtSummary.DataUsageDown = msmt.DataUsageDown
msmtSummary.NetworkName = msmt.NetworkName
msmtSummary.NetworkCountryCode = msmt.Network.CountryCode
msmtSummary.ASN = msmt.ASN
msmtSummary.StartTime = msmt.Measurement.StartTime
}
if msmt.IsAnomaly.Bool == true {
msmtSummary.AnomalyCount++
}
msmtSummary.TotalCount++
output.MeasurementItem(msmt, isFirst, isLast)
}
output.MeasurementSummary(msmtSummary)
} else {
doneResults, incompleteResults, err := database.ListResults(probeCLI.DB())
if err != nil {
log.WithError(err).Error("failed to list results")
return err
}
if len(incompleteResults) > 0 {
output.SectionTitle("Incomplete results")
}
for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: result.Network.ASN,
MeasurementCount: 0,
MeasurementAnomalyCount: 0,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
Done: result.IsDone,
IsUploaded: result.IsUploaded,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
}
resultSummary := output.ResultSummaryData{}
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
testKeys := "{}"
// We only care to expose in the testKeys the value of the ndt test result
if result.TestGroupName == "performance" {
// The test_keys column are concanetated with the "|" character as a separator.
// We consider this to be safe since we only really care about values of the
// performance test_keys where the values are all numbers and none of the keys
// contain the "|" character.
for _, e := range strings.Split(result.TestKeys, "|") {
// We use the presence of the "download" key to indicate we have found the
// ndt test_keys, since the dash result does not contain it.
if strings.Contains(e, "download") {
testKeys = e
}
}
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: result.Network.ASN,
TestKeys: testKeys,
MeasurementCount: result.TotalCount,
MeasurementAnomalyCount: result.AnomalyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
resultSummary.TotalTests++
netCount[result.Network.ASN]++
resultSummary.TotalDataUsageUp += result.DataUsageUp
resultSummary.TotalDataUsageDown += result.DataUsageDown
}
resultSummary.TotalNetworks = int64(len(netCount))
output.ResultSummary(resultSummary)
}
return nil
})
}