c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace summary with test_keys

Browse Source
This commit is contained in:
Arturo Filastò
2018-09-10 12:41:28 +02:00
parent b727aba854
commit b29071f37b
19 changed files with 274 additions and 381 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/cli/list/list.go
+30 -21
View File
@@ -41,19 +41,22 @@ func init() {
output.SectionTitle("Incomplete results")
}
for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
MeasurementCount: 0,
MeasurementAnomalyCount: 0,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
}
@@ -61,16 +64,22 @@ func init() {
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
internal/cli/run/run.go
+1 -1
View File
@@ -73,7 +73,7 @@ func init() {
}
}
if err = result.Finished(ctx.DB, group.Summary); err != nil {
if err = result.Finished(ctx.DB); err != nil {
return err
}
return nil
internal/database/actions.go
+46
View File
@@ -2,6 +2,7 @@ package database
import (
"database/sql"
"encoding/json"
"time"
"github.com/apex/log"
@@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR
return measurements, nil
}
// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
var (
totalCount uint64
anmlyCount uint64
err error
)
col := sess.Collection("measurements")
// XXX these two queries can be done with a single query
totalCount, err = col.Find("result_id", resultID).
Count()
if err != nil {
log.WithError(err).Error("failed to get total count")
return totalCount, anmlyCount, err
}
anmlyCount, err = col.Find("result_id", resultID).
And(db.Cond{"is_anomaly": true}).Count()
if err != nil {
log.WithError(err).Error("failed to get anmly count")
return totalCount, anmlyCount, err
}
return totalCount, anmlyCount, err
}
// ListResults return the list of results
func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
doneResults := []ResultNetwork{}
@@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string
}
urlID = lastID
}
log.Debugf("returning url %d", urlID)
return urlID, nil
}
// AddTestKeys writes the summary to the measurement
func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
tkBytes, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly
msmt.TestKeys = string(tkBytes)
msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true}
err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
internal/database/actions_test.go
+33 -3
View File
@@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) {
}
}
func TestURLCreation(t *testing.T) {
func TestNetworkCreate(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
l1 := utils.LocationInfo{
ASN: 2,
CountryCode: "IT",
NetworkName: "Antaninet",
}
l2 := utils.LocationInfo{
ASN: 3,
CountryCode: "IT",
NetworkName: "Fufnet",
}
_, err = CreateNetwork(sess, &l1)
if err != nil {
t.Fatal(err)
}
_, err = CreateNetwork(sess, &l2)
if err != nil {
t.Fatal(err)
}
}
func TestURLCreation(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {
internal/database/models.go
+5 -17
View File
@@ -6,7 +6,6 @@ import (
"path/filepath"
"time"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
@@ -28,7 +27,7 @@ type MeasurementURLNetwork struct {
// Network represents a network tested by the user
type Network struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
NetworkName string `db:"network_name"`
NetworkType string `db:"network_type"`
IP string `db:"ip"`
@@ -38,7 +37,7 @@ type Network struct {
// URL represents URLs from the testing lists
type URL struct {
ID sql.NullInt64 `db:"id"`
ID sql.NullInt64 `db:"id,omitempty"`
URL sql.NullString `db:"url"`
CategoryCode sql.NullString `db:"category_code"`
CountryCode sql.NullString `db:"country_code"`
@@ -46,7 +45,7 @@ type URL struct {
// Measurement model
type Measurement struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestName string `db:"test_name"`
StartTime time.Time `db:"start_time"`
Runtime float64 `db:"runtime"` // Fractional number of seconds
@@ -69,7 +68,7 @@ type Measurement struct {
// Result model
type Result struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestGroupName string `db:"test_group_name"`
StartTime time.Time `db:"start_time"`
NetworkID int64 `db:"network_id"` // Used to include a Network
@@ -82,7 +81,7 @@ type Result struct {
}
// Finished marks the result as done and sets the runtime
func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error {
func (r *Result) Finished(sess sqlbuilder.Database) error {
if r.IsDone == true || r.Runtime != 0 {
return errors.New("Result is already finished")
}
@@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
return nil
}
// WriteSummary writes the summary to the measurement
func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error {
// XXX remove m.Summary = summary
err := sess.Collection("measurements").Find("id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// AddToResult adds a measurement to a result
func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error {
var err error
internal/log/handlers/cli/results.go
+29 -31
View File
@@ -9,7 +9,6 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/util"
"github.com/ooni/probe-cli/nettests/summary"
)
func formatSpeed(speed int64) string {
@@ -24,55 +23,51 @@ func formatSpeed(speed int64) string {
return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
}
var summarizers = map[string]func(string) []string{
"websites": func(ss string) []string {
var summary summary.WebsitesSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
// PerformanceTestKeys is the result summary for a performance test
type PerformanceTestKeys struct {
Upload int64 `json:"upload"`
Download int64 `json:"download"`
Ping float64 `json:"ping"`
Bitrate int64 `json:"median_bitrate"`
}
var summarizers = map[string]func(uint64, uint64, string) []string{
"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"performance": func(ss string) []string {
var summary summary.PerformanceSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
"performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var tk PerformanceTestKeys
if err := json.Unmarshal([]byte(ss), &tk); err != nil {
return nil
}
return []string{
fmt.Sprintf("Download: %s", formatSpeed(summary.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)),
fmt.Sprintf("Ping: %.2fms", summary.Ping),
fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
fmt.Sprintf("Ping: %.2fms", tk.Ping),
}
},
"im": func(ss string) []string {
var summary summary.IMSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"middlebox": func(ss string) []string {
var summary summary.MiddleboxSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("Detected: %v", summary.Detected),
fmt.Sprintf("Detected: %v", anomalyCount > 0),
"",
"",
}
},
}
func makeSummary(name string, ss string) []string {
return summarizers[name](ss)
func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
return summarizers[name](totalCount, anomalyCount, ss)
}
func logResultItem(w io.Writer, f log.Fields) error {
@@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error {
fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
summary := makeSummary(name, f.Get("summary").(string))
summary := makeSummary(name,
f.Get("measurement_count").(uint64),
f.Get("measurement_anomaly_count").(uint64),
f.Get("test_keys").(string))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
util.RightPad(name, colWidth),
internal/output/output.go
+31 -27
View File
@@ -21,38 +21,42 @@ func Progress(key string, perc float64, msg string) {
// ResultItemData is the metadata about a result
type ResultItemData struct {
ID int64
Name string
StartTime time.Time
Summary string
Runtime float64
Country string
NetworkName string
ASN string
Done bool
DataUsageDown int64
DataUsageUp int64
Index int
TotalCount int
ID int64
Name string
StartTime time.Time
TestKeys string
MeasurementCount uint64
MeasurementAnomalyCount uint64
Runtime float64
Country string
NetworkName string
ASN string
Done bool
DataUsageDown int64
DataUsageUp int64
Index int
TotalCount int
}
// ResultItem logs a progress type event
func ResultItem(result ResultItemData) {
log.WithFields(log.Fields{
"type": "result_item",
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"summary": result.Summary,
"country": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,
"runtime": result.Runtime,
"done": result.Done,
"data_usage_down": result.DataUsageDown,
"data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
"type": "result_item",
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"test_keys": result.TestKeys,
"measurement_count": result.MeasurementCount,
"measurement_anomaly_count": result.MeasurementAnomalyCount,
"country": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,
"runtime": result.Runtime,
"done": result.Done,
"data_usage_down": result.DataUsageDown,
"data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
}).Info("result item")
}