diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 2b44ec9..17d1c17 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -41,7 +41,7 @@ CREATE TABLE `networks` ( -- this with more data in the future. `network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile - `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. + `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. -- The longest ip is an ipv6 address like: -- 0000:0000:0000:0000:0000:0000:0000:0000, -- which is 39 chars. diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index d06693b..66be3e4 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -41,19 +41,22 @@ func init() { output.SectionTitle("Incomplete results") } for idx, result := range incompleteResults { + output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, - Index: idx, - TotalCount: len(incompleteResults), - Name: result.TestGroupName, - StartTime: result.StartTime, - NetworkName: result.Network.NetworkName, - Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), - Summary: "{}", //result.Summary, - Done: result.IsDone, - DataUsageUp: result.DataUsageUp, - DataUsageDown: result.DataUsageDown, + ID: result.Result.ID, + Index: idx, + TotalCount: len(incompleteResults), + Name: result.TestGroupName, + StartTime: result.StartTime, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + MeasurementCount: 0, + MeasurementAnomalyCount: 0, + TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now + Done: result.IsDone, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, }) } @@ -61,16 +64,22 @@ func init() { netCount := make(map[uint]int) output.SectionTitle("Results") for idx, result := range doneResults { + totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID) + if err != nil { + log.WithError(err).Error("failed to list measurement counts") + } output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, - Index: idx, - TotalCount: len(doneResults), - Name: result.TestGroupName, - StartTime: result.StartTime, - NetworkName: result.Network.NetworkName, - Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), - Summary: "{}", //result.Summary, + ID: result.Result.ID, + Index: idx, + TotalCount: len(doneResults), + Name: result.TestGroupName, + StartTime: result.StartTime, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now + MeasurementCount: totalCount, + MeasurementAnomalyCount: anmlyCount, Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 88785c3..47333ff 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -73,7 +73,7 @@ func init() { } } - if err = result.Finished(ctx.DB, group.Summary); err != nil { + if err = result.Finished(ctx.DB); err != nil { return err } return nil diff --git a/internal/database/actions.go b/internal/database/actions.go index 6b9c93f..f17229d 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "encoding/json" "time" "github.com/apex/log" @@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR return measurements, nil } +// GetMeasurementCounts returns the number of anomalous and total measurement for a given result +func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) { + var ( + totalCount uint64 + anmlyCount uint64 + err error + ) + col := sess.Collection("measurements") + + // XXX these two queries can be done with a single query + totalCount, err = col.Find("result_id", resultID). + Count() + if err != nil { + log.WithError(err).Error("failed to get total count") + return totalCount, anmlyCount, err + } + + anmlyCount, err = col.Find("result_id", resultID). + And(db.Cond{"is_anomaly": true}).Count() + if err != nil { + log.WithError(err).Error("failed to get anmly count") + return totalCount, anmlyCount, err + } + + return totalCount, anmlyCount, err +} + // ListResults return the list of results func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) { doneResults := []ResultNetwork{} @@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string } urlID = lastID } + log.Debugf("returning url %d", urlID) return urlID, nil } + +// AddTestKeys writes the summary to the measurement +func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error { + tkBytes, err := json.Marshal(tk) + if err != nil { + log.WithError(err).Error("failed to serialize summary") + } + isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly + msmt.TestKeys = string(tkBytes) + msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true} + + err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt) + if err != nil { + return errors.Wrap(err, "updating measurement") + } + return nil +} diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 0da42c5..a060b40 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) { } } -func TestURLCreation(t *testing.T) { +func TestNetworkCreate(t *testing.T) { tmpfile, err := ioutil.TempFile("", "dbtest") if err != nil { t.Fatal(err) } defer os.Remove(tmpfile.Name()) - tmpdir, err := ioutil.TempDir("", "oonitest") + sess, err := Connect(tmpfile.Name()) if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpdir) + + l1 := utils.LocationInfo{ + ASN: 2, + CountryCode: "IT", + NetworkName: "Antaninet", + } + + l2 := utils.LocationInfo{ + ASN: 3, + CountryCode: "IT", + NetworkName: "Fufnet", + } + + _, err = CreateNetwork(sess, &l1) + if err != nil { + t.Fatal(err) + } + + _, err = CreateNetwork(sess, &l2) + if err != nil { + t.Fatal(err) + } + +} + +func TestURLCreation(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) sess, err := Connect(tmpfile.Name()) if err != nil { diff --git a/internal/database/models.go b/internal/database/models.go index 1327f63..a593f24 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -6,7 +6,6 @@ import ( "path/filepath" "time" - "github.com/ooni/probe-cli/nettests/summary" "github.com/pkg/errors" "upper.io/db.v3/lib/sqlbuilder" ) @@ -28,7 +27,7 @@ type MeasurementURLNetwork struct { // Network represents a network tested by the user type Network struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` NetworkName string `db:"network_name"` NetworkType string `db:"network_type"` IP string `db:"ip"` @@ -38,7 +37,7 @@ type Network struct { // URL represents URLs from the testing lists type URL struct { - ID sql.NullInt64 `db:"id"` + ID sql.NullInt64 `db:"id,omitempty"` URL sql.NullString `db:"url"` CategoryCode sql.NullString `db:"category_code"` CountryCode sql.NullString `db:"country_code"` @@ -46,7 +45,7 @@ type URL struct { // Measurement model type Measurement struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` TestName string `db:"test_name"` StartTime time.Time `db:"start_time"` Runtime float64 `db:"runtime"` // Fractional number of seconds @@ -69,7 +68,7 @@ type Measurement struct { // Result model type Result struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` TestGroupName string `db:"test_group_name"` StartTime time.Time `db:"start_time"` NetworkID int64 `db:"network_id"` // Used to include a Network @@ -82,7 +81,7 @@ type Result struct { } // Finished marks the result as done and sets the runtime -func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error { +func (r *Result) Finished(sess sqlbuilder.Database) error { if r.IsDone == true || r.Runtime != 0 { return errors.New("Result is already finished") } @@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error { return nil } -// WriteSummary writes the summary to the measurement -func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error { - // XXX remove m.Summary = summary - - err := sess.Collection("measurements").Find("id", m.ID).Update(m) - if err != nil { - return errors.Wrap(err, "updating measurement") - } - return nil -} - // AddToResult adds a measurement to a result func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error { var err error diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 630145b..d22398a 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -9,7 +9,6 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/internal/util" - "github.com/ooni/probe-cli/nettests/summary" ) func formatSpeed(speed int64) string { @@ -24,55 +23,51 @@ func formatSpeed(speed int64) string { return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) } -var summarizers = map[string]func(string) []string{ - "websites": func(ss string) []string { - var summary summary.WebsitesSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } +// PerformanceTestKeys is the result summary for a performance test +type PerformanceTestKeys struct { + Upload int64 `json:"upload"` + Download int64 `json:"download"` + Ping float64 `json:"ping"` + Bitrate int64 `json:"median_bitrate"` +} + +var summarizers = map[string]func(uint64, uint64, string) []string{ + "websites": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("%d tested", summary.Tested), - fmt.Sprintf("%d blocked", summary.Blocked), + fmt.Sprintf("%d tested", totalCount), + fmt.Sprintf("%d blocked", anomalyCount), "", } }, - "performance": func(ss string) []string { - var summary summary.PerformanceSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { + "performance": func(totalCount uint64, anomalyCount uint64, ss string) []string { + var tk PerformanceTestKeys + if err := json.Unmarshal([]byte(ss), &tk); err != nil { return nil } return []string{ - fmt.Sprintf("Download: %s", formatSpeed(summary.Download)), - fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)), - fmt.Sprintf("Ping: %.2fms", summary.Ping), + fmt.Sprintf("Download: %s", formatSpeed(tk.Download)), + fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)), + fmt.Sprintf("Ping: %.2fms", tk.Ping), } }, - "im": func(ss string) []string { - var summary summary.IMSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } + "im": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("%d tested", summary.Tested), - fmt.Sprintf("%d blocked", summary.Blocked), + fmt.Sprintf("%d tested", totalCount), + fmt.Sprintf("%d blocked", anomalyCount), "", } }, - "middlebox": func(ss string) []string { - var summary summary.MiddleboxSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } + "middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("Detected: %v", summary.Detected), + fmt.Sprintf("Detected: %v", anomalyCount > 0), "", "", } }, } -func makeSummary(name string, ss string) []string { - return summarizers[name](ss) +func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string { + return summarizers[name](totalCount, anomalyCount, ss) } func logResultItem(w io.Writer, f log.Fields) error { @@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error { fmt.Fprintf(w, "┃ "+firstRow+" ┃\n") fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n") - summary := makeSummary(name, f.Get("summary").(string)) + summary := makeSummary(name, + f.Get("measurement_count").(uint64), + f.Get("measurement_anomaly_count").(uint64), + f.Get("test_keys").(string)) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", util.RightPad(name, colWidth), diff --git a/internal/output/output.go b/internal/output/output.go index 2758fd8..7d6bf77 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -21,38 +21,42 @@ func Progress(key string, perc float64, msg string) { // ResultItemData is the metadata about a result type ResultItemData struct { - ID int64 - Name string - StartTime time.Time - Summary string - Runtime float64 - Country string - NetworkName string - ASN string - Done bool - DataUsageDown int64 - DataUsageUp int64 - Index int - TotalCount int + ID int64 + Name string + StartTime time.Time + TestKeys string + MeasurementCount uint64 + MeasurementAnomalyCount uint64 + Runtime float64 + Country string + NetworkName string + ASN string + Done bool + DataUsageDown int64 + DataUsageUp int64 + Index int + TotalCount int } // ResultItem logs a progress type event func ResultItem(result ResultItemData) { log.WithFields(log.Fields{ - "type": "result_item", - "id": result.ID, - "name": result.Name, - "start_time": result.StartTime, - "summary": result.Summary, - "country": result.Country, - "network_name": result.NetworkName, - "asn": result.ASN, - "runtime": result.Runtime, - "done": result.Done, - "data_usage_down": result.DataUsageDown, - "data_usage_up": result.DataUsageUp, - "index": result.Index, - "total_count": result.TotalCount, + "type": "result_item", + "id": result.ID, + "name": result.Name, + "start_time": result.StartTime, + "test_keys": result.TestKeys, + "measurement_count": result.MeasurementCount, + "measurement_anomaly_count": result.MeasurementAnomalyCount, + "country": result.Country, + "network_name": result.NetworkName, + "asn": result.ASN, + "runtime": result.Runtime, + "done": result.Done, + "data_usage_down": result.DataUsageDown, + "data_usage_up": result.DataUsageUp, + "index": result.Index, + "total_count": result.TotalCount, }).Info("result item") } diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 8b68dac..e209c55 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -1,14 +1,10 @@ package groups import ( - "encoding/json" - - "github.com/apex/log" "github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests/im" "github.com/ooni/probe-cli/nettests/middlebox" "github.com/ooni/probe-cli/nettests/performance" - "github.com/ooni/probe-cli/nettests/summary" "github.com/ooni/probe-cli/nettests/websites" ) @@ -16,7 +12,6 @@ import ( type NettestGroup struct { Label string Nettests []nettests.Nettest - Summary summary.ResultSummaryFunc } // NettestGroups that can be run by the user @@ -26,35 +21,6 @@ var NettestGroups = map[string]NettestGroup{ Nettests: []nettests.Nettest{ websites.WebConnectivity{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - // XXX to generate this I need to create the summary map as a list - var summary summary.WebsitesSummary - summary.Tested = 0 - summary.Blocked = 0 - for _, msmtSummaryStr := range m["WebConnectivity"] { - var wcSummary websites.WebConnectivitySummary - - err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal WebConnectivity summary") - return "", err - } - if wcSummary.Blocked { - summary.Blocked++ - } - summary.Tested++ - } - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "performance": NettestGroup{ Label: "Performance", @@ -62,38 +28,6 @@ var NettestGroups = map[string]NettestGroup{ performance.Dash{}, performance.NDT{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - var ( - err error - ndtSummary performance.NDTSummary - dashSummary performance.DashSummary - summary summary.PerformanceSummary - ) - err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal Dash summary") - return "", err - } - err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal NDT summary") - return "", err - } - summary.Bitrate = dashSummary.Bitrate - summary.Download = ndtSummary.Download - summary.Upload = ndtSummary.Upload - summary.Ping = ndtSummary.AvgRTT - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "middlebox": NettestGroup{ Label: "Middleboxes", @@ -101,35 +35,6 @@ var NettestGroups = map[string]NettestGroup{ middlebox.HTTPInvalidRequestLine{}, middlebox.HTTPHeaderFieldManipulation{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - var ( - err error - hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary - hirlSummary middlebox.HTTPInvalidRequestLineSummary - summary summary.MiddleboxSummary - ) - err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal hhfm summary") - return "", err - } - err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal hirl summary") - return "", err - } - summary.Detected = hirlSummary.Tampering == true || hhfmSummary.Tampering == true - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "im": NettestGroup{ Label: "Instant Messaging", @@ -138,52 +43,5 @@ var NettestGroups = map[string]NettestGroup{ im.Telegram{}, im.WhatsApp{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - var ( - err error - waSummary im.WhatsAppSummary - tgSummary im.TelegramSummary - fbSummary im.FacebookMessengerSummary - summary summary.IMSummary - ) - err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal whatsapp summary") - return "", err - } - err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal telegram summary") - return "", err - } - err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal facebook summary") - return "", err - } - // XXX it could actually be that some are not tested when the - // configuration is changed. - summary.Tested = 3 - summary.Blocked = 0 - if fbSummary.Blocked == true { - summary.Blocked++ - } - if tgSummary.Blocked == true { - summary.Blocked++ - } - if waSummary.Blocked == true { - summary.Blocked++ - } - - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, } diff --git a/nettests/im/facebook_messenger.go b/nettests/im/facebook_messenger.go index d6d97b2..4f7d716 100644 --- a/nettests/im/facebook_messenger.go +++ b/nettests/im/facebook_messenger.go @@ -16,15 +16,15 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error { return mknt.Run() } -// FacebookMessengerSummary for the test -type FacebookMessengerSummary struct { - DNSBlocking bool - TCPBlocking bool - Blocked bool +// FacebookMessengerTestKeys for the test +type FacebookMessengerTestKeys struct { + DNSBlocking bool `json:"facebook_dns_blocking"` + TCPBlocking bool `json:"facebook_tcp_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) interface{} { var ( dnsBlocking bool tcpBlocking bool @@ -41,10 +41,10 @@ func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { tcpBlocking = tk["facebook_tcp_blocking"].(bool) } - return FacebookMessengerSummary{ + return FacebookMessengerTestKeys{ DNSBlocking: dnsBlocking, TCPBlocking: tcpBlocking, - Blocked: dnsBlocking || tcpBlocking, + IsAnomaly: dnsBlocking || tcpBlocking, } } diff --git a/nettests/im/telegram.go b/nettests/im/telegram.go index 6c2984b..0ef4943 100644 --- a/nettests/im/telegram.go +++ b/nettests/im/telegram.go @@ -16,16 +16,16 @@ func (h Telegram) Run(ctl *nettests.Controller) error { return mknt.Run() } -// TelegramSummary for the test -type TelegramSummary struct { - HTTPBlocking bool - TCPBlocking bool - WebBlocking bool - Blocked bool +// TelegramTestKeys for the test +type TelegramTestKeys struct { + HTTPBlocking bool `json:"telegram_http_blocking"` + TCPBlocking bool `json:"telegram_tcp_blocking"` + WebBlocking bool `json:"telegram_web_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h Telegram) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h Telegram) GetTestKeys(tk map[string]interface{}) interface{} { var ( tcpBlocking bool httpBlocking bool @@ -48,11 +48,11 @@ func (h Telegram) Summary(tk map[string]interface{}) interface{} { webBlocking = tk["telegram_web_status"].(string) == "blocked" } - return TelegramSummary{ + return TelegramTestKeys{ TCPBlocking: tcpBlocking, HTTPBlocking: httpBlocking, WebBlocking: webBlocking, - Blocked: webBlocking || httpBlocking || tcpBlocking, + IsAnomaly: webBlocking || httpBlocking || tcpBlocking, } } diff --git a/nettests/im/whatsapp.go b/nettests/im/whatsapp.go index 36a18c3..19b2199 100644 --- a/nettests/im/whatsapp.go +++ b/nettests/im/whatsapp.go @@ -16,16 +16,16 @@ func (h WhatsApp) Run(ctl *nettests.Controller) error { return mknt.Run() } -// WhatsAppSummary for the test -type WhatsAppSummary struct { - RegistrationServerBlocking bool - WebBlocking bool - EndpointsBlocking bool - Blocked bool +// WhatsAppTestKeys for the test +type WhatsAppTestKeys struct { + RegistrationServerBlocking bool `json:"registration_server_blocking"` + WebBlocking bool `json:"whatsapp_web_blocking"` + EndpointsBlocking bool `json:"whatsapp_endpoints_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h WhatsApp) GetTestKeys(tk map[string]interface{}) interface{} { var ( webBlocking bool registrationBlocking bool @@ -46,11 +46,11 @@ func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { webBlocking = computeBlocking("whatsapp_web_status") endpointsBlocking = computeBlocking("whatsapp_endpoints_status") - return WhatsAppSummary{ + return WhatsAppTestKeys{ RegistrationServerBlocking: registrationBlocking, WebBlocking: webBlocking, EndpointsBlocking: endpointsBlocking, - Blocked: registrationBlocking || webBlocking || endpointsBlocking, + IsAnomaly: registrationBlocking || webBlocking || endpointsBlocking, } } diff --git a/nettests/middlebox/http_header_field_manipulation.go b/nettests/middlebox/http_header_field_manipulation.go index 822909d..f5df162 100644 --- a/nettests/middlebox/http_header_field_manipulation.go +++ b/nettests/middlebox/http_header_field_manipulation.go @@ -16,13 +16,13 @@ func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error { return mknt.Run() } -// HTTPHeaderFieldManipulationSummary for the test -type HTTPHeaderFieldManipulationSummary struct { - Tampering bool +// HTTPHeaderFieldManipulationTestKeys for the test +type HTTPHeaderFieldManipulationTestKeys struct { + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys returns a projection of the tests keys needed for the views +func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) interface{} { tampering := false for _, v := range tk["tampering"].(map[string]interface{}) { t, ok := v.(bool) @@ -32,8 +32,8 @@ func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interfac } } - return HTTPHeaderFieldManipulationSummary{ - Tampering: tampering, + return HTTPHeaderFieldManipulationTestKeys{ + IsAnomaly: tampering, } } diff --git a/nettests/middlebox/http_invalid_request_line.go b/nettests/middlebox/http_invalid_request_line.go index 947e64e..b815c94 100644 --- a/nettests/middlebox/http_invalid_request_line.go +++ b/nettests/middlebox/http_invalid_request_line.go @@ -16,17 +16,17 @@ func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error { return mknt.Run() } -// HTTPInvalidRequestLineSummary for the test -type HTTPInvalidRequestLineSummary struct { - Tampering bool +// HTTPInvalidRequestLineTestKeys for the test +type HTTPInvalidRequestLineTestKeys struct { + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h HTTPInvalidRequestLine) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) interface{} { tampering := tk["tampering"].(bool) - return HTTPInvalidRequestLineSummary{ - Tampering: tampering, + return HTTPInvalidRequestLineTestKeys{ + IsAnomaly: tampering, } } diff --git a/nettests/nettests.go b/nettests/nettests.go index 398b60f..5f4e242 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -20,7 +20,7 @@ import ( // Nettest interface. Every Nettest should implement this. type Nettest interface { Run(*Controller) error - Summary(map[string]interface{}) interface{} + GetTestKeys(map[string]interface{}) interface{} LogSummary(string) error } @@ -119,7 +119,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { IncludeIP: c.Ctx.Config.Sharing.IncludeIP, IncludeASN: c.Ctx.Config.Sharing.IncludeASN, IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, - LogLevel: "INFO", + LogLevel: "DEBUG", ProbeCC: c.Ctx.Location.CountryCode, ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN), @@ -139,18 +139,16 @@ func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) nt.On("log", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - level := e.Value.LogLevel msg := e.Value.Message switch level { case "ERROR": - log.Error(msg) + log.Errorf("%v: %s", color.RedString("mklog"), msg) case "INFO": - log.Info(msg) + log.Infof("%v: %s", color.BlueString("mklog"), msg) default: - log.Debug(msg) + log.Debugf("%v: %s", color.WhiteString("mklog"), msg) } }) @@ -285,6 +283,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { } }) + log.Debugf("Registered all the handlers") return nil } @@ -307,13 +306,13 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) { var entry Entry json.Unmarshal([]byte(jsonStr), &entry) - summary := c.nt.Summary(entry.TestKeys) - summaryBytes, err := json.Marshal(summary) - if err != nil { - log.WithError(err).Error("failed to serialize summary") - } + tk := c.nt.GetTestKeys(entry.TestKeys) + log.Debugf("Fetching: %s %v", idx, c.msmts[idx]) - c.msmts[idx].WriteSummary(c.Ctx.DB, string(summaryBytes)) + err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk) + if err != nil { + log.WithError(err).Error("failed to add test keys to summary") + } } // MKStart is the interface for the mk.Nettest Start() function diff --git a/nettests/performance/dash.go b/nettests/performance/dash.go index cced30c..3d63809 100644 --- a/nettests/performance/dash.go +++ b/nettests/performance/dash.go @@ -16,22 +16,24 @@ func (d Dash) Run(ctl *nettests.Controller) error { return dash.Run() } -// DashSummary for the test +// DashTestKeys for the test // TODO: process 'receiver_data' to provide an array of performance for a chart. -type DashSummary struct { - Latency float64 - Bitrate int64 - Delay float64 +type DashTestKeys struct { + Latency float64 `json:"connect_latency"` + Bitrate int64 `json:"median_bitrate"` + Delay float64 `json:"min_playout_delay"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (d Dash) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (d Dash) GetTestKeys(tk map[string]interface{}) interface{} { simple := tk["simple"].(map[string]interface{}) - return DashSummary{ - Latency: simple["connect_latency"].(float64), - Bitrate: int64(simple["median_bitrate"].(float64)), - Delay: simple["min_playout_delay"].(float64), + return DashTestKeys{ + IsAnomaly: false, + Latency: simple["connect_latency"].(float64), + Bitrate: int64(simple["median_bitrate"].(float64)), + Delay: simple["min_playout_delay"].(float64), } } diff --git a/nettests/performance/ndt.go b/nettests/performance/ndt.go index c9e8098..590b779 100644 --- a/nettests/performance/ndt.go +++ b/nettests/performance/ndt.go @@ -16,26 +16,27 @@ func (n NDT) Run(ctl *nettests.Controller) error { return nt.Run() } -// NDTSummary for the test -type NDTSummary struct { - Upload int64 - Download int64 - Ping int64 - MaxRTT float64 - AvgRTT float64 - MinRTT float64 - MSS int64 - OutOfOrder int64 - PacketLoss float64 - Timeouts int64 +// NDTTestKeys for the test +type NDTTestKeys struct { + Upload int64 `json:"upload"` + Download int64 `json:"download"` + Ping int64 `json:"ping"` + MaxRTT float64 `json:"max_rtt"` + AvgRTT float64 `json:"avg_rtt"` + MinRTT float64 `json:"min_rtt"` + MSS int64 `json:"mss"` + OutOfOrder int64 `json:"out_of_order"` + PacketLoss float64 `json:"packet_loss"` + Timeouts int64 `json:"timeouts"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (n NDT) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (n NDT) GetTestKeys(tk map[string]interface{}) interface{} { simple := tk["simple"].(map[string]interface{}) advanced := tk["advanced"].(map[string]interface{}) - return NDTSummary{ + return NDTTestKeys{ Upload: int64(simple["upload"].(float64)), Download: int64(simple["download"].(float64)), Ping: int64(simple["ping"].(float64)), diff --git a/nettests/summary/summary.go b/nettests/summary/summary.go deleted file mode 100644 index f225927..0000000 --- a/nettests/summary/summary.go +++ /dev/null @@ -1,44 +0,0 @@ -package summary - -import "fmt" - -// ResultSummaryFunc is the function used to generate result summaries -type ResultSummaryFunc func(SummaryMap) (string, error) - -// SummaryMap contains a mapping from test name to serialized summary for it -type SummaryMap map[string][]string - -// PerformanceSummary is the result summary for a performance test -type PerformanceSummary struct { - Upload int64 - Download int64 - Ping float64 - Bitrate int64 -} - -// MiddleboxSummary is the summary for the middlebox tests -type MiddleboxSummary struct { - Detected bool -} - -// IMSummary is the summary for the im tests -type IMSummary struct { - Tested uint - Blocked uint -} - -// WebsitesSummary is the summary for the websites test -type WebsitesSummary struct { - Tested uint - Blocked uint -} - -func CheckRequiredKeys(rk []string, m SummaryMap) error { - for _, key := range rk { - if _, ok := m[key]; ok { - continue - } - return fmt.Errorf("missing SummaryMap key '%s'", key) - } - return nil -} diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 32a7bf0..6427bd4 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -29,10 +29,11 @@ const orchestrateBaseURL = "https://events.proteus.test.ooni.io" func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { var ( - parsed = new(URLResponse) - urls []string - urlIDMap map[int64]int64 + parsed = new(URLResponse) + urls []string ) + urlIDMap := make(map[int64]int64) + log.Debug("Looking up URLs") // XXX pass in the configuration for category codes reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", orchestrateBaseURL, @@ -53,6 +54,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { } for idx, url := range parsed.Results { + log.Debugf("Going over URL %d", idx) urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode) if err != nil { log.Error("failed to add to the URL table") @@ -82,15 +84,15 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error { return nt.Run() } -// WebConnectivitySummary for the test -type WebConnectivitySummary struct { - Accessible bool - Blocking string - Blocked bool +// WebConnectivityTestKeys for the test +type WebConnectivityTestKeys struct { + Accessible bool `json:"accessible"` + Blocking string `json:"blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) interface{} { var ( blocked bool blocking string @@ -117,10 +119,10 @@ func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { accessible = tk["accessible"].(bool) } - return WebConnectivitySummary{ + return WebConnectivityTestKeys{ Accessible: accessible, Blocking: blocking, - Blocked: blocked, + IsAnomaly: blocked, } }