c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace summary with test_keys

Browse Source
This commit is contained in:
Arturo Filastò 2018-09-10 12:41:28 +02:00
parent b727aba854
commit b29071f37b
19 changed files with 274 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/migrations/1_create_msmt_results.sql
View File

@ -41,7 +41,7 @@ CREATE TABLE `networks` (
-- this with more data in the future. -- this with more data in the future.
`network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile `network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile
`ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address.
-- The longest ip is an ipv6 address like: -- The longest ip is an ipv6 address like:
-- 0000:0000:0000:0000:0000:0000:0000:0000, -- 0000:0000:0000:0000:0000:0000:0000:0000,
-- which is 39 chars. -- which is 39 chars.

51
internal/cli/list/list.go
View File

@ -41,19 +41,22 @@ func init() {
output.SectionTitle("Incomplete results") output.SectionTitle("Incomplete results")
} }
for idx, result := range incompleteResults { for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{ output.ResultItem(output.ResultItemData{
ID: result.Result.ID, ID: result.Result.ID,
Index: idx, Index: idx,
TotalCount: len(incompleteResults), TotalCount: len(incompleteResults),
Name: result.TestGroupName, Name: result.TestGroupName,
StartTime: result.StartTime, StartTime: result.StartTime,
NetworkName: result.Network.NetworkName, NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode, Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN), ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary, MeasurementCount: 0,
Done: result.IsDone, MeasurementAnomalyCount: 0,
DataUsageUp: result.DataUsageUp, TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
DataUsageDown: result.DataUsageDown, Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
}) })
} }
@ -61,16 +64,22 @@ func init() {
netCount := make(map[uint]int) netCount := make(map[uint]int)
output.SectionTitle("Results") output.SectionTitle("Results")
for idx, result := range doneResults { for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
output.ResultItem(output.ResultItemData{ output.ResultItem(output.ResultItemData{
ID: result.Result.ID, ID: result.Result.ID,
Index: idx, Index: idx,
TotalCount: len(doneResults), TotalCount: len(doneResults),
Name: result.TestGroupName, Name: result.TestGroupName,
StartTime: result.StartTime, StartTime: result.StartTime,
NetworkName: result.Network.NetworkName, NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode, Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN), ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary, TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
Done: result.IsDone, Done: result.IsDone,
DataUsageUp: result.DataUsageUp, DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown, DataUsageDown: result.DataUsageDown,

2
internal/cli/run/run.go
View File

@ -73,7 +73,7 @@ func init() {
} }
} }
if err = result.Finished(ctx.DB, group.Summary); err != nil { if err = result.Finished(ctx.DB); err != nil {
return err return err
} }
return nil return nil

46
internal/database/actions.go
View File

@ -2,6 +2,7 @@ package database
import ( import (
"database/sql" "database/sql"
"encoding/json"
"time" "time"
"github.com/apex/log" "github.com/apex/log"
@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR
return measurements, nil return measurements, nil
} }
// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
var (
totalCount uint64
anmlyCount uint64
err error
)
col := sess.Collection("measurements")
// XXX these two queries can be done with a single query
totalCount, err = col.Find("result_id", resultID).
Count()
if err != nil {
log.WithError(err).Error("failed to get total count")
return totalCount, anmlyCount, err
}
anmlyCount, err = col.Find("result_id", resultID).
And(db.Cond{"is_anomaly": true}).Count()
if err != nil {
log.WithError(err).Error("failed to get anmly count")
return totalCount, anmlyCount, err
}
return totalCount, anmlyCount, err
}
// ListResults return the list of results // ListResults return the list of results
func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) { func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
doneResults := []ResultNetwork{} doneResults := []ResultNetwork{}
@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string
} }
urlID = lastID urlID = lastID
} }
log.Debugf("returning url %d", urlID)
return urlID, nil return urlID, nil
} }
// AddTestKeys writes the summary to the measurement
func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
tkBytes, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly
msmt.TestKeys = string(tkBytes)
msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true}
err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}

36
internal/database/actions_test.go
View File

@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) {
} }
} }
func TestURLCreation(t *testing.T) { func TestNetworkCreate(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest") tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer os.Remove(tmpfile.Name()) defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest") sess, err := Connect(tmpfile.Name())
if err != nil { if err != nil {
t.Fatal(err) t.Fatal(err)
} }
defer os.RemoveAll(tmpdir)
l1 := utils.LocationInfo{
ASN: 2,
CountryCode: "IT",
NetworkName: "Antaninet",
}
l2 := utils.LocationInfo{
ASN: 3,
CountryCode: "IT",
NetworkName: "Fufnet",
}
_, err = CreateNetwork(sess, &l1)
if err != nil {
t.Fatal(err)
}
_, err = CreateNetwork(sess, &l2)
if err != nil {
t.Fatal(err)
}
}
func TestURLCreation(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name()) sess, err := Connect(tmpfile.Name())
if err != nil { if err != nil {

22
internal/database/models.go
View File

@ -6,7 +6,6 @@ import (
"path/filepath" "path/filepath"
"time" "time"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/pkg/errors" "github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder" "upper.io/db.v3/lib/sqlbuilder"
) )
@ -28,7 +27,7 @@ type MeasurementURLNetwork struct {
// Network represents a network tested by the user // Network represents a network tested by the user
type Network struct { type Network struct {
ID int64 `db:"id"` ID int64 `db:"id,omitempty"`
NetworkName string `db:"network_name"` NetworkName string `db:"network_name"`
NetworkType string `db:"network_type"` NetworkType string `db:"network_type"`
IP string `db:"ip"` IP string `db:"ip"`
@ -38,7 +37,7 @@ type Network struct {
// URL represents URLs from the testing lists // URL represents URLs from the testing lists
type URL struct { type URL struct {
ID sql.NullInt64 `db:"id"` ID sql.NullInt64 `db:"id,omitempty"`
URL sql.NullString `db:"url"` URL sql.NullString `db:"url"`
CategoryCode sql.NullString `db:"category_code"` CategoryCode sql.NullString `db:"category_code"`
CountryCode sql.NullString `db:"country_code"` CountryCode sql.NullString `db:"country_code"`
@ -46,7 +45,7 @@ type URL struct {
// Measurement model // Measurement model
type Measurement struct { type Measurement struct {
ID int64 `db:"id"` ID int64 `db:"id,omitempty"`
TestName string `db:"test_name"` TestName string `db:"test_name"`
StartTime time.Time `db:"start_time"` StartTime time.Time `db:"start_time"`
Runtime float64 `db:"runtime"` // Fractional number of seconds Runtime float64 `db:"runtime"` // Fractional number of seconds
@ -69,7 +68,7 @@ type Measurement struct {
// Result model // Result model
type Result struct { type Result struct {
ID int64 `db:"id"` ID int64 `db:"id,omitempty"`
TestGroupName string `db:"test_group_name"` TestGroupName string `db:"test_group_name"`
StartTime time.Time `db:"start_time"` StartTime time.Time `db:"start_time"`
NetworkID int64 `db:"network_id"` // Used to include a Network NetworkID int64 `db:"network_id"` // Used to include a Network
@ -82,7 +81,7 @@ type Result struct {
} }
// Finished marks the result as done and sets the runtime // Finished marks the result as done and sets the runtime
func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error { func (r *Result) Finished(sess sqlbuilder.Database) error {
if r.IsDone == true || r.Runtime != 0 { if r.IsDone == true || r.Runtime != 0 {
return errors.New("Result is already finished") return errors.New("Result is already finished")
} }
@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
return nil return nil
} }
// WriteSummary writes the summary to the measurement
func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error {
// XXX remove m.Summary = summary
err := sess.Collection("measurements").Find("id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// AddToResult adds a measurement to a result // AddToResult adds a measurement to a result
func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error { func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error {
var err error var err error

60
internal/log/handlers/cli/results.go
View File

@ -9,7 +9,6 @@ import (
"github.com/apex/log" "github.com/apex/log"
"github.com/ooni/probe-cli/internal/util" "github.com/ooni/probe-cli/internal/util"
"github.com/ooni/probe-cli/nettests/summary"
) )
func formatSpeed(speed int64) string { func formatSpeed(speed int64) string {
@ -24,55 +23,51 @@ func formatSpeed(speed int64) string {
return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
} }
var summarizers = map[string]func(string) []string{ // PerformanceTestKeys is the result summary for a performance test
"websites": func(ss string) []string { type PerformanceTestKeys struct {
var summary summary.WebsitesSummary Upload int64 `json:"upload"`
if err := json.Unmarshal([]byte(ss), &summary); err != nil { Download int64 `json:"download"`
return nil Ping float64 `json:"ping"`
} Bitrate int64 `json:"median_bitrate"`
}
var summarizers = map[string]func(uint64, uint64, string) []string{
"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{ return []string{
fmt.Sprintf("%d tested", summary.Tested), fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", summary.Blocked), fmt.Sprintf("%d blocked", anomalyCount),
"", "",
} }
}, },
"performance": func(ss string) []string { "performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var summary summary.PerformanceSummary var tk PerformanceTestKeys
if err := json.Unmarshal([]byte(ss), &summary); err != nil { if err := json.Unmarshal([]byte(ss), &tk); err != nil {
return nil return nil
} }
return []string{ return []string{
fmt.Sprintf("Download: %s", formatSpeed(summary.Download)), fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)), fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
fmt.Sprintf("Ping: %.2fms", summary.Ping), fmt.Sprintf("Ping: %.2fms", tk.Ping),
} }
}, },
"im": func(ss string) []string { "im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var summary summary.IMSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
return []string{ return []string{
fmt.Sprintf("%d tested", summary.Tested), fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", summary.Blocked), fmt.Sprintf("%d blocked", anomalyCount),
"", "",
} }
}, },
"middlebox": func(ss string) []string { "middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var summary summary.MiddleboxSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
return []string{ return []string{
fmt.Sprintf("Detected: %v", summary.Detected), fmt.Sprintf("Detected: %v", anomalyCount > 0),
"", "",
"", "",
} }
}, },
} }
func makeSummary(name string, ss string) []string { func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
return summarizers[name](ss) return summarizers[name](totalCount, anomalyCount, ss)
} }
func logResultItem(w io.Writer, f log.Fields) error { func logResultItem(w io.Writer, f log.Fields) error {
@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error {
fmt.Fprintf(w, "┃ "+firstRow+" ┃\n") fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n") fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
summary := makeSummary(name, f.Get("summary").(string)) summary := makeSummary(name,
f.Get("measurement_count").(uint64),
f.Get("measurement_anomaly_count").(uint64),
f.Get("test_keys").(string))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
util.RightPad(name, colWidth), util.RightPad(name, colWidth),

58
internal/output/output.go
View File

@ -21,38 +21,42 @@ func Progress(key string, perc float64, msg string) {
// ResultItemData is the metadata about a result // ResultItemData is the metadata about a result
type ResultItemData struct { type ResultItemData struct {
ID int64 ID int64
Name string Name string
StartTime time.Time StartTime time.Time
Summary string TestKeys string
Runtime float64 MeasurementCount uint64
Country string MeasurementAnomalyCount uint64
NetworkName string Runtime float64
ASN string Country string
Done bool NetworkName string
DataUsageDown int64 ASN string
DataUsageUp int64 Done bool
Index int DataUsageDown int64
TotalCount int DataUsageUp int64
Index int
TotalCount int
} }
// ResultItem logs a progress type event // ResultItem logs a progress type event
func ResultItem(result ResultItemData) { func ResultItem(result ResultItemData) {
log.WithFields(log.Fields{ log.WithFields(log.Fields{
"type": "result_item", "type": "result_item",
"id": result.ID, "id": result.ID,
"name": result.Name, "name": result.Name,
"start_time": result.StartTime, "start_time": result.StartTime,
"summary": result.Summary, "test_keys": result.TestKeys,
"country": result.Country, "measurement_count": result.MeasurementCount,
"network_name": result.NetworkName, "measurement_anomaly_count": result.MeasurementAnomalyCount,
"asn": result.ASN, "country": result.Country,
"runtime": result.Runtime, "network_name": result.NetworkName,
"done": result.Done, "asn": result.ASN,
"data_usage_down": result.DataUsageDown, "runtime": result.Runtime,
"data_usage_up": result.DataUsageUp, "done": result.Done,
"index": result.Index, "data_usage_down": result.DataUsageDown,
"total_count": result.TotalCount, "data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
}).Info("result item") }).Info("result item")
} }

142
nettests/groups/groups.go
View File

@ -1,14 +1,10 @@
package groups package groups
import ( import (
"encoding/json"
"github.com/apex/log"
"github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests"
"github.com/ooni/probe-cli/nettests/im" "github.com/ooni/probe-cli/nettests/im"
"github.com/ooni/probe-cli/nettests/middlebox" "github.com/ooni/probe-cli/nettests/middlebox"
"github.com/ooni/probe-cli/nettests/performance" "github.com/ooni/probe-cli/nettests/performance"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/ooni/probe-cli/nettests/websites" "github.com/ooni/probe-cli/nettests/websites"
) )
@ -16,7 +12,6 @@ import (
type NettestGroup struct { type NettestGroup struct {
Label string Label string
Nettests []nettests.Nettest Nettests []nettests.Nettest
Summary summary.ResultSummaryFunc
} }
// NettestGroups that can be run by the user // NettestGroups that can be run by the user
@ -26,35 +21,6 @@ var NettestGroups = map[string]NettestGroup{
Nettests: []nettests.Nettest{ Nettests: []nettests.Nettest{
websites.WebConnectivity{}, websites.WebConnectivity{},
}, },
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
// XXX to generate this I need to create the summary map as a list
var summary summary.WebsitesSummary
summary.Tested = 0
summary.Blocked = 0
for _, msmtSummaryStr := range m["WebConnectivity"] {
var wcSummary websites.WebConnectivitySummary
err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal WebConnectivity summary")
return "", err
}
if wcSummary.Blocked {
summary.Blocked++
}
summary.Tested++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
}, },
"performance": NettestGroup{ "performance": NettestGroup{
Label: "Performance", Label: "Performance",
@ -62,38 +28,6 @@ var NettestGroups = map[string]NettestGroup{
performance.Dash{}, performance.Dash{},
performance.NDT{}, performance.NDT{},
}, },
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
ndtSummary performance.NDTSummary
dashSummary performance.DashSummary
summary summary.PerformanceSummary
)
err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal Dash summary")
return "", err
}
err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal NDT summary")
return "", err
}
summary.Bitrate = dashSummary.Bitrate
summary.Download = ndtSummary.Download
summary.Upload = ndtSummary.Upload
summary.Ping = ndtSummary.AvgRTT
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
}, },
"middlebox": NettestGroup{ "middlebox": NettestGroup{
Label: "Middleboxes", Label: "Middleboxes",
@ -101,35 +35,6 @@ var NettestGroups = map[string]NettestGroup{
middlebox.HTTPInvalidRequestLine{}, middlebox.HTTPInvalidRequestLine{},
middlebox.HTTPHeaderFieldManipulation{}, middlebox.HTTPHeaderFieldManipulation{},
}, },
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary
hirlSummary middlebox.HTTPInvalidRequestLineSummary
summary summary.MiddleboxSummary
)
err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hhfm summary")
return "", err
}
err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hirl summary")
return "", err
}
summary.Detected = hirlSummary.Tampering == true || hhfmSummary.Tampering == true
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
}, },
"im": NettestGroup{ "im": NettestGroup{
Label: "Instant Messaging", Label: "Instant Messaging",
@ -138,52 +43,5 @@ var NettestGroups = map[string]NettestGroup{
im.Telegram{}, im.Telegram{},
im.WhatsApp{}, im.WhatsApp{},
}, },
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
waSummary im.WhatsAppSummary
tgSummary im.TelegramSummary
fbSummary im.FacebookMessengerSummary
summary summary.IMSummary
)
err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal whatsapp summary")
return "", err
}
err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal telegram summary")
return "", err
}
err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal facebook summary")
return "", err
}
// XXX it could actually be that some are not tested when the
// configuration is changed.
summary.Tested = 3
summary.Blocked = 0
if fbSummary.Blocked == true {
summary.Blocked++
}
if tgSummary.Blocked == true {
summary.Blocked++
}
if waSummary.Blocked == true {
summary.Blocked++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
}, },
} }

18
nettests/im/facebook_messenger.go
View File

@ -16,15 +16,15 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
return mknt.Run() return mknt.Run()
} }
// FacebookMessengerSummary for the test // FacebookMessengerTestKeys for the test
type FacebookMessengerSummary struct { type FacebookMessengerTestKeys struct {
DNSBlocking bool DNSBlocking bool `json:"facebook_dns_blocking"`
TCPBlocking bool TCPBlocking bool `json:"facebook_tcp_blocking"`
Blocked bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) interface{} {
var ( var (
dnsBlocking bool dnsBlocking bool
tcpBlocking bool tcpBlocking bool
@ -41,10 +41,10 @@ func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
tcpBlocking = tk["facebook_tcp_blocking"].(bool) tcpBlocking = tk["facebook_tcp_blocking"].(bool)
} }
return FacebookMessengerSummary{ return FacebookMessengerTestKeys{
DNSBlocking: dnsBlocking, DNSBlocking: dnsBlocking,
TCPBlocking: tcpBlocking, TCPBlocking: tcpBlocking,
Blocked: dnsBlocking || tcpBlocking, IsAnomaly: dnsBlocking || tcpBlocking,
} }
} }

20
nettests/im/telegram.go
View File

@ -16,16 +16,16 @@ func (h Telegram) Run(ctl *nettests.Controller) error {
return mknt.Run() return mknt.Run()
} }
// TelegramSummary for the test // TelegramTestKeys for the test
type TelegramSummary struct { type TelegramTestKeys struct {
HTTPBlocking bool HTTPBlocking bool `json:"telegram_http_blocking"`
TCPBlocking bool TCPBlocking bool `json:"telegram_tcp_blocking"`
WebBlocking bool WebBlocking bool `json:"telegram_web_blocking"`
Blocked bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (h Telegram) Summary(tk map[string]interface{}) interface{} { func (h Telegram) GetTestKeys(tk map[string]interface{}) interface{} {
var ( var (
tcpBlocking bool tcpBlocking bool
httpBlocking bool httpBlocking bool
@ -48,11 +48,11 @@ func (h Telegram) Summary(tk map[string]interface{}) interface{} {
webBlocking = tk["telegram_web_status"].(string) == "blocked" webBlocking = tk["telegram_web_status"].(string) == "blocked"
} }
return TelegramSummary{ return TelegramTestKeys{
TCPBlocking: tcpBlocking, TCPBlocking: tcpBlocking,
HTTPBlocking: httpBlocking, HTTPBlocking: httpBlocking,
WebBlocking: webBlocking, WebBlocking: webBlocking,
Blocked: webBlocking || httpBlocking || tcpBlocking, IsAnomaly: webBlocking || httpBlocking || tcpBlocking,
} }
} }

20
nettests/im/whatsapp.go
View File

@ -16,16 +16,16 @@ func (h WhatsApp) Run(ctl *nettests.Controller) error {
return mknt.Run() return mknt.Run()
} }
// WhatsAppSummary for the test // WhatsAppTestKeys for the test
type WhatsAppSummary struct { type WhatsAppTestKeys struct {
RegistrationServerBlocking bool RegistrationServerBlocking bool `json:"registration_server_blocking"`
WebBlocking bool WebBlocking bool `json:"whatsapp_web_blocking"`
EndpointsBlocking bool EndpointsBlocking bool `json:"whatsapp_endpoints_blocking"`
Blocked bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { func (h WhatsApp) GetTestKeys(tk map[string]interface{}) interface{} {
var ( var (
webBlocking bool webBlocking bool
registrationBlocking bool registrationBlocking bool
@ -46,11 +46,11 @@ func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
webBlocking = computeBlocking("whatsapp_web_status") webBlocking = computeBlocking("whatsapp_web_status")
endpointsBlocking = computeBlocking("whatsapp_endpoints_status") endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
return WhatsAppSummary{ return WhatsAppTestKeys{
RegistrationServerBlocking: registrationBlocking, RegistrationServerBlocking: registrationBlocking,
WebBlocking: webBlocking, WebBlocking: webBlocking,
EndpointsBlocking: endpointsBlocking, EndpointsBlocking: endpointsBlocking,
Blocked: registrationBlocking || webBlocking || endpointsBlocking, IsAnomaly: registrationBlocking || webBlocking || endpointsBlocking,
} }
} }

14
nettests/middlebox/http_header_field_manipulation.go
View File

@ -16,13 +16,13 @@ func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error {
return mknt.Run() return mknt.Run()
} }
// HTTPHeaderFieldManipulationSummary for the test // HTTPHeaderFieldManipulationTestKeys for the test
type HTTPHeaderFieldManipulationSummary struct { type HTTPHeaderFieldManipulationTestKeys struct {
Tampering bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys returns a projection of the tests keys needed for the views
func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interface{} { func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := false tampering := false
for _, v := range tk["tampering"].(map[string]interface{}) { for _, v := range tk["tampering"].(map[string]interface{}) {
t, ok := v.(bool) t, ok := v.(bool)
@ -32,8 +32,8 @@ func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interfac
} }
} }
return HTTPHeaderFieldManipulationSummary{ return HTTPHeaderFieldManipulationTestKeys{
Tampering: tampering, IsAnomaly: tampering,
} }
} }

14
nettests/middlebox/http_invalid_request_line.go
View File

@ -16,17 +16,17 @@ func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error {
return mknt.Run() return mknt.Run()
} }
// HTTPInvalidRequestLineSummary for the test // HTTPInvalidRequestLineTestKeys for the test
type HTTPInvalidRequestLineSummary struct { type HTTPInvalidRequestLineTestKeys struct {
Tampering bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (h HTTPInvalidRequestLine) Summary(tk map[string]interface{}) interface{} { func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := tk["tampering"].(bool) tampering := tk["tampering"].(bool)
return HTTPInvalidRequestLineSummary{ return HTTPInvalidRequestLineTestKeys{
Tampering: tampering, IsAnomaly: tampering,
} }
} }

25
nettests/nettests.go
View File

@ -20,7 +20,7 @@ import (
// Nettest interface. Every Nettest should implement this. // Nettest interface. Every Nettest should implement this.
type Nettest interface { type Nettest interface {
Run(*Controller) error Run(*Controller) error
Summary(map[string]interface{}) interface{} GetTestKeys(map[string]interface{}) interface{}
LogSummary(string) error LogSummary(string) error
} }
@ -119,7 +119,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
IncludeIP: c.Ctx.Config.Sharing.IncludeIP, IncludeIP: c.Ctx.Config.Sharing.IncludeIP,
IncludeASN: c.Ctx.Config.Sharing.IncludeASN, IncludeASN: c.Ctx.Config.Sharing.IncludeASN,
IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
LogLevel: "INFO", LogLevel: "DEBUG",
ProbeCC: c.Ctx.Location.CountryCode, ProbeCC: c.Ctx.Location.CountryCode,
ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN), ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
@ -139,18 +139,16 @@ func (c *Controller) Init(nt *mk.Nettest) error {
log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath)
nt.On("log", func(e mk.Event) { nt.On("log", func(e mk.Event) {
log.Debugf(color.RedString(e.Key))
level := e.Value.LogLevel level := e.Value.LogLevel
msg := e.Value.Message msg := e.Value.Message
switch level { switch level {
case "ERROR": case "ERROR":
log.Error(msg) log.Errorf("%v: %s", color.RedString("mklog"), msg)
case "INFO": case "INFO":
log.Info(msg) log.Infof("%v: %s", color.BlueString("mklog"), msg)
default: default:
log.Debug(msg) log.Debugf("%v: %s", color.WhiteString("mklog"), msg)
} }
}) })
@ -285,6 +283,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
} }
}) })
log.Debugf("Registered all the handlers")
return nil return nil
} }
@ -307,13 +306,13 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) {
var entry Entry var entry Entry
json.Unmarshal([]byte(jsonStr), &entry) json.Unmarshal([]byte(jsonStr), &entry)
summary := c.nt.Summary(entry.TestKeys) tk := c.nt.GetTestKeys(entry.TestKeys)
summaryBytes, err := json.Marshal(summary)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
log.Debugf("Fetching: %s %v", idx, c.msmts[idx]) log.Debugf("Fetching: %s %v", idx, c.msmts[idx])
c.msmts[idx].WriteSummary(c.Ctx.DB, string(summaryBytes)) err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk)
if err != nil {
log.WithError(err).Error("failed to add test keys to summary")
}
} }
// MKStart is the interface for the mk.Nettest Start() function // MKStart is the interface for the mk.Nettest Start() function

24
nettests/performance/dash.go
View File

@ -16,22 +16,24 @@ func (d Dash) Run(ctl *nettests.Controller) error {
return dash.Run() return dash.Run()
} }
// DashSummary for the test // DashTestKeys for the test
// TODO: process 'receiver_data' to provide an array of performance for a chart. // TODO: process 'receiver_data' to provide an array of performance for a chart.
type DashSummary struct { type DashTestKeys struct {
Latency float64 Latency float64 `json:"connect_latency"`
Bitrate int64 Bitrate int64 `json:"median_bitrate"`
Delay float64 Delay float64 `json:"min_playout_delay"`
IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (d Dash) Summary(tk map[string]interface{}) interface{} { func (d Dash) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{}) simple := tk["simple"].(map[string]interface{})
return DashSummary{ return DashTestKeys{
Latency: simple["connect_latency"].(float64), IsAnomaly: false,
Bitrate: int64(simple["median_bitrate"].(float64)), Latency: simple["connect_latency"].(float64),
Delay: simple["min_playout_delay"].(float64), Bitrate: int64(simple["median_bitrate"].(float64)),
Delay: simple["min_playout_delay"].(float64),
} }
} }

31
nettests/performance/ndt.go
View File

@ -16,26 +16,27 @@ func (n NDT) Run(ctl *nettests.Controller) error {
return nt.Run() return nt.Run()
} }
// NDTSummary for the test // NDTTestKeys for the test
type NDTSummary struct { type NDTTestKeys struct {
Upload int64 Upload int64 `json:"upload"`
Download int64 Download int64 `json:"download"`
Ping int64 Ping int64 `json:"ping"`
MaxRTT float64 MaxRTT float64 `json:"max_rtt"`
AvgRTT float64 AvgRTT float64 `json:"avg_rtt"`
MinRTT float64 MinRTT float64 `json:"min_rtt"`
MSS int64 MSS int64 `json:"mss"`
OutOfOrder int64 OutOfOrder int64 `json:"out_of_order"`
PacketLoss float64 PacketLoss float64 `json:"packet_loss"`
Timeouts int64 Timeouts int64 `json:"timeouts"`
IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (n NDT) Summary(tk map[string]interface{}) interface{} { func (n NDT) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{}) simple := tk["simple"].(map[string]interface{})
advanced := tk["advanced"].(map[string]interface{}) advanced := tk["advanced"].(map[string]interface{})
return NDTSummary{ return NDTTestKeys{
Upload: int64(simple["upload"].(float64)), Upload: int64(simple["upload"].(float64)),
Download: int64(simple["download"].(float64)), Download: int64(simple["download"].(float64)),
Ping: int64(simple["ping"].(float64)), Ping: int64(simple["ping"].(float64)),

44
nettests/summary/summary.go
View File

@ -1,44 +0,0 @@
package summary
import "fmt"
// ResultSummaryFunc is the function used to generate result summaries
type ResultSummaryFunc func(SummaryMap) (string, error)
// SummaryMap contains a mapping from test name to serialized summary for it
type SummaryMap map[string][]string
// PerformanceSummary is the result summary for a performance test
type PerformanceSummary struct {
Upload int64
Download int64
Ping float64
Bitrate int64
}
// MiddleboxSummary is the summary for the middlebox tests
type MiddleboxSummary struct {
Detected bool
}
// IMSummary is the summary for the im tests
type IMSummary struct {
Tested uint
Blocked uint
}
// WebsitesSummary is the summary for the websites test
type WebsitesSummary struct {
Tested uint
Blocked uint
}
func CheckRequiredKeys(rk []string, m SummaryMap) error {
for _, key := range rk {
if _, ok := m[key]; ok {
continue
}
return fmt.Errorf("missing SummaryMap key '%s'", key)
}
return nil
}

26
nettests/websites/web_connectivity.go
View File

@ -29,10 +29,11 @@ const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
var ( var (
parsed = new(URLResponse) parsed = new(URLResponse)
urls []string urls []string
urlIDMap map[int64]int64
) )
urlIDMap := make(map[int64]int64)
log.Debug("Looking up URLs")
// XXX pass in the configuration for category codes // XXX pass in the configuration for category codes
reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
orchestrateBaseURL, orchestrateBaseURL,
@ -53,6 +54,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
} }
for idx, url := range parsed.Results { for idx, url := range parsed.Results {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode) urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode)
if err != nil { if err != nil {
log.Error("failed to add to the URL table") log.Error("failed to add to the URL table")
@ -82,15 +84,15 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
return nt.Run() return nt.Run()
} }
// WebConnectivitySummary for the test // WebConnectivityTestKeys for the test
type WebConnectivitySummary struct { type WebConnectivityTestKeys struct {
Accessible bool Accessible bool `json:"accessible"`
Blocking string Blocking string `json:"blocking"`
Blocked bool IsAnomaly bool `json:"-"`
} }
// Summary generates a summary for a test run // GetTestKeys generates a summary for a test run
func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) interface{} {
var ( var (
blocked bool blocked bool
blocking string blocking string
@ -117,10 +119,10 @@ func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
accessible = tk["accessible"].(bool) accessible = tk["accessible"].(bool)
} }
return WebConnectivitySummary{ return WebConnectivityTestKeys{
Accessible: accessible, Accessible: accessible,
Blocking: blocking, Blocking: blocking,
Blocked: blocked, IsAnomaly: blocked,
} }
} }