c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace summary with test_keys

Browse Source
This commit is contained in:
Arturo Filastò 2018-09-10 12:41:28 +02:00
parent b727aba854
commit b29071f37b
19 changed files with 274 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
data/migrations/1_create_msmt_results.sql
View File

@ -41,7 +41,7 @@ CREATE TABLE `networks` (
-- this with more data in the future.
`network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile
`ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address.
`ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address.
-- The longest ip is an ipv6 address like:
-- 0000:0000:0000:0000:0000:0000:0000:0000,
-- which is 39 chars.

51
internal/cli/list/list.go
View File

@ -41,19 +41,22 @@ func init() {
output.SectionTitle("Incomplete results")
}
for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
ID: result.Result.ID,
Index: idx,
TotalCount: len(incompleteResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
MeasurementCount: 0,
MeasurementAnomalyCount: 0,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
})
}
@ -61,16 +64,22 @@ func init() {
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
ID: result.Result.ID,
Index: idx,
TotalCount: len(doneResults),
Name: result.TestGroupName,
StartTime: result.StartTime,
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,

2
internal/cli/run/run.go
View File

@ -73,7 +73,7 @@ func init() {
}
}
if err = result.Finished(ctx.DB, group.Summary); err != nil {
if err = result.Finished(ctx.DB); err != nil {
return err
}
return nil

46
internal/database/actions.go
View File

@ -2,6 +2,7 @@ package database
import (
"database/sql"
"encoding/json"
"time"
"github.com/apex/log"
@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR
return measurements, nil
}
// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
var (
totalCount uint64
anmlyCount uint64
err error
)
col := sess.Collection("measurements")
// XXX these two queries can be done with a single query
totalCount, err = col.Find("result_id", resultID).
Count()
if err != nil {
log.WithError(err).Error("failed to get total count")
return totalCount, anmlyCount, err
}
anmlyCount, err = col.Find("result_id", resultID).
And(db.Cond{"is_anomaly": true}).Count()
if err != nil {
log.WithError(err).Error("failed to get anmly count")
return totalCount, anmlyCount, err
}
return totalCount, anmlyCount, err
}
// ListResults return the list of results
func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
doneResults := []ResultNetwork{}
@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string
}
urlID = lastID
}
log.Debugf("returning url %d", urlID)
return urlID, nil
}
// AddTestKeys writes the summary to the measurement
func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
tkBytes, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly
msmt.TestKeys = string(tkBytes)
msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true}
err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}

36
internal/database/actions_test.go
View File

@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) {
}
}
func TestURLCreation(t *testing.T) {
func TestNetworkCreate(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
l1 := utils.LocationInfo{
ASN: 2,
CountryCode: "IT",
NetworkName: "Antaninet",
}
l2 := utils.LocationInfo{
ASN: 3,
CountryCode: "IT",
NetworkName: "Fufnet",
}
_, err = CreateNetwork(sess, &l1)
if err != nil {
t.Fatal(err)
}
_, err = CreateNetwork(sess, &l2)
if err != nil {
t.Fatal(err)
}
}
func TestURLCreation(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {

22
internal/database/models.go
View File

@ -6,7 +6,6 @@ import (
"path/filepath"
"time"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
@ -28,7 +27,7 @@ type MeasurementURLNetwork struct {
// Network represents a network tested by the user
type Network struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
NetworkName string `db:"network_name"`
NetworkType string `db:"network_type"`
IP string `db:"ip"`
@ -38,7 +37,7 @@ type Network struct {
// URL represents URLs from the testing lists
type URL struct {
ID sql.NullInt64 `db:"id"`
ID sql.NullInt64 `db:"id,omitempty"`
URL sql.NullString `db:"url"`
CategoryCode sql.NullString `db:"category_code"`
CountryCode sql.NullString `db:"country_code"`
@ -46,7 +45,7 @@ type URL struct {
// Measurement model
type Measurement struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestName string `db:"test_name"`
StartTime time.Time `db:"start_time"`
Runtime float64 `db:"runtime"` // Fractional number of seconds
@ -69,7 +68,7 @@ type Measurement struct {
// Result model
type Result struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestGroupName string `db:"test_group_name"`
StartTime time.Time `db:"start_time"`
NetworkID int64 `db:"network_id"` // Used to include a Network
@ -82,7 +81,7 @@ type Result struct {
}
// Finished marks the result as done and sets the runtime
func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error {
func (r *Result) Finished(sess sqlbuilder.Database) error {
if r.IsDone == true || r.Runtime != 0 {
return errors.New("Result is already finished")
}
@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
return nil
}
// WriteSummary writes the summary to the measurement
func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error {
// XXX remove m.Summary = summary
err := sess.Collection("measurements").Find("id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// AddToResult adds a measurement to a result
func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error {
var err error

60
internal/log/handlers/cli/results.go
View File

@ -9,7 +9,6 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/util"
"github.com/ooni/probe-cli/nettests/summary"
)
func formatSpeed(speed int64) string {
@ -24,55 +23,51 @@ func formatSpeed(speed int64) string {
return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
}
var summarizers = map[string]func(string) []string{
"websites": func(ss string) []string {
var summary summary.WebsitesSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
// PerformanceTestKeys is the result summary for a performance test
type PerformanceTestKeys struct {
Upload int64 `json:"upload"`
Download int64 `json:"download"`
Ping float64 `json:"ping"`
Bitrate int64 `json:"median_bitrate"`
}
var summarizers = map[string]func(uint64, uint64, string) []string{
"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"performance": func(ss string) []string {
var summary summary.PerformanceSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
"performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var tk PerformanceTestKeys
if err := json.Unmarshal([]byte(ss), &tk); err != nil {
return nil
}
return []string{
fmt.Sprintf("Download: %s", formatSpeed(summary.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)),
fmt.Sprintf("Ping: %.2fms", summary.Ping),
fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
fmt.Sprintf("Ping: %.2fms", tk.Ping),
}
},
"im": func(ss string) []string {
var summary summary.IMSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"middlebox": func(ss string) []string {
var summary summary.MiddleboxSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("Detected: %v", summary.Detected),
fmt.Sprintf("Detected: %v", anomalyCount > 0),
"",
"",
}
},
}
func makeSummary(name string, ss string) []string {
return summarizers[name](ss)
func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
return summarizers[name](totalCount, anomalyCount, ss)
}
func logResultItem(w io.Writer, f log.Fields) error {
@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error {
fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
summary := makeSummary(name, f.Get("summary").(string))
summary := makeSummary(name,
f.Get("measurement_count").(uint64),
f.Get("measurement_anomaly_count").(uint64),
f.Get("test_keys").(string))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
util.RightPad(name, colWidth),

58
internal/output/output.go
View File

@ -21,38 +21,42 @@ func Progress(key string, perc float64, msg string) {
// ResultItemData is the metadata about a result
type ResultItemData struct {
ID int64
Name string
StartTime time.Time
Summary string
Runtime float64
Country string
NetworkName string
ASN string
Done bool
DataUsageDown int64
DataUsageUp int64
Index int
TotalCount int
ID int64
Name string
StartTime time.Time
TestKeys string
MeasurementCount uint64
MeasurementAnomalyCount uint64
Runtime float64
Country string
NetworkName string
ASN string
Done bool
DataUsageDown int64
DataUsageUp int64
Index int
TotalCount int
}
// ResultItem logs a progress type event
func ResultItem(result ResultItemData) {
log.WithFields(log.Fields{
"type": "result_item",
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"summary": result.Summary,
"country": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,
"runtime": result.Runtime,
"done": result.Done,
"data_usage_down": result.DataUsageDown,
"data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
"type": "result_item",
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"test_keys": result.TestKeys,
"measurement_count": result.MeasurementCount,
"measurement_anomaly_count": result.MeasurementAnomalyCount,
"country": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,
"runtime": result.Runtime,
"done": result.Done,
"data_usage_down": result.DataUsageDown,
"data_usage_up": result.DataUsageUp,
"index": result.Index,
"total_count": result.TotalCount,
}).Info("result item")
}

142
nettests/groups/groups.go
View File

@ -1,14 +1,10 @@
package groups
import (
"encoding/json"
"github.com/apex/log"
"github.com/ooni/probe-cli/nettests"
"github.com/ooni/probe-cli/nettests/im"
"github.com/ooni/probe-cli/nettests/middlebox"
"github.com/ooni/probe-cli/nettests/performance"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/ooni/probe-cli/nettests/websites"
)
@ -16,7 +12,6 @@ import (
type NettestGroup struct {
Label string
Nettests []nettests.Nettest
Summary summary.ResultSummaryFunc
}
// NettestGroups that can be run by the user
@ -26,35 +21,6 @@ var NettestGroups = map[string]NettestGroup{
Nettests: []nettests.Nettest{
websites.WebConnectivity{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
// XXX to generate this I need to create the summary map as a list
var summary summary.WebsitesSummary
summary.Tested = 0
summary.Blocked = 0
for _, msmtSummaryStr := range m["WebConnectivity"] {
var wcSummary websites.WebConnectivitySummary
err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal WebConnectivity summary")
return "", err
}
if wcSummary.Blocked {
summary.Blocked++
}
summary.Tested++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"performance": NettestGroup{
Label: "Performance",
@ -62,38 +28,6 @@ var NettestGroups = map[string]NettestGroup{
performance.Dash{},
performance.NDT{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
ndtSummary performance.NDTSummary
dashSummary performance.DashSummary
summary summary.PerformanceSummary
)
err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal Dash summary")
return "", err
}
err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal NDT summary")
return "", err
}
summary.Bitrate = dashSummary.Bitrate
summary.Download = ndtSummary.Download
summary.Upload = ndtSummary.Upload
summary.Ping = ndtSummary.AvgRTT
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"middlebox": NettestGroup{
Label: "Middleboxes",
@ -101,35 +35,6 @@ var NettestGroups = map[string]NettestGroup{
middlebox.HTTPInvalidRequestLine{},
middlebox.HTTPHeaderFieldManipulation{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary
hirlSummary middlebox.HTTPInvalidRequestLineSummary
summary summary.MiddleboxSummary
)
err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hhfm summary")
return "", err
}
err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hirl summary")
return "", err
}
summary.Detected = hirlSummary.Tampering == true || hhfmSummary.Tampering == true
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"im": NettestGroup{
Label: "Instant Messaging",
@ -138,52 +43,5 @@ var NettestGroups = map[string]NettestGroup{
im.Telegram{},
im.WhatsApp{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
waSummary im.WhatsAppSummary
tgSummary im.TelegramSummary
fbSummary im.FacebookMessengerSummary
summary summary.IMSummary
)
err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal whatsapp summary")
return "", err
}
err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal telegram summary")
return "", err
}
err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal facebook summary")
return "", err
}
// XXX it could actually be that some are not tested when the
// configuration is changed.
summary.Tested = 3
summary.Blocked = 0
if fbSummary.Blocked == true {
summary.Blocked++
}
if tgSummary.Blocked == true {
summary.Blocked++
}
if waSummary.Blocked == true {
summary.Blocked++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
}

18
nettests/im/facebook_messenger.go
View File

@ -16,15 +16,15 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// FacebookMessengerSummary for the test
type FacebookMessengerSummary struct {
DNSBlocking bool
TCPBlocking bool
Blocked bool
// FacebookMessengerTestKeys for the test
type FacebookMessengerTestKeys struct {
DNSBlocking bool `json:"facebook_dns_blocking"`
TCPBlocking bool `json:"facebook_tcp_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) interface{} {
var (
dnsBlocking bool
tcpBlocking bool
@ -41,10 +41,10 @@ func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
tcpBlocking = tk["facebook_tcp_blocking"].(bool)
}
return FacebookMessengerSummary{
return FacebookMessengerTestKeys{
DNSBlocking: dnsBlocking,
TCPBlocking: tcpBlocking,
Blocked: dnsBlocking || tcpBlocking,
IsAnomaly: dnsBlocking || tcpBlocking,
}
}

20
nettests/im/telegram.go
View File

@ -16,16 +16,16 @@ func (h Telegram) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// TelegramSummary for the test
type TelegramSummary struct {
HTTPBlocking bool
TCPBlocking bool
WebBlocking bool
Blocked bool
// TelegramTestKeys for the test
type TelegramTestKeys struct {
HTTPBlocking bool `json:"telegram_http_blocking"`
TCPBlocking bool `json:"telegram_tcp_blocking"`
WebBlocking bool `json:"telegram_web_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h Telegram) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h Telegram) GetTestKeys(tk map[string]interface{}) interface{} {
var (
tcpBlocking bool
httpBlocking bool
@ -48,11 +48,11 @@ func (h Telegram) Summary(tk map[string]interface{}) interface{} {
webBlocking = tk["telegram_web_status"].(string) == "blocked"
}
return TelegramSummary{
return TelegramTestKeys{
TCPBlocking: tcpBlocking,
HTTPBlocking: httpBlocking,
WebBlocking: webBlocking,
Blocked: webBlocking || httpBlocking || tcpBlocking,
IsAnomaly: webBlocking || httpBlocking || tcpBlocking,
}
}

20
nettests/im/whatsapp.go
View File

@ -16,16 +16,16 @@ func (h WhatsApp) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// WhatsAppSummary for the test
type WhatsAppSummary struct {
RegistrationServerBlocking bool
WebBlocking bool
EndpointsBlocking bool
Blocked bool
// WhatsAppTestKeys for the test
type WhatsAppTestKeys struct {
RegistrationServerBlocking bool `json:"registration_server_blocking"`
WebBlocking bool `json:"whatsapp_web_blocking"`
EndpointsBlocking bool `json:"whatsapp_endpoints_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h WhatsApp) GetTestKeys(tk map[string]interface{}) interface{} {
var (
webBlocking bool
registrationBlocking bool
@ -46,11 +46,11 @@ func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
webBlocking = computeBlocking("whatsapp_web_status")
endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
return WhatsAppSummary{
return WhatsAppTestKeys{
RegistrationServerBlocking: registrationBlocking,
WebBlocking: webBlocking,
EndpointsBlocking: endpointsBlocking,
Blocked: registrationBlocking || webBlocking || endpointsBlocking,
IsAnomaly: registrationBlocking || webBlocking || endpointsBlocking,
}
}

14
nettests/middlebox/http_header_field_manipulation.go
View File

@ -16,13 +16,13 @@ func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// HTTPHeaderFieldManipulationSummary for the test
type HTTPHeaderFieldManipulationSummary struct {
Tampering bool
// HTTPHeaderFieldManipulationTestKeys for the test
type HTTPHeaderFieldManipulationTestKeys struct {
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys returns a projection of the tests keys needed for the views
func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := false
for _, v := range tk["tampering"].(map[string]interface{}) {
t, ok := v.(bool)
@ -32,8 +32,8 @@ func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interfac
}
}
return HTTPHeaderFieldManipulationSummary{
Tampering: tampering,
return HTTPHeaderFieldManipulationTestKeys{
IsAnomaly: tampering,
}
}

14
nettests/middlebox/http_invalid_request_line.go
View File

@ -16,17 +16,17 @@ func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// HTTPInvalidRequestLineSummary for the test
type HTTPInvalidRequestLineSummary struct {
Tampering bool
// HTTPInvalidRequestLineTestKeys for the test
type HTTPInvalidRequestLineTestKeys struct {
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h HTTPInvalidRequestLine) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := tk["tampering"].(bool)
return HTTPInvalidRequestLineSummary{
Tampering: tampering,
return HTTPInvalidRequestLineTestKeys{
IsAnomaly: tampering,
}
}

25
nettests/nettests.go
View File

@ -20,7 +20,7 @@ import (
// Nettest interface. Every Nettest should implement this.
type Nettest interface {
Run(*Controller) error
Summary(map[string]interface{}) interface{}
GetTestKeys(map[string]interface{}) interface{}
LogSummary(string) error
}
@ -119,7 +119,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
IncludeIP: c.Ctx.Config.Sharing.IncludeIP,
IncludeASN: c.Ctx.Config.Sharing.IncludeASN,
IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
LogLevel: "INFO",
LogLevel: "DEBUG",
ProbeCC: c.Ctx.Location.CountryCode,
ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
@ -139,18 +139,16 @@ func (c *Controller) Init(nt *mk.Nettest) error {
log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath)
nt.On("log", func(e mk.Event) {
log.Debugf(color.RedString(e.Key))
level := e.Value.LogLevel
msg := e.Value.Message
switch level {
case "ERROR":
log.Error(msg)
log.Errorf("%v: %s", color.RedString("mklog"), msg)
case "INFO":
log.Info(msg)
log.Infof("%v: %s", color.BlueString("mklog"), msg)
default:
log.Debug(msg)
log.Debugf("%v: %s", color.WhiteString("mklog"), msg)
}
})
@ -285,6 +283,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
}
})
log.Debugf("Registered all the handlers")
return nil
}
@ -307,13 +306,13 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) {
var entry Entry
json.Unmarshal([]byte(jsonStr), &entry)
summary := c.nt.Summary(entry.TestKeys)
summaryBytes, err := json.Marshal(summary)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
tk := c.nt.GetTestKeys(entry.TestKeys)
log.Debugf("Fetching: %s %v", idx, c.msmts[idx])
c.msmts[idx].WriteSummary(c.Ctx.DB, string(summaryBytes))
err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk)
if err != nil {
log.WithError(err).Error("failed to add test keys to summary")
}
}
// MKStart is the interface for the mk.Nettest Start() function

24
nettests/performance/dash.go
View File

@ -16,22 +16,24 @@ func (d Dash) Run(ctl *nettests.Controller) error {
return dash.Run()
}
// DashSummary for the test
// DashTestKeys for the test
// TODO: process 'receiver_data' to provide an array of performance for a chart.
type DashSummary struct {
Latency float64
Bitrate int64
Delay float64
type DashTestKeys struct {
Latency float64 `json:"connect_latency"`
Bitrate int64 `json:"median_bitrate"`
Delay float64 `json:"min_playout_delay"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (d Dash) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (d Dash) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{})
return DashSummary{
Latency: simple["connect_latency"].(float64),
Bitrate: int64(simple["median_bitrate"].(float64)),
Delay: simple["min_playout_delay"].(float64),
return DashTestKeys{
IsAnomaly: false,
Latency: simple["connect_latency"].(float64),
Bitrate: int64(simple["median_bitrate"].(float64)),
Delay: simple["min_playout_delay"].(float64),
}
}

31
nettests/performance/ndt.go
View File

@ -16,26 +16,27 @@ func (n NDT) Run(ctl *nettests.Controller) error {
return nt.Run()
}
// NDTSummary for the test
type NDTSummary struct {
Upload int64
Download int64
Ping int64
MaxRTT float64
AvgRTT float64
MinRTT float64
MSS int64
OutOfOrder int64
PacketLoss float64
Timeouts int64
// NDTTestKeys for the test
type NDTTestKeys struct {
Upload int64 `json:"upload"`
Download int64 `json:"download"`
Ping int64 `json:"ping"`
MaxRTT float64 `json:"max_rtt"`
AvgRTT float64 `json:"avg_rtt"`
MinRTT float64 `json:"min_rtt"`
MSS int64 `json:"mss"`
OutOfOrder int64 `json:"out_of_order"`
PacketLoss float64 `json:"packet_loss"`
Timeouts int64 `json:"timeouts"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (n NDT) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (n NDT) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{})
advanced := tk["advanced"].(map[string]interface{})
return NDTSummary{
return NDTTestKeys{
Upload: int64(simple["upload"].(float64)),
Download: int64(simple["download"].(float64)),
Ping: int64(simple["ping"].(float64)),

44
nettests/summary/summary.go
View File

@ -1,44 +0,0 @@
package summary
import "fmt"
// ResultSummaryFunc is the function used to generate result summaries
type ResultSummaryFunc func(SummaryMap) (string, error)
// SummaryMap contains a mapping from test name to serialized summary for it
type SummaryMap map[string][]string
// PerformanceSummary is the result summary for a performance test
type PerformanceSummary struct {
Upload int64
Download int64
Ping float64
Bitrate int64
}
// MiddleboxSummary is the summary for the middlebox tests
type MiddleboxSummary struct {
Detected bool
}
// IMSummary is the summary for the im tests
type IMSummary struct {
Tested uint
Blocked uint
}
// WebsitesSummary is the summary for the websites test
type WebsitesSummary struct {
Tested uint
Blocked uint
}
func CheckRequiredKeys(rk []string, m SummaryMap) error {
for _, key := range rk {
if _, ok := m[key]; ok {
continue
}
return fmt.Errorf("missing SummaryMap key '%s'", key)
}
return nil
}

26
nettests/websites/web_connectivity.go
View File

@ -29,10 +29,11 @@ const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
var (
parsed = new(URLResponse)
urls []string
urlIDMap map[int64]int64
parsed = new(URLResponse)
urls []string
)
urlIDMap := make(map[int64]int64)
log.Debug("Looking up URLs")
// XXX pass in the configuration for category codes
reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
orchestrateBaseURL,
@ -53,6 +54,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
}
for idx, url := range parsed.Results {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode)
if err != nil {
log.Error("failed to add to the URL table")
@ -82,15 +84,15 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
return nt.Run()
}
// WebConnectivitySummary for the test
type WebConnectivitySummary struct {
Accessible bool
Blocking string
Blocked bool
// WebConnectivityTestKeys for the test
type WebConnectivityTestKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) interface{} {
var (
blocked bool
blocking string
@ -117,10 +119,10 @@ func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
accessible = tk["accessible"].(bool)
}
return WebConnectivitySummary{
return WebConnectivityTestKeys{
Accessible: accessible,
Blocking: blocking,
Blocked: blocked,
IsAnomaly: blocked,
}
}