c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Replace summary with test_keys

Browse Source
This commit is contained in:
Arturo Filastò 2018-09-10 12:41:28 +02:00
parent b727aba854
commit b29071f37b
19 changed files with 274 additions and 381 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
internal/cli/list/list.go
View File

@ -41,6 +41,7 @@ func init() {
output.SectionTitle("Incomplete results")
}
for idx, result := range incompleteResults {
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
@ -50,7 +51,9 @@ func init() {
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
MeasurementCount: 0,
MeasurementAnomalyCount: 0,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,
@ -61,6 +64,10 @@ func init() {
netCount := make(map[uint]int)
output.SectionTitle("Results")
for idx, result := range doneResults {
totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID)
if err != nil {
log.WithError(err).Error("failed to list measurement counts")
}
output.ResultItem(output.ResultItemData{
ID: result.Result.ID,
Index: idx,
@ -70,7 +77,9 @@ func init() {
NetworkName: result.Network.NetworkName,
Country: result.Network.CountryCode,
ASN: fmt.Sprintf("AS%d", result.Network.ASN),
Summary: "{}", //result.Summary,
TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now
MeasurementCount: totalCount,
MeasurementAnomalyCount: anmlyCount,
Done: result.IsDone,
DataUsageUp: result.DataUsageUp,
DataUsageDown: result.DataUsageDown,

2
internal/cli/run/run.go
View File

@ -73,7 +73,7 @@ func init() {
}
}
if err = result.Finished(ctx.DB, group.Summary); err != nil {
if err = result.Finished(ctx.DB); err != nil {
return err
}
return nil

46
internal/database/actions.go
View File

@ -2,6 +2,7 @@ package database
import (
"database/sql"
"encoding/json"
"time"
"github.com/apex/log"
@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR
return measurements, nil
}
// GetMeasurementCounts returns the number of anomalous and total measurement for a given result
func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) {
var (
totalCount uint64
anmlyCount uint64
err error
)
col := sess.Collection("measurements")
// XXX these two queries can be done with a single query
totalCount, err = col.Find("result_id", resultID).
Count()
if err != nil {
log.WithError(err).Error("failed to get total count")
return totalCount, anmlyCount, err
}
anmlyCount, err = col.Find("result_id", resultID).
And(db.Cond{"is_anomaly": true}).Count()
if err != nil {
log.WithError(err).Error("failed to get anmly count")
return totalCount, anmlyCount, err
}
return totalCount, anmlyCount, err
}
// ListResults return the list of results
func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) {
doneResults := []ResultNetwork{}
@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string
}
urlID = lastID
}
log.Debugf("returning url %d", urlID)
return urlID, nil
}
// AddTestKeys writes the summary to the measurement
func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error {
tkBytes, err := json.Marshal(tk)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly
msmt.TestKeys = string(tkBytes)
msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true}
err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}

36
internal/database/actions_test.go
View File

@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) {
}
}
func TestURLCreation(t *testing.T) {
func TestNetworkCreate(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
tmpdir, err := ioutil.TempDir("", "oonitest")
sess, err := Connect(tmpfile.Name())
if err != nil {
t.Fatal(err)
}
defer os.RemoveAll(tmpdir)
l1 := utils.LocationInfo{
ASN: 2,
CountryCode: "IT",
NetworkName: "Antaninet",
}
l2 := utils.LocationInfo{
ASN: 3,
CountryCode: "IT",
NetworkName: "Fufnet",
}
_, err = CreateNetwork(sess, &l1)
if err != nil {
t.Fatal(err)
}
_, err = CreateNetwork(sess, &l2)
if err != nil {
t.Fatal(err)
}
}
func TestURLCreation(t *testing.T) {
tmpfile, err := ioutil.TempFile("", "dbtest")
if err != nil {
t.Fatal(err)
}
defer os.Remove(tmpfile.Name())
sess, err := Connect(tmpfile.Name())
if err != nil {

22
internal/database/models.go
View File

@ -6,7 +6,6 @@ import (
"path/filepath"
"time"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/pkg/errors"
"upper.io/db.v3/lib/sqlbuilder"
)
@ -28,7 +27,7 @@ type MeasurementURLNetwork struct {
// Network represents a network tested by the user
type Network struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
NetworkName string `db:"network_name"`
NetworkType string `db:"network_type"`
IP string `db:"ip"`
@ -38,7 +37,7 @@ type Network struct {
// URL represents URLs from the testing lists
type URL struct {
ID sql.NullInt64 `db:"id"`
ID sql.NullInt64 `db:"id,omitempty"`
URL sql.NullString `db:"url"`
CategoryCode sql.NullString `db:"category_code"`
CountryCode sql.NullString `db:"country_code"`
@ -46,7 +45,7 @@ type URL struct {
// Measurement model
type Measurement struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestName string `db:"test_name"`
StartTime time.Time `db:"start_time"`
Runtime float64 `db:"runtime"` // Fractional number of seconds
@ -69,7 +68,7 @@ type Measurement struct {
// Result model
type Result struct {
ID int64 `db:"id"`
ID int64 `db:"id,omitempty"`
TestGroupName string `db:"test_group_name"`
StartTime time.Time `db:"start_time"`
NetworkID int64 `db:"network_id"` // Used to include a Network
@ -82,7 +81,7 @@ type Result struct {
}
// Finished marks the result as done and sets the runtime
func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error {
func (r *Result) Finished(sess sqlbuilder.Database) error {
if r.IsDone == true || r.Runtime != 0 {
return errors.New("Result is already finished")
}
@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error {
return nil
}
// WriteSummary writes the summary to the measurement
func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error {
// XXX remove m.Summary = summary
err := sess.Collection("measurements").Find("id", m.ID).Update(m)
if err != nil {
return errors.Wrap(err, "updating measurement")
}
return nil
}
// AddToResult adds a measurement to a result
func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error {
var err error

60
internal/log/handlers/cli/results.go
View File

@ -9,7 +9,6 @@ import (
"github.com/apex/log"
"github.com/ooni/probe-cli/internal/util"
"github.com/ooni/probe-cli/nettests/summary"
)
func formatSpeed(speed int64) string {
@ -24,55 +23,51 @@ func formatSpeed(speed int64) string {
return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000))
}
var summarizers = map[string]func(string) []string{
"websites": func(ss string) []string {
var summary summary.WebsitesSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
// PerformanceTestKeys is the result summary for a performance test
type PerformanceTestKeys struct {
Upload int64 `json:"upload"`
Download int64 `json:"download"`
Ping float64 `json:"ping"`
Bitrate int64 `json:"median_bitrate"`
}
var summarizers = map[string]func(uint64, uint64, string) []string{
"websites": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"performance": func(ss string) []string {
var summary summary.PerformanceSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
"performance": func(totalCount uint64, anomalyCount uint64, ss string) []string {
var tk PerformanceTestKeys
if err := json.Unmarshal([]byte(ss), &tk); err != nil {
return nil
}
return []string{
fmt.Sprintf("Download: %s", formatSpeed(summary.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)),
fmt.Sprintf("Ping: %.2fms", summary.Ping),
fmt.Sprintf("Download: %s", formatSpeed(tk.Download)),
fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)),
fmt.Sprintf("Ping: %.2fms", tk.Ping),
}
},
"im": func(ss string) []string {
var summary summary.IMSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"im": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("%d tested", summary.Tested),
fmt.Sprintf("%d blocked", summary.Blocked),
fmt.Sprintf("%d tested", totalCount),
fmt.Sprintf("%d blocked", anomalyCount),
"",
}
},
"middlebox": func(ss string) []string {
var summary summary.MiddleboxSummary
if err := json.Unmarshal([]byte(ss), &summary); err != nil {
return nil
}
"middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string {
return []string{
fmt.Sprintf("Detected: %v", summary.Detected),
fmt.Sprintf("Detected: %v", anomalyCount > 0),
"",
"",
}
},
}
func makeSummary(name string, ss string) []string {
return summarizers[name](ss)
func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string {
return summarizers[name](totalCount, anomalyCount, ss)
}
func logResultItem(w io.Writer, f log.Fields) error {
@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error {
fmt.Fprintf(w, "┃ "+firstRow+" ┃\n")
fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n")
summary := makeSummary(name, f.Get("summary").(string))
summary := makeSummary(name,
f.Get("measurement_count").(uint64),
f.Get("measurement_anomaly_count").(uint64),
f.Get("test_keys").(string))
fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n",
util.RightPad(name, colWidth),

8
internal/output/output.go
View File

@ -24,7 +24,9 @@ type ResultItemData struct {
ID int64
Name string
StartTime time.Time
Summary string
TestKeys string
MeasurementCount uint64
MeasurementAnomalyCount uint64
Runtime float64
Country string
NetworkName string
@ -43,7 +45,9 @@ func ResultItem(result ResultItemData) {
"id": result.ID,
"name": result.Name,
"start_time": result.StartTime,
"summary": result.Summary,
"test_keys": result.TestKeys,
"measurement_count": result.MeasurementCount,
"measurement_anomaly_count": result.MeasurementAnomalyCount,
"country": result.Country,
"network_name": result.NetworkName,
"asn": result.ASN,

142
nettests/groups/groups.go
View File

@ -1,14 +1,10 @@
package groups
import (
"encoding/json"
"github.com/apex/log"
"github.com/ooni/probe-cli/nettests"
"github.com/ooni/probe-cli/nettests/im"
"github.com/ooni/probe-cli/nettests/middlebox"
"github.com/ooni/probe-cli/nettests/performance"
"github.com/ooni/probe-cli/nettests/summary"
"github.com/ooni/probe-cli/nettests/websites"
)
@ -16,7 +12,6 @@ import (
type NettestGroup struct {
Label string
Nettests []nettests.Nettest
Summary summary.ResultSummaryFunc
}
// NettestGroups that can be run by the user
@ -26,35 +21,6 @@ var NettestGroups = map[string]NettestGroup{
Nettests: []nettests.Nettest{
websites.WebConnectivity{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
// XXX to generate this I need to create the summary map as a list
var summary summary.WebsitesSummary
summary.Tested = 0
summary.Blocked = 0
for _, msmtSummaryStr := range m["WebConnectivity"] {
var wcSummary websites.WebConnectivitySummary
err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal WebConnectivity summary")
return "", err
}
if wcSummary.Blocked {
summary.Blocked++
}
summary.Tested++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"performance": NettestGroup{
Label: "Performance",
@ -62,38 +28,6 @@ var NettestGroups = map[string]NettestGroup{
performance.Dash{},
performance.NDT{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
ndtSummary performance.NDTSummary
dashSummary performance.DashSummary
summary summary.PerformanceSummary
)
err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal Dash summary")
return "", err
}
err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal NDT summary")
return "", err
}
summary.Bitrate = dashSummary.Bitrate
summary.Download = ndtSummary.Download
summary.Upload = ndtSummary.Upload
summary.Ping = ndtSummary.AvgRTT
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"middlebox": NettestGroup{
Label: "Middleboxes",
@ -101,35 +35,6 @@ var NettestGroups = map[string]NettestGroup{
middlebox.HTTPInvalidRequestLine{},
middlebox.HTTPHeaderFieldManipulation{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary
hirlSummary middlebox.HTTPInvalidRequestLineSummary
summary summary.MiddleboxSummary
)
err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hhfm summary")
return "", err
}
err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal hirl summary")
return "", err
}
summary.Detected = hirlSummary.Tampering == true || hhfmSummary.Tampering == true
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
"im": NettestGroup{
Label: "Instant Messaging",
@ -138,52 +43,5 @@ var NettestGroups = map[string]NettestGroup{
im.Telegram{},
im.WhatsApp{},
},
Summary: func(m summary.SummaryMap) (string, error) {
if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil {
log.WithError(err).Error("missing keys")
return "", err
}
var (
err error
waSummary im.WhatsAppSummary
tgSummary im.TelegramSummary
fbSummary im.FacebookMessengerSummary
summary summary.IMSummary
)
err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal whatsapp summary")
return "", err
}
err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal telegram summary")
return "", err
}
err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary)
if err != nil {
log.WithError(err).Error("failed to unmarshal facebook summary")
return "", err
}
// XXX it could actually be that some are not tested when the
// configuration is changed.
summary.Tested = 3
summary.Blocked = 0
if fbSummary.Blocked == true {
summary.Blocked++
}
if tgSummary.Blocked == true {
summary.Blocked++
}
if waSummary.Blocked == true {
summary.Blocked++
}
summaryBytes, err := json.Marshal(summary)
if err != nil {
return "", err
}
return string(summaryBytes), nil
},
},
}

18
nettests/im/facebook_messenger.go
View File

@ -16,15 +16,15 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// FacebookMessengerSummary for the test
type FacebookMessengerSummary struct {
DNSBlocking bool
TCPBlocking bool
Blocked bool
// FacebookMessengerTestKeys for the test
type FacebookMessengerTestKeys struct {
DNSBlocking bool `json:"facebook_dns_blocking"`
TCPBlocking bool `json:"facebook_tcp_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) interface{} {
var (
dnsBlocking bool
tcpBlocking bool
@ -41,10 +41,10 @@ func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
tcpBlocking = tk["facebook_tcp_blocking"].(bool)
}
return FacebookMessengerSummary{
return FacebookMessengerTestKeys{
DNSBlocking: dnsBlocking,
TCPBlocking: tcpBlocking,
Blocked: dnsBlocking || tcpBlocking,
IsAnomaly: dnsBlocking || tcpBlocking,
}
}

20
nettests/im/telegram.go
View File

@ -16,16 +16,16 @@ func (h Telegram) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// TelegramSummary for the test
type TelegramSummary struct {
HTTPBlocking bool
TCPBlocking bool
WebBlocking bool
Blocked bool
// TelegramTestKeys for the test
type TelegramTestKeys struct {
HTTPBlocking bool `json:"telegram_http_blocking"`
TCPBlocking bool `json:"telegram_tcp_blocking"`
WebBlocking bool `json:"telegram_web_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h Telegram) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h Telegram) GetTestKeys(tk map[string]interface{}) interface{} {
var (
tcpBlocking bool
httpBlocking bool
@ -48,11 +48,11 @@ func (h Telegram) Summary(tk map[string]interface{}) interface{} {
webBlocking = tk["telegram_web_status"].(string) == "blocked"
}
return TelegramSummary{
return TelegramTestKeys{
TCPBlocking: tcpBlocking,
HTTPBlocking: httpBlocking,
WebBlocking: webBlocking,
Blocked: webBlocking || httpBlocking || tcpBlocking,
IsAnomaly: webBlocking || httpBlocking || tcpBlocking,
}
}

20
nettests/im/whatsapp.go
View File

@ -16,16 +16,16 @@ func (h WhatsApp) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// WhatsAppSummary for the test
type WhatsAppSummary struct {
RegistrationServerBlocking bool
WebBlocking bool
EndpointsBlocking bool
Blocked bool
// WhatsAppTestKeys for the test
type WhatsAppTestKeys struct {
RegistrationServerBlocking bool `json:"registration_server_blocking"`
WebBlocking bool `json:"whatsapp_web_blocking"`
EndpointsBlocking bool `json:"whatsapp_endpoints_blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h WhatsApp) GetTestKeys(tk map[string]interface{}) interface{} {
var (
webBlocking bool
registrationBlocking bool
@ -46,11 +46,11 @@ func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
webBlocking = computeBlocking("whatsapp_web_status")
endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
return WhatsAppSummary{
return WhatsAppTestKeys{
RegistrationServerBlocking: registrationBlocking,
WebBlocking: webBlocking,
EndpointsBlocking: endpointsBlocking,
Blocked: registrationBlocking || webBlocking || endpointsBlocking,
IsAnomaly: registrationBlocking || webBlocking || endpointsBlocking,
}
}

14
nettests/middlebox/http_header_field_manipulation.go
View File

@ -16,13 +16,13 @@ func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// HTTPHeaderFieldManipulationSummary for the test
type HTTPHeaderFieldManipulationSummary struct {
Tampering bool
// HTTPHeaderFieldManipulationTestKeys for the test
type HTTPHeaderFieldManipulationTestKeys struct {
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys returns a projection of the tests keys needed for the views
func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := false
for _, v := range tk["tampering"].(map[string]interface{}) {
t, ok := v.(bool)
@ -32,8 +32,8 @@ func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interfac
}
}
return HTTPHeaderFieldManipulationSummary{
Tampering: tampering,
return HTTPHeaderFieldManipulationTestKeys{
IsAnomaly: tampering,
}
}

14
nettests/middlebox/http_invalid_request_line.go
View File

@ -16,17 +16,17 @@ func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error {
return mknt.Run()
}
// HTTPInvalidRequestLineSummary for the test
type HTTPInvalidRequestLineSummary struct {
Tampering bool
// HTTPInvalidRequestLineTestKeys for the test
type HTTPInvalidRequestLineTestKeys struct {
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (h HTTPInvalidRequestLine) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) interface{} {
tampering := tk["tampering"].(bool)
return HTTPInvalidRequestLineSummary{
Tampering: tampering,
return HTTPInvalidRequestLineTestKeys{
IsAnomaly: tampering,
}
}

25
nettests/nettests.go
View File

@ -20,7 +20,7 @@ import (
// Nettest interface. Every Nettest should implement this.
type Nettest interface {
Run(*Controller) error
Summary(map[string]interface{}) interface{}
GetTestKeys(map[string]interface{}) interface{}
LogSummary(string) error
}
@ -119,7 +119,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
IncludeIP: c.Ctx.Config.Sharing.IncludeIP,
IncludeASN: c.Ctx.Config.Sharing.IncludeASN,
IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
LogLevel: "INFO",
LogLevel: "DEBUG",
ProbeCC: c.Ctx.Location.CountryCode,
ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
@ -139,18 +139,16 @@ func (c *Controller) Init(nt *mk.Nettest) error {
log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath)
nt.On("log", func(e mk.Event) {
log.Debugf(color.RedString(e.Key))
level := e.Value.LogLevel
msg := e.Value.Message
switch level {
case "ERROR":
log.Error(msg)
log.Errorf("%v: %s", color.RedString("mklog"), msg)
case "INFO":
log.Info(msg)
log.Infof("%v: %s", color.BlueString("mklog"), msg)
default:
log.Debug(msg)
log.Debugf("%v: %s", color.WhiteString("mklog"), msg)
}
})
@ -285,6 +283,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
}
})
log.Debugf("Registered all the handlers")
return nil
}
@ -307,13 +306,13 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) {
var entry Entry
json.Unmarshal([]byte(jsonStr), &entry)
summary := c.nt.Summary(entry.TestKeys)
summaryBytes, err := json.Marshal(summary)
if err != nil {
log.WithError(err).Error("failed to serialize summary")
}
tk := c.nt.GetTestKeys(entry.TestKeys)
log.Debugf("Fetching: %s %v", idx, c.msmts[idx])
c.msmts[idx].WriteSummary(c.Ctx.DB, string(summaryBytes))
err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk)
if err != nil {
log.WithError(err).Error("failed to add test keys to summary")
}
}
// MKStart is the interface for the mk.Nettest Start() function

18
nettests/performance/dash.go
View File

@ -16,19 +16,21 @@ func (d Dash) Run(ctl *nettests.Controller) error {
return dash.Run()
}
// DashSummary for the test
// DashTestKeys for the test
// TODO: process 'receiver_data' to provide an array of performance for a chart.
type DashSummary struct {
Latency float64
Bitrate int64
Delay float64
type DashTestKeys struct {
Latency float64 `json:"connect_latency"`
Bitrate int64 `json:"median_bitrate"`
Delay float64 `json:"min_playout_delay"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (d Dash) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (d Dash) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{})
return DashSummary{
return DashTestKeys{
IsAnomaly: false,
Latency: simple["connect_latency"].(float64),
Bitrate: int64(simple["median_bitrate"].(float64)),
Delay: simple["min_playout_delay"].(float64),

31
nettests/performance/ndt.go
View File

@ -16,26 +16,27 @@ func (n NDT) Run(ctl *nettests.Controller) error {
return nt.Run()
}
// NDTSummary for the test
type NDTSummary struct {
Upload int64
Download int64
Ping int64
MaxRTT float64
AvgRTT float64
MinRTT float64
MSS int64
OutOfOrder int64
PacketLoss float64
Timeouts int64
// NDTTestKeys for the test
type NDTTestKeys struct {
Upload int64 `json:"upload"`
Download int64 `json:"download"`
Ping int64 `json:"ping"`
MaxRTT float64 `json:"max_rtt"`
AvgRTT float64 `json:"avg_rtt"`
MinRTT float64 `json:"min_rtt"`
MSS int64 `json:"mss"`
OutOfOrder int64 `json:"out_of_order"`
PacketLoss float64 `json:"packet_loss"`
Timeouts int64 `json:"timeouts"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (n NDT) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (n NDT) GetTestKeys(tk map[string]interface{}) interface{} {
simple := tk["simple"].(map[string]interface{})
advanced := tk["advanced"].(map[string]interface{})
return NDTSummary{
return NDTTestKeys{
Upload: int64(simple["upload"].(float64)),
Download: int64(simple["download"].(float64)),
Ping: int64(simple["ping"].(float64)),

44
nettests/summary/summary.go
View File

@ -1,44 +0,0 @@
package summary
import "fmt"
// ResultSummaryFunc is the function used to generate result summaries
type ResultSummaryFunc func(SummaryMap) (string, error)
// SummaryMap contains a mapping from test name to serialized summary for it
type SummaryMap map[string][]string
// PerformanceSummary is the result summary for a performance test
type PerformanceSummary struct {
Upload int64
Download int64
Ping float64
Bitrate int64
}
// MiddleboxSummary is the summary for the middlebox tests
type MiddleboxSummary struct {
Detected bool
}
// IMSummary is the summary for the im tests
type IMSummary struct {
Tested uint
Blocked uint
}
// WebsitesSummary is the summary for the websites test
type WebsitesSummary struct {
Tested uint
Blocked uint
}
func CheckRequiredKeys(rk []string, m SummaryMap) error {
for _, key := range rk {
if _, ok := m[key]; ok {
continue
}
return fmt.Errorf("missing SummaryMap key '%s'", key)
}
return nil
}

22
nettests/websites/web_connectivity.go
View File

@ -31,8 +31,9 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
var (
parsed = new(URLResponse)
urls []string
urlIDMap map[int64]int64
)
urlIDMap := make(map[int64]int64)
log.Debug("Looking up URLs")
// XXX pass in the configuration for category codes
reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
orchestrateBaseURL,
@ -53,6 +54,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
}
for idx, url := range parsed.Results {
log.Debugf("Going over URL %d", idx)
urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode)
if err != nil {
log.Error("failed to add to the URL table")
@ -82,15 +84,15 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
return nt.Run()
}
// WebConnectivitySummary for the test
type WebConnectivitySummary struct {
Accessible bool
Blocking string
Blocked bool
// WebConnectivityTestKeys for the test
type WebConnectivityTestKeys struct {
Accessible bool `json:"accessible"`
Blocking string `json:"blocking"`
IsAnomaly bool `json:"-"`
}
// Summary generates a summary for a test run
func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
// GetTestKeys generates a summary for a test run
func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) interface{} {
var (
blocked bool
blocking string
@ -117,10 +119,10 @@ func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
accessible = tk["accessible"].(bool)
}
return WebConnectivitySummary{
return WebConnectivityTestKeys{
Accessible: accessible,
Blocking: blocking,
Blocked: blocked,
IsAnomaly: blocked,
}
}