c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Update the measurement, network and url creation to the new schema

Browse Source
This commit is contained in:
Arturo Filastò
2018-09-07 12:55:27 +02:00
parent 71ed0e969f
commit 35bd334cfc
6 changed files with 138 additions and 80 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nettests/nettests.go
+27 -14
View File
@@ -6,6 +6,7 @@ import (
"fmt"
"os"
"path/filepath"
"time"
"github.com/apex/log"
"github.com/fatih/color"
@@ -24,7 +25,10 @@ type Nettest interface {
}
// NewController creates a nettest controller
func NewController(nt Nettest, ctx *ooni.Context, res *database.Result, msmtPath string) *Controller {
func NewController(nt Nettest, ctx *ooni.Context, res *database.Result) *Controller {
msmtPath := filepath.Join(ctx.TempDir,
fmt.Sprintf("msmt-%T-%s.jsonl", nt,
time.Now().UTC().Format(utils.ResultTimestamp)))
return &Controller{
Ctx: ctx,
nt: nt,
@@ -36,11 +40,12 @@ func NewController(nt Nettest, ctx *ooni.Context, res *database.Result, msmtPath
// Controller is passed to the run method of every Nettest
// each nettest instance has one controller
type Controller struct {
Ctx *ooni.Context
res *database.Result
nt Nettest
msmts map[int64]*database.Measurement
msmtPath string // XXX maybe we can drop this and just use a temporary file
Ctx *ooni.Context
res *database.Result
nt Nettest
msmts map[int64]*database.Measurement
msmtPath string // XXX maybe we can drop this and just use a temporary file
inputIdxMap map[int64]int64 // Used to map mk idx to database id
}
func getCaBundlePath() string {
@@ -51,6 +56,11 @@ func getCaBundlePath() string {
return "/etc/ssl/cert.pem"
}
func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error {
c.inputIdxMap = inputIdxMap
return nil
}
// Init should be called once to initialise the nettest
func (c *Controller) Init(nt *mk.Nettest) error {
log.Debugf("Init: %v", nt)
@@ -58,12 +68,11 @@ func (c *Controller) Init(nt *mk.Nettest) error {
c.msmts = make(map[int64]*database.Measurement)
msmtTemplate := database.Measurement{
ReportID: sql.NullString{String: "", Valid: false},
TestName: nt.Name,
ResultID: c.res.ID,
ReportFilePath: c.msmtPath,
}
// These values are shared by every measurement
reportID := sql.NullString{String: "", Valid: false}
testName := nt.Name
resultID := c.res.ID
reportFilePath := c.msmtPath
// This is to workaround homedirs having UTF-8 characters in them.
// See: https://github.com/measurement-kit/measurement-kit/issues/1635
@@ -157,7 +166,7 @@ func (c *Controller) Init(nt *mk.Nettest) error {
nt.On("status.report_created", func(e mk.Event) {
log.Debugf("%s", e.Key)
msmtTemplate.ReportID = sql.NullString{String: e.Value.ReportID, Valid: true}
reportID = sql.NullString{String: e.Value.ReportID, Valid: true}
})
nt.On("status.geoip_lookup", func(e mk.Event) {
@@ -175,7 +184,11 @@ func (c *Controller) Init(nt *mk.Nettest) error {
log.Debugf(color.RedString(e.Key))
idx := e.Value.Idx
msmt, err := database.CreateMeasurement(c.Ctx.DB, msmtTemplate, e.Value.Input)
urlID := sql.NullInt64{Int64: 0, Valid: false}
if c.inputIdxMap != nil {
urlID = sql.NullInt64{Int64: c.inputIdxMap[idx], Valid: true}
}
msmt, err := database.CreateMeasurement(c.Ctx.DB, reportID, testName, resultID, reportFilePath, urlID)
if err != nil {
log.WithError(err).Error("Failed to create measurement")
return
nettests/websites/web_connectivity.go
+50 -9
View File
@@ -6,7 +6,9 @@ import (
"io/ioutil"
"net/http"
"github.com/apex/log"
"github.com/measurement-kit/go-measurement-kit"
"github.com/ooni/probe-cli/internal/database"
"github.com/ooni/probe-cli/nettests"
"github.com/pkg/errors"
)
@@ -14,6 +16,7 @@ import (
// URLInfo contains the URL and the citizenlab category code for that URL
type URLInfo struct {
URL string `json:"url"`
CountryCode string `json:"country_code"`
CategoryCode string `json:"category_code"`
}
@@ -24,10 +27,11 @@ type URLResponse struct {
const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
func lookupURLs(ctl *nettests.Controller) ([]string, error) {
func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) {
var (
parsed = new(URLResponse)
urls []string
parsed = new(URLResponse)
urls []string
urlIDMap map[int64]int64
)
// XXX pass in the configuration for category codes
reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
@@ -36,22 +40,58 @@ func lookupURLs(ctl *nettests.Controller) ([]string, error) {
resp, err := http.Get(reqURL)
if err != nil {
return urls, errors.Wrap(err, "failed to perform request")
return urls, urlIDMap, errors.Wrap(err, "failed to perform request")
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return urls, errors.Wrap(err, "failed to read response body")
return urls, urlIDMap, errors.Wrap(err, "failed to read response body")
}
err = json.Unmarshal([]byte(body), &parsed)
if err != nil {
return urls, errors.Wrap(err, "failed to parse json")
return urls, urlIDMap, errors.Wrap(err, "failed to parse json")
}
for _, url := range parsed.Results {
for idx, url := range parsed.Results {
var urlID int64
res, err := ctl.Ctx.DB.Update("urls").Set(
"url", url.URL,
"category_code", url.CategoryCode,
"country_code", url.CountryCode,
).Where("url = ? AND country_code = ?", url.URL, url.CountryCode).Exec()
if err != nil {
log.Error("Failed to write to the URL table")
} else {
affected, err := res.RowsAffected()
if err != nil {
log.Error("Failed to get affected row count")
} else if affected == 0 {
newID, err := ctl.Ctx.DB.Collection("urls").Insert(
database.URL{
URL: url.URL,
CategoryCode: url.CategoryCode,
CountryCode: url.CountryCode,
})
if err != nil {
log.Error("Failed to insert into the URLs table")
}
urlID = newID.(int64)
} else {
lastID, err := res.LastInsertId()
if err != nil {
log.Error("failed to get URL ID")
}
urlID = lastID
}
}
urlIDMap[int64(idx)] = urlID
urls = append(urls, url.URL)
}
return urls, nil
return urls, urlIDMap, nil
}
// WebConnectivity test implementation
@@ -63,10 +103,11 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error {
nt := mk.NewNettest("WebConnectivity")
ctl.Init(nt)
urls, err := lookupURLs(ctl)
urls, urlIDMap, err := lookupURLs(ctl)
if err != nil {
return err
}
ctl.SetInputIdxMap(urlIDMap)
nt.Options.Inputs = urls
return nt.Run()