Merge pull request #9 from OpenObservatory/feature/web_connectivity

Feature/web connectivity

Gopkg.lock

@@ -89,7 +89,7 @@
   branch = "master"
   name = "github.com/measurement-kit/go-measurement-kit"
   packages = ["."]
-  revision = "6ae2401a8e498a90ccdd3edbda1841add079b70e"
+  revision = "cbf1c976aeaa2906f4fda278fc3068f7bde63c47"
 
 [[projects]]
   branch = "master"

Makefile

@@ -5,6 +5,11 @@ build:
 	@$(GO) build -i -o dist/ooni cmd/ooni/main.go
 .PHONY: build
 
+update-mk:
+	@echo "updating mk"
+	@dep ensure -update github.com/measurement-kit/go-measurement-kit
+.PHONY: update-mk
+
 bindata:
 	@$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \
 		-nometadata	\

internal/cli/geoip/geoip.go

@@ -1,8 +1,6 @@
 package geoip
 
 import (
-	"path/filepath"
-
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/openobservatory/gooni/internal/cli/root"
@@ -21,10 +19,10 @@ func init() {
 			return err
 		}
 
-		geoipPath := filepath.Join(ctx.Home, "geoip")
-
+		geoipPath := utils.GeoIPDir(ctx.Home)
 		if *shouldUpdate {
 			utils.DownloadGeoIPDatabaseFiles(geoipPath)
+			utils.DownloadLegacyGeoIPDatabaseFiles(geoipPath)
 		}
 
 		loc, err := utils.GeoIPLookup(geoipPath)

internal/cli/run/run.go

@@ -33,6 +33,12 @@ func init() {
 		}
 		log.Debugf("Running test group %s", group.Label)
 
+		err = ctx.MaybeLocationLookup()
+		if err != nil {
+			log.WithError(err).Error("Failed to lookup the location of the probe")
+			return err
+		}
+
 		result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
 			Name:      *nettestGroup,
 			StartTime: time.Now().UTC(),

internal/database/models.go

@@ -15,7 +15,7 @@ import (
 type ResultSummaryFunc func(SummaryMap) (string, error)
 
 // SummaryMap contains a mapping from test name to serialized summary for it
-type SummaryMap map[string]string
+type SummaryMap map[string][]string
 
 // UpdateOne will run the specified update query and check that it only affected one row
 func UpdateOne(db *sqlx.DB, query string, arg interface{}) error {
@@ -129,13 +129,19 @@ func (m *Measurement) WriteSummary(db *sqlx.DB, summary string) error {
 
 // AddToResult adds a measurement to a result
 func (m *Measurement) AddToResult(db *sqlx.DB, result *Result) error {
+	var err error
+
 	m.ResultID = result.ID
 	finalPath := filepath.Join(result.MeasurementDir,
 		filepath.Base(m.ReportFilePath))
 
-	err := os.Rename(m.ReportFilePath, finalPath)
-	if err != nil {
-		return errors.Wrap(err, "moving report file")
+	// If the finalPath already exists, it means it has already been moved there.
+	// This happens in multi input reports
+	if _, err = os.Stat(finalPath); os.IsNotExist(err) {
+		err = os.Rename(m.ReportFilePath, finalPath)
+		if err != nil {
+			return errors.Wrap(err, "moving report file")
+		}
 	}
 	m.ReportFilePath = finalPath
 
@@ -204,7 +210,12 @@ func MakeSummaryMap(db *sqlx.DB, r *Result) (SummaryMap, error) {
 		return nil, errors.Wrap(err, "failed to get measurements")
 	}
 	for _, msmt := range msmts {
-		summaryMap[msmt.Name] = msmt.Summary
+		val, ok := summaryMap[msmt.Name]
+		if ok {
+			summaryMap[msmt.Name] = append(val, msmt.Summary)
+		} else {
+			summaryMap[msmt.Name] = []string{msmt.Summary}
+		}
 	}
 	return summaryMap, nil
 }

nettests/groups/groups.go

@@ -34,7 +34,14 @@ type MiddleboxSummary struct {
 
 // IMSummary is the summary for the im tests
 type IMSummary struct {
-	Detected bool
+	Tested  uint
+	Blocked uint
+}
+
+// WebsitesSummary is the summary for the websites test
+type WebsitesSummary struct {
+	Tested  uint
+	Blocked uint
 }
 
 // NettestGroups that can be run by the user
@@ -45,7 +52,28 @@ var NettestGroups = map[string]NettestGroup{
 			websites.WebConnectivity{},
 		},
 		Summary: func(m database.SummaryMap) (string, error) {
-			return "{}", nil
+			// XXX to generate this I need to create the summary map as a list
+			var summary WebsitesSummary
+			summary.Tested = 0
+			summary.Blocked = 0
+			for _, msmtSummaryStr := range m["WebConnectivity"] {
+				var wcSummary websites.WebConnectivitySummary
+
+				err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary)
+				if err != nil {
+					log.WithError(err).Error("failed to unmarshal WebConnectivity summary")
+					return "", err
+				}
+				if wcSummary.Blocked {
+					summary.Blocked++
+				}
+				summary.Tested++
+			}
+			summaryBytes, err := json.Marshal(summary)
+			if err != nil {
+				return "", err
+			}
+			return string(summaryBytes), nil
 		},
 	},
 	"performance": NettestGroup{
@@ -61,12 +89,12 @@ var NettestGroups = map[string]NettestGroup{
 				dashSummary performance.DashSummary
 				summary     PerformanceSummary
 			)
-			err = json.Unmarshal([]byte(m["Dash"]), &dashSummary)
+			err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal Dash summary")
 				return "", err
 			}
-			err = json.Unmarshal([]byte(m["Ndt"]), &ndtSummary)
+			err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal NDT summary")
 				return "", err
@@ -95,12 +123,12 @@ var NettestGroups = map[string]NettestGroup{
 				hirlSummary middlebox.HTTPInvalidRequestLineSummary
 				summary     MiddleboxSummary
 			)
-			err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"]), &hhfmSummary)
+			err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal hhfm summary")
 				return "", err
 			}
-			err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"]), &hirlSummary)
+			err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary)
 			if err != nil {
 				log.WithError(err).Error("failed to unmarshal hirl summary")
 				return "", err
@@ -121,7 +149,47 @@ var NettestGroups = map[string]NettestGroup{
 			im.WhatsApp{},
 		},
 		Summary: func(m database.SummaryMap) (string, error) {
-			return "{}", nil
+			var (
+				err       error
+				waSummary im.WhatsAppSummary
+				tgSummary im.TelegramSummary
+				fbSummary im.FacebookMessengerSummary
+				summary   IMSummary
+			)
+			err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal whatsapp summary")
+				return "", err
+			}
+			err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal telegram summary")
+				return "", err
+			}
+			err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary)
+			if err != nil {
+				log.WithError(err).Error("failed to unmarshal facebook summary")
+				return "", err
+			}
+			// XXX it could actually be that some are not tested when the
+			// configuration is changed.
+			summary.Tested = 3
+			summary.Blocked = 0
+			if fbSummary.Blocked == true {
+				summary.Blocked++
+			}
+			if tgSummary.Blocked == true {
+				summary.Blocked++
+			}
+			if waSummary.Blocked == true {
+				summary.Blocked++
+			}
+
+			summaryBytes, err := json.Marshal(summary)
+			if err != nil {
+				return "", err
+			}
+			return string(summaryBytes), nil
 		},
 	},
 }

nettests/im/facebook_messenger.go

@@ -20,13 +20,31 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error {
 type FacebookMessengerSummary struct {
 	DNSBlocking bool
 	TCPBlocking bool
+	Blocked     bool
 }
 
 // Summary generates a summary for a test run
 func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} {
+	var (
+		dnsBlocking bool
+		tcpBlocking bool
+	)
+	if tk["facebook_dns_blocking"] == nil {
+		dnsBlocking = false
+	} else {
+		dnsBlocking = tk["facebook_dns_blocking"].(bool)
+	}
+
+	if tk["facebook_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["facebook_tcp_blocking"].(bool)
+	}
+
 	return FacebookMessengerSummary{
-		DNSBlocking: tk["facebook_dns_blocking"].(bool),
-		TCPBlocking: tk["facebook_tcp_blocking"].(bool),
+		DNSBlocking: dnsBlocking,
+		TCPBlocking: tcpBlocking,
+		Blocked:     dnsBlocking || tcpBlocking,
 	}
 }
 

nettests/im/telegram.go

@@ -21,14 +21,38 @@ type TelegramSummary struct {
 	HTTPBlocking bool
 	TCPBlocking  bool
 	WebBlocking  bool
+	Blocked      bool
 }
 
 // Summary generates a summary for a test run
 func (h Telegram) Summary(tk map[string]interface{}) interface{} {
+	var (
+		tcpBlocking  bool
+		httpBlocking bool
+		webBlocking  bool
+	)
+
+	if tk["telegram_tcp_blocking"] == nil {
+		tcpBlocking = false
+	} else {
+		tcpBlocking = tk["telegram_tcp_blocking"].(bool)
+	}
+	if tk["telegram_http_blocking"] == nil {
+		httpBlocking = false
+	} else {
+		httpBlocking = tk["telegram_http_blocking"].(bool)
+	}
+	if tk["telegram_web_status"] == nil {
+		webBlocking = false
+	} else {
+		webBlocking = tk["telegram_web_status"].(string) == "blocked"
+	}
+
 	return TelegramSummary{
-		TCPBlocking:  tk["telegram_tcp_blocking"].(bool) == true,
-		HTTPBlocking: tk["telegram_http_blocking"].(bool) == true,
-		WebBlocking:  tk["telegram_web_status"].(string) == "blocked",
+		TCPBlocking:  tcpBlocking,
+		HTTPBlocking: httpBlocking,
+		WebBlocking:  webBlocking,
+		Blocked:      webBlocking || httpBlocking || tcpBlocking,
 	}
 }
 

nettests/im/whatsapp.go

@@ -21,16 +21,36 @@ type WhatsAppSummary struct {
 	RegistrationServerBlocking bool
 	WebBlocking                bool
 	EndpointsBlocking          bool
+	Blocked                    bool
 }
 
 // Summary generates a summary for a test run
 func (h WhatsApp) Summary(tk map[string]interface{}) interface{} {
-	const blk = "blocked"
+	var (
+		webBlocking          bool
+		registrationBlocking bool
+		endpointsBlocking    bool
+	)
+
+	var computeBlocking = func(key string) bool {
+		const blk = "blocked"
+		if tk[key] == nil {
+			return false
+		}
+		if tk[key].(string) == blk {
+			return true
+		}
+		return false
+	}
+	registrationBlocking = computeBlocking("registration_server_status")
+	webBlocking = computeBlocking("whatsapp_web_status")
+	endpointsBlocking = computeBlocking("whatsapp_endpoints_status")
 
 	return WhatsAppSummary{
-		RegistrationServerBlocking: tk["registration_server_status"].(string) == blk,
-		WebBlocking:                tk["whatsapp_web_status"].(string) == blk,
-		EndpointsBlocking:          tk["whatsapp_endpoints_status"].(string) == blk,
+		RegistrationServerBlocking: registrationBlocking,
+		WebBlocking:                webBlocking,
+		EndpointsBlocking:          endpointsBlocking,
+		Blocked:                    registrationBlocking || webBlocking || endpointsBlocking,
 	}
 }
 

nettests/nettests.go

@@ -3,6 +3,7 @@ package nettests
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 
 	"github.com/apex/log"
 	"github.com/measurement-kit/go-measurement-kit"
@@ -11,6 +12,7 @@ import (
 	"github.com/openobservatory/gooni/internal/colors"
 	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/output"
+	"github.com/openobservatory/gooni/utils"
 )
 
 // Nettest interface. Every Nettest should implement this.
@@ -43,6 +45,8 @@ type Controller struct {
 // Init should be called once to initialise the nettest
 func (c *Controller) Init(nt *mk.Nettest) error {
 	log.Debugf("Init: %v", nt)
+	c.Ctx.LocationLookup()
+
 	c.msmts = make(map[int64]*database.Measurement)
 
 	msmtTemplate := database.Measurement{
@@ -57,16 +61,21 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 
 	log.Debugf("OutputPath: %s", c.msmtPath)
 	nt.Options = mk.NettestOptions{
-		IncludeIP:        c.Ctx.Config.Sharing.IncludeIP,
-		IncludeASN:       c.Ctx.Config.Sharing.IncludeASN,
-		IncludeCountry:   c.Ctx.Config.Advanced.IncludeCountry,
+		IncludeIP:      c.Ctx.Config.Sharing.IncludeIP,
+		IncludeASN:     c.Ctx.Config.Sharing.IncludeASN,
+		IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry,
+
+		ProbeCC:  c.Ctx.Location.CountryCode,
+		ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN),
+		ProbeIP:  c.Ctx.Location.IP,
+
 		DisableCollector: false,
 		SoftwareName:     "ooniprobe",
 		SoftwareVersion:  version.Version,
 
 		// XXX
-		GeoIPCountryPath: "",
-		GeoIPASNPath:     "",
+		GeoIPCountryPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat"),
+		GeoIPASNPath:     filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat"),
 		OutputPath:       c.msmtPath,
 		CaBundlePath:     "/etc/ssl/cert.pem",
 	}

nettests/websites/web_connectivity.go

@@ -1,10 +1,58 @@
 package websites
 
 import (
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+
 	"github.com/measurement-kit/go-measurement-kit"
 	"github.com/openobservatory/gooni/nettests"
+	"github.com/pkg/errors"
 )
 
+// URLInfo contains the URL and the citizenlab category code for that URL
+type URLInfo struct {
+	URL          string `json:"url"`
+	CategoryCode string `json:"category_code"`
+}
+
+// URLResponse is the orchestrate url response containing a list of URLs
+type URLResponse struct {
+	Results []URLInfo `json:"results"`
+}
+
+const orchestrateBaseURL = "https://events.proteus.test.ooni.io"
+
+func lookupURLs(ctl *nettests.Controller) ([]string, error) {
+	var (
+		parsed = new(URLResponse)
+		urls   []string
+	)
+	reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s",
+		orchestrateBaseURL,
+		ctl.Ctx.Location.CountryCode)
+
+	resp, err := http.Get(reqURL)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to perform request")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to read response body")
+	}
+	err = json.Unmarshal([]byte(body), &parsed)
+	if err != nil {
+		return urls, errors.Wrap(err, "failed to parse json")
+	}
+
+	for _, url := range parsed.Results {
+		urls = append(urls, url.URL)
+	}
+	return urls, nil
+}
+
 // WebConnectivity test implementation
 type WebConnectivity struct {
 }
@@ -13,12 +61,56 @@ type WebConnectivity struct {
 func (n WebConnectivity) Run(ctl *nettests.Controller) error {
 	nt := mk.NewNettest("WebConnectivity")
 	ctl.Init(nt)
+
+	urls, err := lookupURLs(ctl)
+	if err != nil {
+		return err
+	}
+	nt.Options.Inputs = urls
+
 	return nt.Run()
 }
 
+// WebConnectivitySummary for the test
+type WebConnectivitySummary struct {
+	Accessible bool
+	Blocking   string
+	Blocked    bool
+}
+
 // Summary generates a summary for a test run
 func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} {
-	return nil
+	var (
+		blocked    bool
+		blocking   string
+		accessible bool
+	)
+
+	// We need to do these complicated type assertions, because some of the fields
+	// are "nullable" and/or can be of different types
+	switch v := tk["blocking"].(type) {
+	case bool:
+		blocked = false
+		blocking = "none"
+	case string:
+		blocked = true
+		blocking = v
+	default:
+		blocked = false
+		blocking = "none"
+	}
+
+	if tk["accessible"] == nil {
+		accessible = false
+	} else {
+		accessible = tk["accessible"].(bool)
+	}
+
+	return WebConnectivitySummary{
+		Accessible: accessible,
+		Blocking:   blocking,
+		Blocked:    blocked,
+	}
 }
 
 // LogSummary writes the summary to the standard output

ooni.go

@@ -13,6 +13,7 @@ import (
 	"github.com/openobservatory/gooni/config"
 	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/legacy"
+	"github.com/openobservatory/gooni/utils"
 	"github.com/pkg/errors"
 )
 
@@ -35,15 +36,39 @@ func Onboarding(c *Config) error {
 
 // Context for OONI Probe
 type Context struct {
-	Config *Config
-	DB     *sqlx.DB
+	Config   *Config
+	DB       *sqlx.DB
+	Location *utils.LocationInfo
+
+	Home    string
+	TempDir string
 
-	Home       string
-	TempDir    string
 	dbPath     string
 	configPath string
 }
 
+// MaybeLocationLookup will lookup the location of the user unless it's already cached
+func (c *Context) MaybeLocationLookup() error {
+	if c.Location == nil {
+		return c.LocationLookup()
+	}
+	return nil
+}
+
+// LocationLookup lookup the location of the user via geoip
+func (c *Context) LocationLookup() error {
+	var err error
+
+	dbPath := filepath.Join(c.Home, "geoip")
+
+	c.Location, err = utils.GeoIPLookup(dbPath)
+	if err != nil {
+		return err
+	}
+
+	return nil
+}
+
 // Init the OONI manager
 func (c *Context) Init() error {
 	var err error
@@ -52,7 +77,7 @@ func (c *Context) Init() error {
 		return errors.Wrap(err, "migrating home")
 	}
 
-	if err = CreateHomeDirs(c.Home); err != nil {
+	if err = MaybeInitializeHome(c.Home); err != nil {
 		return err
 	}
 
@@ -187,16 +212,28 @@ func ParseConfig(b []byte) (*Config, error) {
 	return c, nil
 }
 
-// CreateHomeDirs creates the OONI home subdirectories
-func CreateHomeDirs(home string) error {
+// MaybeInitializeHome does the setup for a new OONI Home
+func MaybeInitializeHome(home string) error {
+	firstRun := false
 	requiredDirs := []string{"db", "msmts", "geoip"}
 	for _, d := range requiredDirs {
 		if _, e := os.Stat(filepath.Join(home, d)); e != nil {
+			firstRun = true
 			if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil {
 				return err
 			}
 		}
 	}
+	if firstRun == true {
+		log.Info("This is the first time you are running OONI Probe. Downloading some files.")
+		geoipDir := utils.GeoIPDir(home)
+		if err := utils.DownloadGeoIPDatabaseFiles(geoipDir); err != nil {
+			return err
+		}
+		if err := utils.DownloadLegacyGeoIPDatabaseFiles(geoipDir); err != nil {
+			return err
+		}
+	}
 
 	return nil
 }

utils/geoip.go

@@ -31,31 +31,82 @@ var geoipFiles = map[string]string{
 	"GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz",
 }
 
+var legacyGeoipFiles = map[string]string{
+	"GeoIPASNum.dat": "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz",
+	"GeoIP.dat":      "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
+}
+
+// GeoIPDir returns the geoip data dir for the given OONI Home
+func GeoIPDir(home string) string {
+	return filepath.Join(home, "geoip")
+}
+
+// Download the file to a temporary location
+func downloadToTemp(url string) (string, error) {
+	out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create temporary directory")
+	}
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch URL")
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to copy response body")
+	}
+	out.Close()
+	resp.Body.Close()
+	return out.Name(), nil
+}
+
+// DownloadLegacyGeoIPDatabaseFiles into the target directory
+func DownloadLegacyGeoIPDatabaseFiles(dir string) error {
+	for filename, url := range legacyGeoipFiles {
+		dstPath := filepath.Join(dir, filename)
+
+		tmpPath, err := downloadToTemp(url)
+		if err != nil {
+			return err
+		}
+
+		// Extract the tar.gz file
+		f, err := os.Open(tmpPath)
+		defer f.Close()
+		if err != nil {
+			return errors.Wrap(err, "failed to read file")
+		}
+
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return errors.Wrap(err, "failed to create gzip reader")
+		}
+
+		outFile, err := os.Create(dstPath)
+		if err != nil {
+			return errors.Wrap(err, "error creating file")
+		}
+		if _, err := io.Copy(outFile, gzf); err != nil {
+			return errors.Wrap(err, "error reading file from gzip")
+		}
+		outFile.Close()
+	}
+	return nil
+}
+
 // DownloadGeoIPDatabaseFiles into the target directory
 func DownloadGeoIPDatabaseFiles(dir string) error {
 	for filename, url := range geoipFiles {
 		dstPath := filepath.Join(dir, filename)
 
-		// Download the file to a temporary location
-		out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+		tmpPath, err := downloadToTemp(url)
 		if err != nil {
-			return errors.Wrap(err, "failed to create temporary directory")
+			return err
 		}
-		resp, err := http.Get(url)
-		if err != nil {
-			return errors.Wrap(err, "failed to fetch URL")
-		}
-
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			return errors.Wrap(err, "failed to copy response body")
-		}
-		out.Close()
-		resp.Body.Close()
 
 		// Extract the tar.gz file
-
-		f, err := os.Open(out.Name())
+		f, err := os.Open(tmpPath)
 		if err != nil {
 			return errors.Wrap(err, "failed to read file")
 		}