Merge pull request #8 from OpenObservatory/feature/geoiplookup

Feature/geoiplookup

.gitignore

@@ -1,3 +1,3 @@
-vendor/
+/vendor
-/ooni
+/dist
 *.njson

Gopkg.lock

@@ -103,6 +103,18 @@
   packages = ["."]
   revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
 
+[[projects]]
+  name = "github.com/oschwald/geoip2-golang"
+  packages = ["."]
+  revision = "7118115686e16b77967cdbf55d1b944fe14ad312"
+  version = "v1.2.1"
+
+[[projects]]
+  name = "github.com/oschwald/maxminddb-golang"
+  packages = ["."]
+  revision = "c5bec84d1963260297932a1b7a1753c8420717a7"
+  version = "v1.3.0"
+
 [[projects]]
   name = "github.com/pkg/errors"
   packages = ["."]
@@ -164,7 +176,10 @@
 [[projects]]
   branch = "master"
   name = "golang.org/x/sys"
-  packages = ["unix"]
+  packages = [
+    "unix",
+    "windows"
+  ]
   revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
 
 [[projects]]
@@ -186,6 +201,6 @@
 [solve-meta]
   analyzer-name = "dep"
   analyzer-version = 1
-  inputs-digest = "cfd4567b0cd553bc4594b7f8bebcb91be475dfd112fd8740f65b5ba4a8253181"
+  inputs-digest = "46860a32f649dbb2e01b285c0d5581078ba4b28a21e21175d47ccb2725a9c9fb"
   solver-name = "gps-cdcl"
   solver-version = 1

Gopkg.toml

@@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"]
 [[constraint]]
   name = "github.com/shuLhan/go-bindata"
   version = "3.3.0"
+
+[[constraint]]
+  name = "github.com/oschwald/geoip2-golang"
+  version = "1.2.1"

Makefile

@@ -2,7 +2,7 @@ GO ?= go
 
 build:
 	@echo "Building ./ooni"
-	@$(GO) build -i -o ooni cmd/ooni/main.go
+	@$(GO) build -i -o dist/ooni cmd/ooni/main.go
 .PHONY: build
 
 bindata:

cmd/ooni/main.go

@@ -4,6 +4,7 @@ import (
 	// commands
 	"github.com/apex/log"
 
+	_ "github.com/openobservatory/gooni/internal/cli/geoip"
 	_ "github.com/openobservatory/gooni/internal/cli/info"
 	_ "github.com/openobservatory/gooni/internal/cli/list"
 	_ "github.com/openobservatory/gooni/internal/cli/nettest"

internal/cli/geoip/geoip.go

@@ -0,0 +1,44 @@
+package geoip
+
+import (
+	"path/filepath"
+
+	"github.com/alecthomas/kingpin"
+	"github.com/apex/log"
+	"github.com/openobservatory/gooni/internal/cli/root"
+	"github.com/openobservatory/gooni/utils"
+)
+
+func init() {
+	cmd := root.Command("geoip", "Perform a geoip lookup")
+
+	shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool()
+
+	cmd.Action(func(_ *kingpin.ParseContext) error {
+		log.Info("geoip")
+		ctx, err := root.Init()
+		if err != nil {
+			return err
+		}
+
+		geoipPath := filepath.Join(ctx.Home, "geoip")
+
+		if *shouldUpdate {
+			utils.DownloadGeoIPDatabaseFiles(geoipPath)
+		}
+
+		loc, err := utils.GeoIPLookup(geoipPath)
+		if err != nil {
+			return err
+		}
+
+		log.WithFields(log.Fields{
+			"asn":          loc.ASN,
+			"network_name": loc.NetworkName,
+			"country_code": loc.CountryCode,
+			"ip":           loc.IP,
+		}).Info("Looked up your location")
+
+		return nil
+	})
+}

internal/cli/root/root.go

@@ -4,7 +4,6 @@ import (
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	ooni "github.com/openobservatory/gooni"
-	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/log/handlers/batch"
 	"github.com/openobservatory/gooni/internal/log/handlers/cli"
 	"github.com/prometheus/common/version"
@@ -17,7 +16,7 @@ var Cmd = kingpin.New("ooni", "")
 var Command = Cmd.Command
 
 // Init should be called by all subcommand that care to have a ooni.OONI instance
-var Init func() (*ooni.Config, *ooni.Context, error)
+var Init func() (*ooni.Context, error)
 
 func init() {
 	configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
@@ -36,39 +35,21 @@ func init() {
 			log.Debugf("ooni version %s", version.Version)
 		}
 
-		Init = func() (*ooni.Config, *ooni.Context, error) {
+		Init = func() (*ooni.Context, error) {
-			var config *ooni.Config
 			var err error
 
-			if *configPath != "" {
+			homePath, err := ooni.GetOONIHome()
-				log.Debugf("Reading config file from %s", *configPath)
-				config, err = ooni.ReadConfig(*configPath)
-			} else {
-				log.Debug("Reading default config file")
-				config, err = ooni.ReadDefaultConfigPaths()
-			}
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
-			dbPath, err := database.DefaultDatabasePath()
+			ctx := ooni.NewContext(*configPath, homePath)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			log.Debugf("Connecting to database sqlite3://%s", dbPath)
-			db, err := database.Connect(dbPath)
-			if err != nil {
-				return nil, nil, err
-			}
-
-			ctx := ooni.New(config, db)
 			err = ctx.Init()
 			if err != nil {
-				return nil, nil, err
+				return nil, err
 			}
 
-			return config, ctx, nil
+			return ctx, nil
 		}
 
 		return nil

internal/cli/run/run.go

@@ -21,7 +21,7 @@ func init() {
 
 	cmd.Action(func(_ *kingpin.ParseContext) error {
 		log.Infof("Starting %s", *nettestGroup)
-		_, ctx, err := root.Init()
+		ctx, err := root.Init()
 		if err != nil {
 			log.Errorf("%s", err)
 			return err
@@ -33,7 +33,7 @@ func init() {
 		}
 		log.Debugf("Running test group %s", group.Label)
 
-		result, err := database.CreateResult(ctx.DB, database.Result{
+		result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
 			Name:      *nettestGroup,
 			StartTime: time.Now().UTC(),
 		})

internal/database/database.go

@@ -1,14 +1,10 @@
 package database
 
 import (
-	"path/filepath"
-
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
 	_ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver
-	ooni "github.com/openobservatory/gooni"
 	"github.com/openobservatory/gooni/internal/bindata"
-	"github.com/pkg/errors"
 	migrate "github.com/rubenv/sql-migrate"
 )
 
@@ -41,12 +37,3 @@ func Connect(path string) (db *sqlx.DB, err error) {
 	}
 	return
 }
-
-// DefaultDatabasePath for the main database
-func DefaultDatabasePath() (string, error) {
-	home, err := ooni.GetOONIHome()
-	if err != nil {
-		return "", errors.Wrap(err, "default database path")
-	}
-	return filepath.Join(home, "db", "main.sqlite3"), nil
-}

internal/database/models.go

@@ -8,7 +8,6 @@ import (
 
 	"github.com/apex/log"
 	"github.com/jmoiron/sqlx"
-	ooni "github.com/openobservatory/gooni"
 	"github.com/pkg/errors"
 )
 
@@ -238,11 +237,7 @@ func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error {
 }
 
 // MakeResultsPath creates and returns a directory for the result
-func MakeResultsPath(r *Result) (string, error) {
+func MakeResultsPath(home string, r *Result) (string, error) {
-	home, err := ooni.GetOONIHome()
-	if err != nil {
-		return "", errors.Wrap(err, "default measurements path")
-	}
 	p := filepath.Join(home, "msmts",
 		fmt.Sprintf("%s-%s", r.Name, r.StartTime.Format(time.RFC3339Nano)))
 
@@ -251,7 +246,7 @@ func MakeResultsPath(r *Result) (string, error) {
 	if _, e := os.Stat(p); e == nil {
 		return "", errors.New("results path already exists")
 	}
-	err = os.MkdirAll(p, 0700)
+	err := os.MkdirAll(p, 0700)
 	if err != nil {
 		return "", err
 	}
@@ -260,10 +255,10 @@ func MakeResultsPath(r *Result) (string, error) {
 
 // CreateResult writes the Result to the database a returns a pointer
 // to the Result
-func CreateResult(db *sqlx.DB, r Result) (*Result, error) {
+func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) {
 	log.Debugf("Creating result %v", r)
 
-	p, err := MakeResultsPath(&r)
+	p, err := MakeResultsPath(homePath, &r)
 	if err != nil {
 		return nil, err
 	}

ooni.go

@@ -11,6 +11,7 @@ import (
 	"github.com/jmoiron/sqlx"
 	homedir "github.com/mitchellh/go-homedir"
 	"github.com/openobservatory/gooni/config"
+	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/legacy"
 	"github.com/pkg/errors"
 )
@@ -34,22 +35,52 @@ func Onboarding(c *Config) error {
 
 // Context for OONI Probe
 type Context struct {
-	Config  *Config
+	Config *Config
-	DB      *sqlx.DB
+	DB     *sqlx.DB
-	TempDir string
+
+	Home       string
+	TempDir    string
+	dbPath     string
+	configPath string
 }
 
 // Init the OONI manager
 func (c *Context) Init() error {
-	if err := legacy.MaybeMigrateHome(); err != nil {
+	var err error
+
+	if err = legacy.MaybeMigrateHome(); err != nil {
 		return errors.Wrap(err, "migrating home")
 	}
+
+	if err = CreateHomeDirs(c.Home); err != nil {
+		return err
+	}
+
+	if c.configPath != "" {
+		log.Debugf("Reading config file from %s", c.configPath)
+		c.Config, err = ReadConfig(c.configPath)
+	} else {
+		log.Debug("Reading default config file")
+		c.Config, err = ReadDefaultConfigPaths(c.Home)
+	}
+	if err != nil {
+		return err
+	}
+
+	c.dbPath = filepath.Join(c.Home, "db", "main.sqlite3")
 	if c.Config.InformedConsent == false {
-		if err := Onboarding(c.Config); err != nil {
+		if err = Onboarding(c.Config); err != nil {
 			return errors.Wrap(err, "onboarding")
 		}
 	}
 
+	log.Debugf("Connecting to database sqlite3://%s", c.dbPath)
+	db, err := database.Connect(c.dbPath)
+	if err != nil {
+		return err
+	}
+	c.DB = db
+
 	tempDir, err := ioutil.TempDir("", "ooni")
 	if err != nil {
 		return errors.Wrap(err, "creating TempDir")
@@ -59,11 +90,12 @@ func (c *Context) Init() error {
 	return nil
 }
 
-// New Context instance.
+// NewContext instance.
-func New(c *Config, d *sqlx.DB) *Context {
+func NewContext(configPath string, homePath string) *Context {
 	return &Context{
-		Config: c,
+		Home:       homePath,
-		DB:     d,
+		Config:     &Config{},
+		configPath: configPath,
 	}
 }
 
@@ -155,24 +187,18 @@ func ParseConfig(b []byte) (*Config, error) {
 	return c, nil
 }
 
-//EnsureDefaultOONIHomeDir makes sure the paths to the OONI Home exist
+// CreateHomeDirs creates the OONI home subdirectories
-func EnsureDefaultOONIHomeDir() (string, error) {
+func CreateHomeDirs(home string) error {
-	home, err := GetOONIHome()
+	requiredDirs := []string{"db", "msmts", "geoip"}
-	if err != nil {
-		return "", err
-	}
-
-	requiredDirs := []string{"db", "msmts"}
 	for _, d := range requiredDirs {
 		if _, e := os.Stat(filepath.Join(home, d)); e != nil {
-			err = os.MkdirAll(filepath.Join(home, d), 0700)
+			if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil {
-			if err != nil {
+				return err
-				return "", err
 			}
 		}
 	}
 
-	return home, nil
+	return nil
 }
 
 // ReadConfig reads the configuration from the path
@@ -206,11 +232,7 @@ func ReadConfig(path string) (*Config, error) {
 }
 
 // ReadDefaultConfigPaths from common locations.
-func ReadDefaultConfigPaths() (*Config, error) {
+func ReadDefaultConfigPaths(home string) (*Config, error) {
-	home, err := EnsureDefaultOONIHomeDir()
-	if err != nil {
-		return nil, errors.Wrap(err, "reading default config paths")
-	}
 	var paths = []string{
 		filepath.Join(home, "config.json"),
 	}

utils/geoip.go

@@ -0,0 +1,210 @@
+package utils
+
+import (
+	"archive/tar"
+	"compress/gzip"
+	"encoding/json"
+	"io"
+	"io/ioutil"
+	"math/rand"
+	"net"
+	"net/http"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/oschwald/geoip2-golang"
+	"github.com/pkg/errors"
+)
+
+// LocationInfo contains location information
+type LocationInfo struct {
+	IP          string
+	ASN         uint
+	NetworkName string
+	CountryCode string
+}
+
+// XXX consider integration with: https://updates.maxmind.com/app/update_getfilename?product_id=GeoLite2-ASN
+var geoipFiles = map[string]string{
+	"GeoLite2-ASN.mmdb":     "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz",
+	"GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz",
+}
+
+// DownloadGeoIPDatabaseFiles into the target directory
+func DownloadGeoIPDatabaseFiles(dir string) error {
+	for filename, url := range geoipFiles {
+		dstPath := filepath.Join(dir, filename)
+
+		// Download the file to a temporary location
+		out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+		if err != nil {
+			return errors.Wrap(err, "failed to create temporary directory")
+		}
+		resp, err := http.Get(url)
+		if err != nil {
+			return errors.Wrap(err, "failed to fetch URL")
+		}
+
+		_, err = io.Copy(out, resp.Body)
+		if err != nil {
+			return errors.Wrap(err, "failed to copy response body")
+		}
+		out.Close()
+		resp.Body.Close()
+
+		// Extract the tar.gz file
+
+		f, err := os.Open(out.Name())
+		if err != nil {
+			return errors.Wrap(err, "failed to read file")
+		}
+
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return errors.Wrap(err, "failed to create gzip reader")
+		}
+		tarReader := tar.NewReader(gzf)
+
+		// Look inside of the tar for the file we need
+		for {
+			header, err := tarReader.Next()
+			if err == io.EOF {
+				break
+			}
+			if err != nil {
+				return errors.Wrap(err, "error extracting tar.gz")
+			}
+			name := header.Name
+			if filepath.Base(name) == filename {
+				outFile, err := os.Create(dstPath)
+				if err != nil {
+					return errors.Wrap(err, "error creating file")
+				}
+				if _, err := io.Copy(outFile, tarReader); err != nil {
+					return errors.Wrap(err, "error reading file from tar")
+				}
+				outFile.Close()
+				break
+			}
+		}
+		f.Close()
+
+	}
+	return nil
+}
+
+// LookupLocation resolves an IP to a location according to the Maxmind DB
+func LookupLocation(dbPath string, ipStr string) (LocationInfo, error) {
+	loc := LocationInfo{IP: ipStr}
+
+	asnDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-ASN.mmdb"))
+	if err != nil {
+		return loc, errors.Wrap(err, "failed to open ASN db")
+	}
+	defer asnDB.Close()
+
+	countryDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-Country.mmdb"))
+	if err != nil {
+		return loc, errors.Wrap(err, "failed to open country db")
+	}
+	defer countryDB.Close()
+
+	ip := net.ParseIP(ipStr)
+
+	asn, err := asnDB.ASN(ip)
+	if err != nil {
+		return loc, err
+	}
+	country, err := countryDB.Country(ip)
+	if err != nil {
+		return loc, err
+	}
+	loc.ASN = asn.AutonomousSystemNumber
+	loc.NetworkName = asn.AutonomousSystemOrganization
+	loc.CountryCode = country.Country.IsoCode
+
+	return loc, nil
+}
+
+type avastResponse struct {
+	IP string `json:"ip"`
+}
+
+func avastLookup() (string, error) {
+	var parsed = new(avastResponse)
+
+	resp, err := http.Get("https://ip-info.ff.avast.com/v1/info")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to perform request")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read response body")
+	}
+	err = json.Unmarshal([]byte(body), &parsed)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to parse json")
+	}
+
+	return parsed.IP, nil
+}
+
+func akamaiLookup() (string, error) {
+	// This is a domain fronted request to akamai
+	client := &http.Client{}
+	req, err := http.NewRequest("GET", "https://a248.e.akamai.net/", nil)
+	if err != nil {
+		return "", err
+	}
+	req.Host = "whatismyip.akamai.com"
+	resp, err := client.Do(req)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to perform request")
+	}
+
+	body, err := ioutil.ReadAll(resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to read response body")
+	}
+	return string(body), nil
+}
+
+type lookupFunc func() (string, error)
+
+var lookupServices = []lookupFunc{
+	avastLookup,
+	akamaiLookup,
+}
+
+// IPLookup gets the users IP address from a IP lookup service
+func IPLookup() (string, error) {
+	rand.Seed(time.Now().Unix())
+
+	retries := 3
+	for retries > 0 {
+		lookup := lookupServices[rand.Intn(len(lookupServices))]
+		ipStr, err := lookup()
+		if err == nil {
+			return ipStr, nil
+		}
+		retries--
+	}
+	return "", errors.New("exceeded maximum retries")
+}
+
+// GeoIPLookup does a geoip lookup and returns location information
+func GeoIPLookup(dbPath string) (*LocationInfo, error) {
+	ipStr, err := IPLookup()
+	if err != nil {
+		return nil, err
+	}
+
+	location, err := LookupLocation(dbPath, ipStr)
+	if err != nil {
+		return nil, err
+	}
+
+	return &location, nil
+}