diff --git a/.gitignore b/.gitignore index e4c28eb..fc60875 100644 --- a/.gitignore +++ b/.gitignore @@ -1,3 +1,3 @@ -vendor/ -/ooni +/vendor +/dist *.njson diff --git a/Gopkg.lock b/Gopkg.lock index 9a5b79e..719134e 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -103,6 +103,18 @@ packages = ["."] revision = "b8bc1bf767474819792c23f32d8286a45736f1c6" +[[projects]] + name = "github.com/oschwald/geoip2-golang" + packages = ["."] + revision = "7118115686e16b77967cdbf55d1b944fe14ad312" + version = "v1.2.1" + +[[projects]] + name = "github.com/oschwald/maxminddb-golang" + packages = ["."] + revision = "c5bec84d1963260297932a1b7a1753c8420717a7" + version = "v1.3.0" + [[projects]] name = "github.com/pkg/errors" packages = ["."] @@ -164,7 +176,10 @@ [[projects]] branch = "master" name = "golang.org/x/sys" - packages = ["unix"] + packages = [ + "unix", + "windows" + ] revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd" [[projects]] @@ -186,6 +201,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "cfd4567b0cd553bc4594b7f8bebcb91be475dfd112fd8740f65b5ba4a8253181" + inputs-digest = "46860a32f649dbb2e01b285c0d5581078ba4b28a21e21175d47ccb2725a9c9fb" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index a73d836..a68ad73 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"] [[constraint]] name = "github.com/shuLhan/go-bindata" version = "3.3.0" + +[[constraint]] + name = "github.com/oschwald/geoip2-golang" + version = "1.2.1" diff --git a/Makefile b/Makefile index b605179..56f819c 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ GO ?= go build: @echo "Building ./ooni" - @$(GO) build -i -o ooni cmd/ooni/main.go + @$(GO) build -i -o dist/ooni cmd/ooni/main.go .PHONY: build bindata: diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index 9925661..a7a7d20 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -4,6 +4,7 @@ import ( // commands "github.com/apex/log" + _ "github.com/openobservatory/gooni/internal/cli/geoip" _ "github.com/openobservatory/gooni/internal/cli/info" _ "github.com/openobservatory/gooni/internal/cli/list" _ "github.com/openobservatory/gooni/internal/cli/nettest" diff --git a/internal/cli/geoip/geoip.go b/internal/cli/geoip/geoip.go new file mode 100644 index 0000000..803d117 --- /dev/null +++ b/internal/cli/geoip/geoip.go @@ -0,0 +1,44 @@ +package geoip + +import ( + "path/filepath" + + "github.com/alecthomas/kingpin" + "github.com/apex/log" + "github.com/openobservatory/gooni/internal/cli/root" + "github.com/openobservatory/gooni/utils" +) + +func init() { + cmd := root.Command("geoip", "Perform a geoip lookup") + + shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool() + + cmd.Action(func(_ *kingpin.ParseContext) error { + log.Info("geoip") + ctx, err := root.Init() + if err != nil { + return err + } + + geoipPath := filepath.Join(ctx.Home, "geoip") + + if *shouldUpdate { + utils.DownloadGeoIPDatabaseFiles(geoipPath) + } + + loc, err := utils.GeoIPLookup(geoipPath) + if err != nil { + return err + } + + log.WithFields(log.Fields{ + "asn": loc.ASN, + "network_name": loc.NetworkName, + "country_code": loc.CountryCode, + "ip": loc.IP, + }).Info("Looked up your location") + + return nil + }) +} diff --git a/internal/cli/root/root.go b/internal/cli/root/root.go index f508e0e..ab6f4f0 100644 --- a/internal/cli/root/root.go +++ b/internal/cli/root/root.go @@ -4,7 +4,6 @@ import ( "github.com/alecthomas/kingpin" "github.com/apex/log" ooni "github.com/openobservatory/gooni" - "github.com/openobservatory/gooni/internal/database" "github.com/openobservatory/gooni/internal/log/handlers/batch" "github.com/openobservatory/gooni/internal/log/handlers/cli" "github.com/prometheus/common/version" @@ -17,7 +16,7 @@ var Cmd = kingpin.New("ooni", "") var Command = Cmd.Command // Init should be called by all subcommand that care to have a ooni.OONI instance -var Init func() (*ooni.Config, *ooni.Context, error) +var Init func() (*ooni.Context, error) func init() { configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String() @@ -36,39 +35,21 @@ func init() { log.Debugf("ooni version %s", version.Version) } - Init = func() (*ooni.Config, *ooni.Context, error) { - var config *ooni.Config + Init = func() (*ooni.Context, error) { var err error - if *configPath != "" { - log.Debugf("Reading config file from %s", *configPath) - config, err = ooni.ReadConfig(*configPath) - } else { - log.Debug("Reading default config file") - config, err = ooni.ReadDefaultConfigPaths() - } + homePath, err := ooni.GetOONIHome() if err != nil { - return nil, nil, err + return nil, err } - dbPath, err := database.DefaultDatabasePath() - if err != nil { - return nil, nil, err - } - - log.Debugf("Connecting to database sqlite3://%s", dbPath) - db, err := database.Connect(dbPath) - if err != nil { - return nil, nil, err - } - - ctx := ooni.New(config, db) + ctx := ooni.NewContext(*configPath, homePath) err = ctx.Init() if err != nil { - return nil, nil, err + return nil, err } - return config, ctx, nil + return ctx, nil } return nil diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index c5368ba..b959ee2 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -21,7 +21,7 @@ func init() { cmd.Action(func(_ *kingpin.ParseContext) error { log.Infof("Starting %s", *nettestGroup) - _, ctx, err := root.Init() + ctx, err := root.Init() if err != nil { log.Errorf("%s", err) return err @@ -33,7 +33,7 @@ func init() { } log.Debugf("Running test group %s", group.Label) - result, err := database.CreateResult(ctx.DB, database.Result{ + result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ Name: *nettestGroup, StartTime: time.Now().UTC(), }) diff --git a/internal/database/database.go b/internal/database/database.go index b9fb683..e860659 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,14 +1,10 @@ package database import ( - "path/filepath" - "github.com/apex/log" "github.com/jmoiron/sqlx" _ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver - ooni "github.com/openobservatory/gooni" "github.com/openobservatory/gooni/internal/bindata" - "github.com/pkg/errors" migrate "github.com/rubenv/sql-migrate" ) @@ -41,12 +37,3 @@ func Connect(path string) (db *sqlx.DB, err error) { } return } - -// DefaultDatabasePath for the main database -func DefaultDatabasePath() (string, error) { - home, err := ooni.GetOONIHome() - if err != nil { - return "", errors.Wrap(err, "default database path") - } - return filepath.Join(home, "db", "main.sqlite3"), nil -} diff --git a/internal/database/models.go b/internal/database/models.go index d575df6..89d007b 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -8,7 +8,6 @@ import ( "github.com/apex/log" "github.com/jmoiron/sqlx" - ooni "github.com/openobservatory/gooni" "github.com/pkg/errors" ) @@ -238,11 +237,7 @@ func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error { } // MakeResultsPath creates and returns a directory for the result -func MakeResultsPath(r *Result) (string, error) { - home, err := ooni.GetOONIHome() - if err != nil { - return "", errors.Wrap(err, "default measurements path") - } +func MakeResultsPath(home string, r *Result) (string, error) { p := filepath.Join(home, "msmts", fmt.Sprintf("%s-%s", r.Name, r.StartTime.Format(time.RFC3339Nano))) @@ -251,7 +246,7 @@ func MakeResultsPath(r *Result) (string, error) { if _, e := os.Stat(p); e == nil { return "", errors.New("results path already exists") } - err = os.MkdirAll(p, 0700) + err := os.MkdirAll(p, 0700) if err != nil { return "", err } @@ -260,10 +255,10 @@ func MakeResultsPath(r *Result) (string, error) { // CreateResult writes the Result to the database a returns a pointer // to the Result -func CreateResult(db *sqlx.DB, r Result) (*Result, error) { +func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) { log.Debugf("Creating result %v", r) - p, err := MakeResultsPath(&r) + p, err := MakeResultsPath(homePath, &r) if err != nil { return nil, err } diff --git a/ooni.go b/ooni.go index 2b90257..56db62d 100644 --- a/ooni.go +++ b/ooni.go @@ -11,6 +11,7 @@ import ( "github.com/jmoiron/sqlx" homedir "github.com/mitchellh/go-homedir" "github.com/openobservatory/gooni/config" + "github.com/openobservatory/gooni/internal/database" "github.com/openobservatory/gooni/internal/legacy" "github.com/pkg/errors" ) @@ -34,22 +35,52 @@ func Onboarding(c *Config) error { // Context for OONI Probe type Context struct { - Config *Config - DB *sqlx.DB - TempDir string + Config *Config + DB *sqlx.DB + + Home string + TempDir string + dbPath string + configPath string } // Init the OONI manager func (c *Context) Init() error { - if err := legacy.MaybeMigrateHome(); err != nil { + var err error + + if err = legacy.MaybeMigrateHome(); err != nil { return errors.Wrap(err, "migrating home") } + + if err = CreateHomeDirs(c.Home); err != nil { + return err + } + + if c.configPath != "" { + log.Debugf("Reading config file from %s", c.configPath) + c.Config, err = ReadConfig(c.configPath) + } else { + log.Debug("Reading default config file") + c.Config, err = ReadDefaultConfigPaths(c.Home) + } + if err != nil { + return err + } + + c.dbPath = filepath.Join(c.Home, "db", "main.sqlite3") if c.Config.InformedConsent == false { - if err := Onboarding(c.Config); err != nil { + if err = Onboarding(c.Config); err != nil { return errors.Wrap(err, "onboarding") } } + log.Debugf("Connecting to database sqlite3://%s", c.dbPath) + db, err := database.Connect(c.dbPath) + if err != nil { + return err + } + c.DB = db + tempDir, err := ioutil.TempDir("", "ooni") if err != nil { return errors.Wrap(err, "creating TempDir") @@ -59,11 +90,12 @@ func (c *Context) Init() error { return nil } -// New Context instance. -func New(c *Config, d *sqlx.DB) *Context { +// NewContext instance. +func NewContext(configPath string, homePath string) *Context { return &Context{ - Config: c, - DB: d, + Home: homePath, + Config: &Config{}, + configPath: configPath, } } @@ -155,24 +187,18 @@ func ParseConfig(b []byte) (*Config, error) { return c, nil } -//EnsureDefaultOONIHomeDir makes sure the paths to the OONI Home exist -func EnsureDefaultOONIHomeDir() (string, error) { - home, err := GetOONIHome() - if err != nil { - return "", err - } - - requiredDirs := []string{"db", "msmts"} +// CreateHomeDirs creates the OONI home subdirectories +func CreateHomeDirs(home string) error { + requiredDirs := []string{"db", "msmts", "geoip"} for _, d := range requiredDirs { if _, e := os.Stat(filepath.Join(home, d)); e != nil { - err = os.MkdirAll(filepath.Join(home, d), 0700) - if err != nil { - return "", err + if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil { + return err } } } - return home, nil + return nil } // ReadConfig reads the configuration from the path @@ -206,11 +232,7 @@ func ReadConfig(path string) (*Config, error) { } // ReadDefaultConfigPaths from common locations. -func ReadDefaultConfigPaths() (*Config, error) { - home, err := EnsureDefaultOONIHomeDir() - if err != nil { - return nil, errors.Wrap(err, "reading default config paths") - } +func ReadDefaultConfigPaths(home string) (*Config, error) { var paths = []string{ filepath.Join(home, "config.json"), } diff --git a/utils/geoip.go b/utils/geoip.go new file mode 100644 index 0000000..1bb7730 --- /dev/null +++ b/utils/geoip.go @@ -0,0 +1,210 @@ +package utils + +import ( + "archive/tar" + "compress/gzip" + "encoding/json" + "io" + "io/ioutil" + "math/rand" + "net" + "net/http" + "os" + "path/filepath" + "time" + + "github.com/oschwald/geoip2-golang" + "github.com/pkg/errors" +) + +// LocationInfo contains location information +type LocationInfo struct { + IP string + ASN uint + NetworkName string + CountryCode string +} + +// XXX consider integration with: https://updates.maxmind.com/app/update_getfilename?product_id=GeoLite2-ASN +var geoipFiles = map[string]string{ + "GeoLite2-ASN.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz", + "GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz", +} + +// DownloadGeoIPDatabaseFiles into the target directory +func DownloadGeoIPDatabaseFiles(dir string) error { + for filename, url := range geoipFiles { + dstPath := filepath.Join(dir, filename) + + // Download the file to a temporary location + out, err := ioutil.TempFile(os.TempDir(), "maxmind") + if err != nil { + return errors.Wrap(err, "failed to create temporary directory") + } + resp, err := http.Get(url) + if err != nil { + return errors.Wrap(err, "failed to fetch URL") + } + + _, err = io.Copy(out, resp.Body) + if err != nil { + return errors.Wrap(err, "failed to copy response body") + } + out.Close() + resp.Body.Close() + + // Extract the tar.gz file + + f, err := os.Open(out.Name()) + if err != nil { + return errors.Wrap(err, "failed to read file") + } + + gzf, err := gzip.NewReader(f) + if err != nil { + return errors.Wrap(err, "failed to create gzip reader") + } + tarReader := tar.NewReader(gzf) + + // Look inside of the tar for the file we need + for { + header, err := tarReader.Next() + if err == io.EOF { + break + } + if err != nil { + return errors.Wrap(err, "error extracting tar.gz") + } + name := header.Name + if filepath.Base(name) == filename { + outFile, err := os.Create(dstPath) + if err != nil { + return errors.Wrap(err, "error creating file") + } + if _, err := io.Copy(outFile, tarReader); err != nil { + return errors.Wrap(err, "error reading file from tar") + } + outFile.Close() + break + } + } + f.Close() + + } + return nil +} + +// LookupLocation resolves an IP to a location according to the Maxmind DB +func LookupLocation(dbPath string, ipStr string) (LocationInfo, error) { + loc := LocationInfo{IP: ipStr} + + asnDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-ASN.mmdb")) + if err != nil { + return loc, errors.Wrap(err, "failed to open ASN db") + } + defer asnDB.Close() + + countryDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-Country.mmdb")) + if err != nil { + return loc, errors.Wrap(err, "failed to open country db") + } + defer countryDB.Close() + + ip := net.ParseIP(ipStr) + + asn, err := asnDB.ASN(ip) + if err != nil { + return loc, err + } + country, err := countryDB.Country(ip) + if err != nil { + return loc, err + } + loc.ASN = asn.AutonomousSystemNumber + loc.NetworkName = asn.AutonomousSystemOrganization + loc.CountryCode = country.Country.IsoCode + + return loc, nil +} + +type avastResponse struct { + IP string `json:"ip"` +} + +func avastLookup() (string, error) { + var parsed = new(avastResponse) + + resp, err := http.Get("https://ip-info.ff.avast.com/v1/info") + if err != nil { + return "", errors.Wrap(err, "failed to perform request") + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to read response body") + } + err = json.Unmarshal([]byte(body), &parsed) + if err != nil { + return "", errors.Wrap(err, "failed to parse json") + } + + return parsed.IP, nil +} + +func akamaiLookup() (string, error) { + // This is a domain fronted request to akamai + client := &http.Client{} + req, err := http.NewRequest("GET", "https://a248.e.akamai.net/", nil) + if err != nil { + return "", err + } + req.Host = "whatismyip.akamai.com" + resp, err := client.Do(req) + if err != nil { + return "", errors.Wrap(err, "failed to perform request") + } + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + return "", errors.Wrap(err, "failed to read response body") + } + return string(body), nil +} + +type lookupFunc func() (string, error) + +var lookupServices = []lookupFunc{ + avastLookup, + akamaiLookup, +} + +// IPLookup gets the users IP address from a IP lookup service +func IPLookup() (string, error) { + rand.Seed(time.Now().Unix()) + + retries := 3 + for retries > 0 { + lookup := lookupServices[rand.Intn(len(lookupServices))] + ipStr, err := lookup() + if err == nil { + return ipStr, nil + } + retries-- + } + return "", errors.New("exceeded maximum retries") +} + +// GeoIPLookup does a geoip lookup and returns location information +func GeoIPLookup(dbPath string) (*LocationInfo, error) { + ipStr, err := IPLookup() + if err != nil { + return nil, err + } + + location, err := LookupLocation(dbPath, ipStr) + if err != nil { + return nil, err + } + + return &location, nil +}