c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Merge pull request #8 from OpenObservatory/feature/geoiplookup

Browse Source 
Feature/geoiplookup
This commit is contained in:
Arturo Filastò 2018-03-23 11:47:11 +00:00 committed by GitHub
commit e2797faeb2
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 340 additions and 81 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

4
.gitignore vendored
View File

@ -1,3 +1,3 @@
vendor/
/ooni
/vendor
/dist
*.njson

19
Gopkg.lock generated
View File

@ -103,6 +103,18 @@
packages = ["."]
revision = "b8bc1bf767474819792c23f32d8286a45736f1c6"
[[projects]]
name = "github.com/oschwald/geoip2-golang"
packages = ["."]
revision = "7118115686e16b77967cdbf55d1b944fe14ad312"
version = "v1.2.1"
[[projects]]
name = "github.com/oschwald/maxminddb-golang"
packages = ["."]
revision = "c5bec84d1963260297932a1b7a1753c8420717a7"
version = "v1.3.0"
[[projects]]
name = "github.com/pkg/errors"
packages = ["."]
@ -164,7 +176,10 @@
[[projects]]
branch = "master"
name = "golang.org/x/sys"
packages = ["unix"]
packages = [
"unix",
"windows"
]
revision = "37707fdb30a5b38865cfb95e5aab41707daec7fd"
[[projects]]
@ -186,6 +201,6 @@
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "cfd4567b0cd553bc4594b7f8bebcb91be475dfd112fd8740f65b5ba4a8253181"
inputs-digest = "46860a32f649dbb2e01b285c0d5581078ba4b28a21e21175d47ccb2725a9c9fb"
solver-name = "gps-cdcl"
solver-version = 1

4
Gopkg.toml
View File

@ -61,3 +61,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"]
[[constraint]]
name = "github.com/shuLhan/go-bindata"
version = "3.3.0"
[[constraint]]
name = "github.com/oschwald/geoip2-golang"
version = "1.2.1"

2
Makefile
View File

@ -2,7 +2,7 @@ GO ?= go
build:
@echo "Building ./ooni"
@$(GO) build -i -o ooni cmd/ooni/main.go
@$(GO) build -i -o dist/ooni cmd/ooni/main.go
.PHONY: build
bindata:

1
cmd/ooni/main.go
View File

@ -4,6 +4,7 @@ import (
// commands
"github.com/apex/log"
_ "github.com/openobservatory/gooni/internal/cli/geoip"
_ "github.com/openobservatory/gooni/internal/cli/info"
_ "github.com/openobservatory/gooni/internal/cli/list"
_ "github.com/openobservatory/gooni/internal/cli/nettest"

44
internal/cli/geoip/geoip.go Normal file
View File

@ -0,0 +1,44 @@
package geoip
import (
"path/filepath"
"github.com/alecthomas/kingpin"
"github.com/apex/log"
"github.com/openobservatory/gooni/internal/cli/root"
"github.com/openobservatory/gooni/utils"
)
func init() {
cmd := root.Command("geoip", "Perform a geoip lookup")
shouldUpdate := cmd.Flag("update", "Update the geoip database").Bool()
cmd.Action(func(_ *kingpin.ParseContext) error {
log.Info("geoip")
ctx, err := root.Init()
if err != nil {
return err
}
geoipPath := filepath.Join(ctx.Home, "geoip")
if *shouldUpdate {
utils.DownloadGeoIPDatabaseFiles(geoipPath)
}
loc, err := utils.GeoIPLookup(geoipPath)
if err != nil {
return err
}
log.WithFields(log.Fields{
"asn": loc.ASN,
"network_name": loc.NetworkName,
"country_code": loc.CountryCode,
"ip": loc.IP,
}).Info("Looked up your location")
return nil
})
}

33
internal/cli/root/root.go
View File

@ -4,7 +4,6 @@ import (
"github.com/alecthomas/kingpin"
"github.com/apex/log"
ooni "github.com/openobservatory/gooni"
"github.com/openobservatory/gooni/internal/database"
"github.com/openobservatory/gooni/internal/log/handlers/batch"
"github.com/openobservatory/gooni/internal/log/handlers/cli"
"github.com/prometheus/common/version"
@ -17,7 +16,7 @@ var Cmd = kingpin.New("ooni", "")
var Command = Cmd.Command
// Init should be called by all subcommand that care to have a ooni.OONI instance
var Init func() (*ooni.Config, *ooni.Context, error)
var Init func() (*ooni.Context, error)
func init() {
configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
@ -36,39 +35,21 @@ func init() {
log.Debugf("ooni version %s", version.Version)
}
Init = func() (*ooni.Config, *ooni.Context, error) {
var config *ooni.Config
Init = func() (*ooni.Context, error) {
var err error
if *configPath != "" {
log.Debugf("Reading config file from %s", *configPath)
config, err = ooni.ReadConfig(*configPath)
} else {
log.Debug("Reading default config file")
config, err = ooni.ReadDefaultConfigPaths()
}
homePath, err := ooni.GetOONIHome()
if err != nil {
return nil, nil, err
return nil, err
}
dbPath, err := database.DefaultDatabasePath()
if err != nil {
return nil, nil, err
}
log.Debugf("Connecting to database sqlite3://%s", dbPath)
db, err := database.Connect(dbPath)
if err != nil {
return nil, nil, err
}
ctx := ooni.New(config, db)
ctx := ooni.NewContext(*configPath, homePath)
err = ctx.Init()
if err != nil {
return nil, nil, err
return nil, err
}
return config, ctx, nil
return ctx, nil
}
return nil

4
internal/cli/run/run.go
View File

@ -21,7 +21,7 @@ func init() {
cmd.Action(func(_ *kingpin.ParseContext) error {
log.Infof("Starting %s", *nettestGroup)
_, ctx, err := root.Init()
ctx, err := root.Init()
if err != nil {
log.Errorf("%s", err)
return err
@ -33,7 +33,7 @@ func init() {
}
log.Debugf("Running test group %s", group.Label)
result, err := database.CreateResult(ctx.DB, database.Result{
result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{
Name: *nettestGroup,
StartTime: time.Now().UTC(),
})

13
internal/database/database.go
View File

@ -1,14 +1,10 @@
package database
import (
"path/filepath"
"github.com/apex/log"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver
ooni "github.com/openobservatory/gooni"
"github.com/openobservatory/gooni/internal/bindata"
"github.com/pkg/errors"
migrate "github.com/rubenv/sql-migrate"
)
@ -41,12 +37,3 @@ func Connect(path string) (db *sqlx.DB, err error) {
}
return
}
// DefaultDatabasePath for the main database
func DefaultDatabasePath() (string, error) {
home, err := ooni.GetOONIHome()
if err != nil {
return "", errors.Wrap(err, "default database path")
}
return filepath.Join(home, "db", "main.sqlite3"), nil
}

13
internal/database/models.go
View File

@ -8,7 +8,6 @@ import (
"github.com/apex/log"
"github.com/jmoiron/sqlx"
ooni "github.com/openobservatory/gooni"
"github.com/pkg/errors"
)
@ -238,11 +237,7 @@ func (r *Result) Finished(db *sqlx.DB, makeSummary ResultSummaryFunc) error {
}
// MakeResultsPath creates and returns a directory for the result
func MakeResultsPath(r *Result) (string, error) {
home, err := ooni.GetOONIHome()
if err != nil {
return "", errors.Wrap(err, "default measurements path")
}
func MakeResultsPath(home string, r *Result) (string, error) {
p := filepath.Join(home, "msmts",
fmt.Sprintf("%s-%s", r.Name, r.StartTime.Format(time.RFC3339Nano)))
@ -251,7 +246,7 @@ func MakeResultsPath(r *Result) (string, error) {
if _, e := os.Stat(p); e == nil {
return "", errors.New("results path already exists")
}
err = os.MkdirAll(p, 0700)
err := os.MkdirAll(p, 0700)
if err != nil {
return "", err
}
@ -260,10 +255,10 @@ func MakeResultsPath(r *Result) (string, error) {
// CreateResult writes the Result to the database a returns a pointer
// to the Result
func CreateResult(db *sqlx.DB, r Result) (*Result, error) {
func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) {
log.Debugf("Creating result %v", r)
p, err := MakeResultsPath(&r)
p, err := MakeResultsPath(homePath, &r)
if err != nil {
return nil, err
}

66
ooni.go
View File

@ -11,6 +11,7 @@ import (
"github.com/jmoiron/sqlx"
homedir "github.com/mitchellh/go-homedir"
"github.com/openobservatory/gooni/config"
"github.com/openobservatory/gooni/internal/database"
"github.com/openobservatory/gooni/internal/legacy"
"github.com/pkg/errors"
)
@ -36,20 +37,50 @@ func Onboarding(c *Config) error {
type Context struct {
Config *Config
DB *sqlx.DB
Home string
TempDir string
dbPath string
configPath string
}
// Init the OONI manager
func (c *Context) Init() error {
if err := legacy.MaybeMigrateHome(); err != nil {
var err error
if err = legacy.MaybeMigrateHome(); err != nil {
return errors.Wrap(err, "migrating home")
}
if err = CreateHomeDirs(c.Home); err != nil {
return err
}
if c.configPath != "" {
log.Debugf("Reading config file from %s", c.configPath)
c.Config, err = ReadConfig(c.configPath)
} else {
log.Debug("Reading default config file")
c.Config, err = ReadDefaultConfigPaths(c.Home)
}
if err != nil {
return err
}
c.dbPath = filepath.Join(c.Home, "db", "main.sqlite3")
if c.Config.InformedConsent == false {
if err := Onboarding(c.Config); err != nil {
if err = Onboarding(c.Config); err != nil {
return errors.Wrap(err, "onboarding")
}
}
log.Debugf("Connecting to database sqlite3://%s", c.dbPath)
db, err := database.Connect(c.dbPath)
if err != nil {
return err
}
c.DB = db
tempDir, err := ioutil.TempDir("", "ooni")
if err != nil {
return errors.Wrap(err, "creating TempDir")
@ -60,10 +91,11 @@ func (c *Context) Init() error {
}
// NewContext instance.
func New(c *Config, d *sqlx.DB) *Context {
func NewContext(configPath string, homePath string) *Context {
return &Context{
Config: c,
DB: d,
Home: homePath,
Config: &Config{},
configPath: configPath,
}
}
@ -155,24 +187,18 @@ func ParseConfig(b []byte) (*Config, error) {
return c, nil
}
//EnsureDefaultOONIHomeDir makes sure the paths to the OONI Home exist
func EnsureDefaultOONIHomeDir() (string, error) {
home, err := GetOONIHome()
if err != nil {
return "", err
}
requiredDirs := []string{"db", "msmts"}
// CreateHomeDirs creates the OONI home subdirectories
func CreateHomeDirs(home string) error {
requiredDirs := []string{"db", "msmts", "geoip"}
for _, d := range requiredDirs {
if _, e := os.Stat(filepath.Join(home, d)); e != nil {
err = os.MkdirAll(filepath.Join(home, d), 0700)
if err != nil {
return "", err
if err := os.MkdirAll(filepath.Join(home, d), 0700); err != nil {
return err
}
}
}
return home, nil
return nil
}
// ReadConfig reads the configuration from the path
@ -206,11 +232,7 @@ func ReadConfig(path string) (*Config, error) {
}
// ReadDefaultConfigPaths from common locations.
func ReadDefaultConfigPaths() (*Config, error) {
home, err := EnsureDefaultOONIHomeDir()
if err != nil {
return nil, errors.Wrap(err, "reading default config paths")
}
func ReadDefaultConfigPaths(home string) (*Config, error) {
var paths = []string{
filepath.Join(home, "config.json"),
}

210
utils/geoip.go Normal file
View File

@ -0,0 +1,210 @@
package utils
import (
"archive/tar"
"compress/gzip"
"encoding/json"
"io"
"io/ioutil"
"math/rand"
"net"
"net/http"
"os"
"path/filepath"
"time"
"github.com/oschwald/geoip2-golang"
"github.com/pkg/errors"
)
// LocationInfo contains location information
type LocationInfo struct {
IP string
ASN uint
NetworkName string
CountryCode string
}
// XXX consider integration with: https://updates.maxmind.com/app/update_getfilename?product_id=GeoLite2-ASN
var geoipFiles = map[string]string{
"GeoLite2-ASN.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-ASN.tar.gz",
"GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz",
}
// DownloadGeoIPDatabaseFiles into the target directory
func DownloadGeoIPDatabaseFiles(dir string) error {
for filename, url := range geoipFiles {
dstPath := filepath.Join(dir, filename)
// Download the file to a temporary location
out, err := ioutil.TempFile(os.TempDir(), "maxmind")
if err != nil {
return errors.Wrap(err, "failed to create temporary directory")
}
resp, err := http.Get(url)
if err != nil {
return errors.Wrap(err, "failed to fetch URL")
}
_, err = io.Copy(out, resp.Body)
if err != nil {
return errors.Wrap(err, "failed to copy response body")
}
out.Close()
resp.Body.Close()
// Extract the tar.gz file
f, err := os.Open(out.Name())
if err != nil {
return errors.Wrap(err, "failed to read file")
}
gzf, err := gzip.NewReader(f)
if err != nil {
return errors.Wrap(err, "failed to create gzip reader")
}
tarReader := tar.NewReader(gzf)
// Look inside of the tar for the file we need
for {
header, err := tarReader.Next()
if err == io.EOF {
break
}
if err != nil {
return errors.Wrap(err, "error extracting tar.gz")
}
name := header.Name
if filepath.Base(name) == filename {
outFile, err := os.Create(dstPath)
if err != nil {
return errors.Wrap(err, "error creating file")
}
if _, err := io.Copy(outFile, tarReader); err != nil {
return errors.Wrap(err, "error reading file from tar")
}
outFile.Close()
break
}
}
f.Close()
}
return nil
}
// LookupLocation resolves an IP to a location according to the Maxmind DB
func LookupLocation(dbPath string, ipStr string) (LocationInfo, error) {
loc := LocationInfo{IP: ipStr}
asnDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-ASN.mmdb"))
if err != nil {
return loc, errors.Wrap(err, "failed to open ASN db")
}
defer asnDB.Close()
countryDB, err := geoip2.Open(filepath.Join(dbPath, "GeoLite2-Country.mmdb"))
if err != nil {
return loc, errors.Wrap(err, "failed to open country db")
}
defer countryDB.Close()
ip := net.ParseIP(ipStr)
asn, err := asnDB.ASN(ip)
if err != nil {
return loc, err
}
country, err := countryDB.Country(ip)
if err != nil {
return loc, err
}
loc.ASN = asn.AutonomousSystemNumber
loc.NetworkName = asn.AutonomousSystemOrganization
loc.CountryCode = country.Country.IsoCode
return loc, nil
}
type avastResponse struct {
IP string `json:"ip"`
}
func avastLookup() (string, error) {
var parsed = new(avastResponse)
resp, err := http.Get("https://ip-info.ff.avast.com/v1/info")
if err != nil {
return "", errors.Wrap(err, "failed to perform request")
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "failed to read response body")
}
err = json.Unmarshal([]byte(body), &parsed)
if err != nil {
return "", errors.Wrap(err, "failed to parse json")
}
return parsed.IP, nil
}
func akamaiLookup() (string, error) {
// This is a domain fronted request to akamai
client := &http.Client{}
req, err := http.NewRequest("GET", "https://a248.e.akamai.net/", nil)
if err != nil {
return "", err
}
req.Host = "whatismyip.akamai.com"
resp, err := client.Do(req)
if err != nil {
return "", errors.Wrap(err, "failed to perform request")
}
body, err := ioutil.ReadAll(resp.Body)
if err != nil {
return "", errors.Wrap(err, "failed to read response body")
}
return string(body), nil
}
type lookupFunc func() (string, error)
var lookupServices = []lookupFunc{
avastLookup,
akamaiLookup,
}
// IPLookup gets the users IP address from a IP lookup service
func IPLookup() (string, error) {
rand.Seed(time.Now().Unix())
retries := 3
for retries > 0 {
lookup := lookupServices[rand.Intn(len(lookupServices))]
ipStr, err := lookup()
if err == nil {
return ipStr, nil
}
retries--
}
return "", errors.New("exceeded maximum retries")
}
// GeoIPLookup does a geoip lookup and returns location information
func GeoIPLookup(dbPath string) (*LocationInfo, error) {
ipStr, err := IPLookup()
if err != nil {
return nil, err
}
location, err := LookupLocation(dbPath, ipStr)
if err != nil {
return nil, err
}
return &location, nil
}