Add support for downloading legacy geoip databases

Pass the geoip databases to measurement-kit

internal/cli/geoip/geoip.go

@@ -1,8 +1,6 @@
 package geoip
 
 import (
-	"path/filepath"
-
 	"github.com/alecthomas/kingpin"
 	"github.com/apex/log"
 	"github.com/openobservatory/gooni/internal/cli/root"
@@ -21,10 +19,10 @@ func init() {
 			return err
 		}
 
-		geoipPath := filepath.Join(ctx.Home, "geoip")
+		geoipPath := utils.GeoIPDir(ctx.Home)
-
 		if *shouldUpdate {
 			utils.DownloadGeoIPDatabaseFiles(geoipPath)
+			utils.DownloadLegacyGeoIPDatabaseFiles(geoipPath)
 		}
 
 		loc, err := utils.GeoIPLookup(geoipPath)

nettests/nettests.go

@@ -3,6 +3,7 @@ package nettests
 import (
 	"encoding/json"
 	"fmt"
+	"path/filepath"
 
 	"github.com/apex/log"
 	"github.com/measurement-kit/go-measurement-kit"
@@ -11,6 +12,7 @@ import (
 	"github.com/openobservatory/gooni/internal/colors"
 	"github.com/openobservatory/gooni/internal/database"
 	"github.com/openobservatory/gooni/internal/output"
+	"github.com/openobservatory/gooni/utils"
 )
 
 // Nettest interface. Every Nettest should implement this.
@@ -72,8 +74,8 @@ func (c *Controller) Init(nt *mk.Nettest) error {
 		SoftwareVersion:  version.Version,
 
 		// XXX
-		GeoIPCountryPath: "",
+		GeoIPCountryPath: filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat"),
-		GeoIPASNPath:     "",
+		GeoIPASNPath:     filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat"),
 		OutputPath:       c.msmtPath,
 		CaBundlePath:     "/etc/ssl/cert.pem",
 	}

utils/geoip.go

@@ -31,31 +31,82 @@ var geoipFiles = map[string]string{
 	"GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz",
 }
 
+var legacyGeoipFiles = map[string]string{
+	"GeoIPASNum.dat": "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz",
+	"GeoIP.dat":      "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
+}
+
+// GeoIPDir returns the geoip data dir for the given OONI Home
+func GeoIPDir(home string) string {
+	return filepath.Join(home, "geoip")
+}
+
+// Download the file to a temporary location
+func downloadToTemp(url string) (string, error) {
+	out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create temporary directory")
+	}
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch URL")
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to copy response body")
+	}
+	out.Close()
+	resp.Body.Close()
+	return out.Name(), nil
+}
+
+// DownloadLegacyGeoIPDatabaseFiles into the target directory
+func DownloadLegacyGeoIPDatabaseFiles(dir string) error {
+	for filename, url := range legacyGeoipFiles {
+		dstPath := filepath.Join(dir, filename)
+
+		tmpPath, err := downloadToTemp(url)
+		if err != nil {
+			return err
+		}
+
+		// Extract the tar.gz file
+		f, err := os.Open(tmpPath)
+		defer f.Close()
+		if err != nil {
+			return errors.Wrap(err, "failed to read file")
+		}
+
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return errors.Wrap(err, "failed to create gzip reader")
+		}
+
+		outFile, err := os.Create(dstPath)
+		if err != nil {
+			return errors.Wrap(err, "error creating file")
+		}
+		if _, err := io.Copy(outFile, gzf); err != nil {
+			return errors.Wrap(err, "error reading file from gzip")
+		}
+		outFile.Close()
+	}
+	return nil
+}
+
 // DownloadGeoIPDatabaseFiles into the target directory
 func DownloadGeoIPDatabaseFiles(dir string) error {
 	for filename, url := range geoipFiles {
 		dstPath := filepath.Join(dir, filename)
 
-		// Download the file to a temporary location
+		tmpPath, err := downloadToTemp(url)
-		out, err := ioutil.TempFile(os.TempDir(), "maxmind")
 		if err != nil {
-			return errors.Wrap(err, "failed to create temporary directory")
+			return err
 		}
-		resp, err := http.Get(url)
-		if err != nil {
-			return errors.Wrap(err, "failed to fetch URL")
-		}
-
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			return errors.Wrap(err, "failed to copy response body")
-		}
-		out.Close()
-		resp.Body.Close()
 
 		// Extract the tar.gz file
-
+		f, err := os.Open(tmpPath)
-		f, err := os.Open(out.Name())
 		if err != nil {
 			return errors.Wrap(err, "failed to read file")
 		}