Add support for downloading legacy geoip databases

Pass the geoip databases to measurement-kit

utils/geoip.go

@@ -31,31 +31,82 @@ var geoipFiles = map[string]string{
 	"GeoLite2-Country.mmdb": "http://geolite.maxmind.com/download/geoip/database/GeoLite2-Country.tar.gz",
 }
 
+var legacyGeoipFiles = map[string]string{
+	"GeoIPASNum.dat": "http://download.maxmind.com/download/geoip/database/asnum/GeoIPASNum.dat.gz",
+	"GeoIP.dat":      "http://geolite.maxmind.com/download/geoip/database/GeoLiteCountry/GeoIP.dat.gz",
+}
+
+// GeoIPDir returns the geoip data dir for the given OONI Home
+func GeoIPDir(home string) string {
+	return filepath.Join(home, "geoip")
+}
+
+// Download the file to a temporary location
+func downloadToTemp(url string) (string, error) {
+	out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+	if err != nil {
+		return "", errors.Wrap(err, "failed to create temporary directory")
+	}
+	resp, err := http.Get(url)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to fetch URL")
+	}
+
+	_, err = io.Copy(out, resp.Body)
+	if err != nil {
+		return "", errors.Wrap(err, "failed to copy response body")
+	}
+	out.Close()
+	resp.Body.Close()
+	return out.Name(), nil
+}
+
+// DownloadLegacyGeoIPDatabaseFiles into the target directory
+func DownloadLegacyGeoIPDatabaseFiles(dir string) error {
+	for filename, url := range legacyGeoipFiles {
+		dstPath := filepath.Join(dir, filename)
+
+		tmpPath, err := downloadToTemp(url)
+		if err != nil {
+			return err
+		}
+
+		// Extract the tar.gz file
+		f, err := os.Open(tmpPath)
+		defer f.Close()
+		if err != nil {
+			return errors.Wrap(err, "failed to read file")
+		}
+
+		gzf, err := gzip.NewReader(f)
+		if err != nil {
+			return errors.Wrap(err, "failed to create gzip reader")
+		}
+
+		outFile, err := os.Create(dstPath)
+		if err != nil {
+			return errors.Wrap(err, "error creating file")
+		}
+		if _, err := io.Copy(outFile, gzf); err != nil {
+			return errors.Wrap(err, "error reading file from gzip")
+		}
+		outFile.Close()
+	}
+	return nil
+}
+
 // DownloadGeoIPDatabaseFiles into the target directory
 func DownloadGeoIPDatabaseFiles(dir string) error {
 	for filename, url := range geoipFiles {
 		dstPath := filepath.Join(dir, filename)
 
-		// Download the file to a temporary location
-		out, err := ioutil.TempFile(os.TempDir(), "maxmind")
+		tmpPath, err := downloadToTemp(url)
 		if err != nil {
-			return errors.Wrap(err, "failed to create temporary directory")
+			return err
 		}
-		resp, err := http.Get(url)
-		if err != nil {
-			return errors.Wrap(err, "failed to fetch URL")
-		}
-
-		_, err = io.Copy(out, resp.Body)
-		if err != nil {
-			return errors.Wrap(err, "failed to copy response body")
-		}
-		out.Close()
-		resp.Body.Close()
 
 		// Extract the tar.gz file
-
-		f, err := os.Open(out.Name())
+		f, err := os.Open(tmpPath)
 		if err != nil {
 			return errors.Wrap(err, "failed to read file")
 		}