c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: spin geoipx off geolocate (#893)

Browse Source 
A bunch of packages (including oohelperd) just need the ability to
use MaxMind-like databases. They don't need the additional functionality
implemented by the geolocate package. Such a package, in fact, is
mostly (if not only) needed by the engine package.

Therefore, move code to query MaxMind-like databases to a separate
package, and avoid depending on geolocate in all the packages for
which it's sufficient to use geoipx.

Part of https://github.com/ooni/probe/issues/2240
This commit is contained in:
Simone Basso
2022-08-28 20:00:25 +02:00
committed by GitHub
parent 1e7384d1cc
commit 110a11828b
23 changed files with 228 additions and 215 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/geolocate/cloudflare.go
+1 -1
View File
@@ -23,7 +23,7 @@ func cloudflareIPLookup(
UserAgent: model.HTTPHeaderUserAgent,
}).WithBodyLogging().Build().FetchResource(ctx, "/cdn-cgi/trace")
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
r := regexp.MustCompile("(?:ip)=(.*)")
ip := strings.Trim(string(r.Find(data)), "ip=")
internal/engine/geolocate/geolocate.go
+7 -38
View File
@@ -10,37 +10,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/version"
)
const (
// DefaultProbeASN is the default probe ASN as a number.
DefaultProbeASN uint = 0
// DefaultProbeCC is the default probe CC.
DefaultProbeCC = "ZZ"
// DefaultProbeIP is the default probe IP.
DefaultProbeIP = model.DefaultProbeIP
// DefaultProbeNetworkName is the default probe network name.
DefaultProbeNetworkName = ""
// DefaultResolverASN is the default resolver ASN.
DefaultResolverASN uint = 0
// DefaultResolverIP is the default resolver IP.
DefaultResolverIP = "127.0.0.2"
// DefaultResolverNetworkName is the default resolver network name.
DefaultResolverNetworkName = ""
)
var (
// DefaultProbeASNString is the default probe ASN as a string.
DefaultProbeASNString = fmt.Sprintf("AS%d", DefaultProbeASN)
// DefaultResolverASNString is the default resolver ASN as a string.
DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN)
)
// Results contains geolocate results.
type Results struct {
// ASN is the autonomous system number.
@@ -139,13 +108,13 @@ type Task struct {
func (op Task) Run(ctx context.Context) (*Results, error) {
var err error
out := &Results{
ASN: DefaultProbeASN,
CountryCode: DefaultProbeCC,
NetworkName: DefaultProbeNetworkName,
ProbeIP: DefaultProbeIP,
ResolverASN: DefaultResolverASN,
ResolverIP: DefaultResolverIP,
ResolverNetworkName: DefaultResolverNetworkName,
ASN: model.DefaultProbeASN,
CountryCode: model.DefaultProbeCC,
NetworkName: model.DefaultProbeNetworkName,
ProbeIP: model.DefaultProbeIP,
ResolverASN: model.DefaultResolverASN,
ResolverIP: model.DefaultResolverIP,
ResolverNetworkName: model.DefaultResolverNetworkName,
}
ip, err := op.probeIPLookupper.LookupProbeIP(ctx)
if err != nil {
internal/engine/geolocate/geolocate_test.go
+24 -22
View File
@@ -4,6 +4,8 @@ import (
"context"
"errors"
"testing"
"github.com/ooni/probe-cli/v3/internal/model"
)
type taskProbeIPLookupper struct {
@@ -25,25 +27,25 @@ func TestLocationLookupCannotLookupProbeIP(t *testing.T) {
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if out.ASN != DefaultProbeASN {
if out.ASN != model.DefaultProbeASN {
t.Fatal("invalid ASN value")
}
if out.CountryCode != DefaultProbeCC {
if out.CountryCode != model.DefaultProbeCC {
t.Fatal("invalid CountryCode value")
}
if out.NetworkName != DefaultProbeNetworkName {
if out.NetworkName != model.DefaultProbeNetworkName {
t.Fatal("invalid NetworkName value")
}
if out.ProbeIP != DefaultProbeIP {
if out.ProbeIP != model.DefaultProbeIP {
t.Fatal("invalid ProbeIP value")
}
if out.ResolverASN != DefaultResolverASN {
if out.ResolverASN != model.DefaultResolverASN {
t.Fatal("invalid ResolverASN value")
}
if out.ResolverIP != DefaultResolverIP {
if out.ResolverIP != model.DefaultResolverIP {
t.Fatal("invalid ResolverIP value")
}
if out.ResolverNetworkName != DefaultResolverNetworkName {
if out.ResolverNetworkName != model.DefaultResolverNetworkName {
t.Fatal("invalid ResolverNetworkName value")
}
}
@@ -69,25 +71,25 @@ func TestLocationLookupCannotLookupProbeASN(t *testing.T) {
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if out.ASN != DefaultProbeASN {
if out.ASN != model.DefaultProbeASN {
t.Fatal("invalid ASN value")
}
if out.CountryCode != DefaultProbeCC {
if out.CountryCode != model.DefaultProbeCC {
t.Fatal("invalid CountryCode value")
}
if out.NetworkName != DefaultProbeNetworkName {
if out.NetworkName != model.DefaultProbeNetworkName {
t.Fatal("invalid NetworkName value")
}
if out.ProbeIP != "1.2.3.4" {
t.Fatal("invalid ProbeIP value")
}
if out.ResolverASN != DefaultResolverASN {
if out.ResolverASN != model.DefaultResolverASN {
t.Fatal("invalid ResolverASN value")
}
if out.ResolverIP != DefaultResolverIP {
if out.ResolverIP != model.DefaultResolverIP {
t.Fatal("invalid ResolverIP value")
}
if out.ResolverNetworkName != DefaultResolverNetworkName {
if out.ResolverNetworkName != model.DefaultResolverNetworkName {
t.Fatal("invalid ResolverNetworkName value")
}
}
@@ -116,7 +118,7 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) {
if out.ASN != 1234 {
t.Fatal("invalid ASN value")
}
if out.CountryCode != DefaultProbeCC {
if out.CountryCode != model.DefaultProbeCC {
t.Fatal("invalid CountryCode value")
}
if out.NetworkName != "1234.com" {
@@ -125,13 +127,13 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) {
if out.ProbeIP != "1.2.3.4" {
t.Fatal("invalid ProbeIP value")
}
if out.ResolverASN != DefaultResolverASN {
if out.ResolverASN != model.DefaultResolverASN {
t.Fatal("invalid ResolverASN value")
}
if out.ResolverIP != DefaultResolverIP {
if out.ResolverIP != model.DefaultResolverIP {
t.Fatal("invalid ResolverIP value")
}
if out.ResolverNetworkName != DefaultResolverNetworkName {
if out.ResolverNetworkName != model.DefaultResolverNetworkName {
t.Fatal("invalid ResolverNetworkName value")
}
}
@@ -173,13 +175,13 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) {
if out.didResolverLookup != true {
t.Fatal("invalid DidResolverLookup value")
}
if out.ResolverASN != DefaultResolverASN {
if out.ResolverASN != model.DefaultResolverASN {
t.Fatal("invalid ResolverASN value")
}
if out.ResolverIP != DefaultResolverIP {
if out.ResolverIP != model.DefaultResolverIP {
t.Fatal("invalid ResolverIP value")
}
if out.ResolverNetworkName != DefaultResolverNetworkName {
if out.ResolverNetworkName != model.DefaultResolverNetworkName {
t.Fatal("invalid ResolverNetworkName value")
}
}
@@ -213,13 +215,13 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) {
if out.didResolverLookup != true {
t.Fatal("invalid DidResolverLookup value")
}
if out.ResolverASN != DefaultResolverASN {
if out.ResolverASN != model.DefaultResolverASN {
t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN)
}
if out.ResolverIP != "4.3.2.1" {
t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP)
}
if out.ResolverNetworkName != DefaultResolverNetworkName {
if out.ResolverNetworkName != model.DefaultResolverNetworkName {
t.Fatal("invalid ResolverNetworkName value")
}
}
internal/engine/geolocate/iplookup.go
+3 -3
View File
@@ -91,10 +91,10 @@ func (c ipLookupClient) doWithCustomFunc(
defer clnt.CloseIdleConnections()
ip, err := fn(ctx, clnt, c.Logger, c.UserAgent)
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
if net.ParseIP(ip) == nil {
return DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip)
return model.DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip)
}
c.Logger.Debugf("iplookup: IP: %s", ip)
return ip, nil
@@ -110,5 +110,5 @@ func (c ipLookupClient) LookupProbeIP(ctx context.Context) (string, error) {
}
union.Add(err)
}
return DefaultProbeIP, union
return model.DefaultProbeIP, union
}
internal/engine/geolocate/iplookup_test.go
+2 -2
View File
@@ -36,7 +36,7 @@ func TestIPLookupAllFailed(t *testing.T) {
if !errors.Is(err, context.Canceled) {
t.Fatal("expected an error here")
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatal("expected the default IP here")
}
}
@@ -51,7 +51,7 @@ func TestIPLookupInvalidIP(t *testing.T) {
if !errors.Is(err, ErrInvalidIPAddress) {
t.Fatal("expected an error here")
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatal("expected the default IP here")
}
}
internal/engine/geolocate/mmdblookup.go
+5 -44
View File
@@ -1,54 +1,15 @@
package geolocate
import (
"net"
"github.com/ooni/probe-assets/assets"
"github.com/oschwald/geoip2-golang"
"github.com/ooni/probe-cli/v3/internal/geoipx"
)
type mmdbLookupper struct{}
func (mmdbLookupper) LookupASN(ip string) (asn uint, org string, err error) {
asn, org = DefaultProbeASN, DefaultProbeNetworkName
db, err := geoip2.FromBytes(assets.ASNDatabaseData())
if err != nil {
return
}
defer db.Close()
record, err := db.ASN(net.ParseIP(ip))
if err != nil {
return
}
asn = record.AutonomousSystemNumber
if record.AutonomousSystemOrganization != "" {
org = record.AutonomousSystemOrganization
}
return
func (mmdbLookupper) LookupASN(ip string) (uint, string, error) {
return geoipx.LookupASN(ip)
}
// LookupASN returns the ASN and the organization associated with the
// given IP address.
func LookupASN(ip string) (asn uint, org string, err error) {
return (mmdbLookupper{}).LookupASN(ip)
}
func (mmdbLookupper) LookupCC(ip string) (cc string, err error) {
cc = DefaultProbeCC
db, err := geoip2.FromBytes(assets.CountryDatabaseData())
if err != nil {
return
}
defer db.Close()
record, err := db.Country(net.ParseIP(ip))
if err != nil {
return
}
// With MaxMind DB we used record.RegisteredCountry.IsoCode but that does
// not seem to work with the db-ip.com database. The record is empty, at
// least for my own IP address in Italy. --Simone (2020-02-25)
if record.Country.IsoCode != "" {
cc = record.Country.IsoCode
}
return
func (mmdbLookupper) LookupCC(ip string) (string, error) {
return geoipx.LookupCC(ip)
}
internal/engine/geolocate/mmdblookup_test.go
-51
View File
@@ -1,51 +0,0 @@
package geolocate
import "testing"
const ipAddr = "8.8.8.8"
func TestLookupASN(t *testing.T) {
asn, org, err := LookupASN(ipAddr)
if err != nil {
t.Fatal(err)
}
if asn != 15169 {
t.Fatal("unexpected ASN value", asn)
}
if org != "Google LLC" {
t.Fatal("unexpected org value", org)
}
}
func TestLookupASNInvalidIP(t *testing.T) {
asn, org, err := LookupASN("xxx")
if err == nil {
t.Fatal("expected an error here")
}
if asn != DefaultProbeASN {
t.Fatal("expected a zero ASN")
}
if org != DefaultProbeNetworkName {
t.Fatal("expected an empty org")
}
}
func TestLookupCC(t *testing.T) {
cc, err := (mmdbLookupper{}).LookupCC(ipAddr)
if err != nil {
t.Fatal(err)
}
if cc != "US" {
t.Fatal("invalid country code", cc)
}
}
func TestLookupCCInvalidIP(t *testing.T) {
cc, err := (mmdbLookupper{}).LookupCC("xxx")
if err == nil {
t.Fatal("expected an error here")
}
if cc != DefaultProbeCC {
t.Fatal("expected an empty cc")
}
}
internal/engine/geolocate/stun.go
+5 -5
View File
@@ -37,7 +37,7 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) {
}
clnt, err := dial("udp", config.Endpoint)
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
defer clnt.Close()
message := stun.MustBuild(stun.TransactionID, stun.BindingRequest)
@@ -55,20 +55,20 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) {
ipch <- xorAddr.IP.String()
})
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
select {
case err := <-errch:
return DefaultProbeIP, err
return model.DefaultProbeIP, err
case ip := <-ipch:
return ip, nil
case <-ctx.Done():
return DefaultProbeIP, ctx.Err()
return model.DefaultProbeIP, ctx.Err()
}
}()
if err != nil {
config.Logger.Debugf("STUNIPLookup: failure using %s: %+v", config.Endpoint, err)
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
return ip, nil
}
internal/engine/geolocate/stun_test.go
+5 -5
View File
@@ -23,7 +23,7 @@ func TestSTUNIPLookupCanceledContext(t *testing.T) {
if !errors.Is(err, context.Canceled) {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the IP address we expected: %+v", ip)
}
}
@@ -41,7 +41,7 @@ func TestSTUNIPLookupDialFailure(t *testing.T) {
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the IP address we expected: %+v", ip)
}
}
@@ -79,7 +79,7 @@ func TestSTUNIPLookupStartReturnsError(t *testing.T) {
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the IP address we expected: %+v", ip)
}
}
@@ -99,7 +99,7 @@ func TestSTUNIPLookupStunEventContainsError(t *testing.T) {
if !errors.Is(err, expected) {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the IP address we expected: %+v", ip)
}
}
@@ -118,7 +118,7 @@ func TestSTUNIPLookupCannotDecodeMessage(t *testing.T) {
if !errors.Is(err, stun.ErrAttributeNotFound) {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the IP address we expected: %+v", ip)
}
}
internal/engine/geolocate/ubuntu.go
+2 -2
View File
@@ -27,13 +27,13 @@ func ubuntuIPLookup(
UserAgent: userAgent,
}).WithBodyLogging().Build().FetchResource(ctx, "/lookup")
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
logger.Debugf("ubuntu: body: %s", string(data))
var v ubuntuResponse
err = xml.Unmarshal(data, &v)
if err != nil {
return DefaultProbeIP, err
return model.DefaultProbeIP, err
}
return v.IP, nil
}
internal/engine/geolocate/ubuntu_test.go
+1 -1
View File
@@ -27,7 +27,7 @@ func TestUbuntuParseError(t *testing.T) {
if err == nil || !strings.HasPrefix(err.Error(), "XML syntax error") {
t.Fatalf("not the error we expected: %+v", err)
}
if ip != DefaultProbeIP {
if ip != model.DefaultProbeIP {
t.Fatalf("not the expected IP address: %s", ip)
}
}