diff --git a/internal/cmd/oohelperd/ipinfo.go b/internal/cmd/oohelperd/ipinfo.go index 6cbdf98..75fe0c8 100644 --- a/internal/cmd/oohelperd/ipinfo.go +++ b/internal/cmd/oohelperd/ipinfo.go @@ -11,7 +11,7 @@ import ( "strings" "github.com/ooni/probe-cli/v3/internal/engine/experiment/webconnectivity" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/geoipx" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -36,7 +36,7 @@ func newIPInfo(creq *ctrlRequest, addrs []string) map[string]*webconnectivity.Co if netxlite.IsBogon(addr) { // note: we already excluded non-IP addrs above flags |= webconnectivity.ControlIPInfoFlagIsBogon } - asn, _, _ := geolocate.LookupASN(addr) // AS0 on failure + asn, _, _ := geoipx.LookupASN(addr) // AS0 on failure ipinfo[addr] = &webconnectivity.ControlIPInfo{ ASN: int64(asn), Flags: flags, diff --git a/internal/engine/experiment.go b/internal/engine/experiment.go index 069e5b6..d4d1266 100644 --- a/internal/engine/experiment.go +++ b/internal/engine/experiment.go @@ -14,7 +14,6 @@ import ( "time" "github.com/ooni/probe-cli/v3/internal/bytecounter" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/version" @@ -199,7 +198,7 @@ func (e *experiment) newMeasurement(input string) *model.Measurement { Input: model.MeasurementTarget(input), MeasurementStartTime: utctimenow.Format(dateFormat), MeasurementStartTimeSaved: utctimenow, - ProbeIP: geolocate.DefaultProbeIP, + ProbeIP: model.DefaultProbeIP, ProbeASN: e.session.ProbeASNString(), ProbeCC: e.session.ProbeCC(), ProbeNetworkName: e.session.ProbeNetworkName(), diff --git a/internal/engine/experiment/webconnectivity/control.go b/internal/engine/experiment/webconnectivity/control.go index 90b5bdb..639d76d 100644 --- a/internal/engine/experiment/webconnectivity/control.go +++ b/internal/engine/experiment/webconnectivity/control.go @@ -3,7 +3,7 @@ package webconnectivity import ( "context" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/geoipx" "github.com/ooni/probe-cli/v3/internal/httpx" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" @@ -122,9 +122,7 @@ func Control( func (dns *ControlDNSResult) FillASNs(sess model.ExperimentSession) { dns.ASNs = []int64{} for _, ip := range dns.Addrs { - // TODO(bassosimone): this would be more efficient if we'd open just - // once the database and then reuse it for every address. - asn, _, _ := geolocate.LookupASN(ip) + asn, _, _ := geoipx.LookupASN(ip) dns.ASNs = append(dns.ASNs, int64(asn)) } } diff --git a/internal/engine/experiment_test.go b/internal/engine/experiment_test.go index 7860d57..d7b9853 100644 --- a/internal/engine/experiment_test.go +++ b/internal/engine/experiment_test.go @@ -26,7 +26,7 @@ func TestExperimentHonoursSharingDefaults(t *testing.T) { name: "probeIP", locationInfo: &geolocate.Results{ProbeIP: "8.8.8.8"}, expect: func(m *model.Measurement) bool { - return m.ProbeIP == geolocate.DefaultProbeIP + return m.ProbeIP == model.DefaultProbeIP }, }, { name: "probeASN", diff --git a/internal/engine/geolocate/cloudflare.go b/internal/engine/geolocate/cloudflare.go index ad364c3..b98c1e6 100644 --- a/internal/engine/geolocate/cloudflare.go +++ b/internal/engine/geolocate/cloudflare.go @@ -23,7 +23,7 @@ func cloudflareIPLookup( UserAgent: model.HTTPHeaderUserAgent, }).WithBodyLogging().Build().FetchResource(ctx, "/cdn-cgi/trace") if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } r := regexp.MustCompile("(?:ip)=(.*)") ip := strings.Trim(string(r.Find(data)), "ip=") diff --git a/internal/engine/geolocate/geolocate.go b/internal/engine/geolocate/geolocate.go index 5776b14..175c7db 100644 --- a/internal/engine/geolocate/geolocate.go +++ b/internal/engine/geolocate/geolocate.go @@ -10,37 +10,6 @@ import ( "github.com/ooni/probe-cli/v3/internal/version" ) -const ( - // DefaultProbeASN is the default probe ASN as a number. - DefaultProbeASN uint = 0 - - // DefaultProbeCC is the default probe CC. - DefaultProbeCC = "ZZ" - - // DefaultProbeIP is the default probe IP. - DefaultProbeIP = model.DefaultProbeIP - - // DefaultProbeNetworkName is the default probe network name. - DefaultProbeNetworkName = "" - - // DefaultResolverASN is the default resolver ASN. - DefaultResolverASN uint = 0 - - // DefaultResolverIP is the default resolver IP. - DefaultResolverIP = "127.0.0.2" - - // DefaultResolverNetworkName is the default resolver network name. - DefaultResolverNetworkName = "" -) - -var ( - // DefaultProbeASNString is the default probe ASN as a string. - DefaultProbeASNString = fmt.Sprintf("AS%d", DefaultProbeASN) - - // DefaultResolverASNString is the default resolver ASN as a string. - DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN) -) - // Results contains geolocate results. type Results struct { // ASN is the autonomous system number. @@ -139,13 +108,13 @@ type Task struct { func (op Task) Run(ctx context.Context) (*Results, error) { var err error out := &Results{ - ASN: DefaultProbeASN, - CountryCode: DefaultProbeCC, - NetworkName: DefaultProbeNetworkName, - ProbeIP: DefaultProbeIP, - ResolverASN: DefaultResolverASN, - ResolverIP: DefaultResolverIP, - ResolverNetworkName: DefaultResolverNetworkName, + ASN: model.DefaultProbeASN, + CountryCode: model.DefaultProbeCC, + NetworkName: model.DefaultProbeNetworkName, + ProbeIP: model.DefaultProbeIP, + ResolverASN: model.DefaultResolverASN, + ResolverIP: model.DefaultResolverIP, + ResolverNetworkName: model.DefaultResolverNetworkName, } ip, err := op.probeIPLookupper.LookupProbeIP(ctx) if err != nil { diff --git a/internal/engine/geolocate/geolocate_test.go b/internal/engine/geolocate/geolocate_test.go index b1d4608..9d10917 100644 --- a/internal/engine/geolocate/geolocate_test.go +++ b/internal/engine/geolocate/geolocate_test.go @@ -4,6 +4,8 @@ import ( "context" "errors" "testing" + + "github.com/ooni/probe-cli/v3/internal/model" ) type taskProbeIPLookupper struct { @@ -25,25 +27,25 @@ func TestLocationLookupCannotLookupProbeIP(t *testing.T) { if !errors.Is(err, expected) { t.Fatalf("not the error we expected: %+v", err) } - if out.ASN != DefaultProbeASN { + if out.ASN != model.DefaultProbeASN { t.Fatal("invalid ASN value") } - if out.CountryCode != DefaultProbeCC { + if out.CountryCode != model.DefaultProbeCC { t.Fatal("invalid CountryCode value") } - if out.NetworkName != DefaultProbeNetworkName { + if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } - if out.ProbeIP != DefaultProbeIP { + if out.ProbeIP != model.DefaultProbeIP { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != DefaultResolverASN { + if out.ResolverASN != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != DefaultResolverIP { + if out.ResolverIP != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != DefaultResolverNetworkName { + if out.ResolverNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -69,25 +71,25 @@ func TestLocationLookupCannotLookupProbeASN(t *testing.T) { if !errors.Is(err, expected) { t.Fatalf("not the error we expected: %+v", err) } - if out.ASN != DefaultProbeASN { + if out.ASN != model.DefaultProbeASN { t.Fatal("invalid ASN value") } - if out.CountryCode != DefaultProbeCC { + if out.CountryCode != model.DefaultProbeCC { t.Fatal("invalid CountryCode value") } - if out.NetworkName != DefaultProbeNetworkName { + if out.NetworkName != model.DefaultProbeNetworkName { t.Fatal("invalid NetworkName value") } if out.ProbeIP != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != DefaultResolverASN { + if out.ResolverASN != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != DefaultResolverIP { + if out.ResolverIP != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != DefaultResolverNetworkName { + if out.ResolverNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -116,7 +118,7 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) { if out.ASN != 1234 { t.Fatal("invalid ASN value") } - if out.CountryCode != DefaultProbeCC { + if out.CountryCode != model.DefaultProbeCC { t.Fatal("invalid CountryCode value") } if out.NetworkName != "1234.com" { @@ -125,13 +127,13 @@ func TestLocationLookupCannotLookupProbeCC(t *testing.T) { if out.ProbeIP != "1.2.3.4" { t.Fatal("invalid ProbeIP value") } - if out.ResolverASN != DefaultResolverASN { + if out.ResolverASN != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != DefaultResolverIP { + if out.ResolverIP != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != DefaultResolverNetworkName { + if out.ResolverNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -173,13 +175,13 @@ func TestLocationLookupCannotLookupResolverIP(t *testing.T) { if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != DefaultResolverASN { + if out.ResolverASN != model.DefaultResolverASN { t.Fatal("invalid ResolverASN value") } - if out.ResolverIP != DefaultResolverIP { + if out.ResolverIP != model.DefaultResolverIP { t.Fatal("invalid ResolverIP value") } - if out.ResolverNetworkName != DefaultResolverNetworkName { + if out.ResolverNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } @@ -213,13 +215,13 @@ func TestLocationLookupCannotLookupResolverNetworkName(t *testing.T) { if out.didResolverLookup != true { t.Fatal("invalid DidResolverLookup value") } - if out.ResolverASN != DefaultResolverASN { + if out.ResolverASN != model.DefaultResolverASN { t.Fatalf("invalid ResolverASN value: %+v", out.ResolverASN) } if out.ResolverIP != "4.3.2.1" { t.Fatalf("invalid ResolverIP value: %+v", out.ResolverIP) } - if out.ResolverNetworkName != DefaultResolverNetworkName { + if out.ResolverNetworkName != model.DefaultResolverNetworkName { t.Fatal("invalid ResolverNetworkName value") } } diff --git a/internal/engine/geolocate/iplookup.go b/internal/engine/geolocate/iplookup.go index f041474..b5b40b0 100644 --- a/internal/engine/geolocate/iplookup.go +++ b/internal/engine/geolocate/iplookup.go @@ -91,10 +91,10 @@ func (c ipLookupClient) doWithCustomFunc( defer clnt.CloseIdleConnections() ip, err := fn(ctx, clnt, c.Logger, c.UserAgent) if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } if net.ParseIP(ip) == nil { - return DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip) + return model.DefaultProbeIP, fmt.Errorf("%w: %s", ErrInvalidIPAddress, ip) } c.Logger.Debugf("iplookup: IP: %s", ip) return ip, nil @@ -110,5 +110,5 @@ func (c ipLookupClient) LookupProbeIP(ctx context.Context) (string, error) { } union.Add(err) } - return DefaultProbeIP, union + return model.DefaultProbeIP, union } diff --git a/internal/engine/geolocate/iplookup_test.go b/internal/engine/geolocate/iplookup_test.go index 15fb768..2739a95 100644 --- a/internal/engine/geolocate/iplookup_test.go +++ b/internal/engine/geolocate/iplookup_test.go @@ -36,7 +36,7 @@ func TestIPLookupAllFailed(t *testing.T) { if !errors.Is(err, context.Canceled) { t.Fatal("expected an error here") } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatal("expected the default IP here") } } @@ -51,7 +51,7 @@ func TestIPLookupInvalidIP(t *testing.T) { if !errors.Is(err, ErrInvalidIPAddress) { t.Fatal("expected an error here") } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatal("expected the default IP here") } } diff --git a/internal/engine/geolocate/mmdblookup.go b/internal/engine/geolocate/mmdblookup.go index d6e5e86..14e901f 100644 --- a/internal/engine/geolocate/mmdblookup.go +++ b/internal/engine/geolocate/mmdblookup.go @@ -1,54 +1,15 @@ package geolocate import ( - "net" - - "github.com/ooni/probe-assets/assets" - "github.com/oschwald/geoip2-golang" + "github.com/ooni/probe-cli/v3/internal/geoipx" ) type mmdbLookupper struct{} -func (mmdbLookupper) LookupASN(ip string) (asn uint, org string, err error) { - asn, org = DefaultProbeASN, DefaultProbeNetworkName - db, err := geoip2.FromBytes(assets.ASNDatabaseData()) - if err != nil { - return - } - defer db.Close() - record, err := db.ASN(net.ParseIP(ip)) - if err != nil { - return - } - asn = record.AutonomousSystemNumber - if record.AutonomousSystemOrganization != "" { - org = record.AutonomousSystemOrganization - } - return +func (mmdbLookupper) LookupASN(ip string) (uint, string, error) { + return geoipx.LookupASN(ip) } -// LookupASN returns the ASN and the organization associated with the -// given IP address. -func LookupASN(ip string) (asn uint, org string, err error) { - return (mmdbLookupper{}).LookupASN(ip) -} - -func (mmdbLookupper) LookupCC(ip string) (cc string, err error) { - cc = DefaultProbeCC - db, err := geoip2.FromBytes(assets.CountryDatabaseData()) - if err != nil { - return - } - defer db.Close() - record, err := db.Country(net.ParseIP(ip)) - if err != nil { - return - } - // With MaxMind DB we used record.RegisteredCountry.IsoCode but that does - // not seem to work with the db-ip.com database. The record is empty, at - // least for my own IP address in Italy. --Simone (2020-02-25) - if record.Country.IsoCode != "" { - cc = record.Country.IsoCode - } - return +func (mmdbLookupper) LookupCC(ip string) (string, error) { + return geoipx.LookupCC(ip) } diff --git a/internal/engine/geolocate/mmdblookup_test.go b/internal/engine/geolocate/mmdblookup_test.go deleted file mode 100644 index b84299f..0000000 --- a/internal/engine/geolocate/mmdblookup_test.go +++ /dev/null @@ -1,51 +0,0 @@ -package geolocate - -import "testing" - -const ipAddr = "8.8.8.8" - -func TestLookupASN(t *testing.T) { - asn, org, err := LookupASN(ipAddr) - if err != nil { - t.Fatal(err) - } - if asn != 15169 { - t.Fatal("unexpected ASN value", asn) - } - if org != "Google LLC" { - t.Fatal("unexpected org value", org) - } -} - -func TestLookupASNInvalidIP(t *testing.T) { - asn, org, err := LookupASN("xxx") - if err == nil { - t.Fatal("expected an error here") - } - if asn != DefaultProbeASN { - t.Fatal("expected a zero ASN") - } - if org != DefaultProbeNetworkName { - t.Fatal("expected an empty org") - } -} - -func TestLookupCC(t *testing.T) { - cc, err := (mmdbLookupper{}).LookupCC(ipAddr) - if err != nil { - t.Fatal(err) - } - if cc != "US" { - t.Fatal("invalid country code", cc) - } -} - -func TestLookupCCInvalidIP(t *testing.T) { - cc, err := (mmdbLookupper{}).LookupCC("xxx") - if err == nil { - t.Fatal("expected an error here") - } - if cc != DefaultProbeCC { - t.Fatal("expected an empty cc") - } -} diff --git a/internal/engine/geolocate/stun.go b/internal/engine/geolocate/stun.go index 9e98dda..cb3746d 100644 --- a/internal/engine/geolocate/stun.go +++ b/internal/engine/geolocate/stun.go @@ -37,7 +37,7 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) { } clnt, err := dial("udp", config.Endpoint) if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } defer clnt.Close() message := stun.MustBuild(stun.TransactionID, stun.BindingRequest) @@ -55,20 +55,20 @@ func stunIPLookup(ctx context.Context, config stunConfig) (string, error) { ipch <- xorAddr.IP.String() }) if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } select { case err := <-errch: - return DefaultProbeIP, err + return model.DefaultProbeIP, err case ip := <-ipch: return ip, nil case <-ctx.Done(): - return DefaultProbeIP, ctx.Err() + return model.DefaultProbeIP, ctx.Err() } }() if err != nil { config.Logger.Debugf("STUNIPLookup: failure using %s: %+v", config.Endpoint, err) - return DefaultProbeIP, err + return model.DefaultProbeIP, err } return ip, nil } diff --git a/internal/engine/geolocate/stun_test.go b/internal/engine/geolocate/stun_test.go index e5b53af..d89a5bf 100644 --- a/internal/engine/geolocate/stun_test.go +++ b/internal/engine/geolocate/stun_test.go @@ -23,7 +23,7 @@ func TestSTUNIPLookupCanceledContext(t *testing.T) { if !errors.Is(err, context.Canceled) { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the IP address we expected: %+v", ip) } } @@ -41,7 +41,7 @@ func TestSTUNIPLookupDialFailure(t *testing.T) { if !errors.Is(err, expected) { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the IP address we expected: %+v", ip) } } @@ -79,7 +79,7 @@ func TestSTUNIPLookupStartReturnsError(t *testing.T) { if !errors.Is(err, expected) { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the IP address we expected: %+v", ip) } } @@ -99,7 +99,7 @@ func TestSTUNIPLookupStunEventContainsError(t *testing.T) { if !errors.Is(err, expected) { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the IP address we expected: %+v", ip) } } @@ -118,7 +118,7 @@ func TestSTUNIPLookupCannotDecodeMessage(t *testing.T) { if !errors.Is(err, stun.ErrAttributeNotFound) { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the IP address we expected: %+v", ip) } } diff --git a/internal/engine/geolocate/ubuntu.go b/internal/engine/geolocate/ubuntu.go index 1b6b191..c37e8c6 100644 --- a/internal/engine/geolocate/ubuntu.go +++ b/internal/engine/geolocate/ubuntu.go @@ -27,13 +27,13 @@ func ubuntuIPLookup( UserAgent: userAgent, }).WithBodyLogging().Build().FetchResource(ctx, "/lookup") if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } logger.Debugf("ubuntu: body: %s", string(data)) var v ubuntuResponse err = xml.Unmarshal(data, &v) if err != nil { - return DefaultProbeIP, err + return model.DefaultProbeIP, err } return v.IP, nil } diff --git a/internal/engine/geolocate/ubuntu_test.go b/internal/engine/geolocate/ubuntu_test.go index d011f38..22e1d28 100644 --- a/internal/engine/geolocate/ubuntu_test.go +++ b/internal/engine/geolocate/ubuntu_test.go @@ -27,7 +27,7 @@ func TestUbuntuParseError(t *testing.T) { if err == nil || !strings.HasPrefix(err.Error(), "XML syntax error") { t.Fatalf("not the error we expected: %+v", err) } - if ip != DefaultProbeIP { + if ip != model.DefaultProbeIP { t.Fatalf("not the expected IP address: %s", ip) } } diff --git a/internal/engine/session.go b/internal/engine/session.go index 13ec0d0..f71843b 100644 --- a/internal/engine/session.go +++ b/internal/engine/session.go @@ -466,7 +466,7 @@ func (s *Session) ProbeASNString() string { func (s *Session) ProbeASN() uint { defer s.mu.Unlock() s.mu.Lock() - asn := geolocate.DefaultProbeASN + asn := model.DefaultProbeASN if s.location != nil { asn = s.location.ASN } @@ -477,7 +477,7 @@ func (s *Session) ProbeASN() uint { func (s *Session) ProbeCC() string { defer s.mu.Unlock() s.mu.Lock() - cc := geolocate.DefaultProbeCC + cc := model.DefaultProbeCC if s.location != nil { cc = s.location.CountryCode } @@ -488,7 +488,7 @@ func (s *Session) ProbeCC() string { func (s *Session) ProbeNetworkName() string { defer s.mu.Unlock() s.mu.Lock() - nn := geolocate.DefaultProbeNetworkName + nn := model.DefaultProbeNetworkName if s.location != nil { nn = s.location.NetworkName } @@ -499,7 +499,7 @@ func (s *Session) ProbeNetworkName() string { func (s *Session) ProbeIP() string { defer s.mu.Unlock() s.mu.Lock() - ip := geolocate.DefaultProbeIP + ip := model.DefaultProbeIP if s.location != nil { ip = s.location.ProbeIP } @@ -520,7 +520,7 @@ func (s *Session) ResolverASNString() string { func (s *Session) ResolverASN() uint { defer s.mu.Unlock() s.mu.Lock() - asn := geolocate.DefaultResolverASN + asn := model.DefaultResolverASN if s.location != nil { asn = s.location.ResolverASN } @@ -531,7 +531,7 @@ func (s *Session) ResolverASN() uint { func (s *Session) ResolverIP() string { defer s.mu.Unlock() s.mu.Lock() - ip := geolocate.DefaultResolverIP + ip := model.DefaultResolverIP if s.location != nil { ip = s.location.ResolverIP } @@ -542,7 +542,7 @@ func (s *Session) ResolverIP() string { func (s *Session) ResolverNetworkName() string { defer s.mu.Unlock() s.mu.Lock() - nn := geolocate.DefaultResolverNetworkName + nn := model.DefaultResolverNetworkName if s.location != nil { nn = s.location.ResolverNetworkName } diff --git a/internal/engine/session_integration_test.go b/internal/engine/session_integration_test.go index 9fd0c35..80dd25f 100644 --- a/internal/engine/session_integration_test.go +++ b/internal/engine/session_integration_test.go @@ -13,7 +13,6 @@ import ( "github.com/apex/log" "github.com/google/go-cmp/cmp" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" "github.com/ooni/probe-cli/v3/internal/engine/probeservices" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" @@ -276,31 +275,31 @@ func TestSessionLocationLookup(t *testing.T) { if err := sess.MaybeLookupLocation(); err != nil { t.Fatal(err) } - if sess.ProbeASNString() == geolocate.DefaultProbeASNString { + if sess.ProbeASNString() == model.DefaultProbeASNString { t.Fatal("unexpected ProbeASNString") } - if sess.ProbeASN() == geolocate.DefaultProbeASN { + if sess.ProbeASN() == model.DefaultProbeASN { t.Fatal("unexpected ProbeASN") } - if sess.ProbeCC() == geolocate.DefaultProbeCC { + if sess.ProbeCC() == model.DefaultProbeCC { t.Fatal("unexpected ProbeCC") } - if sess.ProbeIP() == geolocate.DefaultProbeIP { + if sess.ProbeIP() == model.DefaultProbeIP { t.Fatal("unexpected ProbeIP") } - if sess.ProbeNetworkName() == geolocate.DefaultProbeNetworkName { + if sess.ProbeNetworkName() == model.DefaultProbeNetworkName { t.Fatal("unexpected ProbeNetworkName") } - if sess.ResolverASN() == geolocate.DefaultResolverASN { + if sess.ResolverASN() == model.DefaultResolverASN { t.Fatal("unexpected ResolverASN") } - if sess.ResolverASNString() == geolocate.DefaultResolverASNString { + if sess.ResolverASNString() == model.DefaultResolverASNString { t.Fatal("unexpected ResolverASNString") } - if sess.ResolverIP() == geolocate.DefaultResolverIP { + if sess.ResolverIP() == model.DefaultResolverIP { t.Fatal("unexpected ResolverIP") } - if sess.ResolverNetworkName() == geolocate.DefaultResolverNetworkName { + if sess.ResolverNetworkName() == model.DefaultResolverNetworkName { t.Fatal("unexpected ResolverNetworkName") } } diff --git a/internal/experiment/webconnectivity/analysisdns.go b/internal/experiment/webconnectivity/analysisdns.go index 0ae9b6e..b1ba1b3 100644 --- a/internal/experiment/webconnectivity/analysisdns.go +++ b/internal/experiment/webconnectivity/analysisdns.go @@ -8,7 +8,7 @@ import ( "net" "net/url" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/geoipx" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -289,11 +289,11 @@ func (tk *TestKeys) analysisDNSDiffASN(probeAddrs, thAddrs []string) (asns []uin ) mapping := make(map[uint]int) for _, addr := range probeAddrs { - asn, _, _ := geolocate.LookupASN(addr) + asn, _, _ := geoipx.LookupASN(addr) mapping[asn] |= inProbe // including the zero ASN that means unknown } for _, addr := range thAddrs { - asn, _, _ := geolocate.LookupASN(addr) + asn, _, _ := geoipx.LookupASN(addr) mapping[asn] |= inTH // including the zero ASN that means unknown } for asn, where := range mapping { diff --git a/internal/geoipx/geoipx.go b/internal/geoipx/geoipx.go new file mode 100644 index 0000000..e35692b --- /dev/null +++ b/internal/geoipx/geoipx.go @@ -0,0 +1,50 @@ +// Package geoipx contains code to use the embedded MaxMind-like databases. +package geoipx + +import ( + "net" + + "github.com/ooni/probe-assets/assets" + "github.com/ooni/probe-cli/v3/internal/model" + "github.com/ooni/probe-cli/v3/internal/runtimex" + "github.com/oschwald/geoip2-golang" +) + +// TODO(bassosimone): this would be more efficient if we'd open just +// once the database and then reuse it for every address. + +// LookupASN maps [ip] to an AS number and an AS organization name. +func LookupASN(ip string) (asn uint, org string, err error) { + asn, org = model.DefaultProbeASN, model.DefaultProbeNetworkName + db, err := geoip2.FromBytes(assets.ASNDatabaseData()) + runtimex.PanicOnError(err, "cannot load embedded geoip2 ASN database") + defer db.Close() + record, err := db.ASN(net.ParseIP(ip)) + if err != nil { + return + } + asn = record.AutonomousSystemNumber + if record.AutonomousSystemOrganization != "" { + org = record.AutonomousSystemOrganization + } + return +} + +// LookupCC maps [ip] to a country code. +func LookupCC(ip string) (cc string, err error) { + cc = model.DefaultProbeCC + db, err := geoip2.FromBytes(assets.CountryDatabaseData()) + runtimex.PanicOnError(err, "cannot load embedded geoip2 country database") + defer db.Close() + record, err := db.Country(net.ParseIP(ip)) + if err != nil { + return + } + // With MaxMind DB we used record.RegisteredCountry.IsoCode but that does + // not seem to work with the db-ip.com database. The record is empty, at + // least for my own IP address in Italy. --Simone (2020-02-25) + if record.Country.IsoCode != "" { + cc = record.Country.IsoCode + } + return +} diff --git a/internal/geoipx/geoipx_test.go b/internal/geoipx/geoipx_test.go new file mode 100644 index 0000000..43582bd --- /dev/null +++ b/internal/geoipx/geoipx_test.go @@ -0,0 +1,59 @@ +package geoipx + +import ( + "testing" + + "github.com/ooni/probe-cli/v3/internal/model" +) + +const ipAddr = "8.8.8.8" + +func TestLookupASN(t *testing.T) { + t.Run("with valid IP address", func(t *testing.T) { + asn, org, err := LookupASN(ipAddr) + if err != nil { + t.Fatal(err) + } + if asn != 15169 { + t.Fatal("unexpected ASN value", asn) + } + if org != "Google LLC" { + t.Fatal("unexpected org value", org) + } + }) + + t.Run("with invalid IP address", func(t *testing.T) { + asn, org, err := LookupASN("xxx") + if err == nil { + t.Fatal("expected an error here") + } + if asn != model.DefaultProbeASN { + t.Fatal("expected a zero ASN") + } + if org != model.DefaultProbeNetworkName { + t.Fatal("expected an empty org") + } + }) +} + +func TestLookupCC(t *testing.T) { + t.Run("with valid IP address", func(t *testing.T) { + cc, err := LookupCC(ipAddr) + if err != nil { + t.Fatal(err) + } + if cc != "US" { + t.Fatal("invalid country code", cc) + } + }) + + t.Run("with invalid IP address", func(t *testing.T) { + cc, err := LookupCC("xxx") + if err == nil { + t.Fatal("expected an error here") + } + if cc != model.DefaultProbeCC { + t.Fatal("expected an empty cc") + } + }) +} diff --git a/internal/measurexlite/dns.go b/internal/measurexlite/dns.go index 72d127e..0e9e241 100644 --- a/internal/measurexlite/dns.go +++ b/internal/measurexlite/dns.go @@ -12,7 +12,7 @@ import ( "time" "github.com/miekg/dns" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/geoipx" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" "github.com/ooni/probe-cli/v3/internal/tracex" @@ -163,7 +163,7 @@ func newArchivalDNSAnswers(addrs []string, resp model.DNSResponse) (out []model. log.Printf("BUG: NewArchivalDNSLookupResult: invalid IP address: %s", addr) continue } - asn, org, _ := geolocate.LookupASN(addr) + asn, org, _ := geoipx.LookupASN(addr) switch ipv6 { case false: out = append(out, model.ArchivalDNSAnswer{ diff --git a/internal/model/measurement.go b/internal/model/measurement.go index 9d3149c..0af0be3 100644 --- a/internal/model/measurement.go +++ b/internal/model/measurement.go @@ -1,20 +1,47 @@ package model -import ( - "bytes" - "encoding/json" - "errors" - "net" - "time" -) - // // Definition of the result of a network measurement. // +import ( + "bytes" + "encoding/json" + "errors" + "fmt" + "net" + "time" +) + const ( + // DefaultProbeASN is the default probe ASN as a number. + DefaultProbeASN uint = 0 + + // DefaultProbeCC is the default probe CC. + DefaultProbeCC = "ZZ" + // DefaultProbeIP is the default probe IP. DefaultProbeIP = "127.0.0.1" + + // DefaultProbeNetworkName is the default probe network name. + DefaultProbeNetworkName = "" + + // DefaultResolverASN is the default resolver ASN. + DefaultResolverASN uint = 0 + + // DefaultResolverIP is the default resolver IP. + DefaultResolverIP = "127.0.0.2" + + // DefaultResolverNetworkName is the default resolver network name. + DefaultResolverNetworkName = "" +) + +var ( + // DefaultProbeASNString is the default probe ASN as a string. + DefaultProbeASNString = fmt.Sprintf("AS%d", DefaultProbeASN) + + // DefaultResolverASNString is the default resolver ASN as a string. + DefaultResolverASNString = fmt.Sprintf("AS%d", DefaultResolverASN) ) // MeasurementTarget is the target of a OONI measurement. diff --git a/internal/tracex/archival.go b/internal/tracex/archival.go index 0351c42..a55f483 100644 --- a/internal/tracex/archival.go +++ b/internal/tracex/archival.go @@ -13,7 +13,7 @@ import ( "strings" "time" - "github.com/ooni/probe-cli/v3/internal/engine/geolocate" + "github.com/ooni/probe-cli/v3/internal/geoipx" "github.com/ooni/probe-cli/v3/internal/model" "github.com/ooni/probe-cli/v3/internal/netxlite" ) @@ -201,7 +201,7 @@ func (qtype dnsQueryType) makeAnswerEntry(addr string) DNSAnswerEntry { answer := DNSAnswerEntry{AnswerType: string(qtype)} // Figuring out the ASN and the org here is not just a service to whoever // is reading a JSON: Web Connectivity also depends on it! - asn, org, _ := geolocate.LookupASN(addr) + asn, org, _ := geoipx.LookupASN(addr) answer.ASN = int64(asn) answer.ASOrgName = org switch qtype {