c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: redesign how we import assets (#260)

Browse Source 
* fix(pkg.go.dev): import a subpackage containing the assets

We're trying to fix this issue that pkg.go.dev does not build.

Thanks to @hellais for this very neat idea! Let's keep our
fingers crossed and see whether it fixes!

* feat: use embedded geoip databases

Closes https://github.com/ooni/probe/issues/1372.

Work done as part of https://github.com/ooni/probe/issues/1369.

* fix(assetsx): add tests

* feat: simplify and just vendor uncompressed DBs

* remove tests that seems not necessary anymore

* fix: run go mod tidy

* Address https://github.com/ooni/probe-cli/pull/260/files#r605181364

* rewrite a test in a better way

* fix: gently cleanup the legacy assetsdir

Do not remove the whole directory with brute force. Just zap the
files whose name we know. Then attempt to delete the legacy directory
as well. If not empty, just fail. This is fine because it means the
user has stored other files inside the directory.

* fix: create .miniooni if missing
This commit is contained in:
Simone Basso
2021-04-01 16:57:31 +02:00
committed by GitHub
parent 7ca32b5ce6
commit 31e478b04e
51 changed files with 243 additions and 808 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/netx/archival/archival.go
+6 -6
View File
@@ -397,7 +397,7 @@ type DNSQueryEntry struct {
type dnsQueryType string
// NewDNSQueriesList returns a list of DNS queries.
func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []DNSQueryEntry {
func NewDNSQueriesList(begin time.Time, events []trace.Event) []DNSQueryEntry {
// TODO(bassosimone): add support for CNAME lookups.
var out []DNSQueryEntry
for _, ev := range events {
@@ -409,7 +409,7 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D
for _, addr := range ev.Addresses {
if qtype.ipoftype(addr) {
entry.Answers = append(
entry.Answers, qtype.makeanswerentry(addr, dbpath))
entry.Answers, qtype.makeanswerentry(addr))
}
}
if len(entry.Answers) <= 0 && ev.Err == nil {
@@ -431,16 +431,16 @@ func NewDNSQueriesList(begin time.Time, events []trace.Event, dbpath string) []D
func (qtype dnsQueryType) ipoftype(addr string) bool {
switch qtype {
case "A":
return strings.Contains(addr, ":") == false
return !strings.Contains(addr, ":")
case "AAAA":
return strings.Contains(addr, ":") == true
return strings.Contains(addr, ":")
}
return false
}
func (qtype dnsQueryType) makeanswerentry(addr string, dbpath string) DNSAnswerEntry {
func (qtype dnsQueryType) makeanswerentry(addr string) DNSAnswerEntry {
answer := DNSAnswerEntry{AnswerType: string(qtype)}
asn, org, _ := geolocate.LookupASN(dbpath, addr)
asn, org, _ := geolocate.LookupASN(addr)
answer.ASN = int64(asn)
answer.ASOrgName = org
switch qtype {
internal/engine/netx/archival/archival_internal_test.go
+41
View File
@@ -0,0 +1,41 @@
package archival
import "testing"
func TestDNSQueryIPOfType(t *testing.T) {
type expectation struct {
qtype dnsQueryType
ip string
output bool
}
var expectations = []expectation{{
qtype: "A",
ip: "8.8.8.8",
output: true,
}, {
qtype: "A",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "AAAA",
ip: "8.8.8.8",
output: false,
}, {
qtype: "AAAA",
ip: "2a00:1450:4002:801::2004",
output: true,
}, {
qtype: "ANTANI",
ip: "2a00:1450:4002:801::2004",
output: false,
}, {
qtype: "ANTANI",
ip: "8.8.8.8",
output: false,
}}
for _, exp := range expectations {
if exp.qtype.ipoftype(exp.ip) != exp.output {
t.Fatalf("failure for %+v", exp)
}
}
}
internal/engine/netx/archival/archival_test.go
+7 -38
View File
@@ -16,7 +16,6 @@ import (
"github.com/ooni/probe-cli/v3/internal/engine/netx/archival"
"github.com/ooni/probe-cli/v3/internal/engine/netx/errorx"
"github.com/ooni/probe-cli/v3/internal/engine/netx/trace"
"github.com/ooni/probe-cli/v3/internal/engine/resourcesmanager"
)
func TestNewTCPConnectList(t *testing.T) {
@@ -285,15 +284,10 @@ func TestNewRequestList(t *testing.T) {
}
func TestNewDNSQueriesList(t *testing.T) {
err := (&resourcesmanager.CopyWorker{DestDir: "../../testdata"}).Ensure()
if err != nil {
t.Fatal(err)
}
begin := time.Now()
type args struct {
begin time.Time
events []trace.Event
dbpath string
}
tests := []struct {
name string
@@ -334,9 +328,13 @@ func TestNewDNSQueriesList(t *testing.T) {
},
want: []archival.DNSQueryEntry{{
Answers: []archival.DNSAnswerEntry{{
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.8.8",
}, {
ASN: 15169,
ASOrgName: "Google LLC",
AnswerType: "A",
IPv4: "8.8.4.4",
}},
@@ -357,27 +355,6 @@ func TestNewDNSQueriesList(t *testing.T) {
Time: begin.Add(200 * time.Millisecond),
}},
},
want: []archival.DNSQueryEntry{{
Answers: []archival.DNSAnswerEntry{{
AnswerType: "AAAA",
IPv6: "2001:4860:4860::8888",
}},
Hostname: "dns.google.com",
QueryType: "AAAA",
T: 0.2,
}},
}, {
name: "run with ASN DB",
args: args{
begin: begin,
events: []trace.Event{{
Addresses: []string{"2001:4860:4860::8888"},
Hostname: "dns.google.com",
Name: "resolve_done",
Time: begin.Add(200 * time.Millisecond),
}},
dbpath: "../../testdata/asn.mmdb",
},
want: []archival.DNSQueryEntry{{
Answers: []archival.DNSAnswerEntry{{
ASN: 15169,
@@ -399,7 +376,6 @@ func TestNewDNSQueriesList(t *testing.T) {
Name: "resolve_done",
Time: begin.Add(200 * time.Millisecond),
}},
dbpath: "../../testdata/asn.mmdb",
},
want: []archival.DNSQueryEntry{{
Answers: nil,
@@ -419,9 +395,9 @@ func TestNewDNSQueriesList(t *testing.T) {
}}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
if got := archival.NewDNSQueriesList(
tt.args.begin, tt.args.events, tt.args.dbpath); !reflect.DeepEqual(got, tt.want) {
t.Error(cmp.Diff(got, tt.want))
got := archival.NewDNSQueriesList(tt.args.begin, tt.args.events)
if diff := cmp.Diff(tt.want, got); diff != "" {
t.Fatal(diff)
}
})
}
@@ -1009,13 +985,6 @@ func TestNewFailure(t *testing.T) {
}
}
func TestDNSQueryTypeInvalidIPOfType(t *testing.T) {
qtype := archival.DNSQueryType("ANTANI")
if qtype.IPOfType("8.8.8.8") != false {
t.Fatal("unexpected return value")
}
}
func TestNewFailedOperation(t *testing.T) {
type args struct {
err error
internal/engine/netx/archival/archival_test_internal.go
-8
View File
@@ -1,8 +0,0 @@
package archival
// DNSQueryType allows to access dnsQueryType from unit tests
type DNSQueryType = dnsQueryType
func (qtype dnsQueryType) IPOfType(addr string) bool {
return qtype.ipoftype(addr)
}