c/ooni-probe-cli
Watch
1
0
Fork
You've already forked ooni-probe-cli
 0
Code Issues Pull requests Projects Releases Packages Wiki Activity

fix(measurex): use same keys of the OONI data format (#572)

Browse source 
This change should simplify the pipeline's job.

Reference issue: https://github.com/ooni/probe/issues/1817.

I previously dismissed this possibility, but now it seems clear it
is simpler to have a very tabular data format internally and to
convert such a format to OONI's data format when serializing.

The OONI data format is what the pipeline expects, but processing
is easier with a more linear/tabular format.
This commit is contained in:
Simone Basso 2021-11-05 10:46:45 +01:00 • committed by GitHub
commit aa27bbe33f
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
35 changed files with 1539 additions and 993 deletions
Download patch file Download diff file Expand all files Collapse all files

16
internal/measurex/db.go
View file

@ -28,7 +28,7 @@ type WritableDB interface {
InsertIntoClose(ev *NetworkEvent)
// InsertIntoTLSHandshake saves a TLS handshake event.
InsertIntoTLSHandshake(ev *TLSHandshakeEvent)
InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent)
// InsertIntoLookupHost saves a lookup host event.
InsertIntoLookupHost(ev *DNSLookupEvent)
@ -46,7 +46,7 @@ type WritableDB interface {
InsertIntoHTTPRedirect(ev *HTTPRedirectEvent)
// InsertIntoQUICHandshake saves a QUIC handshake event.
InsertIntoQUICHandshake(ev *QUICHandshakeEvent)
InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent)
}
// MeasurementDB is a WritableDB that also allows high-level code
@ -56,13 +56,13 @@ type MeasurementDB struct {
dialTable []*NetworkEvent
readWriteTable []*NetworkEvent
closeTable []*NetworkEvent
tlsHandshakeTable []*TLSHandshakeEvent
tlsHandshakeTable []*QUICTLSHandshakeEvent
lookupHostTable []*DNSLookupEvent
lookupHTTPSvcTable []*DNSLookupEvent
dnsRoundTripTable []*DNSRoundTripEvent
httpRoundTripTable []*HTTPRoundTripEvent
httpRedirectTable []*HTTPRedirectEvent
quicHandshakeTable []*QUICHandshakeEvent
quicHandshakeTable []*QUICTLSHandshakeEvent
// mu protects all the fields
mu sync.Mutex
@ -126,14 +126,14 @@ func (db *MeasurementDB) selectAllFromCloseUnlocked() (out []*NetworkEvent) {
}
// InsertIntoTLSHandshake implements EventDB.InsertIntoTLSHandshake.
func (db *MeasurementDB) InsertIntoTLSHandshake(ev *TLSHandshakeEvent) {
func (db *MeasurementDB) InsertIntoTLSHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock()
db.tlsHandshakeTable = append(db.tlsHandshakeTable, ev)
db.mu.Unlock()
}
// selectAllFromTLSHandshakeUnlocked returns all TLS handshake events.
func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*TLSHandshakeEvent) {
func (db *MeasurementDB) selectAllFromTLSHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.tlsHandshakeTable...)
return
}
@ -204,14 +204,14 @@ func (db *MeasurementDB) selectAllFromHTTPRedirectUnlocked() (out []*HTTPRedirec
}
// InsertIntoQUICHandshake implements EventDB.InsertIntoQUICHandshake.
func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICHandshakeEvent) {
func (db *MeasurementDB) InsertIntoQUICHandshake(ev *QUICTLSHandshakeEvent) {
db.mu.Lock()
db.quicHandshakeTable = append(db.quicHandshakeTable, ev)
db.mu.Unlock()
}
// selectAllFromQUICHandshakeUnlocked returns all QUIC handshake events.
func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICHandshakeEvent) {
func (db *MeasurementDB) selectAllFromQUICHandshakeUnlocked() (out []*QUICTLSHandshakeEvent) {
out = append(out, db.quicHandshakeTable...)
return
}