c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

refactor: sync messages with spec draft (#435)

Browse Source 
Work part of: https://github.com/ooni/probe/issues/1733

Spec draft: https://github.com/ooni/spec/pull/219
This commit is contained in:
Simone Basso
2021-08-17 11:56:36 +02:00
committed by GitHub
parent ce854e8ae1
commit f2b6a5972f
8 changed files with 169 additions and 128 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/experiment/websteps/control.go
+1 -13
View File
@@ -8,22 +8,10 @@ import (
"github.com/ooni/probe-cli/v3/internal/errorsx"
)
// CtrlRequest is the request sent by the probe
type CtrlRequest struct {
HTTPRequest string `json:"url"`
HTTPRequestHeaders map[string][]string `json:"headers"`
Addrs []string `json:"addrs"`
}
// ControlResponse is the response from the control service.
type ControlResponse struct {
URLMeasurements []*URLMeasurement `json:"urls"`
}
// Control performs the control request and returns the response.
func Control(
ctx context.Context, sess model.ExperimentSession,
thAddr string, creq CtrlRequest) (out ControlResponse, err error) {
thAddr string, creq CtrlRequest) (out CtrlResponse, err error) {
clnt := httpx.Client{
BaseURL: thAddr,
HTTPClient: sess.DefaultHTTPClient(),
internal/engine/experiment/websteps/model.go
+99 -46
View File
@@ -2,70 +2,104 @@ package websteps
import "net/http"
// RoundTrip describes a specific round trip.
type RoundTrip struct {
// proto is the protocol used, it can be "h2", "http/1.1", "h3", "h3-29"
// Websteps test helper spec messages:
// CtrlRequest is the request sent by the probe to the test helper.
type CtrlRequest struct {
// URL is the mandatory URL to measure.
URL string `json:"url"`
// Headers contains optional headers.
Headers map[string][]string `json:"headers"`
// Addrs contains the optional IP addresses resolved by the
// probe for the domain inside URL.
Addrs []string `json:"addrs"`
}
// CtrlResponse is the response from the test helper.
type CtrlResponse struct {
// URLs contains the URLs we should measure. These URLs
// derive from CtrlRequest.URL.
URLs []*URLMeasurement `json:"urls"`
}
// URLMeasurement contains all the URLs measured by the test helper.
type URLMeasurement struct {
// URL is the URL to which this measurement refers.
URL string `json:"url"`
// DNS contains the domain names resolved by the test helper.
DNS *DNSMeasurement `json:"dns"`
// Endpoints contains endpoint measurements.
Endpoints []*EndpointMeasurement `json:"endpoints"`
// RoundTrip is the related round trip. This field MUST NOT be
// exported as JSON, since it's only used internally by the test
// helper and it's completely ignored by the probe.
RoundTrip *RoundTripInfo `json:"-"`
}
// RoundTripInfo contains info on a specific round trip. This data
// structure is not part of the test helper protocol. We use it
// _inside_ the test helper to describe the discovery phase where
// we gather all the URLs that can derive from a given URL.
type RoundTripInfo struct {
// Proto is the protocol used, it can be "h2", "http/1.1", "h3", "h3-29".
Proto string
// Request is the original HTTP request. The headers
// also include cookies.
// Request is the original HTTP request. Headers also include cookies.
Request *http.Request
// Response is the HTTP response.
Response *http.Response
// sortIndex is an internal field using for sorting.
// SortIndex is the index using for sorting round trips.
SortIndex int
}
// URLMeasurement is a measurement of a given URL that
// includes connectivity measurement for each endpoint
// implied by the given URL.
type URLMeasurement struct {
// URL is the URL we're using
URL string `json:"url"`
// DNS contains the domain names resolved by the helper.
DNS *DNSMeasurement `json:"dns"`
// RoundTrip is the related round trip.
RoundTrip *RoundTrip `json:"-"`
// Endpoints contains endpoint measurements.
Endpoints []*EndpointMeasurement `json:"endpoints"`
}
// DNSMeasurement is a DNS measurement.
type DNSMeasurement struct {
// Domain is the domain we wanted to resolve.
Domain string `json:"domain"`
// Addrs contains the resolved addresses.
Addrs []string `json:"addrs"`
// Failure is the error that occurred.
Failure *string `json:"failure"`
// Addrs contains the resolved addresses.
Addrs []string `json:"addrs"`
}
// HTTPSEndpointMeasurement is the measurement of requesting a specific endpoint via HTTPS.
// EndpointMeasurement is an HTTP measurement where we are using
// a specific TCP/TLS/QUIC endpoint to get the URL.
//
// The specification describes this data structure as the sum of
// three distinct types: HTTPEndpointMeasurement for "http",
// HTTPSEndpointMeasurement for "https", and H3EndpointMeasurement
// for "h3". We don't have sum types here, therefore we use the
// Protocol field to indicate which fields are meaningful.
type EndpointMeasurement struct {
// Endpoint is the endpoint we're measuring.
Endpoint string `json:"endpoint"`
// Protocol is the used protocol. It can be "http", "https", "h3", "h3-29" or other supported QUIC protocols
// Protocol is one of "http", "https", "h3", and "h3-29".
Protocol string `json:"protocol"`
// TCPConnectMeasurement is the related TCP connect measurement, if applicable (nil for h3 requests)
TCPConnectMeasurement *TCPConnectMeasurement `json:"tcp_connect"`
// TCPConnect is the TCP connect measurement. This field
// is only meaningful when protocol is "http" or "https."
TCPConnect *TCPConnectMeasurement `json:"tcp_connect"`
// QUICHandshakeMeasurement is the related QUIC(TLS 1.3) handshake measurement, if applicable (nil for http, https requests)
QUICHandshakeMeasurement *TLSHandshakeMeasurement `json:"quic_handshake"`
// QUICHandshake is the QUIC handshake measurement. This field
// is only meaningful when the protocol is one of "h3" and "h3-29".
QUICHandshake *QUICHandshakeMeasurement `json:"quic_handshake"`
// TLSHandshakeMeasurement is the related TLS handshake measurement, if applicable (nil for http, h3 requests)
TLSHandshakeMeasurement *TLSHandshakeMeasurement `json:"tls_handshake"`
// TLSHandshake is the TLS handshake measurement. This field
// is only meaningful when the protocol is "https".
TLSHandshake *TLSHandshakeMeasurement `json:"tls_handshake"`
// HTTPRoundTripMeasurement is the related HTTP GET measurement.
HTTPRoundTripMeasurement *HTTPRoundTripMeasurement `json:"http_round_trip"`
// HTTPRoundTrip is the related HTTP GET measurement.
HTTPRoundTrip *HTTPRoundTripMeasurement `json:"http_round_trip"`
}
// TCPConnectMeasurement is a TCP connect measurement.
@@ -80,23 +114,42 @@ type TLSHandshakeMeasurement struct {
Failure *string `json:"failure"`
}
// HTTPRoundTripMeasurement contains a measured HTTP request and the corresponding response.
// QUICHandshakeMeasurement is a QUIC handshake measurement.
type QUICHandshakeMeasurement = TLSHandshakeMeasurement
// HTTPRoundTripMeasurement contains a measured HTTP request and
// the corresponding response.
type HTTPRoundTripMeasurement struct {
Request *HTTPRequestMeasurement `json:"request"`
// Request contains request data.
Request *HTTPRequestMeasurement `json:"request"`
// Response contains response data.
Response *HTTPResponseMeasurement `json:"response"`
}
// HTTPRequestMeasurement contains the headers of the measured HTTP Get request.
// HTTPRequestMeasurement contains request data.
type HTTPRequestMeasurement struct {
// Method is the request method.
Method string `json:"method"`
// URL is the request URL.
URL string `json:"url"`
// Headers contains request headers.
Headers http.Header `json:"headers"`
Method string `json:"method"`
URL string `json:"url"`
}
// HTTPResponseMeasurement contains the response of the measured HTTP Get request.
// HTTPResponseMeasurement contains response data.
type HTTPResponseMeasurement struct {
BodyLength int64 `json:"body_length"`
Failure *string `json:"failure"`
Headers http.Header `json:"headers"`
StatusCode int64 `json:"status_code"`
// BodyLength contains the body length in bytes.
BodyLength int64 `json:"body_length"`
// Failure is the error that occurred.
Failure *string `json:"failure"`
// Headers contains response headers.
Headers http.Header `json:"headers"`
// StatusCode is the response status code.
StatusCode int64 `json:"status_code"`
}
internal/engine/experiment/websteps/websteps.go
+18 -18
View File
@@ -113,19 +113,19 @@ func (m Measurer) Run(
}
// 4. Query the testhelper
resp, err := Control(ctx, sess, testhelper.Address, CtrlRequest{
HTTPRequest: URL.String(),
HTTPRequestHeaders: map[string][]string{
URL: URL.String(),
Headers: map[string][]string{
"Accept": {httpheader.Accept()},
"Accept-Language": {httpheader.AcceptLanguage()},
"User-Agent": {httpheader.UserAgent()},
},
Addrs: endpoints,
})
if err != nil || resp.URLMeasurements == nil {
if err != nil || resp.URLs == nil {
return errors.New("no control response")
}
// 5. Go over the Control URL measurements and reproduce them without following redirects, one by one.
for _, controlURLMeasurement := range resp.URLMeasurements {
for _, controlURLMeasurement := range resp.URLs {
urlMeasurement := &URLMeasurement{
URL: controlURLMeasurement.URL,
Endpoints: []*EndpointMeasurement{},
@@ -145,7 +145,7 @@ func (m Measurer) Run(
}
// the testhelper tells us which endpoints to measure
for _, controlEndpoint := range controlURLMeasurement.Endpoints {
rt := controlEndpoint.HTTPRoundTripMeasurement
rt := controlEndpoint.HTTPRoundTrip
if rt == nil || rt.Request == nil {
continue
}
@@ -176,7 +176,7 @@ func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoi
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
@@ -186,7 +186,7 @@ func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoi
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
@@ -197,13 +197,13 @@ func (m *Measurer) measureEndpointHTTP(ctx context.Context, URL *url.URL, endpoi
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
@@ -219,7 +219,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
}
// TCP connect step
conn, err := TCPDo(ctx, TCPConfig{Endpoint: endpoint})
endpointMeasurement.TCPConnectMeasurement = &TCPConnectMeasurement{
endpointMeasurement.TCPConnect = &TCPConnectMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
@@ -229,7 +229,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
// TLS handshake step
tlsconn, err := TLSDo(conn, URL.Hostname())
endpointMeasurement.TLSHandshakeMeasurement = &TLSHandshakeMeasurement{
endpointMeasurement.TLSHandshake = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
@@ -239,7 +239,7 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
@@ -250,13 +250,13 @@ func (m *Measurer) measureEndpointHTTPS(ctx context.Context, URL *url.URL, endpo
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,
@@ -279,7 +279,7 @@ func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint
Endpoint: endpoint,
TLSConf: tlsConf,
})
endpointMeasurement.QUICHandshakeMeasurement = &TLSHandshakeMeasurement{
endpointMeasurement.QUICHandshake = &TLSHandshakeMeasurement{
Failure: archival.NewFailure(err),
}
if err != nil {
@@ -287,7 +287,7 @@ func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint
}
// HTTP roundtrip step
request := NewRequest(ctx, URL, headers)
endpointMeasurement.HTTPRoundTripMeasurement = &HTTPRoundTripMeasurement{
endpointMeasurement.HTTPRoundTrip = &HTTPRoundTripMeasurement{
Request: &HTTPRequestMeasurement{
Headers: request.Header,
Method: "GET",
@@ -298,13 +298,13 @@ func (m *Measurer) measureEndpointH3(ctx context.Context, URL *url.URL, endpoint
resp, body, err := HTTPDo(request, transport)
if err != nil {
// failed Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
Failure: archival.NewFailure(err),
}
return endpointMeasurement
}
// successful Response
endpointMeasurement.HTTPRoundTripMeasurement.Response = &HTTPResponseMeasurement{
endpointMeasurement.HTTPRoundTrip.Response = &HTTPResponseMeasurement{
BodyLength: int64(len(body)),
Failure: nil,
Headers: resp.Header,