2022-05-31 21:53:01 +02:00
|
|
|
package tracex
|
2021-02-02 12:05:47 +01:00
|
|
|
|
2022-06-01 07:44:54 +02:00
|
|
|
//
|
|
|
|
// HTTP
|
|
|
|
//
|
|
|
|
|
2021-02-02 12:05:47 +01:00
|
|
|
import (
|
|
|
|
"bytes"
|
2021-06-15 11:57:40 +02:00
|
|
|
"context"
|
2021-02-02 12:05:47 +01:00
|
|
|
"io"
|
|
|
|
"net/http"
|
|
|
|
"time"
|
|
|
|
|
2022-01-07 18:33:37 +01:00
|
|
|
"github.com/ooni/probe-cli/v3/internal/model"
|
2021-09-28 12:42:01 +02:00
|
|
|
"github.com/ooni/probe-cli/v3/internal/netxlite"
|
2021-02-02 12:05:47 +01:00
|
|
|
)
|
|
|
|
|
|
|
|
// SaverMetadataHTTPTransport is a RoundTripper that saves
|
|
|
|
// events related to HTTP request and response metadata
|
|
|
|
type SaverMetadataHTTPTransport struct {
|
2022-01-07 18:33:37 +01:00
|
|
|
model.HTTPTransport
|
2022-05-31 21:53:01 +02:00
|
|
|
Saver *Saver
|
2021-02-02 12:05:47 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// RoundTrip implements RoundTripper.RoundTrip
|
2022-06-01 15:20:28 +02:00
|
|
|
func (txp *SaverMetadataHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPRequestMetadata{&EventValue{
|
2022-06-01 15:20:28 +02:00
|
|
|
HTTPRequestHeaders: httpCloneRequestHeaders(req),
|
|
|
|
HTTPMethod: req.Method,
|
|
|
|
HTTPURL: req.URL.String(),
|
|
|
|
Transport: txp.HTTPTransport.Network(),
|
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2022-01-07 18:33:37 +01:00
|
|
|
resp, err := txp.HTTPTransport.RoundTrip(req)
|
2021-02-02 12:05:47 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPResponseMetadata{&EventValue{
|
2022-06-01 15:20:28 +02:00
|
|
|
HTTPResponseHeaders: resp.Header,
|
|
|
|
HTTPStatusCode: resp.StatusCode,
|
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2021-02-02 12:05:47 +01:00
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
|
2022-06-01 15:20:28 +02:00
|
|
|
// httpCloneRequestHeaders returns a clone of the headers where we have
|
fix(netxlite): do not mutate outgoing requests (#508)
I have recently seen a data race related our way of
mutating the outgoing request to set the host header.
Unfortunately, I've lost track of the race output,
because I rebooted my Linux box before saving it.
Though, after inspecting why and and where we're mutating
outgoing requets, I've found that:
1. we add the host header when logging to have it logged,
which is not a big deal since we already emit the URL
rather than just the URL path when logging a request, and
so we can safely zap this piece of code;
2. as a result, in measurements we may omit the host header
but again this is pretty much obvious from the URL itself
and so it should not be very important (nonetheless,
avoid surprises and keep the existing behavior);
3. when the User-Agent header is not set, we default to
a `miniooni/0.1.0-dev` user agent, which is probably not
very useful anyway, so we can actually remove it.
Part of https://github.com/ooni/probe/issues/1733 (this diff
has been extracted from https://github.com/ooni/probe-cli/pull/506).
2021-09-27 13:35:47 +02:00
|
|
|
// also set the host header, which normally is not set by
|
|
|
|
// golang until it serializes the request itself.
|
2022-06-01 15:20:28 +02:00
|
|
|
func httpCloneRequestHeaders(req *http.Request) http.Header {
|
fix(netxlite): do not mutate outgoing requests (#508)
I have recently seen a data race related our way of
mutating the outgoing request to set the host header.
Unfortunately, I've lost track of the race output,
because I rebooted my Linux box before saving it.
Though, after inspecting why and and where we're mutating
outgoing requets, I've found that:
1. we add the host header when logging to have it logged,
which is not a big deal since we already emit the URL
rather than just the URL path when logging a request, and
so we can safely zap this piece of code;
2. as a result, in measurements we may omit the host header
but again this is pretty much obvious from the URL itself
and so it should not be very important (nonetheless,
avoid surprises and keep the existing behavior);
3. when the User-Agent header is not set, we default to
a `miniooni/0.1.0-dev` user agent, which is probably not
very useful anyway, so we can actually remove it.
Part of https://github.com/ooni/probe/issues/1733 (this diff
has been extracted from https://github.com/ooni/probe-cli/pull/506).
2021-09-27 13:35:47 +02:00
|
|
|
header := req.Header.Clone()
|
|
|
|
if req.Host != "" {
|
|
|
|
header.Set("Host", req.Host)
|
|
|
|
} else {
|
|
|
|
header.Set("Host", req.URL.Host)
|
|
|
|
}
|
|
|
|
return header
|
|
|
|
}
|
|
|
|
|
2021-02-02 12:05:47 +01:00
|
|
|
// SaverTransactionHTTPTransport is a RoundTripper that saves
|
|
|
|
// events related to the HTTP transaction
|
|
|
|
type SaverTransactionHTTPTransport struct {
|
2022-01-07 18:33:37 +01:00
|
|
|
model.HTTPTransport
|
2022-05-31 21:53:01 +02:00
|
|
|
Saver *Saver
|
2021-02-02 12:05:47 +01:00
|
|
|
}
|
|
|
|
|
|
|
|
// RoundTrip implements RoundTripper.RoundTrip
|
2022-06-01 15:20:28 +02:00
|
|
|
func (txp *SaverTransactionHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPTransactionStart{&EventValue{
|
2021-02-02 12:05:47 +01:00
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2022-01-07 18:33:37 +01:00
|
|
|
resp, err := txp.HTTPTransport.RoundTrip(req)
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPTransactionDone{&EventValue{
|
2021-02-02 12:05:47 +01:00
|
|
|
Err: err,
|
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2021-02-02 12:05:47 +01:00
|
|
|
return resp, err
|
|
|
|
}
|
|
|
|
|
|
|
|
// SaverBodyHTTPTransport is a RoundTripper that saves
|
|
|
|
// body events occurring during the round trip
|
|
|
|
type SaverBodyHTTPTransport struct {
|
2022-01-07 18:33:37 +01:00
|
|
|
model.HTTPTransport
|
2022-05-31 21:53:01 +02:00
|
|
|
Saver *Saver
|
2021-02-02 12:05:47 +01:00
|
|
|
SnapshotSize int
|
|
|
|
}
|
|
|
|
|
|
|
|
// RoundTrip implements RoundTripper.RoundTrip
|
2022-06-01 15:20:28 +02:00
|
|
|
func (txp *SaverBodyHTTPTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
2021-02-02 12:05:47 +01:00
|
|
|
const defaultSnapSize = 1 << 17
|
|
|
|
snapsize := defaultSnapSize
|
|
|
|
if txp.SnapshotSize != 0 {
|
|
|
|
snapsize = txp.SnapshotSize
|
|
|
|
}
|
|
|
|
if req.Body != nil {
|
2022-06-01 07:44:54 +02:00
|
|
|
data, err := httpSaverSnapRead(req.Context(), req.Body, snapsize)
|
2021-02-02 12:05:47 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-06-01 07:44:54 +02:00
|
|
|
req.Body = httpSaverCompose(data, req.Body)
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPRequestBodySnapshot{&EventValue{
|
2021-02-02 12:05:47 +01:00
|
|
|
DataIsTruncated: len(data) >= snapsize,
|
|
|
|
Data: data,
|
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2021-02-02 12:05:47 +01:00
|
|
|
}
|
2022-01-07 18:33:37 +01:00
|
|
|
resp, err := txp.HTTPTransport.RoundTrip(req)
|
2021-02-02 12:05:47 +01:00
|
|
|
if err != nil {
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-06-01 07:44:54 +02:00
|
|
|
data, err := httpSaverSnapRead(req.Context(), resp.Body, snapsize)
|
2021-02-02 12:05:47 +01:00
|
|
|
if err != nil {
|
|
|
|
resp.Body.Close()
|
|
|
|
return nil, err
|
|
|
|
}
|
2022-06-01 07:44:54 +02:00
|
|
|
resp.Body = httpSaverCompose(data, resp.Body)
|
2022-06-01 14:32:16 +02:00
|
|
|
txp.Saver.Write(&EventHTTPResponseBodySnapshot{&EventValue{
|
2021-02-02 12:05:47 +01:00
|
|
|
DataIsTruncated: len(data) >= snapsize,
|
|
|
|
Data: data,
|
|
|
|
Time: time.Now(),
|
2022-06-01 14:32:16 +02:00
|
|
|
}})
|
2021-02-02 12:05:47 +01:00
|
|
|
return resp, nil
|
|
|
|
}
|
|
|
|
|
2022-06-01 07:44:54 +02:00
|
|
|
func httpSaverSnapRead(ctx context.Context, r io.ReadCloser, snapsize int) ([]byte, error) {
|
2021-09-28 12:42:01 +02:00
|
|
|
return netxlite.ReadAllContext(ctx, io.LimitReader(r, int64(snapsize)))
|
2021-02-02 12:05:47 +01:00
|
|
|
}
|
|
|
|
|
2022-06-01 07:44:54 +02:00
|
|
|
func httpSaverCompose(data []byte, r io.ReadCloser) io.ReadCloser {
|
|
|
|
return httpSaverReadCloser{Closer: r, Reader: io.MultiReader(bytes.NewReader(data), r)}
|
2021-02-02 12:05:47 +01:00
|
|
|
}
|
|
|
|
|
2022-06-01 07:44:54 +02:00
|
|
|
type httpSaverReadCloser struct {
|
2021-02-02 12:05:47 +01:00
|
|
|
io.Closer
|
|
|
|
io.Reader
|
|
|
|
}
|
|
|
|
|
2022-06-01 15:20:28 +02:00
|
|
|
var _ model.HTTPTransport = &SaverMetadataHTTPTransport{}
|
|
|
|
var _ model.HTTPTransport = &SaverBodyHTTPTransport{}
|
|
|
|
var _ model.HTTPTransport = &SaverTransactionHTTPTransport{}
|