ooni-probe-cli/internal/measurexlite/http.go

package measurexlite

//
// Support for generating HTTP traces
//

import (
	"net/http"
	"sort"
	"time"

	"github.com/ooni/probe-cli/v3/internal/model"
	"github.com/ooni/probe-cli/v3/internal/tracex"
)

// NewArchivalHTTPRequestResult creates a new model.ArchivalHTTPRequestResult.
//
// Arguments:
//
// - index is the index of the trace;
//
// - started is when we started sending the request;
//
// - network is the underlying network in use ("tcp" or "udp");
//
// - address is the remote endpoint's address;
//
// - alpn is the negotiated ALPN or an empty string when not applicable;
//
// - transport is the HTTP transport's protocol we're using ("udp" or "tcp"): this field
// was introduced a long time ago to support QUIC measurements and we keep it for backwards
// compatibility but network, address, and alpn are much more informative;
//
// - req is the certainly-non-nil HTTP request;
//
// - resp is the possibly-nil HTTP response;
//
// - maxRespBodySize is the maximum body snapshot size;
//
// - body is the possibly-nil HTTP response body;
//
// - err is the possibly-nil error that occurred during the transaction;
//
// - finished is when we finished reading the response's body.
func NewArchivalHTTPRequestResult(index int64, started time.Duration, network, address, alpn string,
	transport string, req *http.Request, resp *http.Response, maxRespBodySize int64, body []byte, err error,
	finished time.Duration) *model.ArchivalHTTPRequestResult {
	return &model.ArchivalHTTPRequestResult{
		Network: network,
		Address: address,
		ALPN:    alpn,
		Failure: tracex.NewFailure(err),
		Request: model.ArchivalHTTPRequest{
			Body:            model.ArchivalMaybeBinaryData{},
			BodyIsTruncated: false,
			HeadersList:     newHTTPRequestHeaderList(req),
			Headers:         newHTTPRequestHeaderMap(req),
			Method:          httpRequestMethod(req),
			Tor:             model.ArchivalHTTPTor{},
			Transport:       transport, // kept for backward compat
			URL:             httpRequestURL(req),
		},
		Response: model.ArchivalHTTPResponse{
			Body:            httpResponseBody(body),
			BodyIsTruncated: httpResponseBodyIsTruncated(body, maxRespBodySize),
			Code:            httpResponseStatusCode(resp),
			HeadersList:     newHTTPResponseHeaderList(resp),
			Headers:         newHTTPResponseHeaderMap(resp),
			Locations:       httpResponseLocations(resp),
		},
		T0:            started.Seconds(),
		T:             finished.Seconds(),
		TransactionID: index,
	}
}

// httpRequestMethod returns the HTTP request method or an empty string
func httpRequestMethod(req *http.Request) (out string) {
	if req != nil {
		out = req.Method
	}
	return
}

// newHTTPRequestHeaderList calls newHTTPHeaderList with the request headers or
// return an empty array in case the request is nil.
func newHTTPRequestHeaderList(req *http.Request) []model.ArchivalHTTPHeader {
	m := http.Header{}
	if req != nil {
		m = req.Header
	}
	return newHTTPHeaderList(m)
}

// newHTTPRequestHeaderMap calls newHTTPHeaderMap with the request headers or
// return an empty map in case the request is nil.
func newHTTPRequestHeaderMap(req *http.Request) map[string]model.ArchivalMaybeBinaryData {
	m := http.Header{}
	if req != nil {
		m = req.Header
	}
	return newHTTPHeaderMap(m)
}

// httpRequestURL returns the req.URL.String() or an empty string.
func httpRequestURL(req *http.Request) (out string) {
	if req != nil && req.URL != nil {
		out = req.URL.String()
	}
	return
}

// httpResponseBody returns the response body, if possible, or an empty body.
func httpResponseBody(body []byte) (out model.ArchivalMaybeBinaryData) {
	if body != nil {
		out.Value = string(body)
	}
	return
}

// httpResponseBodyIsTruncated determines whether the body is truncated (if possible)
func httpResponseBodyIsTruncated(body []byte, maxSnapSize int64) (out bool) {
	if len(body) > 0 && maxSnapSize > 0 {
		out = int64(len(body)) >= maxSnapSize
	}
	return
}

// httpResponseStatusCode returns the status code, if possible
func httpResponseStatusCode(resp *http.Response) (code int64) {
	if resp != nil {
		code = int64(resp.StatusCode)
	}
	return
}

// newHTTPResponseHeaderList calls newHTTPHeaderList with the request headers or
// return an empty array in case the request is nil.
func newHTTPResponseHeaderList(resp *http.Response) (out []model.ArchivalHTTPHeader) {
	m := http.Header{}
	if resp != nil {
		m = resp.Header
	}
	return newHTTPHeaderList(m)
}

// newHTTPResponseHeaderMap calls newHTTPHeaderMap with the request headers or
// return an empty map in case the request is nil.
func newHTTPResponseHeaderMap(resp *http.Response) (out map[string]model.ArchivalMaybeBinaryData) {
	m := http.Header{}
	if resp != nil {
		m = resp.Header
	}
	return newHTTPHeaderMap(m)
}

// httpResponseLocations returns the locations inside the response (if possible)
func httpResponseLocations(resp *http.Response) []string {
	if resp == nil {
		return []string{}
	}
	loc, err := resp.Location()
	if err != nil {
		return []string{}
	}
	return []string{loc.String()}
}

// newHTTPHeaderList creates a list representation of HTTP headers
func newHTTPHeaderList(header http.Header) (out []model.ArchivalHTTPHeader) {
	out = []model.ArchivalHTTPHeader{}
	keys := []string{}
	for key := range header {
		keys = append(keys, key)
	}
	// ensure the output is consistent, which helps with testing
	// for an example of why we need to sort headers, see
	// https://github.com/ooni/probe-engine/pull/751/checks?check_run_id=853562310
	sort.Strings(keys)
	for _, key := range keys {
		for _, value := range header[key] {
			out = append(out, model.ArchivalHTTPHeader{
				Key: key,
				Value: model.ArchivalMaybeBinaryData{
					Value: value,
				},
			})
		}
	}
	return
}

// newHTTPHeaderMap creates a map representation of HTTP headers
func newHTTPHeaderMap(header http.Header) (out map[string]model.ArchivalMaybeBinaryData) {
	out = make(map[string]model.ArchivalMaybeBinaryData)
	for key, values := range header {
		for _, value := range values {
			out[key] = model.ArchivalMaybeBinaryData{
				Value: value,
			}
			break
		}
	}
	return
}