chore: merge probe-engine into probe-cli (#201)
This is how I did it: 1. `git clone https://github.com/ooni/probe-engine internal/engine` 2. ``` (cd internal/engine && git describe --tags) v0.23.0 ``` 3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod` 4. `rm -rf internal/.git internal/engine/go.{mod,sum}` 5. `git add internal/engine` 6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;` 7. `go build ./...` (passes) 8. `go test -race ./...` (temporary failure on RiseupVPN) 9. `go mod tidy` 10. this commit message Once this piece of work is done, we can build a new version of `ooniprobe` that is using `internal/engine` directly. We need to do more work to ensure all the other functionality in `probe-engine` (e.g. making mobile packages) are still WAI. Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
GitHub
@@ -0,0 +1,8 @@
|
||||
package ndt7
|
||||
|
||||
import "time"
|
||||
|
||||
type (
|
||||
callbackJSON func(data []byte) error
|
||||
callbackPerformance func(elapsed time.Duration, count int64)
|
||||
)
|
||||
@@ -0,0 +1,10 @@
|
||||
package ndt7
|
||||
|
||||
import "time"
|
||||
|
||||
func defaultCallbackJSON(data []byte) error {
|
||||
return nil
|
||||
}
|
||||
|
||||
func defaultCallbackPerformance(elapsed time.Duration, count int64) {
|
||||
}
|
||||
@@ -0,0 +1,91 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"crypto/tls"
|
||||
"net/http"
|
||||
"net/url"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/dialer"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/resolver"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/selfcensor"
|
||||
)
|
||||
|
||||
type dialManager struct {
|
||||
ndt7URL string
|
||||
logger model.Logger
|
||||
proxyURL *url.URL
|
||||
readBufferSize int
|
||||
tlsConfig *tls.Config
|
||||
userAgent string
|
||||
writeBufferSize int
|
||||
}
|
||||
|
||||
func newDialManager(ndt7URL string, logger model.Logger, userAgent string) dialManager {
|
||||
return dialManager{
|
||||
ndt7URL: ndt7URL,
|
||||
logger: logger,
|
||||
readBufferSize: paramMaxBufferSize,
|
||||
userAgent: userAgent,
|
||||
writeBufferSize: paramMaxBufferSize,
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr dialManager) dialWithTestName(ctx context.Context, testName string) (*websocket.Conn, error) {
|
||||
var reso resolver.Resolver = resolver.SystemResolver{}
|
||||
reso = resolver.LoggingResolver{Resolver: reso, Logger: mgr.logger}
|
||||
var dlr dialer.Dialer = selfcensor.SystemDialer{}
|
||||
dlr = dialer.TimeoutDialer{Dialer: dlr}
|
||||
dlr = dialer.ErrorWrapperDialer{Dialer: dlr}
|
||||
dlr = dialer.LoggingDialer{Dialer: dlr, Logger: mgr.logger}
|
||||
dlr = dialer.DNSDialer{Dialer: dlr, Resolver: reso}
|
||||
dlr = dialer.ProxyDialer{Dialer: dlr, ProxyURL: mgr.proxyURL}
|
||||
dlr = dialer.ByteCounterDialer{Dialer: dlr}
|
||||
dlr = dialer.ShapingDialer{Dialer: dlr}
|
||||
dialer := websocket.Dialer{
|
||||
NetDialContext: dlr.DialContext,
|
||||
ReadBufferSize: mgr.readBufferSize,
|
||||
TLSClientConfig: mgr.tlsConfig,
|
||||
WriteBufferSize: mgr.writeBufferSize,
|
||||
}
|
||||
headers := http.Header{}
|
||||
headers.Add("Sec-WebSocket-Protocol", "net.measurementlab.ndt.v7")
|
||||
headers.Add("User-Agent", mgr.userAgent)
|
||||
mgr.logrequest(mgr.ndt7URL, headers)
|
||||
conn, _, err := dialer.DialContext(ctx, mgr.ndt7URL, headers)
|
||||
mgr.logresponse(err)
|
||||
return conn, err
|
||||
}
|
||||
|
||||
func (mgr dialManager) logrequest(url string, headers http.Header) {
|
||||
mgr.logger.Debugf("> GET %s", url)
|
||||
for key, values := range headers {
|
||||
for _, v := range values {
|
||||
mgr.logger.Debugf("> %s: %s", key, v)
|
||||
}
|
||||
}
|
||||
mgr.logger.Debug("> Connection: upgrade")
|
||||
mgr.logger.Debug("> Upgrade: websocket")
|
||||
mgr.logger.Debug(">")
|
||||
}
|
||||
|
||||
func (mgr dialManager) logresponse(err error) {
|
||||
if err != nil {
|
||||
mgr.logger.Debugf("< %+v", err)
|
||||
return
|
||||
}
|
||||
mgr.logger.Debug("< 101")
|
||||
mgr.logger.Debug("< Connection: upgrade")
|
||||
mgr.logger.Debug("< Upgrade: websocket")
|
||||
mgr.logger.Debug("<")
|
||||
}
|
||||
|
||||
func (mgr dialManager) dialDownload(ctx context.Context) (*websocket.Conn, error) {
|
||||
return mgr.dialWithTestName(ctx, "download")
|
||||
}
|
||||
|
||||
func (mgr dialManager) dialUpload(ctx context.Context) (*websocket.Conn, error) {
|
||||
return mgr.dialWithTestName(ctx, "upload")
|
||||
}
|
||||
@@ -0,0 +1,70 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"net/http/httptest"
|
||||
"net/url"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestDialDownloadWithCancelledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately halt
|
||||
mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev")
|
||||
conn, err := mgr.dialDownload(ctx)
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Fatal("expected nil conn here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialUploadWithCancelledContext(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately halt
|
||||
mgr := newDialManager("wss://hostname.fake", log.Log, "miniooni/0.1.0-dev")
|
||||
conn, err := mgr.dialUpload(ctx)
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Fatal("expected nil conn here")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDialIncludesUserAgent(t *testing.T) {
|
||||
do := func(testName string) {
|
||||
var userAgent string
|
||||
handler := http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
|
||||
userAgent = r.UserAgent()
|
||||
w.WriteHeader(500)
|
||||
})
|
||||
server := httptest.NewServer(handler)
|
||||
defer server.Close()
|
||||
url, err := url.Parse(server.URL)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
url.Scheme = "ws"
|
||||
mgr := newDialManager(url.String(), log.Log, "miniooni/0.1.0-dev")
|
||||
conn, err := mgr.dialWithTestName(context.Background(), testName)
|
||||
if !errors.Is(err, websocket.ErrBadHandshake) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if conn != nil {
|
||||
t.Fatal("expected nil conn here")
|
||||
}
|
||||
if userAgent != "miniooni/0.1.0-dev" {
|
||||
t.Fatal("User-Agent not sent")
|
||||
}
|
||||
}
|
||||
do("download")
|
||||
do("upload")
|
||||
}
|
||||
@@ -0,0 +1,73 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"io"
|
||||
"io/ioutil"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type downloadManager struct {
|
||||
conn mockableConn
|
||||
maxMessageSize int64
|
||||
maxRuntime time.Duration
|
||||
measureInterval time.Duration
|
||||
onJSON callbackJSON
|
||||
onPerformance callbackPerformance
|
||||
}
|
||||
|
||||
func newDownloadManager(
|
||||
conn mockableConn, onPerformance callbackPerformance,
|
||||
onJSON callbackJSON,
|
||||
) downloadManager {
|
||||
return downloadManager{
|
||||
conn: conn,
|
||||
maxMessageSize: paramMaxMessageSize,
|
||||
maxRuntime: paramMaxRuntime,
|
||||
measureInterval: paramMeasureInterval,
|
||||
onJSON: onJSON,
|
||||
onPerformance: onPerformance,
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr downloadManager) run(ctx context.Context) error {
|
||||
var total int64
|
||||
start := time.Now()
|
||||
if err := mgr.conn.SetReadDeadline(start.Add(mgr.maxRuntime)); err != nil {
|
||||
return err
|
||||
}
|
||||
mgr.conn.SetReadLimit(mgr.maxMessageSize)
|
||||
ticker := time.NewTicker(mgr.measureInterval)
|
||||
defer ticker.Stop()
|
||||
for ctx.Err() == nil {
|
||||
kind, reader, err := mgr.conn.NextReader()
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
if kind == websocket.TextMessage {
|
||||
data, err := ioutil.ReadAll(reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += int64(len(data))
|
||||
if err := mgr.onJSON(data); err != nil {
|
||||
return err
|
||||
}
|
||||
continue
|
||||
}
|
||||
n, err := io.Copy(ioutil.Discard, reader)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
total += int64(n)
|
||||
select {
|
||||
case now := <-ticker.C:
|
||||
mgr.onPerformance(now.Sub(start), total)
|
||||
default:
|
||||
// NOTHING
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,145 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"io"
|
||||
"strings"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestDownloadSetReadDeadlineFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
ReadDeadlineErr: expected,
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
defaultCallbackJSON,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadNextReaderFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
NextReaderErr: expected,
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
defaultCallbackJSON,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadTextMessageReadAllFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
NextReaderMsgType: websocket.TextMessage,
|
||||
NextReaderReader: func() io.Reader {
|
||||
return &alwaysFailingReader{
|
||||
Err: expected,
|
||||
}
|
||||
},
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
defaultCallbackJSON,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
type alwaysFailingReader struct {
|
||||
Err error
|
||||
}
|
||||
|
||||
func (r *alwaysFailingReader) Read(p []byte) (int, error) {
|
||||
return 0, r.Err
|
||||
}
|
||||
|
||||
func TestDownloadBinaryMessageReadAllFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
NextReaderMsgType: websocket.BinaryMessage,
|
||||
NextReaderReader: func() io.Reader {
|
||||
return &alwaysFailingReader{
|
||||
Err: expected,
|
||||
}
|
||||
},
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
defaultCallbackJSON,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadOnJSONCallbackError(t *testing.T) {
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
NextReaderMsgType: websocket.TextMessage,
|
||||
NextReaderReader: func() io.Reader {
|
||||
return &invalidJSONReader{}
|
||||
},
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
func(data []byte) error {
|
||||
var v interface{}
|
||||
return json.Unmarshal(data, &v)
|
||||
},
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "unexpected end of JSON input") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
type invalidJSONReader struct{}
|
||||
|
||||
func (r *invalidJSONReader) Read(p []byte) (int, error) {
|
||||
return copy(p, []byte(`{`)), io.EOF
|
||||
}
|
||||
|
||||
func TestDownloadOnJSONLoop(t *testing.T) {
|
||||
mgr := newDownloadManager(
|
||||
&mockableConnMock{
|
||||
NextReaderMsgType: websocket.TextMessage,
|
||||
NextReaderReader: func() io.Reader {
|
||||
return &goodJSONReader{}
|
||||
},
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
func(data []byte) error {
|
||||
var v interface{}
|
||||
return json.Unmarshal(data, &v)
|
||||
},
|
||||
)
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
err := mgr.run(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
type goodJSONReader struct{}
|
||||
|
||||
func (r *goodJSONReader) Read(p []byte) (int, error) {
|
||||
return copy(p, []byte(`{}`)), io.EOF
|
||||
}
|
||||
@@ -0,0 +1,16 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type mockableConn interface {
|
||||
NextReader() (int, io.Reader, error)
|
||||
SetReadDeadline(time.Time) error
|
||||
SetReadLimit(int64)
|
||||
SetWriteDeadline(time.Time) error
|
||||
WritePreparedMessage(*websocket.PreparedMessage) error
|
||||
}
|
||||
@@ -0,0 +1,39 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"io"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
type mockableConnMock struct {
|
||||
NextReaderMsgType int
|
||||
NextReaderErr error
|
||||
NextReaderReader func() io.Reader
|
||||
ReadDeadlineErr error
|
||||
WriteDeadlineErr error
|
||||
WritePreparedMessageErr error
|
||||
}
|
||||
|
||||
func (c *mockableConnMock) NextReader() (int, io.Reader, error) {
|
||||
var reader io.Reader
|
||||
if c.NextReaderReader != nil {
|
||||
reader = c.NextReaderReader()
|
||||
}
|
||||
return c.NextReaderMsgType, reader, c.NextReaderErr
|
||||
}
|
||||
|
||||
func (c *mockableConnMock) SetReadDeadline(time.Time) error {
|
||||
return c.ReadDeadlineErr
|
||||
}
|
||||
|
||||
func (c *mockableConnMock) SetReadLimit(int64) {}
|
||||
|
||||
func (c *mockableConnMock) SetWriteDeadline(time.Time) error {
|
||||
return c.WriteDeadlineErr
|
||||
}
|
||||
|
||||
func (c *mockableConnMock) WritePreparedMessage(*websocket.PreparedMessage) error {
|
||||
return c.WritePreparedMessageErr
|
||||
}
|
||||
@@ -0,0 +1,301 @@
|
||||
// Package ndt7 contains the ndt7 network experiment.
|
||||
//
|
||||
// See https://github.com/ooni/spec/blob/master/nettests/ts-022-ndt.md
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"net/http"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/humanizex"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/mlablocatev2"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
)
|
||||
|
||||
const (
|
||||
testName = "ndt"
|
||||
testVersion = "0.8.0"
|
||||
)
|
||||
|
||||
// Config contains the experiment settings
|
||||
type Config struct {
|
||||
noDownload bool
|
||||
noUpload bool
|
||||
}
|
||||
|
||||
// Summary is the measurement summary
|
||||
type Summary struct {
|
||||
AvgRTT float64 `json:"avg_rtt"` // Average RTT [ms]
|
||||
Download float64 `json:"download"` // download speed [kbit/s]
|
||||
MSS int64 `json:"mss"` // MSS
|
||||
MaxRTT float64 `json:"max_rtt"` // Max AvgRTT sample seen [ms]
|
||||
MinRTT float64 `json:"min_rtt"` // Min RTT according to kernel [ms]
|
||||
Ping float64 `json:"ping"` // Equivalent to MinRTT [ms]
|
||||
RetransmitRate float64 `json:"retransmit_rate"` // bytes_retrans/bytes_sent [0..1]
|
||||
Upload float64 `json:"upload"` // upload speed [kbit/s]
|
||||
}
|
||||
|
||||
// ServerInfo contains information on the selected server
|
||||
//
|
||||
// Site is currently an extension to the NDT specification
|
||||
// until the data format of the new mlab locate is clear.
|
||||
type ServerInfo struct {
|
||||
Hostname string `json:"hostname"`
|
||||
Site string `json:"site,omitempty"`
|
||||
}
|
||||
|
||||
// TestKeys contains the test keys
|
||||
type TestKeys struct {
|
||||
// Download contains download results
|
||||
Download []Measurement `json:"download"`
|
||||
|
||||
// Failure is the failure string
|
||||
Failure *string `json:"failure"`
|
||||
|
||||
// Protocol contains the version of the ndt protocol
|
||||
Protocol int64 `json:"protocol"`
|
||||
|
||||
// Server contains information on the selected server
|
||||
Server ServerInfo `json:"server"`
|
||||
|
||||
// Summary contains the measurement summary
|
||||
Summary Summary `json:"summary"`
|
||||
|
||||
// Upload contains upload results
|
||||
Upload []Measurement `json:"upload"`
|
||||
}
|
||||
|
||||
// Measurer performs the measurement.
|
||||
type Measurer struct {
|
||||
config Config
|
||||
jsonUnmarshal func(data []byte, v interface{}) error
|
||||
preDownloadHook func()
|
||||
preUploadHook func()
|
||||
}
|
||||
|
||||
func (m *Measurer) discover(
|
||||
ctx context.Context, sess model.ExperimentSession) (mlablocatev2.NDT7Result, error) {
|
||||
httpClient := &http.Client{
|
||||
Transport: netx.NewHTTPTransport(netx.Config{
|
||||
Logger: sess.Logger(),
|
||||
}),
|
||||
}
|
||||
defer httpClient.CloseIdleConnections()
|
||||
client := mlablocatev2.NewClient(httpClient, sess.Logger(), sess.UserAgent())
|
||||
out, err := client.QueryNDT7(ctx)
|
||||
if err != nil {
|
||||
return mlablocatev2.NDT7Result{}, err
|
||||
}
|
||||
return out[0], nil // same as with locate services v1
|
||||
}
|
||||
|
||||
// ExperimentName implements ExperimentMeasurer.ExperiExperimentName.
|
||||
func (m *Measurer) ExperimentName() string {
|
||||
return testName
|
||||
}
|
||||
|
||||
// ExperimentVersion implements ExperimentMeasurer.ExperimentVersion.
|
||||
func (m *Measurer) ExperimentVersion() string {
|
||||
return testVersion
|
||||
}
|
||||
|
||||
func (m *Measurer) doDownload(
|
||||
ctx context.Context, sess model.ExperimentSession,
|
||||
callbacks model.ExperimentCallbacks, tk *TestKeys,
|
||||
URL string,
|
||||
) error {
|
||||
if m.config.noDownload == true {
|
||||
return nil // useful to make tests faster
|
||||
}
|
||||
conn, err := newDialManager(URL,
|
||||
sess.Logger(), sess.UserAgent()).dialDownload(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer callbacks.OnProgress(0.5, " download: done")
|
||||
defer conn.Close()
|
||||
mgr := newDownloadManager(
|
||||
conn,
|
||||
func(timediff time.Duration, count int64) {
|
||||
elapsed := timediff.Seconds()
|
||||
// The percentage of completion of download goes from 0 to
|
||||
// 50% of the whole experiment, hence the `/2.0`.
|
||||
percentage := elapsed / paramMaxRuntimeUpperBound / 2.0
|
||||
speed := float64(count) * 8.0 / elapsed
|
||||
message := fmt.Sprintf(" download: speed %s", humanizex.SI(
|
||||
float64(speed), "bit/s"))
|
||||
tk.Summary.Download = speed / 1e03 /* bit/s => kbit/s */
|
||||
callbacks.OnProgress(percentage, message)
|
||||
tk.Download = append(tk.Download, Measurement{
|
||||
AppInfo: &AppInfo{
|
||||
ElapsedTime: int64(timediff / time.Microsecond),
|
||||
NumBytes: count,
|
||||
},
|
||||
Origin: "client",
|
||||
Test: "download",
|
||||
})
|
||||
},
|
||||
func(data []byte) error {
|
||||
sess.Logger().Debugf("%s", string(data))
|
||||
var measurement Measurement
|
||||
if err := m.jsonUnmarshal(data, &measurement); err != nil {
|
||||
return err
|
||||
}
|
||||
if measurement.TCPInfo != nil {
|
||||
rtt := float64(measurement.TCPInfo.RTT) / 1e03 /* us => ms */
|
||||
tk.Summary.AvgRTT = rtt
|
||||
tk.Summary.MSS = int64(measurement.TCPInfo.AdvMSS)
|
||||
if tk.Summary.MaxRTT < rtt {
|
||||
tk.Summary.MaxRTT = rtt
|
||||
}
|
||||
tk.Summary.MinRTT = float64(measurement.TCPInfo.MinRTT) / 1e03 /* us => ms */
|
||||
tk.Summary.Ping = tk.Summary.MinRTT
|
||||
if measurement.TCPInfo.BytesSent > 0 {
|
||||
tk.Summary.RetransmitRate = (float64(measurement.TCPInfo.BytesRetrans) /
|
||||
float64(measurement.TCPInfo.BytesSent))
|
||||
}
|
||||
measurement.BBRInfo = nil // don't encourage people to use it
|
||||
measurement.ConnectionInfo = nil // do we need to save it?
|
||||
measurement.Origin = "server"
|
||||
measurement.Test = "download"
|
||||
tk.Download = append(tk.Download, measurement)
|
||||
}
|
||||
return nil
|
||||
},
|
||||
)
|
||||
if err := mgr.run(ctx); err != nil && err.Error() != "generic_timeout_error" {
|
||||
sess.Logger().Warnf("download: %s", err)
|
||||
}
|
||||
return nil // failure is only when we cannot connect
|
||||
}
|
||||
|
||||
func (m *Measurer) doUpload(
|
||||
ctx context.Context, sess model.ExperimentSession,
|
||||
callbacks model.ExperimentCallbacks, tk *TestKeys,
|
||||
URL string,
|
||||
) error {
|
||||
if m.config.noUpload == true {
|
||||
return nil // useful to make tests faster
|
||||
}
|
||||
conn, err := newDialManager(URL,
|
||||
sess.Logger(), sess.UserAgent()).dialUpload(ctx)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
defer callbacks.OnProgress(1, " upload: done")
|
||||
defer conn.Close()
|
||||
mgr := newUploadManager(
|
||||
conn,
|
||||
func(timediff time.Duration, count int64) {
|
||||
elapsed := timediff.Seconds()
|
||||
// The percentage of completion of upload goes from 50% to 100% of
|
||||
// the whole experiment, hence `0.5 +` and `/2.0`.
|
||||
percentage := 0.5 + elapsed/paramMaxRuntimeUpperBound/2.0
|
||||
speed := float64(count) * 8.0 / elapsed
|
||||
message := fmt.Sprintf(" upload: speed %s", humanizex.SI(
|
||||
float64(speed), "bit/s"))
|
||||
tk.Summary.Upload = speed / 1e03 /* bit/s => kbit/s */
|
||||
callbacks.OnProgress(percentage, message)
|
||||
tk.Upload = append(tk.Upload, Measurement{
|
||||
AppInfo: &AppInfo{
|
||||
ElapsedTime: int64(timediff / time.Microsecond),
|
||||
NumBytes: count,
|
||||
},
|
||||
Origin: "client",
|
||||
Test: "upload",
|
||||
})
|
||||
},
|
||||
)
|
||||
if err := mgr.run(ctx); err != nil && err.Error() != "generic_timeout_error" {
|
||||
sess.Logger().Warnf("upload: %s", err)
|
||||
}
|
||||
return nil // failure is only when we cannot connect
|
||||
}
|
||||
|
||||
// Run implements ExperimentMeasurer.Run.
|
||||
func (m *Measurer) Run(
|
||||
ctx context.Context, sess model.ExperimentSession,
|
||||
measurement *model.Measurement, callbacks model.ExperimentCallbacks,
|
||||
) error {
|
||||
tk := new(TestKeys)
|
||||
tk.Protocol = 7
|
||||
measurement.TestKeys = tk
|
||||
locateResult, err := m.discover(ctx, sess)
|
||||
if err != nil {
|
||||
tk.Failure = failureFromError(err)
|
||||
return err
|
||||
}
|
||||
tk.Server = ServerInfo{
|
||||
Hostname: locateResult.Hostname,
|
||||
Site: locateResult.Site,
|
||||
}
|
||||
callbacks.OnProgress(0, fmt.Sprintf(" download: url: %s", locateResult.WSSDownloadURL))
|
||||
if m.preDownloadHook != nil {
|
||||
m.preDownloadHook()
|
||||
}
|
||||
if err := m.doDownload(ctx, sess, callbacks, tk, locateResult.WSSDownloadURL); err != nil {
|
||||
tk.Failure = failureFromError(err)
|
||||
return err
|
||||
}
|
||||
callbacks.OnProgress(0.5, fmt.Sprintf(" upload: url: %s", locateResult.WSSUploadURL))
|
||||
if m.preUploadHook != nil {
|
||||
m.preUploadHook()
|
||||
}
|
||||
if err := m.doUpload(ctx, sess, callbacks, tk, locateResult.WSSUploadURL); err != nil {
|
||||
tk.Failure = failureFromError(err)
|
||||
return err
|
||||
}
|
||||
return nil
|
||||
}
|
||||
|
||||
// NewExperimentMeasurer creates a new ExperimentMeasurer.
|
||||
func NewExperimentMeasurer(config Config) model.ExperimentMeasurer {
|
||||
return &Measurer{config: config, jsonUnmarshal: json.Unmarshal}
|
||||
}
|
||||
|
||||
func failureFromError(err error) (failure *string) {
|
||||
if err != nil {
|
||||
s := err.Error()
|
||||
failure = &s
|
||||
}
|
||||
return
|
||||
}
|
||||
|
||||
// SummaryKeys contains summary keys for this experiment.
|
||||
//
|
||||
// Note that this structure is part of the ABI contract with probe-cli
|
||||
// therefore we should be careful when changing it.
|
||||
type SummaryKeys struct {
|
||||
Upload float64 `json:"upload"`
|
||||
Download float64 `json:"download"`
|
||||
Ping float64 `json:"ping"`
|
||||
MaxRTT float64 `json:"max_rtt"`
|
||||
AvgRTT float64 `json:"avg_rtt"`
|
||||
MinRTT float64 `json:"min_rtt"`
|
||||
MSS float64 `json:"mss"`
|
||||
RetransmitRate float64 `json:"retransmit_rate"`
|
||||
IsAnomaly bool `json:"-"`
|
||||
}
|
||||
|
||||
// GetSummaryKeys implements model.ExperimentMeasurer.GetSummaryKeys.
|
||||
func (m Measurer) GetSummaryKeys(measurement *model.Measurement) (interface{}, error) {
|
||||
sk := SummaryKeys{IsAnomaly: false}
|
||||
tk, ok := measurement.TestKeys.(*TestKeys)
|
||||
if !ok {
|
||||
return sk, errors.New("invalid test keys type")
|
||||
}
|
||||
sk.Upload = tk.Summary.Upload
|
||||
sk.Download = tk.Summary.Download
|
||||
sk.Ping = tk.Summary.Ping
|
||||
sk.MaxRTT = tk.Summary.MaxRTT
|
||||
sk.AvgRTT = tk.Summary.AvgRTT
|
||||
sk.MinRTT = tk.Summary.MinRTT
|
||||
sk.MSS = float64(tk.Summary.MSS)
|
||||
sk.RetransmitRate = tk.Summary.RetransmitRate
|
||||
return sk, nil
|
||||
}
|
||||
@@ -0,0 +1,250 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"net/http"
|
||||
"strings"
|
||||
"testing"
|
||||
|
||||
"github.com/apex/log"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/mockable"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/model"
|
||||
)
|
||||
|
||||
func TestNewExperimentMeasurer(t *testing.T) {
|
||||
measurer := NewExperimentMeasurer(Config{})
|
||||
if measurer.ExperimentName() != "ndt" {
|
||||
t.Fatal("unexpected name")
|
||||
}
|
||||
if measurer.ExperimentVersion() != "0.8.0" {
|
||||
t.Fatal("unexpected version")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDiscoverCancelledContext(t *testing.T) {
|
||||
m := new(Measurer)
|
||||
sess := &mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
MockableUserAgent: "miniooni/0.1.0-dev",
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately cancel
|
||||
locateResult, err := m.discover(ctx, sess)
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
if locateResult.Hostname != "" {
|
||||
t.Fatal("not the Hostname we expected")
|
||||
}
|
||||
}
|
||||
|
||||
type verifyRequestTransport struct {
|
||||
ExpectedError error
|
||||
}
|
||||
|
||||
func (txp *verifyRequestTransport) RoundTrip(req *http.Request) (*http.Response, error) {
|
||||
if req.URL.RawQuery != "ip=1.2.3.4" {
|
||||
return nil, errors.New("invalid req.URL.RawQuery")
|
||||
}
|
||||
return nil, txp.ExpectedError
|
||||
}
|
||||
|
||||
func TestDoDownloadWithCancelledContext(t *testing.T) {
|
||||
m := new(Measurer)
|
||||
sess := &mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
MockableUserAgent: "miniooni/0.1.0-dev",
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately cancel
|
||||
err := m.doDownload(
|
||||
ctx, sess, model.NewPrinterCallbacks(log.Log), new(TestKeys),
|
||||
"ws://host.name")
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestDoUploadWithCancelledContext(t *testing.T) {
|
||||
m := new(Measurer)
|
||||
sess := &mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
MockableUserAgent: "miniooni/0.1.0-dev",
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately cancel
|
||||
err := m.doUpload(
|
||||
ctx, sess, model.NewPrinterCallbacks(log.Log), new(TestKeys),
|
||||
"ws://host.name")
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestRunWithCancelledContext(t *testing.T) {
|
||||
m := new(Measurer)
|
||||
sess := &mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
MockableUserAgent: "miniooni/0.1.0-dev",
|
||||
}
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
cancel() // immediately cancel
|
||||
err := m.Run(ctx, sess, new(model.Measurement), model.NewPrinterCallbacks(log.Log))
|
||||
if !errors.Is(err, context.Canceled) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestGood(t *testing.T) {
|
||||
if testing.Short() {
|
||||
t.Skip("skip test in short mode")
|
||||
}
|
||||
measurement := new(model.Measurement)
|
||||
measurer := NewExperimentMeasurer(Config{})
|
||||
err := measurer.Run(
|
||||
context.Background(),
|
||||
&mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
},
|
||||
measurement,
|
||||
model.NewPrinterCallbacks(log.Log),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sk, err := measurer.GetSummaryKeys(measurement)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if _, ok := sk.(SummaryKeys); !ok {
|
||||
t.Fatal("invalid type for summary keys")
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailDownload(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
measurer := NewExperimentMeasurer(Config{}).(*Measurer)
|
||||
measurer.preDownloadHook = func() {
|
||||
cancel()
|
||||
}
|
||||
err := measurer.Run(
|
||||
ctx,
|
||||
&mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
},
|
||||
new(model.Measurement),
|
||||
model.NewPrinterCallbacks(log.Log),
|
||||
)
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestFailUpload(t *testing.T) {
|
||||
ctx, cancel := context.WithCancel(context.Background())
|
||||
defer cancel()
|
||||
measurer := NewExperimentMeasurer(Config{noDownload: true}).(*Measurer)
|
||||
measurer.preUploadHook = func() {
|
||||
cancel()
|
||||
}
|
||||
err := measurer.Run(
|
||||
ctx,
|
||||
&mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
},
|
||||
new(model.Measurement),
|
||||
model.NewPrinterCallbacks(log.Log),
|
||||
)
|
||||
if err == nil || !strings.HasSuffix(err.Error(), "operation was canceled") {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
|
||||
func TestDownloadJSONUnmarshalFail(t *testing.T) {
|
||||
measurer := NewExperimentMeasurer(Config{noUpload: true}).(*Measurer)
|
||||
var seenError bool
|
||||
expected := errors.New("expected error")
|
||||
measurer.jsonUnmarshal = func(data []byte, v interface{}) error {
|
||||
seenError = true
|
||||
return expected
|
||||
}
|
||||
err := measurer.Run(
|
||||
context.Background(),
|
||||
&mockable.Session{
|
||||
MockableHTTPClient: http.DefaultClient,
|
||||
MockableLogger: log.Log,
|
||||
},
|
||||
new(model.Measurement),
|
||||
model.NewPrinterCallbacks(log.Log),
|
||||
)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
if !seenError {
|
||||
t.Fatal("did not see expected error")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummaryKeysInvalidType(t *testing.T) {
|
||||
measurement := new(model.Measurement)
|
||||
m := &Measurer{}
|
||||
_, err := m.GetSummaryKeys(measurement)
|
||||
if err.Error() != "invalid test keys type" {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestSummaryKeysGood(t *testing.T) {
|
||||
measurement := &model.Measurement{TestKeys: &TestKeys{Summary: Summary{
|
||||
RetransmitRate: 1,
|
||||
MSS: 2,
|
||||
MinRTT: 3,
|
||||
AvgRTT: 4,
|
||||
MaxRTT: 5,
|
||||
Ping: 6,
|
||||
Download: 7,
|
||||
Upload: 8,
|
||||
}}}
|
||||
m := &Measurer{}
|
||||
osk, err := m.GetSummaryKeys(measurement)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
sk := osk.(SummaryKeys)
|
||||
if sk.RetransmitRate != 1 {
|
||||
t.Fatal("invalid retransmitRate")
|
||||
}
|
||||
if sk.MSS != 2 {
|
||||
t.Fatal("invalid mss")
|
||||
}
|
||||
if sk.MinRTT != 3 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.AvgRTT != 4 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.MaxRTT != 5 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.Ping != 6 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.Download != 7 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.Upload != 8 {
|
||||
t.Fatal("invalid minRTT")
|
||||
}
|
||||
if sk.IsAnomaly {
|
||||
t.Fatal("invalid isAnomaly")
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,14 @@
|
||||
package ndt7
|
||||
|
||||
import "time"
|
||||
|
||||
const (
|
||||
paramFractionForScaling = 16
|
||||
paramMinMessageSize = 1 << 10
|
||||
paramMaxBufferSize = 1 << 20
|
||||
paramMaxScaledMessageSize = 1 << 20
|
||||
paramMaxMessageSize = 1 << 24
|
||||
paramMaxRuntimeUpperBound = 15.0 // seconds
|
||||
paramMaxRuntime = 10 * time.Second
|
||||
paramMeasureInterval = 250 * time.Millisecond
|
||||
)
|
||||
@@ -0,0 +1,177 @@
|
||||
package ndt7
|
||||
|
||||
// This file vendors data structures from the following repositories:
|
||||
//
|
||||
// - github.com/m-lab/ndt7-client-go
|
||||
// - github.com/m-lab/ndt-server
|
||||
// - github.com/m-lab/tcp-info
|
||||
//
|
||||
// It is available under the Apache License v2.0.
|
||||
//
|
||||
// Because m-lab uses mainly Linux as a development platform, they may
|
||||
// unwillingly break our Windows builds. Also, they use lots of depdencies
|
||||
// that we don't actually need. Hence, vendoring FTW.
|
||||
//
|
||||
// The data structures are supposed to stay constant in time or to not
|
||||
// change dramatically, hence this vendoring shouldn't be too bad.
|
||||
|
||||
type (
|
||||
// OriginKind indicates the origin of a measurement.
|
||||
OriginKind string
|
||||
|
||||
// TestKind indicates the direction of a measurement.
|
||||
TestKind string
|
||||
)
|
||||
|
||||
const (
|
||||
// OriginClient indicates that the measurement origin is the client.
|
||||
OriginClient = OriginKind("client")
|
||||
|
||||
// OriginServer indicates that the measurement origin is the server.
|
||||
OriginServer = OriginKind("server")
|
||||
|
||||
// TestDownload indicates that this is a download.
|
||||
TestDownload = TestKind("download")
|
||||
|
||||
// TestUpload indicates that this is an upload.
|
||||
TestUpload = TestKind("upload")
|
||||
)
|
||||
|
||||
// LinuxTCPInfo is the linux defined structure returned in RouteAttr DIAG_INFO messages.
|
||||
// It corresponds to the struct tcp_info in include/uapi/linux/tcp.h
|
||||
type LinuxTCPInfo struct {
|
||||
State uint8 `csv:"TCP.State"`
|
||||
CAState uint8 `csv:"TCP.CAState"`
|
||||
Retransmits uint8 `csv:"TCP.Retransmits"`
|
||||
Probes uint8 `csv:"TCP.Probes"`
|
||||
Backoff uint8 `csv:"TCP.Backoff"`
|
||||
Options uint8 `csv:"TCP.Options"`
|
||||
WScale uint8 `csv:"TCP.WScale"` //snd_wscale : 4, tcpi_rcv_wscale : 4;
|
||||
AppLimited uint8 `csv:"TCP.AppLimited"` //delivery_rate_app_limited:1;
|
||||
|
||||
RTO uint32 `csv:"TCP.RTO"` // offset 8
|
||||
ATO uint32 `csv:"TCP.ATO"`
|
||||
SndMSS uint32 `csv:"TCP.SndMSS"`
|
||||
RcvMSS uint32 `csv:"TCP.RcvMSS"`
|
||||
|
||||
Unacked uint32 `csv:"TCP.Unacked"` // offset 24
|
||||
Sacked uint32 `csv:"TCP.Sacked"`
|
||||
Lost uint32 `csv:"TCP.Lost"`
|
||||
Retrans uint32 `csv:"TCP.Retrans"`
|
||||
Fackets uint32 `csv:"TCP.Fackets"`
|
||||
|
||||
/* Times. */
|
||||
// These seem to be elapsed time, so they increase on almost every sample.
|
||||
// We can probably use them to get more info about intervals between samples.
|
||||
LastDataSent uint32 `csv:"TCP.LastDataSent"` // offset 44
|
||||
LastAckSent uint32 `csv:"TCP.LastAckSent"` /* Not remembered, sorry. */ // offset 48
|
||||
LastDataRecv uint32 `csv:"TCP.LastDataRecv"` // offset 52
|
||||
LastAckRecv uint32 `csv:"TCP.LastDataRecv"` // offset 56
|
||||
|
||||
/* Metrics. */
|
||||
PMTU uint32 `csv:"TCP.PMTU"`
|
||||
RcvSsThresh uint32 `csv:"TCP.RcvSsThresh"`
|
||||
RTT uint32 `csv:"TCP.RTT"`
|
||||
RTTVar uint32 `csv:"TCP.RTTVar"`
|
||||
SndSsThresh uint32 `csv:"TCP.SndSsThresh"`
|
||||
SndCwnd uint32 `csv:"TCP.SndCwnd"`
|
||||
AdvMSS uint32 `csv:"TCP.AdvMSS"`
|
||||
Reordering uint32 `csv:"TCP.Reordering"`
|
||||
|
||||
RcvRTT uint32 `csv:"TCP.RcvRTT"`
|
||||
RcvSpace uint32 `csv:"TCP.RcvSpace"`
|
||||
|
||||
TotalRetrans uint32 `csv:"TCP.TotalRetrans"`
|
||||
|
||||
PacingRate int64 `csv:"TCP.PacingRate"` // This is often -1, so better for it to be signed
|
||||
MaxPacingRate int64 `csv:"TCP.MaxPacingRate"` // This is often -1, so better to be signed.
|
||||
|
||||
// NOTE: In linux, these are uint64, but we make them int64 here for compatibility with BigQuery
|
||||
BytesAcked int64 `csv:"TCP.BytesAcked"` /* RFC4898 tcpEStatsAppHCThruOctetsAcked */
|
||||
BytesReceived int64 `csv:"TCP.BytesReceived"` /* RFC4898 tcpEStatsAppHCThruOctetsReceived */
|
||||
SegsOut int32 `csv:"TCP.SegsOut"` /* RFC4898 tcpEStatsPerfSegsOut */
|
||||
SegsIn int32 `csv:"TCP.SegsIn"` /* RFC4898 tcpEStatsPerfSegsIn */
|
||||
|
||||
NotsentBytes uint32 `csv:"TCP.NotsentBytes"`
|
||||
MinRTT uint32 `csv:"TCP.MinRTT"`
|
||||
DataSegsIn uint32 `csv:"TCP.DataSegsIn"` /* RFC4898 tcpEStatsDataSegsIn */
|
||||
DataSegsOut uint32 `csv:"TCP.DataSegsOut"` /* RFC4898 tcpEStatsDataSegsOut */
|
||||
|
||||
// NOTE: In linux, this is uint64, but we make it int64 here for compatibility with BigQuery
|
||||
DeliveryRate int64 `csv:"TCP.DeliveryRate"`
|
||||
|
||||
BusyTime int64 `csv:"TCP.BusyTime"` /* Time (usec) busy sending data */
|
||||
RWndLimited int64 `csv:"TCP.RWndLimited"` /* Time (usec) limited by receive window */
|
||||
SndBufLimited int64 `csv:"TCP.SndBufLimited"` /* Time (usec) limited by send buffer */
|
||||
|
||||
Delivered uint32 `csv:"TCP.Delivered"`
|
||||
DeliveredCE uint32 `csv:"TCP.DeliveredCE"`
|
||||
|
||||
// NOTE: In linux, these are uint64, but we make them int64 here for compatibility with BigQuery
|
||||
BytesSent int64 `csv:"TCP.BytesSent"` /* RFC4898 tcpEStatsPerfHCDataOctetsOut */
|
||||
BytesRetrans int64 `csv:"TCP.BytesRetrans"` /* RFC4898 tcpEStatsPerfOctetsRetrans */
|
||||
|
||||
DSackDups uint32 `csv:"TCP.DSackDups"` /* RFC4898 tcpEStatsStackDSACKDups */
|
||||
ReordSeen uint32 `csv:"TCP.ReordSeen"` /* reordering events seen */
|
||||
}
|
||||
|
||||
// AppInfo contains an application level measurement. This structure is
|
||||
// described in the ndt7 specification.
|
||||
type AppInfo struct {
|
||||
NumBytes int64
|
||||
ElapsedTime int64
|
||||
}
|
||||
|
||||
// ConnectionInfo contains connection info. This structure is described
|
||||
// in the ndt7 specification.
|
||||
type ConnectionInfo struct {
|
||||
Client string
|
||||
Server string
|
||||
UUID string `json:",omitempty"`
|
||||
}
|
||||
|
||||
// InetDiagBBRInfo implements the struct associated with INET_DIAG_BBRINFO attribute, corresponding with
|
||||
// linux struct tcp_bbr_info in uapi/linux/inet_diag.h.
|
||||
type InetDiagBBRInfo struct {
|
||||
BW int64 `csv:"BBR.BW"` // Max-filtered BW (app throughput) estimate in bytes/second
|
||||
MinRTT uint32 `csv:"BBR.MinRTT"` // Min-filtered RTT in uSec
|
||||
PacingGain uint32 `csv:"BBR.PacingGain"` // Pacing gain shifted left 8 bits
|
||||
CwndGain uint32 `csv:"BBR.CwndGain"` // Cwnd gain shifted left 8 bits
|
||||
}
|
||||
|
||||
// The BBRInfo struct contains information measured using BBR. This structure is
|
||||
// an extension to the ndt7 specification. Variables here have the same
|
||||
// measurement unit that is used by the Linux kernel.
|
||||
type BBRInfo struct {
|
||||
InetDiagBBRInfo
|
||||
ElapsedTime int64
|
||||
}
|
||||
|
||||
// The TCPInfo struct contains information measured using TCP_INFO. This
|
||||
// structure is described in the ndt7 specification.
|
||||
type TCPInfo struct {
|
||||
LinuxTCPInfo
|
||||
ElapsedTime int64
|
||||
}
|
||||
|
||||
// The Measurement struct contains measurement results. This message is
|
||||
// an extension of the one inside of v0.9.0 of the ndt7 spec.
|
||||
type Measurement struct {
|
||||
// AppInfo contains application level measurements.
|
||||
AppInfo *AppInfo `json:",omitempty"`
|
||||
|
||||
// BBRInfo is the data measured using TCP BBR instrumentation.
|
||||
BBRInfo *BBRInfo `json:",omitempty"`
|
||||
|
||||
// ConnectionInfo contains info on the connection.
|
||||
ConnectionInfo *ConnectionInfo `json:",omitempty"`
|
||||
|
||||
// Origin indicates who performed this measurement.
|
||||
Origin OriginKind `json:",omitempty"`
|
||||
|
||||
// Test contains the test name.
|
||||
Test TestKind `json:",omitempty"`
|
||||
|
||||
// TCPInfo contains metrics measured using TCP_INFO instrumentation.
|
||||
TCPInfo *TCPInfo `json:",omitempty"`
|
||||
}
|
||||
@@ -0,0 +1,75 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func newMessage(n int) (*websocket.PreparedMessage, error) {
|
||||
return websocket.NewPreparedMessage(websocket.BinaryMessage, make([]byte, n))
|
||||
}
|
||||
|
||||
type uploadManager struct {
|
||||
conn mockableConn
|
||||
fractionForScaling int64
|
||||
maxRuntime time.Duration
|
||||
maxMessageSize int
|
||||
maxScaledMessageSize int
|
||||
measureInterval time.Duration
|
||||
minMessageSize int
|
||||
newMessage func(int) (*websocket.PreparedMessage, error)
|
||||
onPerformance callbackPerformance
|
||||
}
|
||||
|
||||
func newUploadManager(
|
||||
conn mockableConn, onPerformance callbackPerformance,
|
||||
) uploadManager {
|
||||
return uploadManager{
|
||||
conn: conn,
|
||||
fractionForScaling: paramFractionForScaling,
|
||||
maxRuntime: paramMaxRuntime,
|
||||
maxMessageSize: paramMaxMessageSize,
|
||||
maxScaledMessageSize: paramMaxScaledMessageSize,
|
||||
measureInterval: paramMeasureInterval,
|
||||
minMessageSize: paramMinMessageSize,
|
||||
newMessage: newMessage,
|
||||
onPerformance: onPerformance,
|
||||
}
|
||||
}
|
||||
|
||||
func (mgr uploadManager) run(ctx context.Context) error {
|
||||
var total int64
|
||||
start := time.Now()
|
||||
if err := mgr.conn.SetWriteDeadline(time.Now().Add(mgr.maxRuntime)); err != nil {
|
||||
return err
|
||||
}
|
||||
size := mgr.minMessageSize
|
||||
message, err := mgr.newMessage(size)
|
||||
if err != nil {
|
||||
return err
|
||||
}
|
||||
ticker := time.NewTicker(mgr.measureInterval)
|
||||
defer ticker.Stop()
|
||||
for ctx.Err() == nil {
|
||||
if err := mgr.conn.WritePreparedMessage(message); err != nil {
|
||||
return err
|
||||
}
|
||||
total += int64(size)
|
||||
select {
|
||||
case now := <-ticker.C:
|
||||
mgr.onPerformance(now.Sub(start), total)
|
||||
default:
|
||||
// NOTHING
|
||||
}
|
||||
if size >= mgr.maxScaledMessageSize || int64(size) >= (total/mgr.fractionForScaling) {
|
||||
continue
|
||||
}
|
||||
size <<= 1
|
||||
if message, err = mgr.newMessage(size); err != nil {
|
||||
return err
|
||||
}
|
||||
}
|
||||
return nil
|
||||
}
|
||||
@@ -0,0 +1,89 @@
|
||||
package ndt7
|
||||
|
||||
import (
|
||||
"context"
|
||||
"errors"
|
||||
"testing"
|
||||
"time"
|
||||
|
||||
"github.com/gorilla/websocket"
|
||||
)
|
||||
|
||||
func TestUploadSetWriteDeadlineFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newUploadManager(
|
||||
&mockableConnMock{
|
||||
WriteDeadlineErr: expected,
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadNewMessageFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newUploadManager(
|
||||
&mockableConnMock{},
|
||||
defaultCallbackPerformance,
|
||||
)
|
||||
mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
|
||||
return nil, expected
|
||||
}
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadWritePreparedMessageFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newUploadManager(
|
||||
&mockableConnMock{
|
||||
WritePreparedMessageErr: expected,
|
||||
},
|
||||
defaultCallbackPerformance,
|
||||
)
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadWritePreparedMessageSubsequentFailure(t *testing.T) {
|
||||
expected := errors.New("mocked error")
|
||||
mgr := newUploadManager(
|
||||
&mockableConnMock{},
|
||||
defaultCallbackPerformance,
|
||||
)
|
||||
var already bool
|
||||
mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
|
||||
if !already {
|
||||
already = true
|
||||
return new(websocket.PreparedMessage), nil
|
||||
}
|
||||
return nil, expected
|
||||
}
|
||||
err := mgr.run(context.Background())
|
||||
if !errors.Is(err, expected) {
|
||||
t.Fatal("not the error we expected")
|
||||
}
|
||||
}
|
||||
|
||||
func TestUploadLoop(t *testing.T) {
|
||||
mgr := newUploadManager(
|
||||
&mockableConnMock{},
|
||||
defaultCallbackPerformance,
|
||||
)
|
||||
mgr.newMessage = func(int) (*websocket.PreparedMessage, error) {
|
||||
return new(websocket.PreparedMessage), nil
|
||||
}
|
||||
ctx, cancel := context.WithTimeout(context.Background(), 1*time.Second)
|
||||
defer cancel()
|
||||
err := mgr.run(ctx)
|
||||
if err != nil {
|
||||
t.Fatal(err)
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user