c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

chore: merge probe-engine into probe-cli (#201)

Browse Source 
This is how I did it:

1. `git clone https://github.com/ooni/probe-engine internal/engine`

2. ```
(cd internal/engine && git describe --tags)
v0.23.0
```

3. `nvim go.mod` (merging `go.mod` with `internal/engine/go.mod`

4. `rm -rf internal/.git internal/engine/go.{mod,sum}`

5. `git add internal/engine`

6. `find . -type f -name \*.go -exec sed -i 's@/ooni/probe-engine@/ooni/probe-cli/v3/internal/engine@g' {} \;`

7. `go build ./...` (passes)

8. `go test -race ./...` (temporary failure on RiseupVPN)

9. `go mod tidy`

10. this commit message

Once this piece of work is done, we can build a new version of `ooniprobe` that
is using `internal/engine` directly. We need to do more work to ensure all the
other functionality in `probe-engine` (e.g. making mobile packages) are still WAI.

Part of https://github.com/ooni/probe/issues/1335
This commit is contained in:
Simone Basso
2021-02-02 12:05:47 +01:00
committed by GitHub
parent b1ce300c8d
commit d57c78bc71
535 changed files with 66182 additions and 23 deletions
Download Patch File Download Diff File Expand all files Collapse all files
internal/engine/netx/errorx/errorx.go
+322
View File
@@ -0,0 +1,322 @@
// Package errorx contains error extensions
package errorx
import (
"context"
"crypto/x509"
"errors"
"fmt"
"regexp"
"strings"
)
const (
// FailureConnectionRefused means ECONNREFUSED.
FailureConnectionRefused = "connection_refused"
// FailureConnectionReset means ECONNRESET.
FailureConnectionReset = "connection_reset"
// FailureDNSBogonError means we detected bogon in DNS reply.
FailureDNSBogonError = "dns_bogon_error"
// FailureDNSNXDOMAINError means we got NXDOMAIN in DNS reply.
FailureDNSNXDOMAINError = "dns_nxdomain_error"
// FailureEOFError means we got unexpected EOF on connection.
FailureEOFError = "eof_error"
// FailureGenericTimeoutError means we got some timer has expired.
FailureGenericTimeoutError = "generic_timeout_error"
// FailureInterrupted means that the user interrupted us.
FailureInterrupted = "interrupted"
// FailureNoCompatibleQUICVersion means that the server does not support the proposed QUIC version
FailureNoCompatibleQUICVersion = "quic_incompatible_version"
// FailureSSLInvalidHostname means we got certificate is not valid for SNI.
FailureSSLInvalidHostname = "ssl_invalid_hostname"
// FailureSSLUnknownAuthority means we cannot find CA validating certificate.
FailureSSLUnknownAuthority = "ssl_unknown_authority"
// FailureSSLInvalidCertificate means certificate experired or other
// sort of errors causing it to be invalid.
FailureSSLInvalidCertificate = "ssl_invalid_certificate"
// FailureJSONParseError indicates that we couldn't parse a JSON
FailureJSONParseError = "json_parse_error"
)
const (
// ResolveOperation is the operation where we resolve a domain name
ResolveOperation = "resolve"
// ConnectOperation is the operation where we do a TCP connect
ConnectOperation = "connect"
// TLSHandshakeOperation is the TLS handshake
TLSHandshakeOperation = "tls_handshake"
// QUICHandshakeOperation is the handshake to setup a QUIC connection
QUICHandshakeOperation = "quic_handshake"
// HTTPRoundTripOperation is the HTTP round trip
HTTPRoundTripOperation = "http_round_trip"
// CloseOperation is when we close a socket
CloseOperation = "close"
// ReadOperation is when we read from a socket
ReadOperation = "read"
// WriteOperation is when we write to a socket
WriteOperation = "write"
// ReadFromOperation is when we read from an UDP socket
ReadFromOperation = "read_from"
// WriteToOperation is when we write to an UDP socket
WriteToOperation = "write_to"
// UnknownOperation is when we cannot determine the operation
UnknownOperation = "unknown"
// TopLevelOperation is used when the failure happens at top level. This
// happens for example with urlgetter with a cancelled context.
TopLevelOperation = "top_level"
)
// ErrDNSBogon indicates that we found a bogon address. This is the
// correct value with which to initialize MeasurementRoot.ErrDNSBogon
// to tell this library to return an error when a bogon is found.
var ErrDNSBogon = errors.New("dns: detected bogon address")
// ErrWrapper is our error wrapper for Go errors. The key objective of
// this structure is to properly set Failure, which is also returned by
// the Error() method, so be one of the OONI defined strings.
type ErrWrapper struct {
// ConnID is the connection ID, or zero if not known.
ConnID int64
// DialID is the dial ID, or zero if not known.
DialID int64
// Failure is the OONI failure string. The failure strings are
// loosely backward compatible with Measurement Kit.
//
// This is either one of the FailureXXX strings or any other
// string like `unknown_failure ...`. The latter represents an
// error that we have not yet mapped to a failure.
Failure string
// Operation is the operation that failed. If possible, it
// SHOULD be a _major_ operation. Major operations are:
//
// - ResolveOperation: resolving a domain name failed
// - ConnectOperation: connecting to an IP failed
// - TLSHandshakeOperation: TLS handshaking failed
// - HTTPRoundTripOperation: other errors during round trip
//
// Because a network connection doesn't necessarily know
// what is the current major operation we also have the
// following _minor_ operations:
//
// - CloseOperation: CLOSE failed
// - ReadOperation: READ failed
// - WriteOperation: WRITE failed
//
// If an ErrWrapper referring to a major operation is wrapping
// another ErrWrapper and such ErrWrapper already refers to
// a major operation, then the new ErrWrapper should use the
// child ErrWrapper major operation. Otherwise, it should use
// its own major operation. This way, the topmost wrapper is
// supposed to refer to the major operation that failed.
Operation string
// TransactionID is the transaction ID, or zero if not known.
TransactionID int64
// WrappedErr is the error that we're wrapping.
WrappedErr error
}
// Error returns a description of the error that occurred.
func (e *ErrWrapper) Error() string {
return e.Failure
}
// Unwrap allows to access the underlying error
func (e *ErrWrapper) Unwrap() error {
return e.WrappedErr
}
// SafeErrWrapperBuilder contains a builder for ErrWrapper that
// is safe, i.e., behaves correctly when the error is nil.
type SafeErrWrapperBuilder struct {
// ConnID is the connection ID, if any
ConnID int64
// DialID is the dial ID, if any
DialID int64
// Error is the error, if any
Error error
// Operation is the operation that failed
Operation string
// TransactionID is the transaction ID, if any
TransactionID int64
}
// MaybeBuild builds a new ErrWrapper, if b.Error is not nil, and returns
// a nil error value, instead, if b.Error is nil.
func (b SafeErrWrapperBuilder) MaybeBuild() (err error) {
if b.Error != nil {
err = &ErrWrapper{
ConnID: b.ConnID,
DialID: b.DialID,
Failure: toFailureString(b.Error),
Operation: toOperationString(b.Error, b.Operation),
TransactionID: b.TransactionID,
WrappedErr: b.Error,
}
}
return
}
func toFailureString(err error) string {
// The list returned here matches the values used by MK unless
// explicitly noted otherwise with a comment.
var errwrapper *ErrWrapper
if errors.As(err, &errwrapper) {
return errwrapper.Error() // we've already wrapped it
}
if errors.Is(err, ErrDNSBogon) {
return FailureDNSBogonError // not in MK
}
if errors.Is(err, context.Canceled) {
return FailureInterrupted
}
var x509HostnameError x509.HostnameError
if errors.As(err, &x509HostnameError) {
// Test case: https://wrong.host.badssl.com/
return FailureSSLInvalidHostname
}
var x509UnknownAuthorityError x509.UnknownAuthorityError
if errors.As(err, &x509UnknownAuthorityError) {
// Test case: https://self-signed.badssl.com/. This error has
// never been among the ones returned by MK.
return FailureSSLUnknownAuthority
}
var x509CertificateInvalidError x509.CertificateInvalidError
if errors.As(err, &x509CertificateInvalidError) {
// Test case: https://expired.badssl.com/
return FailureSSLInvalidCertificate
}
s := err.Error()
if strings.HasSuffix(s, "operation was canceled") {
return FailureInterrupted
}
if strings.HasSuffix(s, "EOF") {
return FailureEOFError
}
if strings.HasSuffix(s, "connection refused") {
return FailureConnectionRefused
}
if strings.HasSuffix(s, "connection reset by peer") {
return FailureConnectionReset
}
if strings.HasSuffix(s, "context deadline exceeded") {
return FailureGenericTimeoutError
}
if strings.HasSuffix(s, "transaction is timed out") {
return FailureGenericTimeoutError
}
if strings.HasSuffix(s, "i/o timeout") {
return FailureGenericTimeoutError
}
if strings.HasSuffix(s, "TLS handshake timeout") {
return FailureGenericTimeoutError
}
if strings.HasSuffix(s, "no such host") {
// This is dns_lookup_error in MK but such error is used as a
// generic "hey, the lookup failed" error. Instead, this error
// that we return here is significantly more specific.
return FailureDNSNXDOMAINError
}
// TODO(kelmenhorst): see whether it is possible to match errors
// from qtls rather than strings for TLS errors below.
//
// TODO(kelmenhorst): make sure we have tests for all errors. Also,
// how to ensure we are robust to changes in other libs?
//
// special QUIC errors
matched, err := regexp.MatchString(`.*x509: certificate is valid for.*not.*`, s)
if matched {
return FailureSSLInvalidHostname
}
if strings.HasSuffix(s, "x509: certificate signed by unknown authority") {
return FailureSSLUnknownAuthority
}
certInvalidErrors := []string{"x509: certificate is not authorized to sign other certificates", "x509: certificate has expired or is not yet valid:", "x509: a root or intermediate certificate is not authorized to sign for this name:", "x509: a root or intermediate certificate is not authorized for an extended key usage:", "x509: too many intermediates for path length constraint", "x509: certificate specifies an incompatible key usage", "x509: issuer name does not match subject from issuing certificate", "x509: issuer has name constraints but leaf doesn't have a SAN extension", "x509: issuer has name constraints but leaf contains unknown or unconstrained name:"}
for _, errstr := range certInvalidErrors {
if strings.Contains(s, errstr) {
return FailureSSLInvalidCertificate
}
}
if strings.HasPrefix(s, "No compatible QUIC version found") {
return FailureNoCompatibleQUICVersion
}
if strings.HasSuffix(s, "Handshake did not complete in time") {
return FailureGenericTimeoutError
}
if strings.HasSuffix(s, "connection_refused") {
return FailureConnectionRefused
}
if strings.Contains(s, "stateless_reset") {
return FailureConnectionReset
}
if strings.Contains(s, "deadline exceeded") {
return FailureGenericTimeoutError
}
formatted := fmt.Sprintf("unknown_failure: %s", s)
return Scrub(formatted) // scrub IP addresses in the error
}
func toOperationString(err error, operation string) string {
var errwrapper *ErrWrapper
if errors.As(err, &errwrapper) {
// Basically, as explained in ErrWrapper docs, let's
// keep the child major operation, if any.
if errwrapper.Operation == ConnectOperation {
return errwrapper.Operation
}
if errwrapper.Operation == HTTPRoundTripOperation {
return errwrapper.Operation
}
if errwrapper.Operation == ResolveOperation {
return errwrapper.Operation
}
if errwrapper.Operation == TLSHandshakeOperation {
return errwrapper.Operation
}
if errwrapper.Operation == QUICHandshakeOperation {
return errwrapper.Operation
}
if errwrapper.Operation == "quic_handshake_start" {
return QUICHandshakeOperation
}
if errwrapper.Operation == "quic_handshake_done" {
return QUICHandshakeOperation
}
// FALLTHROUGH
}
return operation
}
internal/engine/netx/errorx/errorx_test.go
+247
View File
@@ -0,0 +1,247 @@
package errorx
import (
"context"
"crypto/tls"
"crypto/x509"
"errors"
"io"
"net"
"syscall"
"testing"
"github.com/google/go-cmp/cmp"
"github.com/lucas-clemente/quic-go"
"github.com/pion/stun"
)
func TestMaybeBuildFactory(t *testing.T) {
err := SafeErrWrapperBuilder{
ConnID: 1,
DialID: 10,
Error: errors.New("mocked error"),
TransactionID: 100,
}.MaybeBuild()
var target *ErrWrapper
if errors.As(err, &target) == false {
t.Fatal("not the expected error type")
}
if target.ConnID != 1 {
t.Fatal("wrong ConnID")
}
if target.DialID != 10 {
t.Fatal("wrong DialID")
}
if target.Failure != "unknown_failure: mocked error" {
t.Fatal("the failure string is wrong")
}
if target.TransactionID != 100 {
t.Fatal("the transactionID is wrong")
}
if target.WrappedErr.Error() != "mocked error" {
t.Fatal("the wrapped error is wrong")
}
}
func TestToFailureString(t *testing.T) {
t.Run("for already wrapped error", func(t *testing.T) {
err := SafeErrWrapperBuilder{Error: io.EOF}.MaybeBuild()
if toFailureString(err) != FailureEOFError {
t.Fatal("unexpected result")
}
})
t.Run("for ErrDNSBogon", func(t *testing.T) {
if toFailureString(ErrDNSBogon) != FailureDNSBogonError {
t.Fatal("unexpected result")
}
})
t.Run("for context.Canceled", func(t *testing.T) {
if toFailureString(context.Canceled) != FailureInterrupted {
t.Fatal("unexpected result")
}
})
t.Run("for x509.HostnameError", func(t *testing.T) {
var err x509.HostnameError
if toFailureString(err) != FailureSSLInvalidHostname {
t.Fatal("unexpected result")
}
})
t.Run("for x509.UnknownAuthorityError", func(t *testing.T) {
var err x509.UnknownAuthorityError
if toFailureString(err) != FailureSSLUnknownAuthority {
t.Fatal("unexpected result")
}
})
t.Run("for x509.CertificateInvalidError", func(t *testing.T) {
var err x509.CertificateInvalidError
if toFailureString(err) != FailureSSLInvalidCertificate {
t.Fatal("unexpected result")
}
})
t.Run("for operation was canceled error", func(t *testing.T) {
if toFailureString(errors.New("operation was canceled")) != FailureInterrupted {
t.Fatal("unexpected result")
}
})
t.Run("for EOF", func(t *testing.T) {
if toFailureString(io.EOF) != FailureEOFError {
t.Fatal("unexpected results")
}
})
t.Run("for connection_refused", func(t *testing.T) {
if toFailureString(syscall.ECONNREFUSED) != FailureConnectionRefused {
t.Fatal("unexpected results")
}
})
t.Run("for connection_reset", func(t *testing.T) {
if toFailureString(syscall.ECONNRESET) != FailureConnectionReset {
t.Fatal("unexpected results")
}
})
t.Run("for context deadline exceeded", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1)
defer cancel()
<-ctx.Done()
if toFailureString(ctx.Err()) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
t.Run("for stun's transaction is timed out", func(t *testing.T) {
if toFailureString(stun.ErrTransactionTimeOut) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
t.Run("for i/o error", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1)
defer cancel() // fail immediately
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "www.google.com:80")
if err == nil {
t.Fatal("expected an error here")
}
if conn != nil {
t.Fatal("expected nil connection here")
}
if toFailureString(err) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
t.Run("for TLS handshake timeout error", func(t *testing.T) {
err := errors.New("net/http: TLS handshake timeout")
if toFailureString(err) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
t.Run("for no such host", func(t *testing.T) {
if toFailureString(&net.DNSError{
Err: "no such host",
}) != FailureDNSNXDOMAINError {
t.Fatal("unexpected results")
}
})
t.Run("for errors including IPv4 address", func(t *testing.T) {
input := errors.New("read tcp 10.0.2.15:56948->93.184.216.34:443: use of closed network connection")
expected := "unknown_failure: read tcp [scrubbed]->[scrubbed]: use of closed network connection"
out := toFailureString(input)
if out != expected {
t.Fatal(cmp.Diff(expected, out))
}
})
t.Run("for errors including IPv6 address", func(t *testing.T) {
input := errors.New("read tcp [::1]:56948->[::1]:443: use of closed network connection")
expected := "unknown_failure: read tcp [scrubbed]->[scrubbed]: use of closed network connection"
out := toFailureString(input)
if out != expected {
t.Fatal(cmp.Diff(expected, out))
}
})
// QUIC failures
t.Run("for connection_refused", func(t *testing.T) {
if toFailureString(errors.New("connection_refused")) != FailureConnectionRefused {
t.Fatal("unexpected results")
}
})
t.Run("for connection_reset", func(t *testing.T) {
if toFailureString(errors.New("stateless_reset")) != FailureConnectionReset {
t.Fatal("unexpected results")
}
})
t.Run("for incompatible quic version", func(t *testing.T) {
if toFailureString(errors.New("No compatible QUIC version found")) != FailureNoCompatibleQUICVersion {
t.Fatal("unexpected results")
}
})
t.Run("for i/o error", func(t *testing.T) {
ctx, cancel := context.WithTimeout(context.Background(), 1)
defer cancel() // fail immediately
udpAddr := &net.UDPAddr{IP: net.ParseIP("216.58.212.164"), Port: 80, Zone: ""}
udpConn, err := net.ListenUDP("udp", &net.UDPAddr{IP: net.IPv4zero, Port: 0})
sess, err := quic.DialEarlyContext(ctx, udpConn, udpAddr, "google.com:80", &tls.Config{}, &quic.Config{})
if err == nil {
t.Fatal("expected an error here")
}
if sess != nil {
t.Fatal("expected nil session here")
}
if toFailureString(err) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
t.Run("for QUIC handshake timeout error", func(t *testing.T) {
err := errors.New("Handshake did not complete in time")
if toFailureString(err) != FailureGenericTimeoutError {
t.Fatal("unexpected results")
}
})
}
func TestToOperationString(t *testing.T) {
t.Run("for connect", func(t *testing.T) {
// You're doing HTTP and connect fails. You want to know
// that connect failed not that HTTP failed.
err := &ErrWrapper{Operation: ConnectOperation}
if toOperationString(err, HTTPRoundTripOperation) != ConnectOperation {
t.Fatal("unexpected result")
}
})
t.Run("for http_round_trip", func(t *testing.T) {
// You're doing DoH and something fails inside HTTP. You want
// to know about the internal HTTP error, not resolve.
err := &ErrWrapper{Operation: HTTPRoundTripOperation}
if toOperationString(err, ResolveOperation) != HTTPRoundTripOperation {
t.Fatal("unexpected result")
}
})
t.Run("for resolve", func(t *testing.T) {
// You're doing HTTP and the DNS fails. You want to
// know that resolve failed.
err := &ErrWrapper{Operation: ResolveOperation}
if toOperationString(err, HTTPRoundTripOperation) != ResolveOperation {
t.Fatal("unexpected result")
}
})
t.Run("for tls_handshake", func(t *testing.T) {
// You're doing HTTP and the TLS handshake fails. You want
// to know about a TLS handshake error.
err := &ErrWrapper{Operation: TLSHandshakeOperation}
if toOperationString(err, HTTPRoundTripOperation) != TLSHandshakeOperation {
t.Fatal("unexpected result")
}
})
t.Run("for minor operation", func(t *testing.T) {
// You just noticed that TLS handshake failed and you
// have a child error telling you that read failed. Here
// you want to know about a TLS handshake error.
err := &ErrWrapper{Operation: ReadOperation}
if toOperationString(err, TLSHandshakeOperation) != TLSHandshakeOperation {
t.Fatal("unexpected result")
}
})
t.Run("for quic_handshake", func(t *testing.T) {
// You're doing HTTP and the TLS handshake fails. You want
// to know about a TLS handshake error.
err := &ErrWrapper{Operation: QUICHandshakeOperation}
if toOperationString(err, HTTPRoundTripOperation) != QUICHandshakeOperation {
t.Fatal("unexpected result")
}
})
}
internal/engine/netx/errorx/sanitizer.go
+70
View File
@@ -0,0 +1,70 @@
package errorx
import "regexp"
// The code in this file is adapted from github.com/keroserene/snowflake's
// common/safelog/safelog.go implementation <https://git.io/JfO9w>.
//
// ================================================================================
// Copyright (c) 2016, Serene Han, Arlo Breault
// Copyright (c) 2019-2020, The Tor Project, Inc
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// * Neither the names of the copyright owners nor the names of its
// contributors may be used to endorse or promote products derived from this
// software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ================================================================================
const ipv4Address = `\d{1,3}\.\d{1,3}\.\d{1,3}\.\d{1,3}`
const ipv6Address = `([0-9a-fA-F]{0,4}:){5,7}([0-9a-fA-F]{0,4})?`
const ipv6Compressed = `([0-9a-fA-F]{0,4}:){0,5}([0-9a-fA-F]{0,4})?(::)([0-9a-fA-F]{0,4}:){0,5}([0-9a-fA-F]{0,4})?`
const ipv6Full = `(` + ipv6Address + `(` + ipv4Address + `))` +
`|(` + ipv6Compressed + `(` + ipv4Address + `))` +
`|(` + ipv6Address + `)` + `|(` + ipv6Compressed + `)`
const optionalPort = `(:\d{1,5})?`
const addressPattern = `((` + ipv4Address + `)|(\[(` + ipv6Full + `)\])|(` + ipv6Full + `))` + optionalPort
const fullAddrPattern = `(^|\s|[^\w:])` + addressPattern + `(\s|(:\s)|[^\w:]|$)`
var scrubberPatterns = []*regexp.Regexp{
regexp.MustCompile(fullAddrPattern),
}
var addressRegexp = regexp.MustCompile(addressPattern)
func scrub(b []byte) []byte {
scrubbedBytes := b
for _, pattern := range scrubberPatterns {
// this is a workaround since go does not yet support look ahead or look
// behind for regular expressions.
scrubbedBytes = pattern.ReplaceAllFunc(scrubbedBytes, func(b []byte) []byte {
return addressRegexp.ReplaceAll(b, []byte("[scrubbed]"))
})
}
return scrubbedBytes
}
// Scrub sanitizes a string containing an error such that
// any occurrence of IP endpoints is scrubbed
func Scrub(s string) string {
return string(scrub([]byte(s)))
}
internal/engine/netx/errorx/sanitizer_test.go
+129
View File
@@ -0,0 +1,129 @@
package errorx
import (
"testing"
"github.com/google/go-cmp/cmp"
)
// The code in this file is adapted from github.com/keroserene/snowflake's
// common/safelog/safelog.go implementation <https://git.io/JfO9w>.
//
// ================================================================================
// Copyright (c) 2016, Serene Han, Arlo Breault
// Copyright (c) 2019-2020, The Tor Project, Inc
//
// Redistribution and use in source and binary forms, with or without modification,
// are permitted provided that the following conditions are met:
//
// * Redistributions of source code must retain the above copyright notice, this
// list of conditions and the following disclaimer.
//
// * Redistributions in binary form must reproduce the above copyright notice,
// this list of conditions and the following disclaimer in the documentation and/or
// other materials provided with the distribution.
//
// * Neither the names of the copyright owners nor the names of its
// contributors may be used to endorse or promote products derived from this
// software without specific prior written permission.
//
// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" AND
// ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED
// WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE
// DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT HOLDER OR CONTRIBUTORS BE LIABLE FOR
// ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES
// (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES;
// LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON
// ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
// (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS
// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
// ================================================================================
//Test the log scrubber on known problematic log messages
func TestLogScrubberMessages(t *testing.T) {
for _, test := range []struct {
input, expected string
}{
{
"http: TLS handshake error from 129.97.208.23:38310: ",
"http: TLS handshake error from [scrubbed]: ",
},
{
"http2: panic serving [2620:101:f000:780:9097:75b1:519f:dbb8]:58344: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack",
"http2: panic serving [scrubbed]: interface conversion: *http2.responseWriter is not http.Hijacker: missing method Hijack",
},
{
//Make sure it doesn't scrub fingerprint
"a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74",
"a=fingerprint:sha-256 33:B6:FA:F6:94:CA:74:61:45:4A:D2:1F:2C:2F:75:8A:D9:EB:23:34:B2:30:E9:1B:2A:A6:A9:E0:44:72:CC:74",
},
{
//try with enclosing parens
"(1:2:3:4:c:d:e:f) {1:2:3:4:c:d:e:f}",
"([scrubbed]) {[scrubbed]}",
},
{
//Make sure it doesn't scrub timestamps
"2019/05/08 15:37:31 starting",
"2019/05/08 15:37:31 starting",
},
} {
if Scrub(test.input) != test.expected {
t.Error(cmp.Diff(test.input, test.expected))
}
}
}
func TestLogScrubberGoodFormats(t *testing.T) {
for _, addr := range []string{
// IPv4
"1.2.3.4",
"255.255.255.255",
// IPv4 with port
"1.2.3.4:55",
"255.255.255.255:65535",
// IPv6
"1:2:3:4:c:d:e:f",
"1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF",
// IPv6 with brackets
"[1:2:3:4:c:d:e:f]",
"[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]",
// IPv6 with brackets and port
"[1:2:3:4:c:d:e:f]:55",
"[1111:2222:3333:4444:CCCC:DDDD:EEEE:FFFF]:65535",
// compressed IPv6
"::f",
"::d:e:f",
"1:2:3::",
"1:2:3::d:e:f",
"1:2:3:d:e:f::",
"::1:2:3:d:e:f",
"1111:2222:3333::DDDD:EEEE:FFFF",
// compressed IPv6 with brackets
"[::d:e:f]",
"[1:2:3::]",
"[1:2:3::d:e:f]",
"[1111:2222:3333::DDDD:EEEE:FFFF]",
"[1:2:3:4:5:6::8]",
"[1::7:8]",
// compressed IPv6 with brackets and port
"[1::]:58344",
"[::d:e:f]:55",
"[1:2:3::]:55",
"[1:2:3::d:e:f]:55",
"[1111:2222:3333::DDDD:EEEE:FFFF]:65535",
// IPv4-compatible and IPv4-mapped
"::255.255.255.255",
"::ffff:255.255.255.255",
"[::255.255.255.255]",
"[::ffff:255.255.255.255]",
"[::255.255.255.255]:65535",
"[::ffff:255.255.255.255]:65535",
"[::ffff:0:255.255.255.255]",
"[2001:db8:3:4::192.0.2.33]",
} {
if Scrub(addr) != "[scrubbed]" {
t.Error(cmp.Diff(addr, "[scrubbed]"))
}
}
}