2021-09-28 12:42:01 +02:00
|
|
|
package netxlite
|
2021-06-15 11:57:40 +02:00
|
|
|
|
|
|
|
import (
|
|
|
|
"context"
|
|
|
|
"errors"
|
2021-06-15 13:44:28 +02:00
|
|
|
"io"
|
2021-06-15 11:57:40 +02:00
|
|
|
"strings"
|
2021-09-07 22:41:34 +02:00
|
|
|
"sync"
|
2021-06-15 11:57:40 +02:00
|
|
|
"testing"
|
2021-09-05 14:49:38 +02:00
|
|
|
|
2022-01-03 13:53:23 +01:00
|
|
|
"github.com/ooni/probe-cli/v3/internal/model/mocks"
|
2021-06-15 11:57:40 +02:00
|
|
|
)
|
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
func TestReadAllContext(t *testing.T) {
|
|
|
|
t.Run("with success and background context", func(t *testing.T) {
|
|
|
|
r := strings.NewReader("deadbeef")
|
|
|
|
ctx := context.Background()
|
|
|
|
out, err := ReadAllContext(ctx, r)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if len(out) != 8 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
})
|
2021-06-15 11:57:40 +02:00
|
|
|
|
fix(netxlite): robust {ReadAll,Copy}Context with wrapped io.EOF (#661)
* chore(netxlite): add currently failing test case
This diff introduces a test cases that will fail because of the reason
explained in https://github.com/ooni/probe/issues/1965.
* chore(netxlite/iox_test.go): add failing unit tests
These tests directly show how the Go implementation of ReadAll
and Copy has the issue of checking for io.EOF equality.
* fix(netxlite): make {ReadAll,Copy}Context robust to wrapped io.EOF
The fix is simple: we just need to check for `errors.Is(err, io.EOF)`
after either io.ReadAll or io.Copy has returned. When this condition is
true, we need to convert the error back to `nil` as it ought to be.
While there, observe that the unit tests I committed in the previous
commit are wrongly asserting that the error must be wrapped. This
assertion is not correct, because in both cases we have just ensured
that the returned error is `nil` (i.e., success).
See https://github.com/ooni/probe/issues/1965.
* cleanup: remove previous workaround for wrapped io.EOF
These workarounds were partial, meaning that they would cover some
cases in which the issue occurred but not all of them.
Handling the problem in `netxlite.{ReadAll,Copy}Context` is the
right thing to do _as long as_ we always use these functions instead
of `io.{ReadAll,Copy}`.
This is why it's now important to ensure we clearly mention that
inside of the `CONTRIBUTING.md` guide and to also ensure that we're
not using these functions in the code base.
* fix(urlgetter): repair tests who assumed to see EOF error
Now that we have established that we should normalize EOF when
reading bodies like the stdlib does and now that it's clear why
our behavior diverged from the stdlib, we also need to repair
all the tests that assumed this incorrect behavior.
* fix(all): don't use io{,util}.{Copy,ReadAll}
* feat: add checks to ensure we don't use io.{Copy,ReadAll}
* doc(netxlite): document we know how to deal w/ wrapped io.EOF
* fix(nocopyreadall.bash): add exception for i/n/iox.go
2022-01-12 14:26:10 +01:00
|
|
|
t.Run("with success and wrapped io.EOF", func(t *testing.T) {
|
|
|
|
// See https://github.com/ooni/probe/issues/1965
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
// "When Read encounters an error or end-of-file condition
|
|
|
|
// after successfully reading n > 0 bytes, it returns
|
|
|
|
// the number of bytes read. It may return the (non-nil)
|
|
|
|
// error from the same call or return the error (and n == 0)
|
|
|
|
// from a subsequent call.""
|
|
|
|
//
|
|
|
|
// See https://pkg.go.dev/io#Reader
|
|
|
|
//
|
|
|
|
// Note: Returning a wrapped error to ensure we address
|
|
|
|
// https://github.com/ooni/probe/issues/1965
|
2022-07-01 12:22:22 +02:00
|
|
|
return len(b), NewErrWrapper(ClassifyGenericError,
|
fix(netxlite): robust {ReadAll,Copy}Context with wrapped io.EOF (#661)
* chore(netxlite): add currently failing test case
This diff introduces a test cases that will fail because of the reason
explained in https://github.com/ooni/probe/issues/1965.
* chore(netxlite/iox_test.go): add failing unit tests
These tests directly show how the Go implementation of ReadAll
and Copy has the issue of checking for io.EOF equality.
* fix(netxlite): make {ReadAll,Copy}Context robust to wrapped io.EOF
The fix is simple: we just need to check for `errors.Is(err, io.EOF)`
after either io.ReadAll or io.Copy has returned. When this condition is
true, we need to convert the error back to `nil` as it ought to be.
While there, observe that the unit tests I committed in the previous
commit are wrongly asserting that the error must be wrapped. This
assertion is not correct, because in both cases we have just ensured
that the returned error is `nil` (i.e., success).
See https://github.com/ooni/probe/issues/1965.
* cleanup: remove previous workaround for wrapped io.EOF
These workarounds were partial, meaning that they would cover some
cases in which the issue occurred but not all of them.
Handling the problem in `netxlite.{ReadAll,Copy}Context` is the
right thing to do _as long as_ we always use these functions instead
of `io.{ReadAll,Copy}`.
This is why it's now important to ensure we clearly mention that
inside of the `CONTRIBUTING.md` guide and to also ensure that we're
not using these functions in the code base.
* fix(urlgetter): repair tests who assumed to see EOF error
Now that we have established that we should normalize EOF when
reading bodies like the stdlib does and now that it's clear why
our behavior diverged from the stdlib, we also need to repair
all the tests that assumed this incorrect behavior.
* fix(all): don't use io{,util}.{Copy,ReadAll}
* feat: add checks to ensure we don't use io.{Copy,ReadAll}
* doc(netxlite): document we know how to deal w/ wrapped io.EOF
* fix(nocopyreadall.bash): add exception for i/n/iox.go
2022-01-12 14:26:10 +01:00
|
|
|
ReadOperation, io.EOF)
|
|
|
|
},
|
|
|
|
}
|
|
|
|
out, err := ReadAllContext(context.Background(), r)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if len(out) <= 0 {
|
|
|
|
t.Fatal("we expected to see a positive number of bytes here")
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
})
|
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with failure and background context", func(t *testing.T) {
|
|
|
|
expected := errors.New("mocked error")
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
return 0, expected
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
out, err := ReadAllContext(ctx, r)
|
|
|
|
if !errors.Is(err, expected) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if len(out) != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
})
|
2021-06-15 11:57:40 +02:00
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with success and cancelled context", func(t *testing.T) {
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
sigch := make(chan interface{})
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
<-sigch
|
|
|
|
// "When Read encounters an error or end-of-file condition
|
|
|
|
// after successfully reading n > 0 bytes, it returns
|
|
|
|
// the number of bytes read. It may return the (non-nil)
|
|
|
|
// error from the same call or return the error (and n == 0)
|
|
|
|
// from a subsequent call.""
|
|
|
|
//
|
|
|
|
// See https://pkg.go.dev/io#Reader
|
|
|
|
return len(b), io.EOF
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
cancel() // fail immediately
|
|
|
|
out, err := ReadAllContext(ctx, r)
|
|
|
|
if !errors.Is(err, context.Canceled) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if len(out) != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
close(sigch)
|
|
|
|
wg.Wait()
|
|
|
|
})
|
2021-06-15 11:57:40 +02:00
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with failure and cancelled context", func(t *testing.T) {
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
sigch := make(chan interface{})
|
|
|
|
expected := errors.New("mocked error")
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
<-sigch
|
|
|
|
return 0, expected
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
cancel() // fail immediately
|
|
|
|
out, err := ReadAllContext(ctx, r)
|
|
|
|
if !errors.Is(err, context.Canceled) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if len(out) != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
close(sigch)
|
|
|
|
wg.Wait()
|
|
|
|
})
|
2021-06-15 11:57:40 +02:00
|
|
|
}
|
2021-06-15 13:44:28 +02:00
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
func TestCopyContext(t *testing.T) {
|
|
|
|
t.Run("with success and background context", func(t *testing.T) {
|
|
|
|
r := strings.NewReader("deadbeef")
|
|
|
|
ctx := context.Background()
|
|
|
|
out, err := CopyContext(ctx, io.Discard, r)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if out != 8 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
})
|
2021-06-15 13:44:28 +02:00
|
|
|
|
fix(netxlite): robust {ReadAll,Copy}Context with wrapped io.EOF (#661)
* chore(netxlite): add currently failing test case
This diff introduces a test cases that will fail because of the reason
explained in https://github.com/ooni/probe/issues/1965.
* chore(netxlite/iox_test.go): add failing unit tests
These tests directly show how the Go implementation of ReadAll
and Copy has the issue of checking for io.EOF equality.
* fix(netxlite): make {ReadAll,Copy}Context robust to wrapped io.EOF
The fix is simple: we just need to check for `errors.Is(err, io.EOF)`
after either io.ReadAll or io.Copy has returned. When this condition is
true, we need to convert the error back to `nil` as it ought to be.
While there, observe that the unit tests I committed in the previous
commit are wrongly asserting that the error must be wrapped. This
assertion is not correct, because in both cases we have just ensured
that the returned error is `nil` (i.e., success).
See https://github.com/ooni/probe/issues/1965.
* cleanup: remove previous workaround for wrapped io.EOF
These workarounds were partial, meaning that they would cover some
cases in which the issue occurred but not all of them.
Handling the problem in `netxlite.{ReadAll,Copy}Context` is the
right thing to do _as long as_ we always use these functions instead
of `io.{ReadAll,Copy}`.
This is why it's now important to ensure we clearly mention that
inside of the `CONTRIBUTING.md` guide and to also ensure that we're
not using these functions in the code base.
* fix(urlgetter): repair tests who assumed to see EOF error
Now that we have established that we should normalize EOF when
reading bodies like the stdlib does and now that it's clear why
our behavior diverged from the stdlib, we also need to repair
all the tests that assumed this incorrect behavior.
* fix(all): don't use io{,util}.{Copy,ReadAll}
* feat: add checks to ensure we don't use io.{Copy,ReadAll}
* doc(netxlite): document we know how to deal w/ wrapped io.EOF
* fix(nocopyreadall.bash): add exception for i/n/iox.go
2022-01-12 14:26:10 +01:00
|
|
|
t.Run("with success and wrapped io.EOF", func(t *testing.T) {
|
|
|
|
// See https://github.com/ooni/probe/issues/1965
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
// "When Read encounters an error or end-of-file condition
|
|
|
|
// after successfully reading n > 0 bytes, it returns
|
|
|
|
// the number of bytes read. It may return the (non-nil)
|
|
|
|
// error from the same call or return the error (and n == 0)
|
|
|
|
// from a subsequent call.""
|
|
|
|
//
|
|
|
|
// See https://pkg.go.dev/io#Reader
|
|
|
|
//
|
|
|
|
// Note: Returning a wrapped error to ensure we address
|
|
|
|
// https://github.com/ooni/probe/issues/1965
|
2022-07-01 12:22:22 +02:00
|
|
|
return len(b), NewErrWrapper(ClassifyGenericError,
|
fix(netxlite): robust {ReadAll,Copy}Context with wrapped io.EOF (#661)
* chore(netxlite): add currently failing test case
This diff introduces a test cases that will fail because of the reason
explained in https://github.com/ooni/probe/issues/1965.
* chore(netxlite/iox_test.go): add failing unit tests
These tests directly show how the Go implementation of ReadAll
and Copy has the issue of checking for io.EOF equality.
* fix(netxlite): make {ReadAll,Copy}Context robust to wrapped io.EOF
The fix is simple: we just need to check for `errors.Is(err, io.EOF)`
after either io.ReadAll or io.Copy has returned. When this condition is
true, we need to convert the error back to `nil` as it ought to be.
While there, observe that the unit tests I committed in the previous
commit are wrongly asserting that the error must be wrapped. This
assertion is not correct, because in both cases we have just ensured
that the returned error is `nil` (i.e., success).
See https://github.com/ooni/probe/issues/1965.
* cleanup: remove previous workaround for wrapped io.EOF
These workarounds were partial, meaning that they would cover some
cases in which the issue occurred but not all of them.
Handling the problem in `netxlite.{ReadAll,Copy}Context` is the
right thing to do _as long as_ we always use these functions instead
of `io.{ReadAll,Copy}`.
This is why it's now important to ensure we clearly mention that
inside of the `CONTRIBUTING.md` guide and to also ensure that we're
not using these functions in the code base.
* fix(urlgetter): repair tests who assumed to see EOF error
Now that we have established that we should normalize EOF when
reading bodies like the stdlib does and now that it's clear why
our behavior diverged from the stdlib, we also need to repair
all the tests that assumed this incorrect behavior.
* fix(all): don't use io{,util}.{Copy,ReadAll}
* feat: add checks to ensure we don't use io.{Copy,ReadAll}
* doc(netxlite): document we know how to deal w/ wrapped io.EOF
* fix(nocopyreadall.bash): add exception for i/n/iox.go
2022-01-12 14:26:10 +01:00
|
|
|
ReadOperation, io.EOF)
|
|
|
|
},
|
|
|
|
}
|
|
|
|
out, err := CopyContext(context.Background(), io.Discard, r)
|
|
|
|
if err != nil {
|
|
|
|
t.Fatal(err)
|
|
|
|
}
|
|
|
|
if out <= 0 {
|
|
|
|
t.Fatal("we expected to see a positive number of bytes here")
|
|
|
|
}
|
|
|
|
wg.Wait()
|
|
|
|
})
|
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with failure and background context", func(t *testing.T) {
|
|
|
|
expected := errors.New("mocked error")
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
return 0, expected
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx := context.Background()
|
|
|
|
out, err := CopyContext(ctx, io.Discard, r)
|
|
|
|
if !errors.Is(err, expected) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if out != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
})
|
2021-06-15 13:44:28 +02:00
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with success and cancelled context", func(t *testing.T) {
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
sigch := make(chan interface{})
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
<-sigch
|
|
|
|
// "When Read encounters an error or end-of-file condition
|
|
|
|
// after successfully reading n > 0 bytes, it returns
|
|
|
|
// the number of bytes read. It may return the (non-nil)
|
|
|
|
// error from the same call or return the error (and n == 0)
|
|
|
|
// from a subsequent call.""
|
|
|
|
//
|
|
|
|
// See https://pkg.go.dev/io#Reader
|
|
|
|
return len(b), io.EOF
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
cancel() // fail immediately
|
|
|
|
out, err := CopyContext(ctx, io.Discard, r)
|
|
|
|
if !errors.Is(err, context.Canceled) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if out != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
close(sigch)
|
|
|
|
wg.Wait()
|
|
|
|
})
|
2021-06-15 13:44:28 +02:00
|
|
|
|
2021-09-07 22:41:34 +02:00
|
|
|
t.Run("with failure and cancelled context", func(t *testing.T) {
|
|
|
|
wg := &sync.WaitGroup{}
|
|
|
|
wg.Add(1)
|
|
|
|
sigch := make(chan interface{})
|
|
|
|
expected := errors.New("mocked error")
|
|
|
|
r := &mocks.Reader{
|
|
|
|
MockRead: func(b []byte) (int, error) {
|
|
|
|
defer wg.Done()
|
|
|
|
<-sigch
|
|
|
|
return 0, expected
|
|
|
|
},
|
|
|
|
}
|
|
|
|
ctx, cancel := context.WithCancel(context.Background())
|
|
|
|
cancel() // fail immediately
|
|
|
|
out, err := CopyContext(ctx, io.Discard, r)
|
|
|
|
if !errors.Is(err, context.Canceled) {
|
|
|
|
t.Fatal("not the error we expected", err)
|
|
|
|
}
|
2021-11-06 17:49:58 +01:00
|
|
|
var errWrapper *ErrWrapper
|
|
|
|
if !errors.As(err, &errWrapper) {
|
|
|
|
t.Fatal("the returned error is not wrapped")
|
|
|
|
}
|
2021-09-07 22:41:34 +02:00
|
|
|
if out != 0 {
|
|
|
|
t.Fatal("not the expected number of bytes")
|
|
|
|
}
|
|
|
|
close(sigch)
|
|
|
|
wg.Wait()
|
|
|
|
})
|
2021-06-15 13:44:28 +02:00
|
|
|
}
|