refactor(sessionresolver): adapt to changing network conditions (#238)
* feat(sessionresolver): try many and use what works * fix(sessionresolver): make sure we can use quic * fix: the config struct is unnecessary * fix: make kvstore optional * feat: write simple integration test * feat: start adding tests * feat: continue writing tests * fix(sessionresolver): add more unit tests * fix(sessionresolver): finish adding tests * refactor(sessionresolver): changes after code review
This commit is contained in:
Simone Basso
GitHub
parent
12e1164940
commit
034db78f94
19 changed files with 1260 additions and 66 deletions
|
|
@ -1,85 +1,134 @@
|
|||
// Package sessionresolver contains the resolver used by the session. This
|
||||
// resolver uses Powerdns DoH by default and falls back on the system
|
||||
// provided resolver if Powerdns DoH is not working.
|
||||
// resolver will try to figure out which is the best service for running
|
||||
// domain name resolutions and will consistently use it.
|
||||
//
|
||||
// Occasionally this code will also swap the best resolver with other
|
||||
// ~good resolvers to give them a chance to perform.
|
||||
//
|
||||
// The penalty/reward mechanism is strongly derivative, so the code should
|
||||
// adapt ~quickly to changing network conditions. Occasionally, we will
|
||||
// have longer resolutions when trying out other resolvers.
|
||||
//
|
||||
// At the beginning we randomize the known resolvers so that we do not
|
||||
// have any preferential ordering. The initial resolutions may be slower
|
||||
// if there are many issues with resolvers.
|
||||
//
|
||||
// The system resolver is given the lowest priority at the beginning
|
||||
// but it will of course be the most popular resolver if anything else
|
||||
// is failing us. (We will still occasionally probe for other working
|
||||
// resolvers and increase their score on success.)
|
||||
package sessionresolver
|
||||
|
||||
import (
|
||||
"context"
|
||||
"encoding/json"
|
||||
"errors"
|
||||
"fmt"
|
||||
"math/rand"
|
||||
"sync"
|
||||
"time"
|
||||
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/atomicx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/internal/multierror"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/netx/bytecounter"
|
||||
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
|
||||
)
|
||||
|
||||
// Resolver is the session resolver.
|
||||
// Resolver is the session resolver. You should create an instance of
|
||||
// this structure and use it in session.go.
|
||||
type Resolver struct {
|
||||
Primary netx.DNSClient
|
||||
PrimaryFailure *atomicx.Int64
|
||||
PrimaryQuery *atomicx.Int64
|
||||
Fallback netx.DNSClient
|
||||
FallbackFailure *atomicx.Int64
|
||||
FallbackQuery *atomicx.Int64
|
||||
ByteCounter *bytecounter.Counter // optional
|
||||
KVStore KVStore // optional
|
||||
Logger Logger // optional
|
||||
codec codec
|
||||
dnsClientMaker dnsclientmaker
|
||||
mu sync.Mutex
|
||||
once sync.Once
|
||||
res map[string]childResolver
|
||||
}
|
||||
|
||||
// New creates a new session resolver.
|
||||
func New(config netx.Config) *Resolver {
|
||||
primary, err := netx.NewDNSClientWithOverrides(config,
|
||||
"https://cloudflare.com/dns-query", "dns.cloudflare.com", "", "")
|
||||
runtimex.PanicOnError(err, "cannot create dns over https resolver")
|
||||
fallback, err := netx.NewDNSClient(config, "system:///")
|
||||
runtimex.PanicOnError(err, "cannot create system resolver")
|
||||
return &Resolver{
|
||||
Primary: primary,
|
||||
PrimaryFailure: atomicx.NewInt64(),
|
||||
PrimaryQuery: atomicx.NewInt64(),
|
||||
Fallback: fallback,
|
||||
FallbackFailure: atomicx.NewInt64(),
|
||||
FallbackQuery: atomicx.NewInt64(),
|
||||
}
|
||||
}
|
||||
|
||||
// CloseIdleConnections closes the idle connections, if any
|
||||
// CloseIdleConnections closes the idle connections, if any. This
|
||||
// function is guaranteed to be idempotent.
|
||||
func (r *Resolver) CloseIdleConnections() {
|
||||
r.Primary.CloseIdleConnections()
|
||||
r.Fallback.CloseIdleConnections()
|
||||
r.once.Do(r.closeall)
|
||||
}
|
||||
|
||||
// Stats returns stats about the session resolver.
|
||||
func (r *Resolver) Stats() string {
|
||||
return fmt.Sprintf("sessionresolver: failure rate: primary: %d/%d; fallback: %d/%d",
|
||||
r.PrimaryFailure.Load(), r.PrimaryQuery.Load(),
|
||||
r.FallbackFailure.Load(), r.FallbackQuery.Load())
|
||||
data, err := json.Marshal(r.readstatedefault())
|
||||
runtimex.PanicOnError(err, "json.Marshal should not fail here")
|
||||
return fmt.Sprintf("sessionresolver: %s", string(data))
|
||||
}
|
||||
|
||||
// LookupHost implements Resolver.LookupHost
|
||||
// ErrLookupHost indicates that LookupHost failed.
|
||||
var ErrLookupHost = errors.New("sessionresolver: LookupHost failed")
|
||||
|
||||
// LookupHost implements Resolver.LookupHost. This function returns a
|
||||
// multierror.Union error on failure, so you can see individual errors
|
||||
// and get a better picture of what's been going wrong.
|
||||
func (r *Resolver) LookupHost(ctx context.Context, hostname string) ([]string, error) {
|
||||
// Algorithm similar to Firefox TRR2 mode. See:
|
||||
// https://wiki.mozilla.org/Trusted_Recursive_Resolver#DNS-over-HTTPS_Prefs_in_Firefox
|
||||
// We use a higher timeout than Firefox's timeout (1.5s) to be on the safe side
|
||||
// and therefore see to use DoH more often.
|
||||
r.PrimaryQuery.Add(1)
|
||||
trr2, cancel := context.WithTimeout(ctx, 4*time.Second)
|
||||
defer cancel()
|
||||
addrs, err := r.Primary.LookupHost(trr2, hostname)
|
||||
if err != nil {
|
||||
r.PrimaryFailure.Add(1)
|
||||
r.FallbackQuery.Add(1)
|
||||
addrs, err = r.Fallback.LookupHost(ctx, hostname)
|
||||
if err != nil {
|
||||
r.FallbackFailure.Add(1)
|
||||
state := r.readstatedefault()
|
||||
r.maybeConfusion(state, time.Now().UnixNano())
|
||||
defer r.writestate(state)
|
||||
me := multierror.New(ErrLookupHost)
|
||||
for _, e := range state {
|
||||
addrs, err := r.lookupHost(ctx, e, hostname)
|
||||
if err == nil {
|
||||
return addrs, nil
|
||||
}
|
||||
me.Add(&errwrapper{error: err, URL: e.URL})
|
||||
}
|
||||
return addrs, err
|
||||
return nil, me
|
||||
}
|
||||
|
||||
// Network implements Resolver.Network
|
||||
func (r *Resolver) lookupHost(ctx context.Context, ri *resolverinfo, hostname string) ([]string, error) {
|
||||
const ewma = 0.9 // the last sample is very important
|
||||
re, err := r.getresolver(ri.URL)
|
||||
if err != nil {
|
||||
r.logger().Warnf("sessionresolver: getresolver: %s", err.Error())
|
||||
ri.Score = 0 // this is a hard error
|
||||
return nil, err
|
||||
}
|
||||
addrs, err := r.timeLimitedLookup(ctx, re, hostname)
|
||||
if err == nil {
|
||||
r.logger().Infof("sessionresolver: %s... %v", ri.URL, nil)
|
||||
ri.Score = ewma*1.0 + (1-ewma)*ri.Score // increase score
|
||||
return addrs, nil
|
||||
}
|
||||
r.logger().Warnf("sessionresolver: %s... %s", ri.URL, err.Error())
|
||||
ri.Score = ewma*0.0 + (1-ewma)*ri.Score // decrease score
|
||||
return nil, err
|
||||
}
|
||||
|
||||
// maybeConfusion will rearrange the first elements of the vector
|
||||
// with low probability, so giving other resolvers a chance
|
||||
// to run and show that they are also viable. We do not fully
|
||||
// reorder the vector because that could lead to long runtimes.
|
||||
//
|
||||
// The return value is only meaningful for testing.
|
||||
func (r *Resolver) maybeConfusion(state []*resolverinfo, seed int64) int {
|
||||
rng := rand.New(rand.NewSource(seed))
|
||||
const confusion = 0.3
|
||||
if rng.Float64() >= confusion {
|
||||
return -1
|
||||
}
|
||||
switch len(state) {
|
||||
case 0, 1: // nothing to do
|
||||
return 0
|
||||
case 2:
|
||||
state[0], state[1] = state[1], state[0]
|
||||
return 2
|
||||
default:
|
||||
state[0], state[2] = state[2], state[0]
|
||||
return 3
|
||||
}
|
||||
}
|
||||
|
||||
// Network implements Resolver.Network.
|
||||
func (r *Resolver) Network() string {
|
||||
return "sessionresolver"
|
||||
}
|
||||
|
||||
// Address implements Resolver.Address
|
||||
// Address implements Resolver.Address.
|
||||
func (r *Resolver) Address() string {
|
||||
return ""
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue