refactor: move more commands to internal/cmd (#207)

* refactor: move more commands to internal/cmd

Part of https://github.com/ooni/probe/issues/1335.

We would like all commands to be at the same level of engine
rather than inside engine (now that we can do it).

* fix: update .gitignore

* refactor: also move jafar outside engine

* We should be good now?
This commit is contained in:
Simone Basso
2021-02-03 12:23:15 +01:00
committed by GitHub
parent 6351d898d6
commit 4eeadd06a5
85 changed files with 72 additions and 65 deletions
+98
View File
@@ -0,0 +1,98 @@
// Package iptables contains code for managing firewall rules. This package
// really only works reliably on Linux. In all other systems the functionality
// in here is just a set of stubs returning errors.
package iptables
import (
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
)
type shell interface {
createChains() error
dropIfDestinationEquals(ip string) error
rstIfDestinationEqualsAndIsTCP(ip string) error
dropIfContainsKeywordHex(keyword string) error
dropIfContainsKeyword(keyword string) error
rstIfContainsKeywordHexAndIsTCP(keyword string) error
rstIfContainsKeywordAndIsTCP(keyword string) error
hijackDNS(address string) error
hijackHTTPS(address string) error
hijackHTTP(address string) error
waive() error
}
// CensoringPolicy implements a censoring policy.
type CensoringPolicy struct {
DropIPs []string // drop IP traffic to these IPs
DropKeywordsHex []string // drop IP packets with these hex keywords
DropKeywords []string // drop IP packets with these keywords
HijackDNSAddress string // where to hijack DNS to
HijackHTTPSAddress string // where to hijack HTTPS to
HijackHTTPAddress string // where to hijack HTTP to
ResetIPs []string // RST TCP/IP traffic to these IPs
ResetKeywordsHex []string // RST TCP/IP flows with these hex keywords
ResetKeywords []string // RST TCP/IP flows with these keywords
sh shell
}
// NewCensoringPolicy returns a new censoring policy.
func NewCensoringPolicy() *CensoringPolicy {
return &CensoringPolicy{
sh: newShell(),
}
}
// Apply applies the censorship policy
func (c *CensoringPolicy) Apply() (err error) {
defer func() {
if recover() != nil {
// JUST KNOW WE'VE BEEN HERE
}
}()
err = c.sh.createChains()
runtimex.PanicOnError(err, "c.sh.createChains failed")
// Implementation note: we want the RST rules to be first such
// that we end up enforcing them before the drop rules.
for _, keyword := range c.ResetKeywordsHex {
err = c.sh.rstIfContainsKeywordHexAndIsTCP(keyword)
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordHexAndIsTCP failed")
}
for _, keyword := range c.ResetKeywords {
err = c.sh.rstIfContainsKeywordAndIsTCP(keyword)
runtimex.PanicOnError(err, "c.sh.rstIfContainsKeywordAndIsTCP failed")
}
for _, ip := range c.ResetIPs {
err = c.sh.rstIfDestinationEqualsAndIsTCP(ip)
runtimex.PanicOnError(err, "c.sh.rstIfDestinationEqualsAndIsTCP failed")
}
for _, keyword := range c.DropKeywordsHex {
err = c.sh.dropIfContainsKeywordHex(keyword)
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeywordHex failed")
}
for _, keyword := range c.DropKeywords {
err = c.sh.dropIfContainsKeyword(keyword)
runtimex.PanicOnError(err, "c.sh.dropIfContainsKeyword failed")
}
for _, ip := range c.DropIPs {
err = c.sh.dropIfDestinationEquals(ip)
runtimex.PanicOnError(err, "c.sh.dropIfDestinationEquals failed")
}
if c.HijackDNSAddress != "" {
err = c.sh.hijackDNS(c.HijackDNSAddress)
runtimex.PanicOnError(err, "c.sh.hijackDNS failed")
}
if c.HijackHTTPSAddress != "" {
err = c.sh.hijackHTTPS(c.HijackHTTPSAddress)
runtimex.PanicOnError(err, "c.sh.hijackHTTPS failed")
}
if c.HijackHTTPAddress != "" {
err = c.sh.hijackHTTP(c.HijackHTTPAddress)
runtimex.PanicOnError(err, "c.sh.hijackHTTP failed")
}
return
}
// Waive removes any censorship policy
func (c *CensoringPolicy) Waive() error {
return c.sh.waive()
}
@@ -0,0 +1,345 @@
package iptables
import (
"context"
"errors"
"net"
"net/http"
"net/http/httptest"
"net/url"
"os/exec"
"runtime"
"strings"
"testing"
"time"
"github.com/apex/log"
"github.com/ooni/probe-cli/v3/internal/cmd/jafar/resolver"
"github.com/ooni/probe-cli/v3/internal/cmd/jafar/uncensored"
"github.com/ooni/probe-cli/v3/internal/engine/shellx"
)
func init() {
log.SetLevel(log.ErrorLevel)
}
func newCensoringPolicy() *CensoringPolicy {
policy := NewCensoringPolicy()
policy.Waive() // start over to allow for repeated tests on failure
return policy
}
func TestCannotApplyPolicy(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropIPs = []string{"antani"}
if err := policy.Apply(); err == nil {
t.Fatal("expected an error here")
}
}
func TestCreateChainsError(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
// you should not be able to apply the policy when there is
// already a policy, you need to waive it first
if err := policy.Apply(); err == nil {
t.Fatal("expected an error here")
}
}
func TestDropIP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropIPs = []string{"1.1.1.1"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
conn, err := (&net.Dialer{}).DialContext(ctx, "tcp", "1.1.1.1:853")
if err == nil {
t.Fatalf("expected an error here")
}
if err.Error() != "dial tcp 1.1.1.1:853: i/o timeout" {
t.Fatal("unexpected error occurred")
}
if conn != nil {
t.Fatal("expected nil connection here")
}
}
func TestDropKeyword(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropKeywords = []string{"ooni.io"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err == nil {
t.Fatal("expected an error here")
}
if !strings.HasSuffix(err.Error(), "context deadline exceeded") {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestDropKeywordHex(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.DropKeywordsHex = []string{"|6f 6f 6e 69|"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
ctx, cancel := context.WithTimeout(context.Background(), time.Second)
defer cancel()
req, err := http.NewRequest("GET", "http://www.ooni.io", nil)
if err != nil {
t.Fatal(err)
}
resp, err := http.DefaultClient.Do(req.WithContext(ctx))
if err == nil {
t.Fatal("expected an error here")
}
// the error we see with GitHub Actions is different from the error
// we see when testing locally on Fedora
if !strings.HasSuffix(err.Error(), "operation not permitted") &&
!strings.HasSuffix(err.Error(), "Temporary failure in name resolution") &&
!strings.HasSuffix(err.Error(), "no such host") {
t.Fatalf("unexpected error occurred: %+v", err)
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestResetIP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetIPs = []string{"1.1.1.1"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
conn, err := (&net.Dialer{}).Dial("tcp", "1.1.1.1:853")
if err == nil {
t.Fatalf("expected an error here")
}
if err.Error() != "dial tcp 1.1.1.1:853: connect: connection refused" {
t.Fatal("unexpected error occurred")
}
if conn != nil {
t.Fatal("expected nil connection here")
}
}
func TestResetKeyword(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetKeywords = []string{"ooni.io"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
resp, err := http.Get("http://www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestResetKeywordHex(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
policy := newCensoringPolicy()
defer policy.Waive()
policy.ResetKeywordsHex = []string{"|6f 6f 6e 69|"}
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
resp, err := http.Get("http://www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "read: connection reset by peer") == false {
t.Fatal("unexpected error occurred")
}
if resp != nil {
t.Fatal("expected nil response here")
}
}
func TestHijackDNS(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
resolver := resolver.NewCensoringResolver(
[]string{"ooni.io"}, nil, nil,
uncensored.Must(uncensored.NewClient("dot://1.1.1.1:853")),
)
server, err := resolver.Start("127.0.0.1:0")
if err != nil {
t.Fatal(err)
}
defer server.Shutdown()
policy := newCensoringPolicy()
defer policy.Waive()
policy.HijackDNSAddress = server.PacketConn.LocalAddr().String()
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
addrs, err := net.LookupHost("www.ooni.io")
if err == nil {
t.Fatal("expected an error here")
}
if strings.Contains(err.Error(), "no such host") == false {
t.Fatal("unexpected error occurred")
}
if addrs != nil {
t.Fatal("expected nil addrs here")
}
}
func TestHijackHTTP(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
// Implementation note: this test is complicated by the fact
// that we are running as root and so we're whitelisted.
server := httptest.NewServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(451)
}),
)
defer server.Close()
policy := newCensoringPolicy()
defer policy.Waive()
pu, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
policy.HijackHTTPAddress = pu.Host
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
err = shellx.Run("sudo", "-u", "nobody", "--",
"curl", "-sf", "http://example.com")
if err == nil {
t.Fatal("expected an error here")
}
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatal("not the error type we expected")
}
if exitErr.ExitCode() != 22 {
t.Fatal("not the exit code we expected")
}
}
func TestHijackHTTPS(t *testing.T) {
if runtime.GOOS != "linux" {
t.Skip("not implemented on this platform")
}
if testing.Short() {
t.Skip("skip test in short mode")
}
// Implementation note: this test is complicated by the fact
// that we are running as root and so we're whitelisted.
server := httptest.NewTLSServer(http.HandlerFunc(
func(w http.ResponseWriter, r *http.Request) {
w.WriteHeader(451)
}),
)
defer server.Close()
policy := newCensoringPolicy()
defer policy.Waive()
pu, err := url.Parse(server.URL)
if err != nil {
t.Fatal(err)
}
policy.HijackHTTPSAddress = pu.Host
if err := policy.Apply(); err != nil {
t.Fatal(err)
}
err = shellx.Run("sudo", "-u", "nobody", "--",
"curl", "-sf", "https://example.com")
if err == nil {
t.Fatal("expected an error here")
}
t.Log(err)
var exitErr *exec.ExitError
if !errors.As(err, &exitErr) {
t.Fatal("not the error type we expected")
}
if exitErr.ExitCode() != 60 {
t.Fatal("not the exit code we expected")
}
}
@@ -0,0 +1,117 @@
// +build linux
package iptables
import (
"github.com/ooni/probe-cli/v3/internal/engine/runtimex"
"github.com/ooni/probe-cli/v3/internal/engine/shellx"
)
type linuxShell struct{}
func (s *linuxShell) createChains() (err error) {
defer func() {
if recover() != nil {
// JUST KNOW WE'VE BEEN HERE
}
}()
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_INPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_INPUT chain")
err = shellx.Run("sudo", "iptables", "-N", "JAFAR_OUTPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_OUTPUT chain")
err = shellx.Run("sudo", "iptables", "-t", "nat", "-N", "JAFAR_NAT_OUTPUT")
runtimex.PanicOnError(err, "cannot create JAFAR_NAT_OUTPUT chain")
err = shellx.Run("sudo", "iptables", "-I", "OUTPUT", "-j", "JAFAR_OUTPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_OUTPUT")
err = shellx.Run("sudo", "iptables", "-I", "INPUT", "-j", "JAFAR_INPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_INPUT")
err = shellx.Run("sudo", "iptables", "-t", "nat", "-I", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
runtimex.PanicOnError(err, "cannot insert jump to JAFAR_NAT_OUTPUT")
return nil
}
func (s *linuxShell) dropIfDestinationEquals(ip string) error {
return shellx.Run("sudo", "iptables", "-A", "JAFAR_OUTPUT", "-d", ip, "-j", "DROP")
}
func (s *linuxShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "--proto", "tcp", "-d", ip,
"-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) dropIfContainsKeywordHex(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
"--hex-string", keyword, "-j", "DROP",
)
}
func (s *linuxShell) dropIfContainsKeyword(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--algo", "kmp",
"--string", keyword, "-j", "DROP",
)
}
func (s *linuxShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
"kmp", "--hex-string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
return shellx.Run(
"sudo", "iptables", "-A", "JAFAR_OUTPUT", "-m", "string", "--proto", "tcp", "--algo",
"kmp", "--string", keyword, "-j", "REJECT", "--reject-with", "tcp-reset",
)
}
func (s *linuxShell) hijackDNS(address string) error {
// Hijack any DNS query, like the Vodafone station does when using the
// secure network feature. Our transparent proxies will use DoT, in order
// to bypass this restriction and avoid routing loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "udp",
"--dport", "53", "-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) hijackHTTPS(address string) error {
// We need to whitelist root otherwise the traffic sent by Jafar
// itself will match the rule and loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
"--dport", "443", "-m", "owner", "!", "--uid-owner", "0",
"-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) hijackHTTP(address string) error {
// We need to whitelist root otherwise the traffic sent by Jafar
// itself will match the rule and loop.
return shellx.Run(
"sudo", "iptables", "-t", "nat", "-A", "JAFAR_NAT_OUTPUT", "-p", "tcp",
"--dport", "80", "-m", "owner", "!", "--uid-owner", "0",
"-j", "DNAT", "--to", address,
)
}
func (s *linuxShell) waive() error {
shellx.RunQuiet("sudo", "iptables", "-D", "OUTPUT", "-j", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-D", "INPUT", "-j", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-D", "OUTPUT", "-j", "JAFAR_NAT_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_INPUT")
shellx.RunQuiet("sudo", "iptables", "-F", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-X", "JAFAR_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-F", "JAFAR_NAT_OUTPUT")
shellx.RunQuiet("sudo", "iptables", "-t", "nat", "-X", "JAFAR_NAT_OUTPUT")
return nil
}
func newShell() *linuxShell {
return &linuxShell{}
}
@@ -0,0 +1,45 @@
// +build !linux
package iptables
import "errors"
type otherwiseShell struct{}
func (*otherwiseShell) createChains() error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfDestinationEquals(ip string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfDestinationEqualsAndIsTCP(ip string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfContainsKeywordHex(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) dropIfContainsKeyword(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfContainsKeywordHexAndIsTCP(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) rstIfContainsKeywordAndIsTCP(keyword string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackDNS(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackHTTPS(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) hijackHTTP(address string) error {
return errors.New("not implemented")
}
func (*otherwiseShell) waive() error {
return errors.New("not implemented")
}
func newShell() *otherwiseShell {
return &otherwiseShell{}
}