refactor: now utils is an internal package (#166)

Let us make internal everything because this is not a library.

internal/utils/homedir/homedir.go

@@ -0,0 +1,211 @@
+package homedir
+
+// Stolen from: https://github.com/puma/puma-dev/blob/master/homedir/homedir.go
+
+import (
+	"bytes"
+	"errors"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"strconv"
+	"strings"
+	"sync"
+)
+
+// DisableCache will disable caching of the home directory. Caching is enabled
+// by default.
+var DisableCache bool
+
+var homedirCache string
+var cacheLock sync.RWMutex
+
+// ErrNoHomeDir when no home dir could be found
+var ErrNoHomeDir = errors.New("no home directory available")
+
+// Dir returns the home directory for the executing user.
+//
+// This uses an OS-specific method for discovering the home directory.
+// An error is returned if a home directory cannot be detected.
+func Dir() (string, error) {
+	if !DisableCache {
+		cacheLock.RLock()
+		cached := homedirCache
+		cacheLock.RUnlock()
+		if cached != "" {
+			return cached, nil
+		}
+	}
+
+	cacheLock.Lock()
+	defer cacheLock.Unlock()
+
+	var result string
+	var err error
+	switch runtime.GOOS {
+	case "windows":
+		result, err = dirWindows()
+	case "darwin":
+		result, err = dirDarwin()
+	default:
+		// Unix-like system, so just assume Unix
+		result, err = dirUnix()
+	}
+
+	if err != nil {
+		return "", err
+	}
+
+	homedirCache = result
+	return result, nil
+}
+
+// Expand expands the path to include the home directory if the path
+// is prefixed with `~`. If it isn't prefixed with `~`, the path is
+// returned as-is.
+func Expand(path string) (string, error) {
+	if len(path) == 0 {
+		return path, nil
+	}
+
+	if path[0] != '~' {
+		return path, nil
+	}
+
+	if len(path) > 1 && path[1] != '/' && path[1] != '\\' {
+		return "", errors.New("cannot expand user-specific home dir")
+	}
+
+	dir, err := Dir()
+	if err != nil {
+		return "", err
+	}
+
+	return filepath.Join(dir, path[1:]), nil
+}
+
+func MustExpand(path string) string {
+	str, err := Expand(path)
+	if err != nil {
+		panic(err)
+	}
+
+	return str
+}
+
+func dirDarwin() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	var stdout bytes.Buffer
+
+	// If that fails, try OS specific commands
+	cmd := exec.Command("sh", "-c", `dscl -q . -read /Users/"$(whoami)" NFSHomeDirectory | sed 's/^[^ ]*: //'`)
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result != "" {
+			return result, nil
+		}
+	}
+
+	// try the shell
+	stdout.Reset()
+	cmd = exec.Command("sh", "-c", "cd && pwd")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result != "" {
+			return result, nil
+		}
+	}
+
+	// try to figure out the user and check the default location
+	stdout.Reset()
+	cmd = exec.Command("whoami")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		user := strings.TrimSpace(stdout.String())
+
+		path := "/Users/" + user
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			return path, nil
+		}
+	}
+
+	return "", ErrNoHomeDir
+}
+
+func dirUnix() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	var stdout bytes.Buffer
+
+	// If that fails, try OS specific commands
+	cmd := exec.Command("getent", "passwd", strconv.Itoa(os.Getuid()))
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		if passwd := strings.TrimSpace(stdout.String()); passwd != "" {
+			// username:password:uid:gid:gecos:home:shell
+			passwdParts := strings.SplitN(passwd, ":", 7)
+			if len(passwdParts) > 5 {
+				return passwdParts[5], nil
+			}
+		}
+	}
+
+	// If all else fails, try the shell
+	stdout.Reset()
+	cmd = exec.Command("sh", "-c", "cd && pwd")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		result := strings.TrimSpace(stdout.String())
+		if result == "" {
+			return "", errors.New("blank output when reading home directory")
+		}
+	}
+
+	// try to figure out the user and check the default location
+	stdout.Reset()
+	cmd = exec.Command("whoami")
+	cmd.Stdout = &stdout
+	if err := cmd.Run(); err == nil {
+		user := strings.TrimSpace(stdout.String())
+
+		path := "/home/" + user
+
+		stat, err := os.Stat(path)
+		if err == nil && stat.IsDir() {
+			return path, nil
+		}
+	}
+
+	return "", ErrNoHomeDir
+}
+
+func dirWindows() (string, error) {
+	// First prefer the HOME environmental variable
+	if home := os.Getenv("HOME"); home != "" {
+		return home, nil
+	}
+
+	drive := os.Getenv("HOMEDRIVE")
+	path := os.Getenv("HOMEPATH")
+	home := drive + path
+	if drive == "" || path == "" {
+		home = os.Getenv("USERPROFILE")
+	}
+	if home == "" {
+		return "", errors.New("HOMEDRIVE, HOMEPATH, and USERPROFILE are blank")
+	}
+
+	return home, nil
+}

internal/utils/paths.go

@@ -0,0 +1,77 @@
+package utils
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"path/filepath"
+	"time"
+
+	"github.com/ooni/probe-cli/internal/utils/homedir"
+)
+
+// RequiredDirs returns the required ooni home directories
+func RequiredDirs(home string) []string {
+	requiredDirs := []string{}
+	requiredSubdirs := []string{"assets", "db", "msmts"}
+	for _, d := range requiredSubdirs {
+		requiredDirs = append(requiredDirs, filepath.Join(home, d))
+	}
+	return requiredDirs
+}
+
+// ConfigPath returns the default path to the config file
+func ConfigPath(home string) string {
+	return filepath.Join(home, "config.json")
+}
+
+// AssetsDir returns the assets data dir for the given OONI Home
+func AssetsDir(home string) string {
+	return filepath.Join(home, "assets")
+}
+
+// EngineDir returns the directory where ooni/probe-engine should
+// store its private data given a specific OONI Home.
+func EngineDir(home string) string {
+	return filepath.Join(home, "engine")
+}
+
+// DBDir returns the database dir for the given name
+func DBDir(home string, name string) string {
+	return filepath.Join(home, "db", fmt.Sprintf("%s.sqlite3", name))
+}
+
+// ResultTimestamp is a windows friendly timestamp
+const ResultTimestamp = "2006-01-02T150405.999999999Z0700"
+
+// MakeResultsDir creates and returns a directory for the result
+func MakeResultsDir(home string, name string, ts time.Time) (string, error) {
+	p := filepath.Join(home, "msmts",
+		fmt.Sprintf("%s-%s", name, ts.Format(ResultTimestamp)))
+
+	// If the path already exists, this is a problem. It should not clash, because
+	// we are using nanosecond precision for the starttime.
+	if _, e := os.Stat(p); e == nil {
+		return "", errors.New("results path already exists")
+	}
+	err := os.MkdirAll(p, 0700)
+	if err != nil {
+		return "", err
+	}
+	return p, nil
+}
+
+// GetOONIHome returns the path to the OONI Home
+func GetOONIHome() (string, error) {
+	if ooniHome := os.Getenv("OONI_HOME"); ooniHome != "" {
+		return ooniHome, nil
+	}
+
+	home, err := homedir.Dir()
+	if err != nil {
+		return "", err
+	}
+
+	path := filepath.Join(home, ".ooniprobe")
+	return path, nil
+}

internal/utils/shutil/shutil.go

@@ -0,0 +1,326 @@
+package shutil
+
+import (
+  "fmt"
+  "io"
+  "io/ioutil"
+  "os"
+  "path/filepath"
+)
+
+
+type SameFileError struct {
+  Src string
+  Dst string
+}
+
+func (e SameFileError) Error() string {
+  return fmt.Sprintf("%s and %s are the same file", e.Src, e.Dst)
+}
+
+type SpecialFileError struct {
+  File string
+  FileInfo os.FileInfo
+}
+
+func (e SpecialFileError) Error() string {
+  return fmt.Sprintf("`%s` is a named pipe", e.File)
+}
+
+type NotADirectoryError struct {
+  Src string
+}
+
+func (e NotADirectoryError) Error() string {
+  return fmt.Sprintf("`%s` is not a directory", e.Src)
+}
+
+
+type AlreadyExistsError struct {
+  Dst string
+}
+
+func (e AlreadyExistsError) Error() string {
+  return fmt.Sprintf("`%s` already exists", e.Dst)
+}
+
+
+func samefile(src string, dst string) bool {
+  srcInfo, _ := os.Stat(src)
+  dstInfo, _ := os.Stat(dst)
+  return os.SameFile(srcInfo, dstInfo)
+}
+
+func specialfile(fi os.FileInfo) bool {
+  return (fi.Mode() & os.ModeNamedPipe) == os.ModeNamedPipe
+}
+
+func stringInSlice(a string, list []string) bool {
+    for _, b := range list {
+        if b == a {
+            return true
+        }
+    }
+    return false
+}
+
+func IsSymlink(fi os.FileInfo) bool {
+  return (fi.Mode() & os.ModeSymlink) == os.ModeSymlink
+}
+
+
+// Copy data from src to dst
+//
+// If followSymlinks is not set and src is a symbolic link, a
+// new symlink will be created instead of copying the file it points
+// to.
+func CopyFile(src, dst string, followSymlinks bool) (error) {
+  if samefile(src, dst) {
+    return &SameFileError{src, dst}
+  }
+
+  // Make sure src exists and neither are special files
+  srcStat, err := os.Lstat(src)
+  if err != nil {
+    return err
+  }
+  if specialfile(srcStat) {
+    return &SpecialFileError{src, srcStat}
+  }
+
+  dstStat, err := os.Stat(dst)
+  if err != nil && !os.IsNotExist(err) {
+    return err
+  } else if err == nil {
+    if specialfile(dstStat) {
+      return &SpecialFileError{dst, dstStat}
+    }
+  }
+
+  // If we don't follow symlinks and it's a symlink, just link it and be done
+  if !followSymlinks && IsSymlink(srcStat) {
+    return os.Symlink(src, dst)
+  }
+
+  // If we are a symlink, follow it
+  if IsSymlink(srcStat) {
+    src, err = os.Readlink(src)
+    if err != nil {
+      return err
+    }
+    srcStat, err = os.Stat(src)
+    if err != nil {
+      return err
+    }
+  }
+
+  // Do the actual copy
+  fsrc, err := os.Open(src)
+  if err != nil {
+    return err
+  }
+  defer fsrc.Close()
+
+  fdst, err := os.Create(dst)
+  if err != nil {
+    return err
+  }
+  defer fdst.Close()
+
+  size, err := io.Copy(fdst, fsrc)
+  if err != nil {
+    return err
+  }
+
+  if size != srcStat.Size() {
+    return fmt.Errorf("%s: %d/%d copied", src, size, srcStat.Size())
+  }
+
+  return nil
+}
+
+
+// Copy mode bits from src to dst.
+//
+// If followSymlinks is false, symlinks aren't followed if and only
+// if both `src` and `dst` are symlinks. If `lchmod` isn't available
+// and both are symlinks this does nothing. (I don't think lchmod is
+// available in Go)
+func CopyMode(src, dst string, followSymlinks bool) error {
+  srcStat, err := os.Lstat(src)
+  if err != nil {
+    return err
+  }
+
+  dstStat, err := os.Lstat(dst)
+  if err != nil {
+    return err
+  }
+
+  // They are both symlinks and we can't change mode on symlinks.
+  if !followSymlinks && IsSymlink(srcStat) && IsSymlink(dstStat) {
+    return nil
+  }
+
+  // Atleast one is not a symlink, get the actual file stats
+  srcStat, _ = os.Stat(src)
+  err = os.Chmod(dst, srcStat.Mode())
+  return err
+}
+
+
+// Copy data and mode bits ("cp src dst"). Return the file's destination.
+//
+// The destination may be a directory.
+//
+// If followSymlinks is false, symlinks won't be followed. This
+// resembles GNU's "cp -P src dst".
+//
+// If source and destination are the same file, a SameFileError will be
+// rased.
+func Copy(src, dst string, followSymlinks bool) (string, error){
+  dstInfo, err := os.Stat(dst)
+
+  if err == nil && dstInfo.Mode().IsDir() {
+    dst = filepath.Join(dst, filepath.Base(src))
+  }
+
+  if err != nil && !os.IsNotExist(err) {
+    return dst, err
+  }
+
+  err = CopyFile(src, dst, followSymlinks)
+  if err != nil {
+    return dst, err
+  }
+
+  err = CopyMode(src, dst, followSymlinks)
+  if err != nil {
+    return dst, err
+  }
+
+  return dst, nil
+}
+
+type CopyTreeOptions struct {
+  Symlinks bool
+  IgnoreDanglingSymlinks bool
+  CopyFunction func (string, string, bool) (string, error)
+  Ignore func (string, []os.FileInfo) []string
+}
+
+// Recursively copy a directory tree.
+//
+// The destination directory must not already exist.
+//
+// If the optional Symlinks flag is true, symbolic links in the
+// source tree result in symbolic links in the destination tree; if
+// it is false, the contents of the files pointed to by symbolic
+// links are copied. If the file pointed by the symlink doesn't
+// exist, an error will be returned.
+//
+// You can set the optional IgnoreDanglingSymlinks flag to true if you
+// want to silence this error. Notice that this has no effect on
+// platforms that don't support os.Symlink.
+//
+// The optional ignore argument is a callable. If given, it
+// is called with the `src` parameter, which is the directory
+// being visited by CopyTree(), and `names` which is the list of
+// `src` contents, as returned by ioutil.ReadDir():
+//
+//   callable(src, entries) -> ignoredNames
+//
+// Since CopyTree() is called recursively, the callable will be
+// called once for each directory that is copied. It returns a
+// list of names relative to the `src` directory that should
+// not be copied.
+//
+// The optional copyFunction argument is a callable that will be used
+// to copy each file. It will be called with the source path and the
+// destination path as arguments. By default, Copy() is used, but any
+// function that supports the same signature (like Copy2() when it
+// exists) can be used.
+func CopyTree(src, dst string, options *CopyTreeOptions) error {
+  if options == nil {
+    options = &CopyTreeOptions{Symlinks:false,
+                               Ignore:nil,
+                               CopyFunction:Copy,
+                               IgnoreDanglingSymlinks:false}
+  }
+
+
+  srcFileInfo, err := os.Stat(src)
+  if err != nil {
+    return err
+  }
+
+  if !srcFileInfo.IsDir() {
+    return &NotADirectoryError{src}
+  }
+
+  _, err = os.Open(dst)
+  if !os.IsNotExist(err) {
+    return &AlreadyExistsError{dst}
+  }
+
+  entries, err := ioutil.ReadDir(src)
+  if err != nil {
+    return err
+  }
+
+  err = os.MkdirAll(dst, srcFileInfo.Mode())
+  if err != nil {
+    return err
+  }
+
+  ignoredNames := []string{}
+  if options.Ignore != nil {
+    ignoredNames = options.Ignore(src, entries)
+  }
+
+  for _, entry := range entries {
+    if stringInSlice(entry.Name(), ignoredNames) {
+      continue
+    }
+    srcPath := filepath.Join(src, entry.Name())
+    dstPath := filepath.Join(dst, entry.Name())
+
+    entryFileInfo, err := os.Lstat(srcPath)
+    if err != nil {
+      return err
+    }
+
+    // Deal with symlinks
+    if IsSymlink(entryFileInfo) {
+      linkTo, err := os.Readlink(srcPath)
+      if err != nil {
+        return err
+      }
+      if options.Symlinks {
+        os.Symlink(linkTo, dstPath)
+        //CopyStat(srcPath, dstPath, false)
+      } else {
+        // ignore dangling symlink if flag is on
+        _, err = os.Stat(linkTo)
+        if os.IsNotExist(err) && options.IgnoreDanglingSymlinks {
+          continue
+        }
+        _, err = options.CopyFunction(srcPath, dstPath, false)
+        if err != nil {
+          return err
+        }
+      }
+    } else if entryFileInfo.IsDir() {
+      err = CopyTree(srcPath, dstPath, options)
+      if err != nil {
+        return err
+      }
+    } else {
+      _, err = options.CopyFunction(srcPath, dstPath, false)
+      if err != nil {
+        return err
+      }
+    }
+  }
+  return nil
+}

internal/utils/util_test.go

@@ -0,0 +1,18 @@
+package utils
+
+import (
+	"testing"
+
+	"github.com/fatih/color"
+)
+
+func TestEscapeAwareRuneCountInString(t *testing.T) {
+	var bold = color.New(color.Bold)
+	var myColor = color.New(color.FgBlue)
+
+	s := myColor.Sprintf("•ABC%s%s", bold.Sprintf("DEF"), "\x1B[00;38;5;244m\x1B[m\x1B[00;38;5;33mGHI\x1B[0m")
+	count := EscapeAwareRuneCountInString(s)
+	if count != 10 {
+		t.Errorf("Count was incorrect, got: %d, want: %d.", count, 10)
+	}
+}

internal/utils/utils.go

@@ -0,0 +1,117 @@
+package utils
+
+import (
+	"bytes"
+	"fmt"
+	"os"
+	"regexp"
+	"strings"
+	"unicode"
+	"unicode/utf8"
+
+	"github.com/fatih/color"
+)
+
+// Log outputs a log message.
+func Log(msg string, v ...interface{}) {
+	fmt.Printf("     %s\n", color.CyanString(msg, v...))
+}
+
+// Fatal error
+func Fatal(err error) {
+	fmt.Fprintf(os.Stderr, "\n     %s %s\n\n", color.RedString("Error:"), err)
+	os.Exit(1)
+}
+
+// Finds the ansi escape sequences (like colors)
+// Taken from: https://github.com/chalk/ansi-regex/blob/d9d806ecb45d899cf43408906a4440060c5c50e5/index.js
+var ansiEscapes = regexp.MustCompile(`[\x1B\x9B][[\]()#;?]*` +
+	`(?:(?:(?:[a-zA-Z\d]*(?:;[a-zA-Z\\d]*)*)?\x07)` +
+	`|(?:(?:\d{1,4}(?:;\d{0,4})*)?[\dA-PRZcf-ntqry=><~]))`)
+
+// EscapeAwareRuneCountInString counts the number of runes in a
+// string taking into account escape sequences.
+func EscapeAwareRuneCountInString(s string) int {
+	n := utf8.RuneCountInString(s)
+	for _, sm := range ansiEscapes.FindAllString(s, -1) {
+		n -= utf8.RuneCountInString(sm)
+	}
+	return n
+}
+
+// RightPadd adds right padding in from of a string
+func RightPad(str string, length int) string {
+	c := length - EscapeAwareRuneCountInString(str)
+	if c < 0 {
+		c = 0
+	}
+	return str + strings.Repeat(" ", c)
+}
+
+// WrapString wraps the given string within lim width in characters.
+//
+// Wrapping is currently naive and only happens at white-space. A future
+// version of the library will implement smarter wrapping. This means that
+// pathological cases can dramatically reach past the limit, such as a very
+// long word.
+// This is taken from: https://github.com/mitchellh/go-wordwrap/tree/f253961a26562056904822f2a52d4692347db1bd
+func WrapString(s string, lim uint) string {
+	// Initialize a buffer with a slightly larger size to account for breaks
+	init := make([]byte, 0, len(s))
+	buf := bytes.NewBuffer(init)
+
+	var current uint
+	var wordBuf, spaceBuf bytes.Buffer
+
+	for _, char := range s {
+		if char == '\n' {
+			if wordBuf.Len() == 0 {
+				if current+uint(spaceBuf.Len()) > lim {
+					current = 0
+				} else {
+					current += uint(spaceBuf.Len())
+					spaceBuf.WriteTo(buf)
+				}
+				spaceBuf.Reset()
+			} else {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+			buf.WriteRune(char)
+			current = 0
+		} else if unicode.IsSpace(char) {
+			if spaceBuf.Len() == 0 || wordBuf.Len() > 0 {
+				current += uint(spaceBuf.Len() + wordBuf.Len())
+				spaceBuf.WriteTo(buf)
+				spaceBuf.Reset()
+				wordBuf.WriteTo(buf)
+				wordBuf.Reset()
+			}
+
+			spaceBuf.WriteRune(char)
+		} else {
+
+			wordBuf.WriteRune(char)
+
+			if current+uint(spaceBuf.Len()+wordBuf.Len()) > lim && uint(wordBuf.Len()) < lim {
+				buf.WriteRune('\n')
+				current = 0
+				spaceBuf.Reset()
+			}
+		}
+	}
+
+	if wordBuf.Len() == 0 {
+		if current+uint(spaceBuf.Len()) <= lim {
+			spaceBuf.WriteTo(buf)
+		}
+	} else {
+		spaceBuf.WriteTo(buf)
+		wordBuf.WriteTo(buf)
+	}
+
+	return buf.String()
+}