c/ooni-probe-cli
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Start fleshing out database related functions

Browse Source
This commit is contained in:
Arturo Filastò 2018-02-12 18:29:03 +02:00
parent 6008b5c7c5
commit ca231bca9f
8 changed files with 511 additions and 12 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

84
Gopkg.lock generated
View File

@ -37,6 +37,27 @@
packages = ["."]
revision = "cc83f3b3ce5911279513a46d6d3316d67bedaa54"
[[projects]]
branch = "master"
name = "github.com/beorn7/perks"
packages = ["quantile"]
revision = "4c0e84591b9aa9e6dcfdf3e020114cd81f89d5f9"
[[projects]]
name = "github.com/golang/protobuf"
packages = ["proto"]
revision = "925541529c1fa6821df4e44ce2723319eb2be768"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/jmoiron/sqlx"
packages = [
".",
"reflectx"
]
revision = "05cef0741ade10ca668982355b3f3f0bcf0ff0a8"
[[projects]]
name = "github.com/mattn/go-colorable"
packages = ["."]
@ -49,6 +70,18 @@
revision = "0360b2af4f38e8d38c7fce2a9f4e702702d73a39"
version = "v0.0.3"
[[projects]]
name = "github.com/mattn/go-sqlite3"
packages = ["."]
revision = "6c771bb9887719704b210e87e934f08be014bdb1"
version = "v1.6.0"
[[projects]]
name = "github.com/matttproud/golang_protobuf_extensions"
packages = ["pbutil"]
revision = "3247c84500bff8d9fb6d579d800f20b3e091582c"
version = "v1.0.0"
[[projects]]
branch = "master"
name = "github.com/mgutz/ansi"
@ -67,6 +100,49 @@
revision = "645ef00459ed84a119197bfb8d8205042c6df63d"
version = "v0.8.0"
[[projects]]
name = "github.com/prometheus/client_golang"
packages = ["prometheus"]
revision = "c5b7fccd204277076155f10851dad72b76a49317"
version = "v0.8.0"
[[projects]]
branch = "master"
name = "github.com/prometheus/client_model"
packages = ["go"]
revision = "99fa1f4be8e564e8a6b613da7fa6f46c9edafc6c"
[[projects]]
branch = "master"
name = "github.com/prometheus/common"
packages = [
"expfmt",
"internal/bitbucket.org/ww/goautoneg",
"model",
"version"
]
revision = "89604d197083d4781071d3c65855d24ecfb0a563"
[[projects]]
branch = "master"
name = "github.com/prometheus/procfs"
packages = [
".",
"internal/util",
"nfs",
"xfs"
]
revision = "282c8707aa210456a825798969cc27edda34992a"
[[projects]]
branch = "master"
name = "github.com/rubenv/sql-migrate"
packages = [
".",
"sqlparse"
]
revision = "f33734611e84d5fe45f35eccf0174f4836af4542"
[[projects]]
branch = "master"
name = "golang.org/x/sys"
@ -83,9 +159,15 @@
revision = "0aa8b6a162b391fe2d95648b7677d1d6ac2090a6"
version = "v1.4.1"
[[projects]]
name = "gopkg.in/gorp.v1"
packages = ["."]
revision = "c87af80f3cc5036b55b83d77171e156791085e2e"
version = "v1.7.1"
[solve-meta]
analyzer-name = "dep"
analyzer-version = 1
inputs-digest = "b014c3dfbccece13286bfddc98c8ec74007629647cd54df8d9af9d94985cdb17"
inputs-digest = "d1ff03c816b12576a88c6ab2c3ffab44bf0fd40a67d765965f06fd9ac9e58d99"
solver-name = "gps-cdcl"
solver-version = 1

12
Makefile
View File

@ -1,6 +1,18 @@
GO_BINDATA_VERSION = $(shell go-bindata --version | cut -d ' ' -f2 | head -n 1 || echo "missing")
REQ_GO_BINDATA_VERSION = 3.2.0
GO ?= go
build:
@echo "Building ./ooni"
@$(GO) build -o ooni cmd/ooni/main.go
.PHONY: build
bindata:
ifneq ($(GO_BINDATA_VERSION),$(REQ_GO_BINDATA_VERSION))
go get -u github.com/shuLhan/go-bindata/...;
endif
@go-bindata \
-nometadata \
-o internal/bindata/bindata.go -pkg bindata \
data;
.PHONY: bindata

46
data/migrations/1_create_msmt_results.sql Normal file
View File

@ -0,0 +1,46 @@
-- +migrate Down
-- +migrate StatementBegin
DROP TABLE `results`;
DROP TABLE `measurements`;
-- +migrate StatementEnd
-- +migrate Up
-- +migrate StatementBegin
CREATE TABLE `results` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`name` VARCHAR(255),
`startTime` DATETIME,
`endTime` DATETIME,
`summary` JSON,
`done` TINYINT(1),
`dataUsageUp` INTEGER,
`dataUsageDown` INTEGER,
`createdAt` DATETIME NOT NULL,
`updatedAt` DATETIME NOT NULL
);
CREATE TABLE `measurements` (
`id` INTEGER PRIMARY KEY AUTOINCREMENT,
`name` VARCHAR(255),
`startTime` DATETIME,
`endTime` DATETIME,
`summary` JSON,
`ip` VARCHAR(255),
`asn` INTEGER,
`country` VARCHAR(2),
`networkName` VARCHAR(255),
`state` TEXT,
`failure` VARCHAR(255),
`reportFile` VARCHAR(255),
`reportId` VARCHAR(255),
`input` VARCHAR(255),
`measurementId` VARCHAR(255),
`createdAt` DATETIME NOT NULL,
`updatedAt` DATETIME NOT NULL,
`resultId` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE
);
-- +migrate StatementEnd

263
internal/bindata/bindata.go Normal file
View File

@ -0,0 +1,263 @@
// Code generated by go-bindata. DO NOT EDIT.
// sources:
// data/default-config.json
package bindata
import (
"bytes"
"compress/gzip"
"fmt"
"io"
"io/ioutil"
"os"
"path/filepath"
"strings"
"time"
)
func bindataRead(data []byte, name string) ([]byte, error) {
gz, err := gzip.NewReader(bytes.NewBuffer(data))
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
}
var buf bytes.Buffer
_, err = io.Copy(&buf, gz)
clErr := gz.Close()
if err != nil {
return nil, fmt.Errorf("Read %q: %v", name, err)
}
if clErr != nil {
return nil, err
}
return buf.Bytes(), nil
}
type asset struct {
bytes []byte
info fileInfoEx
}
type fileInfoEx interface {
os.FileInfo
MD5Checksum() string
}
type bindataFileInfo struct {
name string
size int64
mode os.FileMode
modTime time.Time
md5checksum string
}
func (fi bindataFileInfo) Name() string {
return fi.name
}
func (fi bindataFileInfo) Size() int64 {
return fi.size
}
func (fi bindataFileInfo) Mode() os.FileMode {
return fi.mode
}
func (fi bindataFileInfo) ModTime() time.Time {
return fi.modTime
}
func (fi bindataFileInfo) MD5Checksum() string {
return fi.md5checksum
}
func (fi bindataFileInfo) IsDir() bool {
return false
}
func (fi bindataFileInfo) Sys() interface{} {
return nil
}
var _dataDefaultConfigJson = []byte(
"\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\xcd\x6e\xe3\x38\x0c\xc7\xef\x7d\x0a\x41\xe7\x3a\xcd\x62\x6f\x39" +
"\xee\x6d\x0f\xbb\x1d\x60\xe6\x56\x14\x82\x62\xd1\x36\x31\x32\xa9\x11\xe9\x64\x82\x41\xdf\x7d\x20\x35\x89\xed\x7e" +
"\x4d\x8f\xe2\x9f\xa2\xc8\x1f\x29\xfe\xba\x31\xc6\x3a\xbb\x33\xf6\xdb\x80\x62\x50\xcc\x89\xa7\x6c\xee\xef\xff\xff" +
"\xd7\x7c\xc9\xbc\x07\xd3\x32\x75\xd8\x9b\x0e\x23\x6c\xcc\x57\x00\x33\xa8\x26\xd9\xdd\xdd\x31\x13\x6e\x90\xef\x06" +
"\x88\xa9\x1e\x52\xf1\x6f\xda\x88\xa6\xe3\x6c\x8a\xd9\xde\x96\xf0\x7e\x52\x76\x53\x0a\x5e\xc1\xee\x8c\xe6\x09\xaa" +
"\x59\x06\x9f\x91\x7a\xbb\x33\x25\x09\x63\x2c\x52\x1b\xa7\x00\x0e\x93\xdd\x99\xce\x47\xa9\x7e\x0b\xc1\x0b\x2d\x02" +
"\x2c\x84\x3e\xc9\x5a\x98\x52\x64\x1f\x5c\x06\x99\xa2\xbe\xd0\x04\x28\xb8\x36\x7b\x19\x5c\x86\xc4\xf9\xaa\xdf\x18" +
"\xf3\x54\x33\x23\x56\xec\xb0\xf5\x8a\x4c\x32\xe7\x07\xe4\xf7\x11\xc2\x3a\x5a\xf5\x3d\x39\x26\xa7\x20\xea\x5a\x1e" +
"\x53\x84\x72\xf1\x3d\x37\x82\xa3\x5c\xea\xbb\xbe\x58\x10\x8d\x5e\x21\xd4\x28\x2b\x2a\xf3\xab\x4b\x24\x67\x6b\x75" +
"\x2f\xe1\x1e\xaa\xd9\x18\x7b\x84\x7d\xd3\x32\x11\xb4\x8a\x07\xd4\x93\xbd\xbd\x28\x9d\x6f\x61\xcf\xfc\xbd\x19\x41" +
"\x04\xa8\x87\x3c\x6b\xc7\xc1\xab\xf8\x94\x66\x8b\x42\x84\x3e\xfb\x71\xb6\x04\x2f\xc3\x7c\xa2\xa0\xf3\xa1\x8c\x44" +
"\x83\x74\xf0\x11\x43\x93\xe1\xc7\x04\xa2\x4d\x44\x82\x17\x2e\x03\xf8\x00\xb9\xe9\x10\x62\x68\x46\x4f\x98\xa6\x58" +
"\x29\xdb\xea\xf6\x78\x2e\x6e\x64\xd2\x21\x9e\x9c\x8f\x91\x8f\x9e\xda\x32\x36\xf6\xef\xed\xf6\xbf\x7f\xec\x95\x58" +
"\xa5\x2d\xa0\x05\xd6\xa2\x47\x47\xd8\x0b\x2a\xcc\x96\x05\xab\xd6\x2b\xf4\x9c\xb1\xaa\x0f\x8f\x55\x7e\xba\x4e\x92" +
"\xa8\x27\x75\x85\x8d\xef\x97\x0d\xf8\x00\xf6\xc7\x50\xdf\xc2\xba\x04\x7b\x36\xad\xf3\x48\x90\x3b\xce\xe3\xb9\xe8" +
"\xcf\x64\x50\x1a\x71\x09\xb5\xec\x8e\x13\xc8\x07\xc8\x05\x5d\x99\x2e\xfb\x86\xe6\xca\xf4\xbf\x76\x28\x8d\x7e\xf7" +
"\xf6\x42\x5c\x5f\x5f\x95\x31\x62\x08\x11\xf6\xfc\xf3\x93\x45\xfc\x79\x80\x3e\x39\x42\x57\x9e\xf3\xd7\x0a\x87\x42" +
"\x33\xbc\xde\x33\x2d\x4f\xa4\xf9\xf4\x62\x73\x08\xb8\xc0\xa3\x47\x72\x5d\x66\x3a\xff\xc5\xd5\x7a\x70\xcf\x2b\xd1" +
"\x1d\x20\xcb\xf3\x47\xb7\xdb\xcd\x76\xf3\xd7\xf3\xb6\x73\x48\xa5\x83\x65\xde\x98\x04\x48\x2f\xd7\x9f\x6e\x7e\x07" +
"\x00\x00\xff\xff\x0f\x7e\x15\xb3\x6d\x05\x00\x00")
func dataDefaultConfigJsonBytes() ([]byte, error) {
return bindataRead(
_dataDefaultConfigJson,
"data/default-config.json",
)
}
func dataDefaultConfigJson() (*asset, error) {
bytes, err := dataDefaultConfigJsonBytes()
if err != nil {
return nil, err
}
info := bindataFileInfo{name: "data/default-config.json", size: 0, md5checksum: "", mode: os.FileMode(0), modTime: time.Unix(0, 0)}
a := &asset{bytes: bytes, info: info}
return a, nil
}
// Asset loads and returns the asset for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func Asset(name string) ([]byte, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("Asset %s can't read by error: %v", name, err)
}
return a.bytes, nil
}
return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
}
// MustAsset is like Asset but panics when Asset would return an error.
// It simplifies safe initialization of global variables.
// nolint: deadcode
func MustAsset(name string) []byte {
a, err := Asset(name)
if err != nil {
panic("asset: Asset(" + name + "): " + err.Error())
}
return a
}
// AssetInfo loads and returns the asset info for the given name.
// It returns an error if the asset could not be found or
// could not be loaded.
func AssetInfo(name string) (os.FileInfo, error) {
cannonicalName := strings.Replace(name, "\\", "/", -1)
if f, ok := _bindata[cannonicalName]; ok {
a, err := f()
if err != nil {
return nil, fmt.Errorf("AssetInfo %s can't read by error: %v", name, err)
}
return a.info, nil
}
return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
}
// AssetNames returns the names of the assets.
// nolint: deadcode
func AssetNames() []string {
names := make([]string, 0, len(_bindata))
for name := range _bindata {
names = append(names, name)
}
return names
}
// _bindata is a table, holding each asset generator, mapped to its name.
var _bindata = map[string]func() (*asset, error){
"data/default-config.json": dataDefaultConfigJson,
}
// AssetDir returns the file names below a certain
// directory embedded in the file by go-bindata.
// For example if you run go-bindata on data/... and data contains the
// following hierarchy:
// data/
// foo.txt
// img/
// a.png
// b.png
// then AssetDir("data") would return []string{"foo.txt", "img"}
// AssetDir("data/img") would return []string{"a.png", "b.png"}
// AssetDir("foo.txt") and AssetDir("notexist") would return an error
// AssetDir("") will return []string{"data"}.
func AssetDir(name string) ([]string, error) {
node := _bintree
if len(name) != 0 {
cannonicalName := strings.Replace(name, "\\", "/", -1)
pathList := strings.Split(cannonicalName, "/")
for _, p := range pathList {
node = node.Children[p]
if node == nil {
return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
}
}
}
if node.Func != nil {
return nil, &os.PathError{Op: "open", Path: name, Err: os.ErrNotExist}
}
rv := make([]string, 0, len(node.Children))
for childName := range node.Children {
rv = append(rv, childName)
}
return rv, nil
}
type bintree struct {
Func func() (*asset, error)
Children map[string]*bintree
}
var _bintree = &bintree{nil, map[string]*bintree{
"data": {nil, map[string]*bintree{
"default-config.json": {dataDefaultConfigJson, map[string]*bintree{}},
}},
}}
// RestoreAsset restores an asset under the given directory
func RestoreAsset(dir, name string) error {
data, err := Asset(name)
if err != nil {
return err
}
info, err := AssetInfo(name)
if err != nil {
return err
}
err = os.MkdirAll(_filePath(dir, filepath.Dir(name)), os.FileMode(0755))
if err != nil {
return err
}
err = ioutil.WriteFile(_filePath(dir, name), data, info.Mode())
if err != nil {
return err
}
return os.Chtimes(_filePath(dir, name), info.ModTime(), info.ModTime())
}
// RestoreAssets restores an asset under the given directory recursively
func RestoreAssets(dir, name string) error {
children, err := AssetDir(name)
// File
if err != nil {
return RestoreAsset(dir, name)
}
// Dir
for _, child := range children {
err = RestoreAssets(dir, filepath.Join(name, child))
if err != nil {
return err
}
}
return nil
}
func _filePath(dir, name string) string {
cannonicalName := strings.Replace(name, "\\", "/", -1)
return filepath.Join(append([]string{dir}, strings.Split(cannonicalName, "/")...)...)
}

21
internal/cli/root/root.go
View File

@ -5,6 +5,7 @@ import (
"github.com/apex/log"
"github.com/apex/log/handlers/cli"
ooni "github.com/openobservatory/gooni"
"github.com/openobservatory/gooni/internal/database"
"github.com/prometheus/common/version"
)
@ -15,7 +16,7 @@ var Cmd = kingpin.New("ooni", "")
var Command = Cmd.Command
// Init should be called by all subcommand that care to have a ooni.OONI instance
var Init func() (*ooni.Config, *ooni.OONI, error)
var Init func() (*ooni.Config, *ooni.Context, error)
func init() {
configPath := Cmd.Flag("config", "Set a custom config file path").Short('c').String()
@ -28,19 +29,33 @@ func init() {
log.Debugf("ooni version %s", version.Version)
}
Init = func() (*ooni.Config, *ooni.OONI, error) {
Init = func() (*ooni.Config, *ooni.Context, error) {
var c *ooni.Config
var err error
if *configPath != "" {
log.Debugf("Reading config file from %s", *configPath)
c, err = ooni.ReadConfig(*configPath)
} else {
log.Debug("Reading default config file")
c, err = ooni.ReadDefaultConfigPaths()
}
if err != nil {
return nil, nil, err
}
o := ooni.New(c)
dbPath, err := DefaultDatabasePath()
if err != nil {
return nil, nil, err
}
log.Debugf("Connecting to database sqlite3://%s", dbPath)
db, err := database.Connect(dbPath)
if err != nil {
return nil, nil, err
}
o := ooni.New(c, db)
o.Init()
return c, o, nil
}

44
internal/database/database.go Normal file
View File

@ -0,0 +1,44 @@
package database
import (
"path/filepath"
"github.com/apex/log"
"github.com/jmoiron/sqlx"
_ "github.com/mattn/go-sqlite3"
"github.com/openobservatory/gooni/internal/bindata"
"github.com/pkg/errors"
migrate "github.com/rubenv/sql-migrate"
)
// RunMigrations runs the database migrations
func RunMigrations(db *sqlx.DB) error {
migrations := &migrate.AssetMigrationSource{
Asset: bindata.Asset,
AssetDir: bindata.AssetDir,
Dir: "data/migrations",
}
n, err := migrate.Exec(db.DB, "sqlite3", migrations, migrate.Up)
if err != nil {
return err
}
log.Debugf("performed %d migrations", n)
return nil
}
func Connect(path string) (*sqlx.DB, error) {
db, err := sqlx.Connect("sqlite3", path)
if err != nil {
return nil, err
}
// XXX RunMigrations(db)
return db, nil
}
func DefaultDatabasePath() (string, error) {
home, err := GetOONIHome()
if err != nil {
return errors.Wrap(err, "default database path")
}
return filepath.Join(home, "db", "main.db"), nil
}

34
internal/database/models.go Normal file
View File

@ -0,0 +1,34 @@
package database
import "time"
// Measurement model
type Measurement struct {
ID int `db:"id"`
Name string `db:"name"`
StartTime time.Time `db:"startTime"`
EndTime time.Time `db:"endTime"`
Summary string `db:"summary"` // XXX this should be JSON
ASN int `db:"asn"`
IP string `db:"ip"`
CountryCode string `db:"country"`
State string `db:"state"`
Failure string `db:"failure"`
ReportFilePath string `db:"reportFile"`
ReportID string `db:"reportId"`
Input string `db:"input"`
MeasurementID string `db:"measurementId"`
ResultID string `db:"resultId"`
}
// Result model
type Result struct {
ID int `db:"id"`
Name int `db:"name"`
StartTime time.Time `db:"startTime"`
EndTime time.Time `db:"endTime"`
Summary string `db:"summary"` // XXX this should be JSON
Done bool `db:"done"`
DataUsageUp int `db:"dataUsageUp"`
DataUsageDown int `db:"dataUsageDown"`
}

19
ooni.go
View File

@ -8,6 +8,7 @@ import (
"sync"
"github.com/apex/log"
"github.com/jmoiron/sqlx"
homedir "github.com/mitchellh/go-homedir"
"github.com/openobservatory/gooni/config"
"github.com/openobservatory/gooni/internal/legacy"
@ -30,28 +31,30 @@ func Onboarding(c *Config) error {
return nil
}
// OONI manager.
type OONI struct {
// Context for OONI Probe
type Context struct {
config *Config
db *sqlx.DB
}
// Init the OONI manager
func (o *OONI) Init() error {
func (c *Context) Init() error {
if err := legacy.MaybeMigrateHome(); err != nil {
return errors.Wrap(err, "migrating home")
}
if o.config.InformedConsent == false {
if err := Onboarding(o.config); err != nil {
if c.config.InformedConsent == false {
if err := Onboarding(c.config); err != nil {
return errors.Wrap(err, "onboarding")
}
}
return nil
}
// New OONI manager instance.
func New(c *Config) *OONI {
return &OONI{
// New Context instance.
func New(c *Config, d *sqlx.DB) *Context {
return &Context{
config: c,
db: d,
}
}