From 8efb71dce835b4f62b6a69a7c2f215c9dc785c20 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 7 Aug 2018 17:51:05 +0200 Subject: [PATCH 01/65] Update the schema for the measurements and results tables Based on discussions with @xanscale and @lorenzoPrimi in: https://github.com/ooni/probe-cli/issues/14 --- data/migrations/1_create_msmt_results.sql | 173 ++++++++++++++++++---- 1 file changed, 147 insertions(+), 26 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index ebe2576..4992704 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -3,6 +3,8 @@ DROP TABLE `results`; DROP TABLE `measurements`; +DROP TABLE `urls`; +DROP TABLE `networks`; -- +migrate StatementEnd @@ -11,36 +13,155 @@ DROP TABLE `measurements`; CREATE TABLE `results` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `name` VARCHAR(255), - `start_time` DATETIME, - `runtime` REAL, - `summary` JSON, - `done` TINYINT(1), - `country` VARCHAR(2), - `asn` VARCHAR(16), - `network_name` VARCHAR(255), - `data_usage_up` INTEGER, - `data_usage_down` INTEGER + -- This can be one of "websites", "im", "performance", "middlebox". + `test_group_name` VARCHAR(16) NOT NULL, + -- We use a different start_time and runtime, because we want to also have + -- data to measure the overhead of creating a report and other factors that + -- go into the test. + -- That is to say: `SUM(runtime) FROM measurements` will always be <= + -- `runtime FROM results` (most times <) + `start_time` DATETIME NOT NULL, + `runtime` REAL NOT NULL, + + -- Used to indicate if the user has seen this result + `is_viewed` TINYINT(1) NOT NULL, + + -- This is a flag used to indicate if the result is done or is currently running. + `is_done` TINYINT(1) NOT NULL, + `data_usage_up` INTEGER NOT NULL, + `data_usage_down` INTEGER NOT NULL, + -- It's probably reasonable to set the maximum length to 260 as this is the + -- maximum length of file paths on windows. + `log_file_path` VARCHAR(260) NOT NULL ); CREATE TABLE `measurements` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `name` VARCHAR(255), - `start_time` DATETIME, - `runtime` REAL, - `summary` JSON, - `ip` VARCHAR(255), - `asn` VARCHAR(16), - `country` VARCHAR(2), - `network_name` VARCHAR(255), - `state` TEXT, - `failure` VARCHAR(255), - `upload_failure` VARCHAR(255), - `uploaded` TINYINT(1), - `report_file` VARCHAR(255), - `report_id` VARCHAR(255), - `input` VARCHAR(255), - `result_id` INTEGER REFERENCES `results` (`id`) ON DELETE SET NULL ON UPDATE CASCADE + -- This can be one of: + -- facebook_messenger + -- telegram + -- whatsapp + -- http_header_field_manipulation + -- http_invalid_request_line + -- dash + -- ndt + `test_name` VARCHAR(64) NOT NULL, + `start_time` DATETIME NOT NULL, + `runtime` REAL NOT NULL, + + -- For the purpose of populating the probe information in the results + -- views, you should pick the first measurement in the JOIN sorted by + -- start_time. + -- You don't have the guarantee that every (ip, asn, country, network_name) + -- is the same in a "measurement set" associated to a "result". + `network_id` INTEGER NOT NULL, + FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`), + + -- Note for golang: we used to have state be one of `done` and `active`, so + -- this is equivalent to done being true or false. + -- `state` TEXT, + `is_done` TINYINT(1) NOT NULL, + -- The reason to have a dedicated is_uploaded flag, instead of just using + -- is_upload_failed, is that we may not have uploaded the measurement due + -- to a setting. + `is_uploaded` TINYINT(1) NOT NULL, + + -- This is the measurement failed to run and the user should be offerred to + -- re-run it. + `is_failed` TINYINT(1) NOT NULL, + `failure_msg` VARCHAR(255), + + `is_upload_failed` TINYINT(1) NOT NULL, + `upload_failure_msg` VARCHAR(255), + + -- Is used to indicate that this particular measurement has been re-run and + -- therefore the UI can take this into account to either hide it from the + -- result view or at the very least disable the ability to re-run it. + -- XXX do we also want to have a reference to the re-run measurement? + `is_rerun` TINYINT(1) NOT NULL, + + -- This is the server-side report_id returned by the collector. By using + -- report_id & input, you can query the api to fetch this measurement. + -- Ex. + -- GET https://api.ooni.io/api/v1/measurements?input=$INPUT&report_id=$REPORT_ID + -- Extract the first item from the `result[]` list and then fetch: + -- `measurement_url` to get the JSON of this measurement row. + -- These two values (`report_id`, `input`) are useful to fetch a + -- measurement that has already been processed by the pipeline, to + -- implement cleanup of already uploaded measurements. + `report_id` VARCHAR(255), -- This can be NULL when no report file has been + -- created. + + `url_id` INTEGER NOT NULL, + FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`), + + -- This is not yet a feature of the collector, but we are planning to add + -- this at some point in the near future. + -- See: https://github.com/ooni/pipeline/blob/master/docs/ooni-uuid.md & + -- https://github.com/ooni/pipeline/issues/48 + `measurement_id` INT(64), + + -- This indicates in the case of a websites test, that a site is likely + -- blocked, or for an IM test if the IM tests says the app is likely + -- blocked, or if a middlebox was detected. + -- You can `JOIN` a `COUNT()` of this value in the results view to get a count of + -- blocked sites or blocked IM apps + `is_anomaly` TINYINT(1), + + -- This is an opaque JSON structure, where we store some of the test_keys + -- we need for the measurement details views and some result views (ex. the + -- upload/download speed of NDT, the reason for blocking of a site, + -- etc.) + `test_keys` JSON, + + -- The cross table reference to JOIN the two tables together. + `result_id` INTEGER NOT NULL, + FOREIGN KEY (`result_id`) REFERENCES `results`(`id`) + ON DELETE CASCADE ON UPDATE CASCADE, -- If we delete a result we also want + -- all the measurements to be deleted as well. + + -- This is a variable used internally to track the path to the on-disk + -- measurements.json. It may make sense to write one file per entry by + -- hooking MK and preventing it from writing to a file on disk which may + -- have many measurements per file. + `report_file_path` VARCHAR(260) NOT NULL, +); + +CREATE TABLE `urls` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `input` VARCHAR(255) NOT NULL, + `category_code` VARCHAR(5) NOT NULL, -- The citizenlab category code for the + -- site. We use the string NONE to denote + -- no known category code. + + `country_code` VARCHAR(2) NOT NULL -- The two letter country code which this + -- URL belongs to +) + +-- We create a separate table for networks for 2 reasons: +-- 1. For some of the views where need the total number of measured networks, +-- it's going to be much more efficient to just lookup the count of rows in this +-- table. +-- 2. (most important) We want to avoid duplicating a bunch of information that +-- is going to be common to several networks the user is on. +-- Example: +-- We may wish to add to this table the location from of the probe from the GPS +-- or add support for allowing the user to "correct" a misclassified measurement +-- or distinguishing between wifi and mobile. +CREATE TABLE `networks` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `network_name` VARCHAR(255), -- String name representing the network_name which by default is populated based + -- on the ASN. + -- We use a separate key to reference the rows in + -- this tables, because we may wish to "enrich" + -- this with more data in the future. + + `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. + -- The longest ip is an ipv6 address like: + -- 0000:0000:0000:0000:0000:0000:0000:0000, + -- which is 39 chars. + `asn` INT(4) NOT NULL, + `country` VARCHAR(2) NOT NULL, -- The two letter country code ); -- +migrate StatementEnd From 6d2ca54590314b2442a60901d33e6d429bb9ff84 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 7 Aug 2018 19:32:59 +0200 Subject: [PATCH 02/65] Minor stylistic changes to schema --- data/migrations/1_create_msmt_results.sql | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 4992704..3d06340 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -129,7 +129,7 @@ CREATE TABLE `measurements` ( CREATE TABLE `urls` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `input` VARCHAR(255) NOT NULL, + `url` VARCHAR(255) NOT NULL, -- XXX is this long enough? `category_code` VARCHAR(5) NOT NULL, -- The citizenlab category code for the -- site. We use the string NONE to denote -- no known category code. @@ -161,7 +161,7 @@ CREATE TABLE `networks` ( -- 0000:0000:0000:0000:0000:0000:0000:0000, -- which is 39 chars. `asn` INT(4) NOT NULL, - `country` VARCHAR(2) NOT NULL, -- The two letter country code + `country_code` VARCHAR(2) NOT NULL, -- The two letter country code ); -- +migrate StatementEnd From 5c807e1161116c81f7fc5e5496a45527d389d6ba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 5 Sep 2018 14:29:47 +0200 Subject: [PATCH 03/65] Add network_type to the schema --- data/migrations/1_create_msmt_results.sql | 1 + 1 file changed, 1 insertion(+) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 3d06340..c1c2bc1 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -155,6 +155,7 @@ CREATE TABLE `networks` ( -- We use a separate key to reference the rows in -- this tables, because we may wish to "enrich" -- this with more data in the future. + `network_type` VARCHAR(16), -- One of wifi, mobile `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. -- The longest ip is an ipv6 address like: From a512421033cdbcbc4c641263b5b6a15bafd8ef7f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 5 Sep 2018 17:58:06 +0200 Subject: [PATCH 04/65] Bugfixing to the create msmt schema --- data/migrations/1_create_msmt_results.sql | 86 ++++++++++++----------- 1 file changed, 44 insertions(+), 42 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index c1c2bc1..d4464e8 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -11,6 +11,44 @@ DROP TABLE `networks`; -- +migrate Up -- +migrate StatementBegin +CREATE TABLE `urls` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `url` VARCHAR(255) NOT NULL, -- XXX is this long enough? + `category_code` VARCHAR(5) NOT NULL, -- The citizenlab category code for the + -- site. We use the string NONE to denote + -- no known category code. + + `country_code` VARCHAR(2) NOT NULL -- The two letter country code which this + -- URL belongs to +); + +-- We create a separate table for networks for 2 reasons: +-- 1. For some of the views where need the total number of measured networks, +-- it's going to be much more efficient to just lookup the count of rows in this +-- table. +-- 2. (most important) We want to avoid duplicating a bunch of information that +-- is going to be common to several networks the user is on. +-- Example: +-- We may wish to add to this table the location from of the probe from the GPS +-- or add support for allowing the user to "correct" a misclassified measurement +-- or distinguishing between wifi and mobile. +CREATE TABLE `networks` ( + `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `network_name` VARCHAR(255), -- String name representing the network_name which by default is populated based + -- on the ASN. + -- We use a separate key to reference the rows in + -- this tables, because we may wish to "enrich" + -- this with more data in the future. + `network_type` VARCHAR(16), -- One of wifi, mobile + + `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. + -- The longest ip is an ipv6 address like: + -- 0000:0000:0000:0000:0000:0000:0000:0000, + -- which is 39 chars. + `asn` INT(4) NOT NULL, + `country_code` VARCHAR(2) NOT NULL -- The two letter country code +); + CREATE TABLE `results` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, -- This can be one of "websites", "im", "performance", "middlebox". @@ -55,7 +93,6 @@ CREATE TABLE `measurements` ( -- You don't have the guarantee that every (ip, asn, country, network_name) -- is the same in a "measurement set" associated to a "result". `network_id` INTEGER NOT NULL, - FOREIGN KEY (`network_id`) REFERENCES `networks`(`id`), -- Note for golang: we used to have state be one of `done` and `active`, so -- this is equivalent to done being true or false. @@ -93,7 +130,6 @@ CREATE TABLE `measurements` ( -- created. `url_id` INTEGER NOT NULL, - FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`), -- This is not yet a feature of the collector, but we are planning to add -- this at some point in the near future. @@ -116,53 +152,19 @@ CREATE TABLE `measurements` ( -- The cross table reference to JOIN the two tables together. `result_id` INTEGER NOT NULL, - FOREIGN KEY (`result_id`) REFERENCES `results`(`id`) - ON DELETE CASCADE ON UPDATE CASCADE, -- If we delete a result we also want - -- all the measurements to be deleted as well. + -- This is a variable used internally to track the path to the on-disk -- measurements.json. It may make sense to write one file per entry by -- hooking MK and preventing it from writing to a file on disk which may -- have many measurements per file. `report_file_path` VARCHAR(260) NOT NULL, -); -CREATE TABLE `urls` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `url` VARCHAR(255) NOT NULL, -- XXX is this long enough? - `category_code` VARCHAR(5) NOT NULL, -- The citizenlab category code for the - -- site. We use the string NONE to denote - -- no known category code. - - `country_code` VARCHAR(2) NOT NULL -- The two letter country code which this - -- URL belongs to -) - --- We create a separate table for networks for 2 reasons: --- 1. For some of the views where need the total number of measured networks, --- it's going to be much more efficient to just lookup the count of rows in this --- table. --- 2. (most important) We want to avoid duplicating a bunch of information that --- is going to be common to several networks the user is on. --- Example: --- We may wish to add to this table the location from of the probe from the GPS --- or add support for allowing the user to "correct" a misclassified measurement --- or distinguishing between wifi and mobile. -CREATE TABLE `networks` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `network_name` VARCHAR(255), -- String name representing the network_name which by default is populated based - -- on the ASN. - -- We use a separate key to reference the rows in - -- this tables, because we may wish to "enrich" - -- this with more data in the future. - `network_type` VARCHAR(16), -- One of wifi, mobile - - `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. - -- The longest ip is an ipv6 address like: - -- 0000:0000:0000:0000:0000:0000:0000:0000, - -- which is 39 chars. - `asn` INT(4) NOT NULL, - `country_code` VARCHAR(2) NOT NULL, -- The two letter country code + FOREIGN KEY (`result_id`) REFERENCES `results`(`id`) + ON DELETE CASCADE ON UPDATE CASCADE, -- If we delete a result we also want + -- all the measurements to be deleted as well. + FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`), + FOREIGN KEY(`network_id`) REFERENCES `networks` (`id`) ); -- +migrate StatementEnd From ec459a7f1590da7f03f5ad43dbecec64f830cab9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 5 Sep 2018 17:58:27 +0200 Subject: [PATCH 05/65] Update bindata --- internal/bindata/bindata.go | 113 +++++++++++++++++++++++++++++++----- 1 file changed, 99 insertions(+), 14 deletions(-) diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 812dc69..922a2c5 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -130,20 +130,105 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xcc\x93\xcf\x8e\x9b\x30\x10\xc6\xef\x3c\xc5\x1c\x13\xb5\x7b\xd8\x4a\xdb" + - "\xcb\x9e\xbc\x30\x6d\x69\x89\x89\x8c\xa9\x9a\x13\xb6\x8a\x37\xb2\x0a\x06\x19\x5b\x51\xdf\xbe\x72\x02\x69\x92\x42" + - "\x73\xdd\xeb\xf7\x9b\x3f\xf6\x7c\x33\x0f\x0f\xf0\xae\xd5\x7b\x2b\x9d\x82\xa4\x3b\x98\xe8\x52\x28\x9c\x74\xaa\x55" + - "\xc6\xbd\xa8\xbd\x36\x51\x94\xb0\x7c\x0b\x9c\xbc\x64\x08\xc2\xaa\xc1\x37\x6e\x10\xcf\x57\x6a\xab\xe4\xe0\xed\x31" + - "\x27\xa0\xf9\x6a\x68\xea\x6b\x52\xf6\xff\x6d\x1b\x33\x24\x1c\x6f\x1b\xc3\x2a\x02\x00\x10\xba\x16\x90\x52\x8e\x9f" + - "\x91\xc1\x96\xa5\x1b\xc2\x76\xf0\x0d\x77\x40\x4a\x9e\xa7\x34\x66\xb8\x41\xca\xdf\x9f\x62\x8d\x6c\x95\x80\xef\x84" + - "\xc5\x5f\x08\x5b\x7d\x78\x7a\x5a\x8f\x60\x70\xd2\xba\xca\xe9\x80\x13\xc2\x91\xa7\x1b\x1c\x91\xf5\xe6\xa4\x33\x24" + - "\xd9\x14\xee\xdb\x56\xda\xdf\x02\xbe\x16\x39\x1d\xb5\xba\x33\x4a\x00\x4f\xe9\x2e\xa5\x7c\xf5\x38\x55\xfe\xd9\x79" + - "\xe3\x42\xe8\xb9\xeb\x44\xe4\x60\xfe\xaa\x8f\x1f\x27\xd9\x28\x77\xe8\xec\xaf\x6a\xf1\xad\xb5\x74\xb2\xf2\x83\xdc" + - "\xab\xca\xf7\xe7\xbf\xff\x0b\xeb\xee\x60\xce\x38\x5a\x3f\xdf\x0e\xf2\xca\xab\xb7\x36\x4d\xdd\xcf\x56\x5e\x98\xd9" + - "\xf2\x90\xef\x4e\x73\x08\xcb\x26\x80\xe3\x8f\xe9\x5b\xaf\x52\x37\xde\xce\x47\xfb\xbe\xe9\x64\x5d\xdd\x0f\x51\xf5" + - "\xcc\x2e\x58\xd5\x77\xd6\x55\xaf\xba\x99\x4f\x1d\x79\xf0\x60\x86\x6a\xd3\x7b\xb7\x90\x17\x4e\xa2\xba\xf4\x8e\xe1" + - "\x27\x64\x48\x63\x2c\x2e\x2f\x26\xd8\xbb\x86\x9c\x42\x82\x19\x72\x84\x02\x39\xd0\x32\xcb\x82\x54\x6e\x83\x55\x10" + - "\x93\x22\x26\x09\x1e\xf7\x65\xf1\x7a\xff\x04\x00\x00\xff\xff\xdf\xf0\xa4\xca\x36\x04\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x6f\xdb\x46\xf2\x7f\xef\x4f\x31\x30\x8a\xd6\xc6\x5f\x92\x93" + + "\xfc\xd3\xe0\xce\xd7\xa2\x70\x13\x25\xa7\x36\x96\x03\x59\xbe\x26\x38\x1c\xc4\x15\x39\x94\xb6\x5e\xee\x32\xfb\x20" + + "\x46\xf7\xe9\x0f\x33\xbb\xa4\x48\xe5\xc9\x01\x9a\x17\x8e\x44\xee\xce\xce\xe3\x6f\x7e\xb3\x1a\x8f\xe1\xff\x2a\xb9" + + "\xb1\xc2\x23\xbc\x30\x8d\x3e\xe9\x3f\xb8\xf5\xc2\x63\x85\xda\xff\x8a\x1b\xa9\x4f\x4e\x5e\x2c\x6e\xde\xc0\xf2\xea" + + "\xd7\xd7\x53\xc8\x2c\xba\xa0\xbc\xcb\xfe\x31\x78\x5a\xa1\x70\xc1\xf2\x9e\xe3\x57\xc1\xaa\xe3\x47\x1a\x7d\x63\xec" + + "\x3d\x3d\xfe\xf4\xb9\x53\x5d\x0c\xdf\xdc\xd5\x5f\x54\xf0\xf9\x62\x7a\xb5\x9c\x0e\x4e\x84\xb3\x13\x80\x4c\x16\x19" + + "\xcc\xe6\xcb\xe9\xab\xe9\x02\xde\x2c\x66\xd7\x57\x8b\x77\xf0\xfb\xf4\x1d\x5c\xdd\x2d\x6f\x66\xf3\xe7\x8b\xe9\xf5" + + "\x74\xbe\x1c\xd1\xca\x60\x55\x06\xff\xba\x5a\x3c\xff\xe7\xd5\xe2\xec\xc9\x8f\x3f\x9e\xc3\xfc\x66\x09\xf3\xbb\xd7" + + "\xaf\x47\x30\x1e\xc3\xdb\xb7\x6f\x41\x3a\xf0\x5b\xe9\x40\x19\xbd\x01\xd4\x26\x6c\xb6\xbf\xd0\xd6\x5c\x78\xdc\x18" + + "\xbb\x5f\xe5\xa6\xc0\x83\x90\x63\x11\xcb\x2d\x42\x2e\xbd\xfc\x2f\x6a\x25\xd6\xd0\xee\x02\xda\x05\xa5\xb1\xe0\xb7" + + "\x78\x02\x0f\xfb\x37\x1e\x83\x93\x1e\x27\xf0\x07\x42\x70\x48\x5b\xc1\x79\x2b\xf5\x06\xe6\x37\xf3\x29\x78\x03\x05" + + "\x6a\xe3\xbf\x45\xa0\x36\x70\xaf\x4d\xa3\x87\x9a\x4d\x4e\xd8\x44\x13\xb4\xff\xc8\xc2\x27\x07\x0b\x5b\x03\x7d\x63" + + "\x40\xa1\xf7\x68\x21\xed\x89\xf6\x35\x5b\x99\x6f\xd9\x7d\x0f\xd3\x68\x3c\x86\xbb\xc5\x6b\x58\x23\x39\xdb\x81\x37" + + "\x27\xe7\x31\x59\xfe\x40\xc8\x2d\x52\x12\x08\x70\x58\x0b\xce\x07\x2f\xd6\x2a\xfa\xb0\x4d\x2d\xfe\xf2\x04\x2c\x0a" + + "\x67\xb4\xbb\xa4\x9d\x8f\x27\xf0\xd2\x58\x70\xa6\x42\x30\x25\xbb\x6c\x27\xb1\x71\xd0\x6c\xd1\x22\x68\xc4\x82\x1f" + + "\x7a\xe3\x85\x02\x1d\xaa\x35\x5a\x5a\x98\x72\xbb\xe8\x64\x8f\x48\x9a\xf4\x3f\x38\xd8\x18\xf2\xb8\x37\xb0\x46\xa8" + + "\x42\xbe\x85\xca\x58\x04\x2c\x4b\x99\x4b\xd4\x9e\xde\xfc\x19\x9c\x07\x65\xcc\x7d\xa8\x59\x3a\x7b\x85\xc4\x5a\xd3" + + "\x38\x90\x3a\xfa\x64\x3c\x8e\x36\x4c\xe8\xd3\x93\x09\x9c\x55\xc6\x79\x90\x55\x6d\xac\x17\xda\x9f\x93\xd9\x8d\x88" + + "\x12\xc5\xce\xc8\x02\x8a\x50\x2b\x99\x0b\x4f\x0a\x08\x58\x07\x9d\x6f\x49\xaa\xd4\xa5\xb1\x95\xf0\xd2\x90\x64\xe1" + + "\x59\xd5\xa1\xa2\xb9\xa9\x2a\x7a\x6b\xc0\xe1\x0e\x2d\xd9\xda\x3a\x8d\x14\x0c\x0e\x2d\x6d\x31\x9a\x95\x99\x7e\x10" + + "\x55\xad\xf0\x32\xf9\xbe\x12\x7b\x68\xa4\xdb\xb2\x22\x45\x41\xff\x71\x4d\xc4\x08\xd0\x7e\x65\xf2\x78\x7c\x69\x4d" + + "\xd5\x3a\xba\xb6\x66\x8d\xf1\x09\x7d\x7d\xf5\xe6\x96\xe4\x19\xcb\x32\x5c\xa8\xc9\x4e\x0e\x99\x50\xca\x34\xac\x6b" + + "\xab\x8a\x37\x70\x9a\x1b\x6b\x31\xf7\xa7\x20\xa0\x92\x2e\x57\xc2\x39\x59\x4a\x2c\xa0\x87\x3b\x49\x60\x21\x1d\xf9" + + "\x24\x48\xb7\x25\x31\x6b\xf4\x0d\xa2\x86\x46\x96\x12\x84\x2e\xa0\x32\x6b\x49\x7e\x1e\x42\x46\x87\x48\xdf\x0a\x1b" + + "\x69\xe3\x4a\x8b\x0a\x87\xf8\xc1\x35\x7f\x1b\x6b\x92\xde\x82\xc5\xda\xa2\x43\xed\x5b\xf3\xfa\x7b\x53\x81\xac\xf7" + + "\x50\x60\x29\x82\xf2\x14\x82\xda\xd4\x41\x09\x8f\x05\xac\x85\xc3\xe2\x6b\x95\x43\x0e\xd0\x2c\xf9\xea\x76\x3e\x79" + + "\xc0\xea\x04\x1e\xbd\x42\xba\xc7\x3d\x39\xdc\x62\x89\x16\x75\x1e\x23\x9a\x32\xf5\x01\x02\x0f\xa9\xe0\x46\xb0\xc6" + + "\x5c\x90\xf8\x66\x98\x35\xa7\xa8\xad\xcc\xb7\xa7\x0f\x15\xd7\x48\x9f\xea\xaa\x10\x5e\xc4\x8a\x41\x28\x83\x0f\x16" + + "\x27\xfd\x10\xf8\x7d\xdd\x0b\xc1\xe3\x67\x31\x02\x37\x9a\xab\x9d\xe2\x3f\x4a\xc1\x27\x44\x83\x4c\xd6\x87\xc5\x4f" + + "\x1f\xf5\xb1\x3a\x06\xce\x58\x74\xe4\x9a\x18\xc1\x2e\x78\x31\xb7\x4d\x09\x42\x83\xac\x77\x4f\x29\xe7\x64\xbd\x7b" + + "\x46\x99\x6c\xd1\xb9\x87\xf8\x7d\xc9\x65\xa2\x37\x48\x35\x5e\x53\xa4\xa3\xb0\x4e\x08\x28\x79\x8f\x97\x0f\x90\xf4" + + "\xe8\xd1\xa3\x47\x97\x5f\xff\x33\x7a\x80\xa8\x98\x80\xd2\xc1\xff\xff\x1d\xf2\xad\xb0\x6c\x49\x26\x9c\xe6\x52\x38" + + "\x7b\xda\xf3\xd0\x5f\xd1\x11\x18\xcf\x87\x45\xd8\x52\x0b\xae\xc1\x6f\xa9\xc2\xe4\x54\xe9\x20\x17\x9a\x20\xce\xc4" + + "\xa0\x9f\x36\xb8\xa6\x36\xe9\x4e\x47\x70\x2a\x2b\xfa\x5b\xa3\x65\x80\xd4\x39\xd2\xd7\x4a\x16\x85\xc2\xb5\xf9\x70" + + "\x1a\xe3\x96\x79\x74\x7e\xb5\xb1\x26\xd4\x47\x25\xfd\xf8\xd9\xd0\x01\xc3\x02\x2a\x64\xc9\x15\xe3\xc1\x79\x61\xfd" + + "\xca\xcb\x0a\x19\x6e\x6c\xd0\xf4\x79\x50\x0d\x1d\x90\x2b\x67\x60\x2b\x76\xd8\x8a\xe3\x04\xf7\xa6\x45\x35\x4e\x74" + + "\xb3\x43\xbb\x45\x51\x90\x3d\xdc\xf8\x22\xe0\x5b\x64\xc8\xa4\x23\x8c\xdf\xa2\x85\x52\xe4\xde\x58\x17\x41\x3f\xc9" + + "\xdb\x18\x90\x9a\x11\x1a\x81\x0c\x9b\x1c\x7c\x25\x18\x60\xa8\x07\x88\xfd\x25\x64\xb7\x77\xd7\x67\x49\xd5\x73\x78" + + "\xb9\xb8\xb9\x86\x01\xa3\x83\x46\x2a\x05\x42\x35\x62\xef\xc8\xbf\x3f\xfd\xdc\x4a\xca\xd2\xae\xb8\xe9\x10\x41\xee" + + "\x5f\xf4\xc2\xc1\x4f\xe7\xd1\xb5\x07\xcf\x64\xf0\xe2\x6a\x39\x5d\xce\xae\xa7\x47\x2e\x6d\xa5\x65\xb0\x98\x5e\xbd" + + "\xee\xbd\x6c\x8f\xbb\x73\xc8\x3d\x47\xea\x82\x9a\x1f\x82\x2c\x0f\x9d\x62\x2b\x1c\x38\x02\x7b\xc6\x8d\xa8\x4b\xca" + + "\x24\xb7\xa2\x56\x8f\x45\x06\xcb\xd9\xfc\x1d\xe5\xf3\xe3\xf3\x4f\x88\xe7\x1c\xa2\x72\x84\x52\x89\x0d\x49\xfd\xe4" + + "\x69\x51\x34\x2d\x2c\x38\xd3\xb8\x5f\xe6\xc1\x52\x02\xa8\x3d\xc5\x5c\x4b\xbd\x99\x74\x67\xd3\xaa\xcf\x9c\xcc\x4b" + + "\x28\xee\xab\xe0\xc4\x06\x57\xa1\x3e\xe4\xfc\xe7\x57\x15\xa6\xd1\x9f\x5b\x37\x1e\xc3\x8c\xb8\x09\xb5\x5c\xb1\x26" + + "\x75\x98\x03\xc5\xfe\x4c\x3d\xdf\xb3\x0d\x95\xf8\x20\xab\x50\x81\x42\xbd\xf1\x0c\xcc\x4f\x9e\x3d\x02\x91\x28\x2e" + + "\x53\xdd\x2e\x2f\x8f\xd6\x9a\x12\x4a\xa9\x10\x6a\xe1\xb7\xc4\x13\xa0\x91\xba\x30\x4d\x82\xbe\x4c\x99\xcd\x8a\xde" + + "\xaf\xe8\x7d\x0f\x1a\x9e\xf5\x40\xf6\x13\xd5\x3f\x4c\xb8\xbf\x08\x02\x2e\xdb\x57\xa5\xc8\x71\x6d\xcc\xfd\xaa\x42" + + "\xe7\x50\x6f\xd0\xb6\x6f\x3c\x2a\xdc\x58\x51\x9d\x74\x38\x28\xbc\x13\x75\xdd\x7e\xdf\x7a\x5f\xaf\xa8\x02\xd1\xae" + + "\x4a\x89\xaa\x58\x55\x42\x4b\x6e\xcc\xd2\xe8\xc1\x2a\xa9\x77\x42\xc9\x62\x65\xf1\x7d\x20\x1c\x51\x52\xf7\x6a\xdb" + + "\x6d\xdb\xcf\xba\xf0\x3d\xb4\x19\xe2\xcc\xb3\xa7\x1f\xa5\xc7\x5f\x51\x38\x2f\xe3\x7c\x01\x75\xb0\xb5\x71\x8c\x8e" + + "\x89\x5d\xb4\x6c\x24\x52\xb4\x3e\x7f\x4c\xad\x36\x15\x75\x2b\x89\x39\xf3\x08\xf6\x26\x80\xdb\x9a\xa0\x0a\xa8\x65" + + "\x7e\x1f\x9b\xb2\xb4\xce\xf7\x91\xa3\x15\xf1\xdb\xcd\x6c\x0e\xce\x58\xa6\x32\xfb\x56\xd2\xc1\xae\x0e\x98\xde\x99" + + "\x40\x35\xf5\x83\x67\x5c\xe4\xbd\x9b\x20\xac\xd0\x1e\x91\xa1\x0d\x88\xb2\xee\xe1\x4c\xd6\x23\x10\x4e\x8f\xda\x9e" + + "\x32\x1a\xb0\xa9\xf3\x56\x5e\xcc\x63\x70\xc4\xb0\xa4\x06\x01\xa7\x7d\xed\x1c\x12\xa5\x74\xce\xe4\x92\x59\x16\x61" + + "\x32\x9c\x46\x7b\xdb\x86\xd0\x8a\xed\x67\xe2\xc7\xee\x9d\x1b\x1f\xe7\x8f\x8d\x51\x42\x6f\x2e\x09\xe6\x5b\xf4\x60" + + "\x4b\x1c\x4d\xac\xbd\xd6\x94\x45\x4c\x20\xfc\xce\x44\xee\xe5\x0e\xb3\x11\x38\x73\xd2\x67\x3e\xd2\x01\xbe\x0f\x72" + + "\x27\x54\x9a\x25\x18\x6d\xd6\xc8\x11\xb3\x81\x81\xa7\x14\xca\x1d\xdc\x97\xf1\x31\x19\x2c\xa7\x6f\x53\x55\x3c\x00" + + "\x7e\x52\x9f\x8e\x30\xd1\x29\x2c\xa0\xc0\x88\x7a\x05\x48\xb7\x0a\xb5\x32\xa2\xc0\x82\x81\x71\x04\x52\x3b\x9f\x9a" + + "\x12\x0f\x38\xc1\x49\xbd\x39\x38\x3d\x2d\x5f\x95\x42\x2a\x2c\x46\x31\x0c\xc2\xb7\x54\x50\x9b\x14\xdf\x4e\x2a\x23" + + "\x52\x2f\x32\x45\xe8\x0a\x87\x83\xe2\xd0\xfb\x01\xa4\xb6\x3b\x1f\x08\xe8\xc7\xf2\xa3\x62\x4c\x75\x83\xe6\x28\x74" + + "\x5d\x24\x25\x35\x85\x8a\x7a\xba\xe5\x65\xad\x3c\x8b\x63\xda\x20\xfd\x41\x93\x28\xea\x4b\xf0\x4e\x2b\x82\xc5\x55" + + "\xe5\x36\x47\x23\xc2\xc9\x91\x3d\x0f\x10\xd6\x5b\xf8\x25\x99\xd4\x05\xdc\xc7\x0d\x8c\xa3\xc0\xc9\x55\x0b\xeb\x65" + + "\x1e\x94\xb0\x03\xc7\x50\x0f\x5d\x53\x0f\x4d\x96\x0a\x5d\x1c\x72\x12\x2d\x96\x26\xf1\x92\xbb\x19\x23\xad\x17\xf7" + + "\x98\xb2\x95\x98\x86\xc8\xe3\x7c\xeb\x0d\xa0\x64\x5e\xb2\x95\x05\x82\xf4\xdd\xec\x77\xf0\x24\xf7\x50\x42\x13\x9e" + + "\x03\x63\x57\xe2\xe2\x56\x28\x9c\xa7\x41\xae\x9b\x29\xc5\x5a\x2a\xe9\xd3\x68\x32\x88\x40\xba\x9a\x29\x0c\xe5\x16" + + "\x13\xaa\x96\x5d\xa5\x2c\xee\x8d\x32\x26\xc1\x19\x0b\xe8\x19\xfd\x4b\x17\x05\x8b\x36\xe8\x6f\x48\x29\x87\x76\x87" + + "\x76\xec\xc8\xc6\xc8\xc8\x56\xb2\x00\x8b\x3e\x58\xcd\x50\x97\x46\x7e\xa5\x90\xd8\xd9\x04\x7e\xdd\x0f\x4b\xe5\xb0" + + "\xe9\x7b\x90\xba\x0e\x3e\x02\x2b\x79\xf6\x7d\x20\x5f\xb0\xf5\xb5\x24\xe5\x4b\xf4\xe9\x0a\xa5\xaf\x7c\xe7\x86\xe9" + + "\x87\xee\xe3\xab\xe9\x92\x1b\x92\xbb\xbc\xb8\x10\xb5\x9c\x18\xa3\xe5\x44\x1a\xfa\x7c\xb1\x7b\x7c\xd1\xef\xb4\xbf" + + "\xf0\xa9\x3f\x7f\x37\x9b\xbf\xb9\x5b\x7e\xdf\xa9\xf3\xf3\x77\x8b\xe9\x9b\x9b\xc5\x72\x35\x7b\x71\x90\xef\xad\xc8" + + "\x7d\x0f\xe8\xa5\xc7\xea\x30\xd3\x27\xfa\xfe\xef\xff\x64\xa0\xa4\xf3\x6d\x51\xe9\xa8\x77\xd7\x88\xfb\x7d\x7e\xc5" + + "\x97\x6e\xde\xc0\x26\x91\x92\xdf\x6e\x6f\xe6\xf1\xca\x60\x68\x24\x8d\xa0\x3d\xf2\x8a\x2e\x8e\x15\x3b\xa1\x02\x3a" + + "\x38\xcb\x3a\xbd\xb3\x11\x64\x6c\x51\x76\x0e\xc2\x72\x45\x97\x41\x1d\xbc\x27\x3a\x4a\xd3\x13\xce\x45\x41\x89\x2f" + + "\x94\x45\x51\xec\x63\x01\xd4\xd6\xe4\xc4\x15\xba\x30\xd6\xb2\x46\xea\xe8\xa3\x1e\x1e\xc8\xaa\x56\x51\x48\xae\x50" + + "\xe8\x50\xf3\x64\x98\xc4\x74\xe8\xd6\x77\x78\x02\x8e\x83\xc6\x1f\x5f\x1a\xf4\x69\x0c\x4f\x52\x0d\xb9\x51\x9b\x96" + + "\xf4\x33\xf9\x6a\x0b\xf5\x2b\x93\xdd\x78\x9c\xae\xcb\x8a\x49\x02\x9b\x60\xd5\x57\x9a\x59\x9b\xe1\x04\xd3\x7b\xf4" + + "\xc4\x86\x51\xd0\xb8\xdd\xde\xe6\x74\x09\x3d\x82\x75\x60\x54\x27\x5f\xd7\x4a\x30\xef\x4d\x57\x43\x83\x56\x26\x7c" + + "\xbc\x77\xab\x8d\x3c\xb0\x02\x8d\xc2\xf6\x06\xf9\x38\x77\x23\x5e\x76\xb9\xbb\x91\x7e\x1b\xd6\x93\xdc\x54\x17\x94" + + "\xc2\x17\x6d\x04\x2e\xd6\xca\xac\x2f\x2a\xe1\x3c\xda\x8b\xc2\xe4\x8e\x5f\x8f\x43\x90\xc5\xa4\x2a\xe0\xfb\x3e\x29" + + "\xfb\xa2\x1c\xe9\x5c\x40\x77\xf1\xf4\x6f\xd1\x35\xfd\xd4\x4c\x2e\x22\x3e\x76\xec\x99\x04\xa6\xae\xb5\x23\x17\x91" + + "\x50\x09\x68\xe7\x4d\x9e\xb6\x46\x31\xb1\x04\xdf\xd4\x92\x3f\x69\xa8\x57\x1d\xfb\x59\x2b\x93\xdf\x53\x73\xa4\x2e" + + "\x4e\x08\xa8\x61\x76\xcd\x1b\xdb\x31\x23\x7d\x75\x34\xa3\xb9\x84\x04\xf5\x97\x05\xc9\x92\xaf\xc8\xd2\x50\x0b\x8d" + + "\x70\x50\xa0\xc7\x9c\xe3\xdf\xe3\x58\x94\x5d\x19\xb1\xb2\x0c\x04\x64\xcf\x6f\xee\xe6\xcb\xb3\xf3\xac\x2b\x3d\x2e" + + "\xac\x23\xfe\x17\xa1\x3a\x15\xab\xe8\xee\x31\x8f\xb4\x80\x68\xbf\xb1\xdd\x83\xd9\x35\xa9\xed\x3a\x8c\x15\xda\x54" + + "\x42\xed\xfb\x28\xfb\x89\x01\x4c\x83\xa9\xc5\xfb\x90\x20\xc1\x79\x1b\x72\xca\x93\x51\xba\xac\x6d\x88\x51\x51\x2b" + + "\xea\xdf\xe6\x32\x9b\xbe\xc7\x7d\x47\x55\x9b\x74\xab\x9b\x2e\xd7\x87\x0c\x03\xbd\x90\xca\xa5\x2b\x60\x02\x2b\x16" + + "\xd5\x6b\x4b\x0e\xce\xf0\xc3\xa4\xdf\xb3\x62\x41\x5f\xd0\xf4\x45\x1f\xc0\xd5\x24\xdd\x94\x30\x7f\xb1\x1c\x25\x5f" + + "\x31\x89\x2a\x5b\xfb\xa9\x1c\x38\x33\xc8\x2d\x1d\xdd\x42\x9f\x4f\xce\x7b\x13\x00\xe9\x9c\xb1\xa5\x7d\x4f\x20\xe4" + + "\xd6\xb8\xf6\x6a\x75\xd0\xc7\x98\x4f\xfb\x74\xb9\x12\xef\xdb\xc0\x9b\x0d\x52\xc7\xed\x00\x86\x0c\xf9\x5c\xa5\x7f" + + "\x3c\xf0\xee\x84\x95\x7c\x10\x73\x06\xa9\x3d\x5a\x2d\x94\xe2\x9e\x4b\xc0\x1f\x19\x3e\x8d\x74\x6d\x23\x35\x7a\x5c" + + "\x48\x77\xff\x09\x44\x75\x93\x3f\x9d\xd1\x13\x98\x79\xa6\x7b\x15\x71\x04\x87\xda\xb1\xee\x8d\xa5\x72\x20\x26\x1b" + + "\x87\x48\xb4\x80\x7c\x2b\x74\x18\x0c\xb6\xc6\xb0\xe7\xae\x7f\xe7\xc0\xd4\x16\x77\xe9\xda\xb4\x25\x12\x24\xa4\x85" + + "\x9a\x28\xc7\x68\x62\x0c\xf7\xe9\x1a\xab\x12\x07\x61\xc4\x03\x2a\xa1\xf7\x03\x0d\xf9\xdc\x92\x6f\x82\xfb\x78\xfc" + + "\xb5\xb9\x35\xc5\xe7\xe5\xcd\x62\x3a\x7b\x35\xe7\x51\xf4\xac\xe7\xea\x73\x58\x4c\x5f\x4e\x17\xd3\xf9\xf3\xe9\xed" + + "\xe1\x3e\xeb\x8c\xc6\xd8\xf3\x04\xd4\x37\x73\x78\x31\x7d\x3d\x5d\x4e\xe1\xf9\xd5\xed\xf3\xab\x17\x53\x7a\x72\xf7" + + "\x86\xe6\xba\xf6\x09\x37\x81\x59\x49\xe9\x5b\xa0\x42\x1f\x69\x0c\xe7\x65\x9f\xe4\x3c\xf4\xa7\x9d\xe4\x07\xa1\xd4" + + "\x71\x11\xb8\xf4\x8b\x40\x3c\xa5\xa0\xe9\xbf\x41\xa5\x26\x9f\xb0\x31\x75\x8d\xa1\x81\xfc\x43\x5b\xb4\x6e\x74\xbc" + + "\xe7\xac\x3f\x39\x0d\xb7\xf5\x2e\xdb\xa3\x67\xce\xbf\xf4\x3b\xe0\xff\x02\x00\x00\xff\xff\x29\x5f\x48\x5d\xaa\x1c" + + "\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( From ff2f973523f5e64b42c9b7275b97ca7a4aa0d339 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 5 Sep 2018 18:40:37 +0200 Subject: [PATCH 06/65] Start integrating upper/db as a new ORM --- Gopkg.lock | 19 +- Gopkg.toml | 4 + internal/cli/list/list.go | 27 +-- internal/database/actions.go | 156 +++++++++++++ internal/database/database.go | 22 +- internal/database/database_test.go | 31 +++ internal/database/models.go | 353 +++++++---------------------- nettests/nettests.go | 8 +- ooni.go | 4 +- 9 files changed, 317 insertions(+), 307 deletions(-) create mode 100644 internal/database/actions.go create mode 100644 internal/database/database_test.go diff --git a/Gopkg.lock b/Gopkg.lock index 3114935..26e99af 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -146,9 +146,26 @@ revision = "c87af80f3cc5036b55b83d77171e156791085e2e" version = "v1.7.1" +[[projects]] + name = "upper.io/db.v3" + packages = [ + ".", + "internal/cache", + "internal/cache/hashstructure", + "internal/immutable", + "internal/sqladapter", + "internal/sqladapter/compat", + "internal/sqladapter/exql", + "lib/reflectx", + "lib/sqlbuilder", + "sqlite" + ] + revision = "199d13d76c7cfba05ea0327375056fdabc8bea80" + version = "v3.5.4" + [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "b2f5c39222a1fb405e3f48d2ae3b4758757fe708e12dbd23743c19135e225579" + inputs-digest = "bb552d1e6530dab8cdd5cc7a6c60bf2b9afbe77e6ea20a31b6d60fc44ad05e26" solver-name = "gps-cdcl" solver-version = 1 diff --git a/Gopkg.toml b/Gopkg.toml index 7bc1e5b..1a01b9c 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -65,3 +65,7 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"] [[constraint]] branch = "master" name = "github.com/getsentry/raven-go" + +[[constraint]] + name = "upper.io/db.v3" + version = "3.5.4" diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index b52882b..73bc901 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -45,13 +45,13 @@ func init() { ID: result.ID, Index: idx, TotalCount: len(incompleteResults), - Name: result.Name, + Name: result.TestGroupName, StartTime: result.StartTime, - NetworkName: result.NetworkName, - Country: result.Country, - ASN: result.ASN, - Summary: result.Summary, - Done: result.Done, + NetworkName: "FIXME", //result.NetworkName, + Country: "FIXME", //result.Country, + ASN: "FIXME", //result.ASN, + Summary: "{}", //result.Summary, + Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, }) @@ -65,18 +65,19 @@ func init() { ID: result.ID, Index: idx, TotalCount: len(doneResults), - Name: result.Name, + Name: result.TestGroupName, StartTime: result.StartTime, - NetworkName: result.NetworkName, - Country: result.Country, - ASN: result.ASN, - Summary: result.Summary, - Done: result.Done, + NetworkName: "FIXME", //result.NetworkName, + Country: "FIXME", //result.Country, + ASN: "FIXME", //result.ASN, + Summary: "{}", //result.Summary, + Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, }) resultSummary.TotalTests++ - netCount[result.ASN]++ + // FIXME + // netCount[result.ASN]++ resultSummary.TotalDataUsageUp += result.DataUsageUp resultSummary.TotalDataUsageDown += result.DataUsageDown } diff --git a/internal/database/actions.go b/internal/database/actions.go new file mode 100644 index 0000000..adc1ce2 --- /dev/null +++ b/internal/database/actions.go @@ -0,0 +1,156 @@ +package database + +import ( + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/utils" + "github.com/pkg/errors" + "upper.io/db.v3/lib/sqlbuilder" +) + +// ListMeasurements given a result ID +func ListMeasurements(db sqlbuilder.Database, resultID int64) ([]*Measurement, error) { + measurements := []*Measurement{} + + /* + FIXME + rows, err := db.Query(`SELECT id, name, + start_time, runtime, + country, + asn, + summary, + input + FROM measurements + WHERE result_id = ? + ORDER BY start_time;`, resultID) + if err != nil { + return measurements, errors.Wrap(err, "failed to get measurement list") + } + + for rows.Next() { + msmt := Measurement{} + err = rows.Scan(&msmt.ID, &msmt.Name, + &msmt.StartTime, &msmt.Runtime, + &msmt.CountryCode, + &msmt.ASN, + &msmt.Summary, &msmt.Input, + //&result.DataUsageUp, &result.DataUsageDown) + ) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + measurements = append(measurements, &msmt) + } + */ + + return measurements, nil +} + +// ListResults return the list of results +func ListResults(db sqlbuilder.Database) ([]*Result, []*Result, error) { + doneResults := []*Result{} + incompleteResults := []*Result{} + + /* + FIXME + rows, err := db.Query(`SELECT id, name, + start_time, runtime, + network_name, country, + asn, + summary, done + FROM results + WHERE done = 1 + ORDER BY start_time;`) + if err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } + for rows.Next() { + result := Result{} + err = rows.Scan(&result.ID, &result.Name, + &result.StartTime, &result.Runtime, + &result.NetworkName, &result.Country, + &result.ASN, + &result.Summary, &result.Done, + //&result.DataUsageUp, &result.DataUsageDown) + ) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + doneResults = append(doneResults, &result) + } + */ + + /* + FIXME + rows, err := db.Query(`SELECT + id, name, + start_time, + network_name, country, + asn + FROM results + WHERE done != 1 + ORDER BY start_time;`) + if err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } + */ + + /* + for rows.Next() { + result := Result{Done: false} + err = rows.Scan(&result.ID, &result.Name, &result.StartTime, + &result.NetworkName, &result.Country, + &result.ASN) + if err != nil { + log.WithError(err).Error("failed to fetch a row") + continue + } + incompleteResults = append(incompleteResults, &result) + } + */ + + return doneResults, incompleteResults, nil +} + +// CreateMeasurement writes the measurement to the database a returns a pointer +// to the Measurement +func CreateMeasurement(sess sqlbuilder.Database, m Measurement, i string) (*Measurement, error) { + col := sess.Collection("measurements") + + // XXX Do we want to have this be part of something else? + m.StartTime = time.Now().UTC() + + // XXX insert also the URL and stuff + //m.Input = i + //m.State = "active" + + newID, err := col.Insert(m) + if err != nil { + return nil, errors.Wrap(err, "creating measurement") + } + m.ID = newID.(int64) + return &m, nil +} + +// CreateResult writes the Result to the database a returns a pointer +// to the Result +func CreateResult(sess sqlbuilder.Database, homePath string, r Result) (*Result, error) { + log.Debugf("Creating result %v", r) + + col := sess.Collection("results") + + p, err := utils.MakeResultsDir(homePath, r.TestGroupName, r.StartTime) + if err != nil { + return nil, err + } + r.MeasurementDir = p + newID, err := col.Insert(r) + if err != nil { + return nil, errors.Wrap(err, "creating result") + } + r.ID = newID.(int64) + return &r, nil +} diff --git a/internal/database/database.go b/internal/database/database.go index f2f0de6..8da63f7 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -1,22 +1,24 @@ package database import ( + "database/sql" + "github.com/apex/log" - "github.com/jmoiron/sqlx" - _ "github.com/mattn/go-sqlite3" // this is needed to load the sqlite3 driver "github.com/ooni/probe-cli/internal/bindata" migrate "github.com/rubenv/sql-migrate" + "upper.io/db.v3/lib/sqlbuilder" + "upper.io/db.v3/sqlite" ) // RunMigrations runs the database migrations -func RunMigrations(db *sqlx.DB) error { +func RunMigrations(db *sql.DB) error { log.Debugf("running migrations") migrations := &migrate.AssetMigrationSource{ Asset: bindata.Asset, AssetDir: bindata.AssetDir, Dir: "data/migrations", } - n, err := migrate.Exec(db.DB, "sqlite3", migrations, migrate.Up) + n, err := migrate.Exec(db, "sqlite3", migrations, migrate.Up) if err != nil { return err } @@ -25,15 +27,15 @@ func RunMigrations(db *sqlx.DB) error { } // Connect to the database -func Connect(path string) (db *sqlx.DB, err error) { - db, err = sqlx.Connect("sqlite3", path) - if err != nil { - return +func Connect(path string) (db sqlbuilder.Database, err error) { + settings := sqlite.ConnectionURL{ + Database: path, } + sess, err := sqlite.Open(settings) - err = RunMigrations(db) + err = RunMigrations(sess.Driver().(*sql.DB)) if err != nil { db = nil } - return + return sess, err } diff --git a/internal/database/database_test.go b/internal/database/database_test.go new file mode 100644 index 0000000..0641d7d --- /dev/null +++ b/internal/database/database_test.go @@ -0,0 +1,31 @@ +package database + +import ( + "io/ioutil" + "os" + "testing" + + "github.com/apex/log" +) + +func TestConnect(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Error(err) + } + sess, err := Connect(tmpfile.Name()) + if err != nil { + t.Error(err) + } + + colls, err := sess.Collections() + if err != nil { + t.Error(err) + } + + if len(colls) < 1 { + log.Fatal("missing tables") + } + + defer os.Remove(tmpfile.Name()) +} diff --git a/internal/database/models.go b/internal/database/models.go index 689d806..06a4a53 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -5,84 +5,76 @@ import ( "path/filepath" "time" - "github.com/apex/log" - "github.com/jmoiron/sqlx" "github.com/ooni/probe-cli/nettests/summary" - "github.com/ooni/probe-cli/utils" "github.com/pkg/errors" + "upper.io/db.v3/lib/sqlbuilder" ) -// UpdateOne will run the specified update query and check that it only affected one row -func UpdateOne(db *sqlx.DB, query string, arg interface{}) error { - res, err := db.NamedExec(query, arg) - - if err != nil { - return errors.Wrap(err, "updating table") - } - count, err := res.RowsAffected() - if err != nil { - return errors.Wrap(err, "updating table") - } - if count != 1 { - return errors.New("inconsistent update count") - } - return nil +// Network represents a network tested by the user +type Network struct { + ID int64 `db:"id"` + NetworkName string `db:"network_name"` + IP string `db:"ip"` + ASN int `db:"asn"` + CountryCode string `db:"country_code"` } -// ListMeasurements given a result ID -func ListMeasurements(db *sqlx.DB, resultID int64) ([]*Measurement, error) { - measurements := []*Measurement{} - - rows, err := db.Query(`SELECT id, name, - start_time, runtime, - country, - asn, - summary, - input - FROM measurements - WHERE result_id = ? - ORDER BY start_time;`, resultID) - if err != nil { - return measurements, errors.Wrap(err, "failed to get measurement list") - } - - for rows.Next() { - msmt := Measurement{} - err = rows.Scan(&msmt.ID, &msmt.Name, - &msmt.StartTime, &msmt.Runtime, - &msmt.CountryCode, - &msmt.ASN, - &msmt.Summary, &msmt.Input, - //&result.DataUsageUp, &result.DataUsageDown) - ) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - measurements = append(measurements, &msmt) - } - - return measurements, nil +type URL struct { + ID int64 `db:"id"` + URL int64 `db:"url"` + CategoryCode string `db:"category_code"` + CountryCode string `db:"country_code"` } // Measurement model type Measurement struct { + ID int64 `db:"id"` + TestName string `db:"test_name"` + StartTime time.Time `db:"start_time"` + Runtime float64 `db:"runtime"` // Fractional number of seconds + NetworkID int64 `db:"network_id"` // Used to include a Network + IsDone bool `db:"is_done"` + IsUploaded bool `db:"is_uploaded"` + IsFailed string `db:"is_failed"` + FailureMsg string `db:"failure_msg"` + IsUploadFailed bool `db:"is_upload_failed"` + UploadFailureMsg string `db:"upload_failure_msg"` + IsRerun bool `db:"is_rerun"` + ReportID string `db:"report_id"` + URLID string `db:"url_id"` // Used to reference URL + MeasurementID int64 `db:"measurement_id"` + IsAnomaly bool `db:"is_anomaly"` + TestKeys struct{} `db:"test_keys"` + ResultID int64 `db:"result_id"` + ReportFilePath string `db:"report_file_path"` +} + +// Result model +type Result struct { ID int64 `db:"id"` - Name string `db:"name"` + TestGroupName string `db:"test_group_name"` StartTime time.Time `db:"start_time"` - Runtime float64 `db:"runtime"` // Fractional number of seconds - Summary string `db:"summary"` // XXX this should be JSON - ASN string `db:"asn"` - IP string `db:"ip"` - CountryCode string `db:"country"` - State string `db:"state"` - Failure string `db:"failure"` - UploadFailure string `db:"upload_failure"` - Uploaded bool `db:"uploaded"` - ReportFilePath string `db:"report_file"` - ReportID string `db:"report_id"` - Input string `db:"input"` - ResultID int64 `db:"result_id"` + Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds + IsViewed bool `db:"is_viewed"` + IsDone bool `db:"is_done"` + DataUsageUp int64 `db:"data_usage_up"` + DataUsageDown int64 `db:"data_usage_down"` + MeasurementDir string `db:"measurement_dir"` +} + +// Finished marks the result as done and sets the runtime +func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error { + if r.IsDone == true || r.Runtime != 0 { + return errors.New("Result is already finished") + } + r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds() + r.IsDone = true + + err := sess.Collection("results").Find("id", r.ID).Update(r) + if err != nil { + return errors.Wrap(err, "updating finished result") + } + return nil } // SetGeoIPInfo for the Measurement @@ -91,12 +83,9 @@ func (m *Measurement) SetGeoIPInfo() error { } // Failed writes the error string to the measurement -func (m *Measurement) Failed(db *sqlx.DB, failure string) error { - m.Failure = failure - - err := UpdateOne(db, `UPDATE measurements - SET failure = :failure, state = :state - WHERE id = :id`, m) +func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error { + m.FailureMsg = failure + err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -104,14 +93,12 @@ func (m *Measurement) Failed(db *sqlx.DB, failure string) error { } // Done marks the measurement as completed -func (m *Measurement) Done(db *sqlx.DB) error { +func (m *Measurement) Done(sess sqlbuilder.Database) error { runtime := time.Now().UTC().Sub(m.StartTime) m.Runtime = runtime.Seconds() - m.State = "done" + m.IsDone = true - err := UpdateOne(db, `UPDATE measurements - SET state = :state, runtime = :runtime - WHERE id = :id`, m) + err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -119,13 +106,11 @@ func (m *Measurement) Done(db *sqlx.DB) error { } // UploadFailed writes the error string for the upload failure to the measurement -func (m *Measurement) UploadFailed(db *sqlx.DB, failure string) error { - m.UploadFailure = failure - m.Uploaded = false +func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) error { + m.UploadFailureMsg = failure + m.IsUploaded = false - err := UpdateOne(db, `UPDATE measurements - SET upload_failure = :upload_failure - WHERE id = :id`, m) + err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -133,12 +118,10 @@ func (m *Measurement) UploadFailed(db *sqlx.DB, failure string) error { } // UploadSucceeded writes the error string for the upload failure to the measurement -func (m *Measurement) UploadSucceeded(db *sqlx.DB) error { - m.Uploaded = true +func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error { + m.IsUploaded = true - err := UpdateOne(db, `UPDATE measurements - SET uploaded = :uploaded - WHERE id = :id`, m) + err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -146,12 +129,10 @@ func (m *Measurement) UploadSucceeded(db *sqlx.DB) error { } // WriteSummary writes the summary to the measurement -func (m *Measurement) WriteSummary(db *sqlx.DB, summary string) error { - m.Summary = summary +func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error { + // XXX remove m.Summary = summary - err := UpdateOne(db, `UPDATE measurements - SET summary = :summary - WHERE id = :id`, m) + err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -159,7 +140,7 @@ func (m *Measurement) WriteSummary(db *sqlx.DB, summary string) error { } // AddToResult adds a measurement to a result -func (m *Measurement) AddToResult(db *sqlx.DB, result *Result) error { +func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error { var err error m.ResultID = result.ID @@ -176,191 +157,9 @@ func (m *Measurement) AddToResult(db *sqlx.DB, result *Result) error { } m.ReportFilePath = finalPath - err = UpdateOne(db, `UPDATE measurements - SET result_id = :result_id, report_file = :report_file - WHERE id = :id`, m) + err = sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } return nil } - -// CreateMeasurement writes the measurement to the database a returns a pointer -// to the Measurement -func CreateMeasurement(db *sqlx.DB, m Measurement, i string) (*Measurement, error) { - // XXX Do we want to have this be part of something else? - m.StartTime = time.Now().UTC() - m.Input = i - m.State = "active" - - res, err := db.NamedExec(`INSERT INTO measurements - (name, start_time, - asn, ip, country, - state, failure, report_file, - report_id, input, - result_id) - VALUES (:name,:start_time, - :asn,:ip,:country, - :state,:failure,:report_file, - :report_id,:input, - :result_id)`, - m) - if err != nil { - return nil, errors.Wrap(err, "creating measurement") - } - id, err := res.LastInsertId() - if err != nil { - return nil, errors.Wrap(err, "creating measurement") - } - m.ID = id - return &m, nil -} - -// Result model -type Result struct { - ID int64 `db:"id"` - Name string `db:"name"` - StartTime time.Time `db:"start_time"` - Country string `db:"country"` - ASN string `db:"asn"` - NetworkName string `db:"network_name"` - Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds - Summary string `db:"summary"` // XXX this should be JSON - Done bool `db:"done"` - DataUsageUp int64 `db:"data_usage_up"` - DataUsageDown int64 `db:"data_usage_down"` - MeasurementDir string `db:"measurement_dir"` -} - -// ListResults return the list of results -func ListResults(db *sqlx.DB) ([]*Result, []*Result, error) { - doneResults := []*Result{} - incompleteResults := []*Result{} - - rows, err := db.Query(`SELECT id, name, - start_time, runtime, - network_name, country, - asn, - summary, done - FROM results - WHERE done = 1 - ORDER BY start_time;`) - if err != nil { - return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") - } - - for rows.Next() { - result := Result{} - err = rows.Scan(&result.ID, &result.Name, - &result.StartTime, &result.Runtime, - &result.NetworkName, &result.Country, - &result.ASN, - &result.Summary, &result.Done, - //&result.DataUsageUp, &result.DataUsageDown) - ) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - doneResults = append(doneResults, &result) - } - - rows, err = db.Query(`SELECT - id, name, - start_time, - network_name, country, - asn - FROM results - WHERE done != 1 - ORDER BY start_time;`) - if err != nil { - return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") - } - - for rows.Next() { - result := Result{Done: false} - err = rows.Scan(&result.ID, &result.Name, &result.StartTime, - &result.NetworkName, &result.Country, - &result.ASN) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - incompleteResults = append(incompleteResults, &result) - } - return doneResults, incompleteResults, nil -} - -// MakeSummaryMap return a mapping of test names to summaries for the given -// result -func MakeSummaryMap(db *sqlx.DB, r *Result) (summary.SummaryMap, error) { - summaryMap := summary.SummaryMap{} - - msmts := []Measurement{} - // XXX maybe we only want to select some of the columns - err := db.Select(&msmts, "SELECT name, summary FROM measurements WHERE result_id = $1", r.ID) - if err != nil { - return nil, errors.Wrap(err, "failed to get measurements") - } - for _, msmt := range msmts { - val, ok := summaryMap[msmt.Name] - if ok { - summaryMap[msmt.Name] = append(val, msmt.Summary) - } else { - summaryMap[msmt.Name] = []string{msmt.Summary} - } - } - return summaryMap, nil -} - -// Finished marks the result as done and sets the runtime -func (r *Result) Finished(db *sqlx.DB, makeSummary summary.ResultSummaryFunc) error { - if r.Done == true || r.Runtime != 0 { - return errors.New("Result is already finished") - } - r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds() - r.Done = true - // XXX add in here functionality to compute the summary - summaryMap, err := MakeSummaryMap(db, r) - if err != nil { - return err - } - - r.Summary, err = makeSummary(summaryMap) - if err != nil { - return err - } - - err = UpdateOne(db, `UPDATE results - SET done = :done, runtime = :runtime, summary = :summary - WHERE id = :id`, r) - if err != nil { - return errors.Wrap(err, "updating finished result") - } - return nil -} - -// CreateResult writes the Result to the database a returns a pointer -// to the Result -func CreateResult(db *sqlx.DB, homePath string, r Result) (*Result, error) { - log.Debugf("Creating result %v", r) - - p, err := utils.MakeResultsDir(homePath, r.Name, r.StartTime) - if err != nil { - return nil, err - } - r.MeasurementDir = p - res, err := db.NamedExec(`INSERT INTO results - (name, start_time, country, network_name, asn) - VALUES (:name,:start_time,:country,:network_name,:asn)`, - r) - if err != nil { - return nil, errors.Wrap(err, "creating result") - } - id, err := res.LastInsertId() - if err != nil { - return nil, errors.Wrap(err, "creating result") - } - r.ID = id - return &r, nil -} diff --git a/nettests/nettests.go b/nettests/nettests.go index 0243515..66d720e 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -58,11 +58,8 @@ func (c *Controller) Init(nt *mk.Nettest) error { c.msmts = make(map[int64]*database.Measurement) msmtTemplate := database.Measurement{ - ASN: "", - IP: "", - CountryCode: "", ReportID: "", - Name: nt.Name, + TestName: nt.Name, ResultID: c.res.ID, ReportFilePath: c.msmtPath, } @@ -165,9 +162,12 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.geoip_lookup", func(e mk.Event) { log.Debugf(color.RedString(e.Key)) + /* FIXME + Put this into the network table msmtTemplate.ASN = e.Value.ProbeASN msmtTemplate.IP = e.Value.ProbeIP msmtTemplate.CountryCode = e.Value.ProbeCC + */ }) nt.On("status.measurement_start", func(e mk.Event) { diff --git a/ooni.go b/ooni.go index b17c57c..fdd740a 100644 --- a/ooni.go +++ b/ooni.go @@ -6,7 +6,6 @@ import ( "path" "github.com/apex/log" - "github.com/jmoiron/sqlx" "github.com/ooni/probe-cli/config" "github.com/ooni/probe-cli/internal/bindata" "github.com/ooni/probe-cli/internal/database" @@ -14,6 +13,7 @@ import ( "github.com/ooni/probe-cli/internal/onboard" "github.com/ooni/probe-cli/utils" "github.com/pkg/errors" + "upper.io/db.v3/lib/sqlbuilder" ) const Version = "3.0.0-dev.0" @@ -21,7 +21,7 @@ const Version = "3.0.0-dev.0" // Context for OONI Probe type Context struct { Config *config.Config - DB *sqlx.DB + DB sqlbuilder.Database Location *utils.LocationInfo Home string From a518ca79da687d9541078e0fea88345d68a9b088 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Sep 2018 15:34:56 +0200 Subject: [PATCH 07/65] Fix DB schema, write a basic unittest for the DB operations --- data/migrations/1_create_msmt_results.sql | 5 +- internal/bindata/bindata.go | 198 +++++++++++----------- internal/database/actions.go | 1 + internal/database/actions_test.go | 56 ++++++ internal/database/database_test.go | 3 +- internal/database/models.go | 44 ++--- 6 files changed, 183 insertions(+), 124 deletions(-) create mode 100644 internal/database/actions_test.go diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index d4464e8..2341e09 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -70,7 +70,7 @@ CREATE TABLE `results` ( `data_usage_down` INTEGER NOT NULL, -- It's probably reasonable to set the maximum length to 260 as this is the -- maximum length of file paths on windows. - `log_file_path` VARCHAR(260) NOT NULL + `measurement_dir` VARCHAR(260) NOT NULL ); CREATE TABLE `measurements` ( @@ -148,7 +148,7 @@ CREATE TABLE `measurements` ( -- we need for the measurement details views and some result views (ex. the -- upload/download speed of NDT, the reason for blocking of a site, -- etc.) - `test_keys` JSON, + `test_keys` JSON NOT NULL, -- The cross table reference to JOIN the two tables together. `result_id` INTEGER NOT NULL, @@ -166,5 +166,4 @@ CREATE TABLE `measurements` ( FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`), FOREIGN KEY(`network_id`) REFERENCES `networks` (`id`) ); - -- +migrate StatementEnd diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 922a2c5..8c087c5 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -130,105 +130,105 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x6f\xdb\x46\xf2\x7f\xef\x4f\x31\x30\x8a\xd6\xc6\x5f\x92\x93" + - "\xfc\xd3\xe0\xce\xd7\xa2\x70\x13\x25\xa7\x36\x96\x03\x59\xbe\x26\x38\x1c\xc4\x15\x39\x94\xb6\x5e\xee\x32\xfb\x20" + - "\x46\xf7\xe9\x0f\x33\xbb\xa4\x48\xe5\xc9\x01\x9a\x17\x8e\x44\xee\xce\xce\xe3\x6f\x7e\xb3\x1a\x8f\xe1\xff\x2a\xb9" + - "\xb1\xc2\x23\xbc\x30\x8d\x3e\xe9\x3f\xb8\xf5\xc2\x63\x85\xda\xff\x8a\x1b\xa9\x4f\x4e\x5e\x2c\x6e\xde\xc0\xf2\xea" + - "\xd7\xd7\x53\xc8\x2c\xba\xa0\xbc\xcb\xfe\x31\x78\x5a\xa1\x70\xc1\xf2\x9e\xe3\x57\xc1\xaa\xe3\x47\x1a\x7d\x63\xec" + - "\x3d\x3d\xfe\xf4\xb9\x53\x5d\x0c\xdf\xdc\xd5\x5f\x54\xf0\xf9\x62\x7a\xb5\x9c\x0e\x4e\x84\xb3\x13\x80\x4c\x16\x19" + - "\xcc\xe6\xcb\xe9\xab\xe9\x02\xde\x2c\x66\xd7\x57\x8b\x77\xf0\xfb\xf4\x1d\x5c\xdd\x2d\x6f\x66\xf3\xe7\x8b\xe9\xf5" + - "\x74\xbe\x1c\xd1\xca\x60\x55\x06\xff\xba\x5a\x3c\xff\xe7\xd5\xe2\xec\xc9\x8f\x3f\x9e\xc3\xfc\x66\x09\xf3\xbb\xd7" + - "\xaf\x47\x30\x1e\xc3\xdb\xb7\x6f\x41\x3a\xf0\x5b\xe9\x40\x19\xbd\x01\xd4\x26\x6c\xb6\xbf\xd0\xd6\x5c\x78\xdc\x18" + - "\xbb\x5f\xe5\xa6\xc0\x83\x90\x63\x11\xcb\x2d\x42\x2e\xbd\xfc\x2f\x6a\x25\xd6\xd0\xee\x02\xda\x05\xa5\xb1\xe0\xb7" + - "\x78\x02\x0f\xfb\x37\x1e\x83\x93\x1e\x27\xf0\x07\x42\x70\x48\x5b\xc1\x79\x2b\xf5\x06\xe6\x37\xf3\x29\x78\x03\x05" + - "\x6a\xe3\xbf\x45\xa0\x36\x70\xaf\x4d\xa3\x87\x9a\x4d\x4e\xd8\x44\x13\xb4\xff\xc8\xc2\x27\x07\x0b\x5b\x03\x7d\x63" + - "\x40\xa1\xf7\x68\x21\xed\x89\xf6\x35\x5b\x99\x6f\xd9\x7d\x0f\xd3\x68\x3c\x86\xbb\xc5\x6b\x58\x23\x39\xdb\x81\x37" + - "\x27\xe7\x31\x59\xfe\x40\xc8\x2d\x52\x12\x08\x70\x58\x0b\xce\x07\x2f\xd6\x2a\xfa\xb0\x4d\x2d\xfe\xf2\x04\x2c\x0a" + - "\x67\xb4\xbb\xa4\x9d\x8f\x27\xf0\xd2\x58\x70\xa6\x42\x30\x25\xbb\x6c\x27\xb1\x71\xd0\x6c\xd1\x22\x68\xc4\x82\x1f" + - "\x7a\xe3\x85\x02\x1d\xaa\x35\x5a\x5a\x98\x72\xbb\xe8\x64\x8f\x48\x9a\xf4\x3f\x38\xd8\x18\xf2\xb8\x37\xb0\x46\xa8" + - "\x42\xbe\x85\xca\x58\x04\x2c\x4b\x99\x4b\xd4\x9e\xde\xfc\x19\x9c\x07\x65\xcc\x7d\xa8\x59\x3a\x7b\x85\xc4\x5a\xd3" + - "\x38\x90\x3a\xfa\x64\x3c\x8e\x36\x4c\xe8\xd3\x93\x09\x9c\x55\xc6\x79\x90\x55\x6d\xac\x17\xda\x9f\x93\xd9\x8d\x88" + - "\x12\xc5\xce\xc8\x02\x8a\x50\x2b\x99\x0b\x4f\x0a\x08\x58\x07\x9d\x6f\x49\xaa\xd4\xa5\xb1\x95\xf0\xd2\x90\x64\xe1" + - "\x59\xd5\xa1\xa2\xb9\xa9\x2a\x7a\x6b\xc0\xe1\x0e\x2d\xd9\xda\x3a\x8d\x14\x0c\x0e\x2d\x6d\x31\x9a\x95\x99\x7e\x10" + - "\x55\xad\xf0\x32\xf9\xbe\x12\x7b\x68\xa4\xdb\xb2\x22\x45\x41\xff\x71\x4d\xc4\x08\xd0\x7e\x65\xf2\x78\x7c\x69\x4d" + - "\xd5\x3a\xba\xb6\x66\x8d\xf1\x09\x7d\x7d\xf5\xe6\x96\xe4\x19\xcb\x32\x5c\xa8\xc9\x4e\x0e\x99\x50\xca\x34\xac\x6b" + - "\xab\x8a\x37\x70\x9a\x1b\x6b\x31\xf7\xa7\x20\xa0\x92\x2e\x57\xc2\x39\x59\x4a\x2c\xa0\x87\x3b\x49\x60\x21\x1d\xf9" + - "\x24\x48\xb7\x25\x31\x6b\xf4\x0d\xa2\x86\x46\x96\x12\x84\x2e\xa0\x32\x6b\x49\x7e\x1e\x42\x46\x87\x48\xdf\x0a\x1b" + - "\x69\xe3\x4a\x8b\x0a\x87\xf8\xc1\x35\x7f\x1b\x6b\x92\xde\x82\xc5\xda\xa2\x43\xed\x5b\xf3\xfa\x7b\x53\x81\xac\xf7" + - "\x50\x60\x29\x82\xf2\x14\x82\xda\xd4\x41\x09\x8f\x05\xac\x85\xc3\xe2\x6b\x95\x43\x0e\xd0\x2c\xf9\xea\x76\x3e\x79" + - "\xc0\xea\x04\x1e\xbd\x42\xba\xc7\x3d\x39\xdc\x62\x89\x16\x75\x1e\x23\x9a\x32\xf5\x01\x02\x0f\xa9\xe0\x46\xb0\xc6" + - "\x5c\x90\xf8\x66\x98\x35\xa7\xa8\xad\xcc\xb7\xa7\x0f\x15\xd7\x48\x9f\xea\xaa\x10\x5e\xc4\x8a\x41\x28\x83\x0f\x16" + - "\x27\xfd\x10\xf8\x7d\xdd\x0b\xc1\xe3\x67\x31\x02\x37\x9a\xab\x9d\xe2\x3f\x4a\xc1\x27\x44\x83\x4c\xd6\x87\xc5\x4f" + - "\x1f\xf5\xb1\x3a\x06\xce\x58\x74\xe4\x9a\x18\xc1\x2e\x78\x31\xb7\x4d\x09\x42\x83\xac\x77\x4f\x29\xe7\x64\xbd\x7b" + - "\x46\x99\x6c\xd1\xb9\x87\xf8\x7d\xc9\x65\xa2\x37\x48\x35\x5e\x53\xa4\xa3\xb0\x4e\x08\x28\x79\x8f\x97\x0f\x90\xf4" + - "\xe8\xd1\xa3\x47\x97\x5f\xff\x33\x7a\x80\xa8\x98\x80\xd2\xc1\xff\xff\x1d\xf2\xad\xb0\x6c\x49\x26\x9c\xe6\x52\x38" + - "\x7b\xda\xf3\xd0\x5f\xd1\x11\x18\xcf\x87\x45\xd8\x52\x0b\xae\xc1\x6f\xa9\xc2\xe4\x54\xe9\x20\x17\x9a\x20\xce\xc4" + - "\xa0\x9f\x36\xb8\xa6\x36\xe9\x4e\x47\x70\x2a\x2b\xfa\x5b\xa3\x65\x80\xd4\x39\xd2\xd7\x4a\x16\x85\xc2\xb5\xf9\x70" + - "\x1a\xe3\x96\x79\x74\x7e\xb5\xb1\x26\xd4\x47\x25\xfd\xf8\xd9\xd0\x01\xc3\x02\x2a\x64\xc9\x15\xe3\xc1\x79\x61\xfd" + - "\xca\xcb\x0a\x19\x6e\x6c\xd0\xf4\x79\x50\x0d\x1d\x90\x2b\x67\x60\x2b\x76\xd8\x8a\xe3\x04\xf7\xa6\x45\x35\x4e\x74" + - "\xb3\x43\xbb\x45\x51\x90\x3d\xdc\xf8\x22\xe0\x5b\x64\xc8\xa4\x23\x8c\xdf\xa2\x85\x52\xe4\xde\x58\x17\x41\x3f\xc9" + - "\xdb\x18\x90\x9a\x11\x1a\x81\x0c\x9b\x1c\x7c\x25\x18\x60\xa8\x07\x88\xfd\x25\x64\xb7\x77\xd7\x67\x49\xd5\x73\x78" + - "\xb9\xb8\xb9\x86\x01\xa3\x83\x46\x2a\x05\x42\x35\x62\xef\xc8\xbf\x3f\xfd\xdc\x4a\xca\xd2\xae\xb8\xe9\x10\x41\xee" + - "\x5f\xf4\xc2\xc1\x4f\xe7\xd1\xb5\x07\xcf\x64\xf0\xe2\x6a\x39\x5d\xce\xae\xa7\x47\x2e\x6d\xa5\x65\xb0\x98\x5e\xbd" + - "\xee\xbd\x6c\x8f\xbb\x73\xc8\x3d\x47\xea\x82\x9a\x1f\x82\x2c\x0f\x9d\x62\x2b\x1c\x38\x02\x7b\xc6\x8d\xa8\x4b\xca" + - "\x24\xb7\xa2\x56\x8f\x45\x06\xcb\xd9\xfc\x1d\xe5\xf3\xe3\xf3\x4f\x88\xe7\x1c\xa2\x72\x84\x52\x89\x0d\x49\xfd\xe4" + - "\x69\x51\x34\x2d\x2c\x38\xd3\xb8\x5f\xe6\xc1\x52\x02\xa8\x3d\xc5\x5c\x4b\xbd\x99\x74\x67\xd3\xaa\xcf\x9c\xcc\x4b" + - "\x28\xee\xab\xe0\xc4\x06\x57\xa1\x3e\xe4\xfc\xe7\x57\x15\xa6\xd1\x9f\x5b\x37\x1e\xc3\x8c\xb8\x09\xb5\x5c\xb1\x26" + - "\x75\x98\x03\xc5\xfe\x4c\x3d\xdf\xb3\x0d\x95\xf8\x20\xab\x50\x81\x42\xbd\xf1\x0c\xcc\x4f\x9e\x3d\x02\x91\x28\x2e" + - "\x53\xdd\x2e\x2f\x8f\xd6\x9a\x12\x4a\xa9\x10\x6a\xe1\xb7\xc4\x13\xa0\x91\xba\x30\x4d\x82\xbe\x4c\x99\xcd\x8a\xde" + - "\xaf\xe8\x7d\x0f\x1a\x9e\xf5\x40\xf6\x13\xd5\x3f\x4c\xb8\xbf\x08\x02\x2e\xdb\x57\xa5\xc8\x71\x6d\xcc\xfd\xaa\x42" + - "\xe7\x50\x6f\xd0\xb6\x6f\x3c\x2a\xdc\x58\x51\x9d\x74\x38\x28\xbc\x13\x75\xdd\x7e\xdf\x7a\x5f\xaf\xa8\x02\xd1\xae" + - "\x4a\x89\xaa\x58\x55\x42\x4b\x6e\xcc\xd2\xe8\xc1\x2a\xa9\x77\x42\xc9\x62\x65\xf1\x7d\x20\x1c\x51\x52\xf7\x6a\xdb" + - "\x6d\xdb\xcf\xba\xf0\x3d\xb4\x19\xe2\xcc\xb3\xa7\x1f\xa5\xc7\x5f\x51\x38\x2f\xe3\x7c\x01\x75\xb0\xb5\x71\x8c\x8e" + - "\x89\x5d\xb4\x6c\x24\x52\xb4\x3e\x7f\x4c\xad\x36\x15\x75\x2b\x89\x39\xf3\x08\xf6\x26\x80\xdb\x9a\xa0\x0a\xa8\x65" + - "\x7e\x1f\x9b\xb2\xb4\xce\xf7\x91\xa3\x15\xf1\xdb\xcd\x6c\x0e\xce\x58\xa6\x32\xfb\x56\xd2\xc1\xae\x0e\x98\xde\x99" + - "\x40\x35\xf5\x83\x67\x5c\xe4\xbd\x9b\x20\xac\xd0\x1e\x91\xa1\x0d\x88\xb2\xee\xe1\x4c\xd6\x23\x10\x4e\x8f\xda\x9e" + - "\x32\x1a\xb0\xa9\xf3\x56\x5e\xcc\x63\x70\xc4\xb0\xa4\x06\x01\xa7\x7d\xed\x1c\x12\xa5\x74\xce\xe4\x92\x59\x16\x61" + - "\x32\x9c\x46\x7b\xdb\x86\xd0\x8a\xed\x67\xe2\xc7\xee\x9d\x1b\x1f\xe7\x8f\x8d\x51\x42\x6f\x2e\x09\xe6\x5b\xf4\x60" + - "\x4b\x1c\x4d\xac\xbd\xd6\x94\x45\x4c\x20\xfc\xce\x44\xee\xe5\x0e\xb3\x11\x38\x73\xd2\x67\x3e\xd2\x01\xbe\x0f\x72" + - "\x27\x54\x9a\x25\x18\x6d\xd6\xc8\x11\xb3\x81\x81\xa7\x14\xca\x1d\xdc\x97\xf1\x31\x19\x2c\xa7\x6f\x53\x55\x3c\x00" + - "\x7e\x52\x9f\x8e\x30\xd1\x29\x2c\xa0\xc0\x88\x7a\x05\x48\xb7\x0a\xb5\x32\xa2\xc0\x82\x81\x71\x04\x52\x3b\x9f\x9a" + - "\x12\x0f\x38\xc1\x49\xbd\x39\x38\x3d\x2d\x5f\x95\x42\x2a\x2c\x46\x31\x0c\xc2\xb7\x54\x50\x9b\x14\xdf\x4e\x2a\x23" + - "\x52\x2f\x32\x45\xe8\x0a\x87\x83\xe2\xd0\xfb\x01\xa4\xb6\x3b\x1f\x08\xe8\xc7\xf2\xa3\x62\x4c\x75\x83\xe6\x28\x74" + - "\x5d\x24\x25\x35\x85\x8a\x7a\xba\xe5\x65\xad\x3c\x8b\x63\xda\x20\xfd\x41\x93\x28\xea\x4b\xf0\x4e\x2b\x82\xc5\x55" + - "\xe5\x36\x47\x23\xc2\xc9\x91\x3d\x0f\x10\xd6\x5b\xf8\x25\x99\xd4\x05\xdc\xc7\x0d\x8c\xa3\xc0\xc9\x55\x0b\xeb\x65" + - "\x1e\x94\xb0\x03\xc7\x50\x0f\x5d\x53\x0f\x4d\x96\x0a\x5d\x1c\x72\x12\x2d\x96\x26\xf1\x92\xbb\x19\x23\xad\x17\xf7" + - "\x98\xb2\x95\x98\x86\xc8\xe3\x7c\xeb\x0d\xa0\x64\x5e\xb2\x95\x05\x82\xf4\xdd\xec\x77\xf0\x24\xf7\x50\x42\x13\x9e" + - "\x03\x63\x57\xe2\xe2\x56\x28\x9c\xa7\x41\xae\x9b\x29\xc5\x5a\x2a\xe9\xd3\x68\x32\x88\x40\xba\x9a\x29\x0c\xe5\x16" + - "\x13\xaa\x96\x5d\xa5\x2c\xee\x8d\x32\x26\xc1\x19\x0b\xe8\x19\xfd\x4b\x17\x05\x8b\x36\xe8\x6f\x48\x29\x87\x76\x87" + - "\x76\xec\xc8\xc6\xc8\xc8\x56\xb2\x00\x8b\x3e\x58\xcd\x50\x97\x46\x7e\xa5\x90\xd8\xd9\x04\x7e\xdd\x0f\x4b\xe5\xb0" + - "\xe9\x7b\x90\xba\x0e\x3e\x02\x2b\x79\xf6\x7d\x20\x5f\xb0\xf5\xb5\x24\xe5\x4b\xf4\xe9\x0a\xa5\xaf\x7c\xe7\x86\xe9" + - "\x87\xee\xe3\xab\xe9\x92\x1b\x92\xbb\xbc\xb8\x10\xb5\x9c\x18\xa3\xe5\x44\x1a\xfa\x7c\xb1\x7b\x7c\xd1\xef\xb4\xbf" + - "\xf0\xa9\x3f\x7f\x37\x9b\xbf\xb9\x5b\x7e\xdf\xa9\xf3\xf3\x77\x8b\xe9\x9b\x9b\xc5\x72\x35\x7b\x71\x90\xef\xad\xc8" + - "\x7d\x0f\xe8\xa5\xc7\xea\x30\xd3\x27\xfa\xfe\xef\xff\x64\xa0\xa4\xf3\x6d\x51\xe9\xa8\x77\xd7\x88\xfb\x7d\x7e\xc5" + - "\x97\x6e\xde\xc0\x26\x91\x92\xdf\x6e\x6f\xe6\xf1\xca\x60\x68\x24\x8d\xa0\x3d\xf2\x8a\x2e\x8e\x15\x3b\xa1\x02\x3a" + - "\x38\xcb\x3a\xbd\xb3\x11\x64\x6c\x51\x76\x0e\xc2\x72\x45\x97\x41\x1d\xbc\x27\x3a\x4a\xd3\x13\xce\x45\x41\x89\x2f" + - "\x94\x45\x51\xec\x63\x01\xd4\xd6\xe4\xc4\x15\xba\x30\xd6\xb2\x46\xea\xe8\xa3\x1e\x1e\xc8\xaa\x56\x51\x48\xae\x50" + - "\xe8\x50\xf3\x64\x98\xc4\x74\xe8\xd6\x77\x78\x02\x8e\x83\xc6\x1f\x5f\x1a\xf4\x69\x0c\x4f\x52\x0d\xb9\x51\x9b\x96" + - "\xf4\x33\xf9\x6a\x0b\xf5\x2b\x93\xdd\x78\x9c\xae\xcb\x8a\x49\x02\x9b\x60\xd5\x57\x9a\x59\x9b\xe1\x04\xd3\x7b\xf4" + - "\xc4\x86\x51\xd0\xb8\xdd\xde\xe6\x74\x09\x3d\x82\x75\x60\x54\x27\x5f\xd7\x4a\x30\xef\x4d\x57\x43\x83\x56\x26\x7c" + - "\xbc\x77\xab\x8d\x3c\xb0\x02\x8d\xc2\xf6\x06\xf9\x38\x77\x23\x5e\x76\xb9\xbb\x91\x7e\x1b\xd6\x93\xdc\x54\x17\x94" + - "\xc2\x17\x6d\x04\x2e\xd6\xca\xac\x2f\x2a\xe1\x3c\xda\x8b\xc2\xe4\x8e\x5f\x8f\x43\x90\xc5\xa4\x2a\xe0\xfb\x3e\x29" + - "\xfb\xa2\x1c\xe9\x5c\x40\x77\xf1\xf4\x6f\xd1\x35\xfd\xd4\x4c\x2e\x22\x3e\x76\xec\x99\x04\xa6\xae\xb5\x23\x17\x91" + - "\x50\x09\x68\xe7\x4d\x9e\xb6\x46\x31\xb1\x04\xdf\xd4\x92\x3f\x69\xa8\x57\x1d\xfb\x59\x2b\x93\xdf\x53\x73\xa4\x2e" + - "\x4e\x08\xa8\x61\x76\xcd\x1b\xdb\x31\x23\x7d\x75\x34\xa3\xb9\x84\x04\xf5\x97\x05\xc9\x92\xaf\xc8\xd2\x50\x0b\x8d" + - "\x70\x50\xa0\xc7\x9c\xe3\xdf\xe3\x58\x94\x5d\x19\xb1\xb2\x0c\x04\x64\xcf\x6f\xee\xe6\xcb\xb3\xf3\xac\x2b\x3d\x2e" + - "\xac\x23\xfe\x17\xa1\x3a\x15\xab\xe8\xee\x31\x8f\xb4\x80\x68\xbf\xb1\xdd\x83\xd9\x35\xa9\xed\x3a\x8c\x15\xda\x54" + - "\x42\xed\xfb\x28\xfb\x89\x01\x4c\x83\xa9\xc5\xfb\x90\x20\xc1\x79\x1b\x72\xca\x93\x51\xba\xac\x6d\x88\x51\x51\x2b" + - "\xea\xdf\xe6\x32\x9b\xbe\xc7\x7d\x47\x55\x9b\x74\xab\x9b\x2e\xd7\x87\x0c\x03\xbd\x90\xca\xa5\x2b\x60\x02\x2b\x16" + - "\xd5\x6b\x4b\x0e\xce\xf0\xc3\xa4\xdf\xb3\x62\x41\x5f\xd0\xf4\x45\x1f\xc0\xd5\x24\xdd\x94\x30\x7f\xb1\x1c\x25\x5f" + - "\x31\x89\x2a\x5b\xfb\xa9\x1c\x38\x33\xc8\x2d\x1d\xdd\x42\x9f\x4f\xce\x7b\x13\x00\xe9\x9c\xb1\xa5\x7d\x4f\x20\xe4" + - "\xd6\xb8\xf6\x6a\x75\xd0\xc7\x98\x4f\xfb\x74\xb9\x12\xef\xdb\xc0\x9b\x0d\x52\xc7\xed\x00\x86\x0c\xf9\x5c\xa5\x7f" + - "\x3c\xf0\xee\x84\x95\x7c\x10\x73\x06\xa9\x3d\x5a\x2d\x94\xe2\x9e\x4b\xc0\x1f\x19\x3e\x8d\x74\x6d\x23\x35\x7a\x5c" + - "\x48\x77\xff\x09\x44\x75\x93\x3f\x9d\xd1\x13\x98\x79\xa6\x7b\x15\x71\x04\x87\xda\xb1\xee\x8d\xa5\x72\x20\x26\x1b" + - "\x87\x48\xb4\x80\x7c\x2b\x74\x18\x0c\xb6\xc6\xb0\xe7\xae\x7f\xe7\xc0\xd4\x16\x77\xe9\xda\xb4\x25\x12\x24\xa4\x85" + - "\x9a\x28\xc7\x68\x62\x0c\xf7\xe9\x1a\xab\x12\x07\x61\xc4\x03\x2a\xa1\xf7\x03\x0d\xf9\xdc\x92\x6f\x82\xfb\x78\xfc" + - "\xb5\xb9\x35\xc5\xe7\xe5\xcd\x62\x3a\x7b\x35\xe7\x51\xf4\xac\xe7\xea\x73\x58\x4c\x5f\x4e\x17\xd3\xf9\xf3\xe9\xed" + - "\xe1\x3e\xeb\x8c\xc6\xd8\xf3\x04\xd4\x37\x73\x78\x31\x7d\x3d\x5d\x4e\xe1\xf9\xd5\xed\xf3\xab\x17\x53\x7a\x72\xf7" + - "\x86\xe6\xba\xf6\x09\x37\x81\x59\x49\xe9\x5b\xa0\x42\x1f\x69\x0c\xe7\x65\x9f\xe4\x3c\xf4\xa7\x9d\xe4\x07\xa1\xd4" + - "\x71\x11\xb8\xf4\x8b\x40\x3c\xa5\xa0\xe9\xbf\x41\xa5\x26\x9f\xb0\x31\x75\x8d\xa1\x81\xfc\x43\x5b\xb4\x6e\x74\xbc" + - "\xe7\xac\x3f\x39\x0d\xb7\xf5\x2e\xdb\xa3\x67\xce\xbf\xf4\x3b\xe0\xff\x02\x00\x00\xff\xff\x29\x5f\x48\x5d\xaa\x1c" + - "\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x6f\xdb\xc6\xb2\xfe\xee\x5f\x31\x30\x8a\xd6\xc6\x95\xe4\x24" + + "\x37\x0d\xee\x75\x5b\x14\xae\xad\xe4\xa8\x8d\xe5\x40\x96\x4f\x13\x1c\x1c\x88\x2b\x72\x28\x6d\xbd\xdc\x65\xf6\x45" + + "\x8a\xce\xaf\x3f\x98\xd9\x25\x45\xca\x4e\xe2\x00\xcd\x07\x47\x22\x77\x67\xe7\xf5\x99\x67\x56\xc3\x21\xfc\x4f\x25" + + "\x57\x56\x78\x84\x2b\xb3\xd5\x47\xdd\x07\xb7\x5e\x78\xac\x50\xfb\xdf\x70\x25\xf5\xd1\xd1\xd5\xec\xe6\x1d\xcc\x2f" + + "\x7e\x7b\x3b\x86\xcc\xa2\x0b\xca\xbb\xec\xa7\xde\xd3\x0a\x85\x0b\x96\xf7\x1c\xbe\x0a\x56\x1d\x3e\xd2\xe8\xb7\xc6" + + "\xde\xd3\xe3\xc7\xcf\x1d\xeb\xa2\xff\xe6\xae\xfe\xa2\x82\x97\xb3\xf1\xc5\x7c\xdc\x3b\x11\x4e\x8e\x00\x32\x59\x64" + + "\x30\x99\xce\xc7\x6f\xc6\x33\x78\x37\x9b\x5c\x5f\xcc\x3e\xc0\x1f\xe3\x0f\x70\x71\x37\xbf\x99\x4c\x2f\x67\xe3\xeb" + + "\xf1\x74\x3e\xa0\x95\xc1\xaa\x0c\xfe\x79\x31\xbb\xfc\xc7\xc5\xec\xe4\xc5\x8f\x3f\x9e\xc2\xf4\x66\x0e\xd3\xbb\xb7" + + "\x6f\x07\x30\x1c\xc2\xfb\xf7\xef\x41\x3a\xf0\x6b\xe9\x40\x19\xbd\x02\xd4\x26\xac\xd6\xbf\xd2\xd6\x5c\x78\x5c\x19" + + "\xbb\x5b\xe4\xa6\xc0\xbd\x90\x43\x11\xf3\x35\x42\x2e\xbd\xfc\x0f\x6a\x25\x96\xd0\xec\x02\xda\x05\xa5\xb1\xe0\xd7" + + "\x78\x04\x4f\xfb\x37\x1c\x82\x93\x1e\x47\xf0\x27\x42\x70\x48\x5b\xc1\x79\x2b\xf5\x0a\xa6\x37\xd3\x31\x78\x03\x05" + + "\x6a\xe3\xbf\x45\xa0\x36\x70\xaf\xcd\x56\xf7\x35\x1b\x1d\xb1\x89\x26\x68\xff\xc0\xc2\x17\x7b\x0b\x1b\x03\xfd\xd6" + + "\x80\x42\xef\xd1\x42\xda\x13\xed\xdb\xae\x65\xbe\x66\xf7\x3d\x4d\xa3\xe1\x10\xee\x66\x6f\x61\x89\xe4\x6c\x07\xde" + + "\x1c\x9d\xc6\x64\xf9\x13\x21\xb7\x48\x49\x20\xc0\x61\x2d\x38\x1f\xbc\x58\xaa\xe8\xc3\x26\xb5\xf8\xcb\x0b\xb0\x28" + + "\x9c\xd1\xee\x9c\x76\x3e\x1f\xc1\x6b\x63\xc1\x99\x0a\xc1\x94\xec\xb2\x8d\xc4\xad\x83\xed\x1a\x2d\x82\x46\x2c\xf8" + + "\xa1\x37\x5e\x28\xd0\xa1\x5a\xa2\xa5\x85\x29\xb7\x8b\x56\xf6\x80\xa4\x49\xff\x83\x83\x95\x21\x8f\x7b\x03\x4b\x84" + + "\x2a\xe4\x6b\xa8\x8c\x45\xc0\xb2\x94\xb9\x44\xed\xe9\xcd\x5f\xc1\x79\x50\xc6\xdc\x87\x9a\xa5\xb3\x57\x48\xac\x35" + + "\x5b\x07\x52\x47\x9f\x0c\x87\xd1\x86\x11\x7d\x7a\x31\x82\x93\xca\x38\x0f\xb2\xaa\x8d\xf5\x42\xfb\x53\x32\x7b\x2b" + + "\xa2\x44\xb1\x31\xb2\x80\x22\xd4\x4a\xe6\xc2\x93\x02\x02\x96\x41\xe7\x6b\x92\x2a\x75\x69\x6c\x25\xbc\x34\x24\x59" + + "\x78\x56\xb5\xaf\x68\x6e\xaa\x8a\xde\x1a\x70\xb8\x41\x4b\xb6\x36\x4e\x23\x05\x83\x43\x4b\x5b\x8c\x66\x65\xc6\x9f" + + "\x44\x55\x2b\x3c\x4f\xbe\xaf\xc4\x0e\xb6\xd2\xad\x59\x91\xa2\xa0\xff\xb8\x26\x62\x04\x68\xbf\x32\x79\x3c\xbe\xb4" + + "\xa6\x6a\x1c\x5d\x5b\xb3\xc4\xf8\x84\xbe\xbe\x79\x77\x4b\xf2\x8c\x65\x19\x2e\xd4\x64\x27\x87\x4c\x28\x65\xb6\xac" + + "\x6b\xa3\x8a\x37\x70\x9c\x1b\x6b\x31\xf7\xc7\x20\xa0\x92\x2e\x57\xc2\x39\x59\x4a\x2c\xa0\x83\x3b\x49\x60\x21\x1d" + + "\xf9\x24\x48\xb7\x26\x31\x4b\xf4\x5b\x44\x0d\x5b\x59\x4a\x10\xba\x80\xca\x2c\x25\xf9\xb9\x0f\x19\x2d\x22\x7d\x2b" + + "\x6c\xa4\x8d\x0b\x2d\x2a\xec\xe3\x07\xd7\xfc\x6d\xac\x49\x7a\x0b\x16\x6b\x8b\x0e\xb5\x6f\xcc\xeb\xee\x4d\x05\xb2" + + "\xdc\x41\x81\xa5\x08\xca\x53\x08\x6a\x53\x07\x25\x3c\x16\xb0\x14\x0e\x8b\xaf\x55\x0e\x39\x40\xb3\xe4\x8b\xdb\xe9" + + "\xe8\x09\xab\x13\x78\x74\x0a\xe9\x1e\x77\xe4\x70\x8b\x25\x5a\xd4\x79\x8c\x68\xca\xd4\x27\x08\xdc\xa7\x82\x1b\xc0" + + "\x12\x73\x41\xe2\xb7\xfd\xac\x39\x46\x6d\x65\xbe\x3e\x7e\xaa\xb8\xad\xf4\xa9\xae\x0a\xe1\x45\xac\x18\x84\x32\xf8" + + "\x60\x71\xd4\x0d\x81\xdf\xd5\x9d\x10\x3c\x7f\x15\x23\x70\xa3\xb9\xda\x29\xfe\x83\x14\x7c\x42\x34\xc8\x64\xbd\x5f" + + "\xfc\xf2\x59\x17\xab\x63\xe0\x8c\x45\x47\xae\x89\x11\x6c\x83\x17\x73\xdb\x94\x20\x34\xc8\x7a\xf3\x92\x72\x4e\xd6" + + "\x9b\x57\x94\xc9\x16\x9d\x7b\x8a\xdf\xe7\x5c\x26\x7a\x85\x54\xe3\x35\x45\x3a\x0a\x6b\x85\x80\x92\xf7\x78\xfe\x04" + + "\x49\xcf\x9e\x3d\x7b\x76\xfe\xf5\x3f\x83\x27\x88\x8a\x09\x28\x1d\xfc\xef\xff\x43\xbe\x16\x96\x2d\xc9\x84\xd3\x5c" + + "\x0a\x27\x2f\x3b\x1e\xfa\x3b\x3a\x02\xe3\x79\xbf\x08\x1b\x6a\xc1\x35\xf8\x2d\x55\x98\x9c\x2a\x1d\xe4\x42\x13\xc4" + + "\x99\x18\xf4\xe3\x2d\x2e\xa9\x4d\xba\xe3\x01\x1c\xcb\x8a\xfe\xd6\x68\x19\x20\x75\x8e\xf4\xb5\x92\x45\xa1\x70\x69" + + "\x3e\x1d\xc7\xb8\x65\x1e\x9d\x5f\xac\xac\x09\xf5\x41\x49\x3f\x7f\xd5\x77\x40\xbf\x80\x0a\x59\x72\xc5\x78\x70\x5e" + + "\x58\xbf\xf0\xb2\x42\x86\x1b\x1b\x34\x7d\xee\x55\x43\x0b\xe4\xca\x19\x58\x8b\x0d\x36\xe2\x38\xc1\xbd\x69\x50\x8d" + + "\x13\xdd\x6c\xd0\xae\x51\x14\x64\x0f\x37\xbe\x08\xf8\x16\x19\x32\xe9\x08\xe3\xd7\x68\xa1\x14\xb9\x37\xd6\x45\xd0" + + "\x4f\xf2\x56\x06\xa4\x66\x84\x46\x20\xc3\x46\x7b\x5f\x09\x06\x18\xea\x01\x62\x77\x0e\xd9\xed\xdd\xf5\x49\x52\xf5" + + "\x14\x5e\xcf\x6e\xae\xa1\xc7\xe8\x60\x2b\x95\x02\xa1\xb6\x62\xe7\xc8\xbf\x3f\xff\xd2\x48\xca\xd2\xae\xb8\x69\x1f" + + "\x41\xee\x5f\xf4\xc2\xc1\xcf\xa7\xd1\xb5\x7b\xcf\x64\x70\x75\x31\x1f\xcf\x27\xd7\xe3\x03\x97\x36\xd2\x32\x98\x8d" + + "\x2f\xde\x76\x5e\x36\xc7\xdd\x39\xe4\x9e\x23\x75\x41\xcd\x0f\x41\x96\xfb\x4e\xb1\x16\x0e\x1c\x81\x3d\xe3\x46\xd4" + + "\x25\x65\x92\x5b\x50\xab\xc7\x22\x83\xf9\x64\xfa\x81\xf2\xf9\xf9\xe9\x23\xe2\x39\x87\xa8\x1c\xa1\x54\x62\x45\x52" + + "\x1f\x3d\x2d\x8a\xa6\x85\x05\x67\x1a\xf7\xcb\x3c\x58\x4a\x00\xb5\xa3\x98\x6b\xa9\x57\xa3\xf6\x6c\x5a\xf5\x99\x93" + + "\x79\x09\xc5\x7d\x11\x9c\x58\xe1\x22\xd4\xfb\x9c\xff\xfc\xaa\xc2\x6c\xf5\xe7\xd6\x0d\x87\x30\x21\x6e\x42\x2d\x57" + + "\x2c\x49\x1d\xe6\x40\xb1\x3f\x53\xcf\xf7\x6c\x43\x25\x3e\xc9\x2a\x54\xa0\x50\xaf\x3c\x03\xf3\x8b\x57\xcf\x40\x24" + + "\x8a\xcb\x54\xb7\xcd\xcb\x83\xb5\xa6\x84\x52\x2a\x84\x5a\xf8\x35\xf1\x04\xd8\x4a\x5d\x98\x6d\x82\xbe\xee\x2c\xb0" + + "\x28\xa4\xed\x80\xc3\xab\x0e\xcc\x3e\x52\xff\xfd\x94\xfb\x9b\x40\xe0\xbc\x79\x55\x8a\x1c\x97\xc6\xdc\x2f\x2a\x74" + + "\x0e\xf5\x0a\x6d\xf3\xc6\xa3\xc2\x95\x15\xd5\x51\x8b\x84\xc2\x3b\x51\xd7\xcd\xf7\xb5\xf7\xf5\x82\x6a\x10\xed\xa2" + + "\x94\xa8\x8a\x45\x25\xb4\xe4\xd6\x2c\x8d\xee\xad\x92\x7a\x23\x94\x2c\x16\x16\x3f\x06\x42\x12\x25\x75\xa7\xba\xdd" + + "\xba\xf9\xac\x0b\xdf\xc1\x9b\x3e\xd2\xbc\x7a\xf9\x20\x41\xfe\x8e\xd2\x79\x1d\x27\x0c\xa8\x83\xad\x8d\x63\x7c\x4c" + + "\xfc\xa2\xe1\x23\x91\xa4\x75\x19\x64\x6a\xb6\xa9\xac\x1b\x49\xcc\x9a\x07\xb0\x33\x01\xdc\xda\x04\x55\x40\x2d\xf3" + + "\xfb\xd8\x96\xa5\x75\xbe\x8b\x1d\x8d\x88\xdf\x6f\x26\x53\x70\xc6\x32\x99\xd9\x35\x92\xf6\x76\xb5\xd0\xf4\xc1\x04" + + "\xaa\xaa\x1f\x3c\x23\x23\xef\x5d\x05\x61\x85\xf6\x88\x0c\x6e\x40\xa4\x75\x07\x27\xb2\x1e\x80\x70\x7a\xd0\x74\x95" + + "\x41\x8f\x4f\x9d\x36\xf2\x62\x26\x83\x23\x8e\x25\x35\x08\x38\xee\x6a\xe7\x90\x48\xa5\x73\x26\x97\xcc\xb3\x08\x95" + + "\xe1\x38\xda\xdb\xb4\x84\x46\x6c\x37\x13\x1f\xba\x77\x6a\x7c\x9c\x40\x56\x46\x09\xbd\x3a\x27\xa0\x6f\xf0\x83\x2d" + + "\x71\x34\xb3\x76\x9a\x53\x16\x51\x81\x10\x3c\x13\xb9\x97\x1b\xcc\x06\xe0\xcc\x51\x97\xfb\x48\x07\xf8\x31\xc8\x8d" + + "\x50\x69\x9a\x60\xbc\x59\x22\x47\xcc\x06\x86\x9e\x52\x28\xb7\x77\x5f\xc6\xc7\x64\x30\x1f\xbf\x4f\x55\xf1\x04\x00" + + "\x4a\x9d\x3a\x02\x45\xab\xb0\x80\x02\x23\xee\x15\x20\xdd\x22\xd4\xca\x88\x02\x0b\x86\xc6\x01\x48\xed\x7c\x6a\x4b" + + "\x3c\xe2\x04\x27\xf5\x6a\xef\xf4\xb4\x7c\x51\x0a\xa9\xb0\x18\xc4\x30\x08\xdf\x90\x41\x6d\x52\x7c\x5b\xa9\x8c\x49" + + "\x9d\xc8\x14\xa1\x2d\x1c\x0e\x8a\x43\xef\x7b\xa0\xda\xec\x7c\x22\xa4\x1f\xca\x8f\x8a\x31\xd9\x0d\x9a\xa3\xd0\xf6" + + "\x91\x94\xd4\x14\x2a\xea\xea\x96\x97\x35\xf2\x2c\x0e\x69\x83\xf4\x7b\x4d\xa2\xa8\x2f\x01\x3c\xad\x08\x16\x17\x95" + + "\x5b\x1d\x0c\x09\x47\x07\xf6\x3c\x41\x58\x67\xe1\x97\x64\x52\x1f\x70\x0f\x5b\x18\x47\x81\x93\xab\x16\xd6\xcb\x3c" + + "\x28\x61\x7b\x8e\xa1\x2e\xba\xa4\x2e\x9a\x2c\x15\xba\xd8\xe7\x24\x5a\x2c\x4d\x62\x26\x77\x13\x46\x5a\x2f\xee\x31" + + "\x65\x2b\x71\x0d\x91\xc7\x09\xd7\x1b\x40\xc9\xcc\x64\x2d\x0b\x04\xe9\xdb\xe9\x6f\xef\x49\xee\xa2\x84\x26\x3c\x09" + + "\xc6\xbe\xc4\xc5\xad\x50\x38\x4f\xa3\x5c\x3b\x55\x8a\xa5\x54\xd2\xa7\xe1\xa4\x17\x81\x74\x39\x53\x18\xca\x2d\xa6" + + "\x54\x0d\xbf\x4a\x59\xdc\x19\x66\x4c\x82\x33\x16\xd0\x31\xfa\xd7\x36\x0a\x16\x6d\xd0\xdf\x90\x52\x0e\xed\x06\xed" + + "\xd0\x91\x8d\x91\x93\x2d\x64\x01\x16\x7d\xb0\x9a\xa1\x2e\x0d\xfd\x4a\x21\xf1\xb3\x11\xfc\xb6\xeb\x97\xca\x7e\xd3" + + "\xf7\x20\x75\x1d\x7c\x04\x56\xf2\xec\xc7\x40\xbe\x60\xeb\x6b\x49\xca\x97\xe8\xd3\x25\x4a\x57\xf9\xd6\x0d\xe3\x4f" + + "\xed\xc7\x37\xe3\x39\x37\x24\x77\x7e\x76\x26\x6a\x39\x32\x46\xcb\x91\x34\xf4\xf9\x6c\xf3\xfc\xac\xdb\x69\x7f\xe5" + + "\x53\x7f\xf9\x6e\x32\x7d\x77\x37\xff\xbe\x55\xe7\x97\xef\x66\xe3\x77\x37\xb3\xf9\x62\x72\xb5\x97\xef\xad\xc8\x7d" + + "\x07\xe8\xa5\xc7\x6a\x3f\xd5\x27\x02\xff\xaf\x7f\x67\xa0\xa4\xf3\x4d\x51\xe9\xa8\x77\xdb\x88\x7b\x04\x81\xaf\xdd" + + "\xbc\x81\x55\xa2\x25\xbf\xdf\xde\x4c\xe3\xa5\x41\xdf\x48\x1a\x42\x3b\xf4\x15\x5d\x1c\x2c\x36\x42\x05\x74\x70\x92" + + "\xb5\x7a\x67\x03\xc8\xd8\xa2\xec\x14\x84\xe5\x8a\x2e\x83\xda\x7b\x4f\xb4\xa4\xa6\x23\x9c\x8b\x82\x12\x5f\x28\x8b" + + "\xa2\xd8\xc5\x02\xa8\xad\xc9\x89\x2b\xb4\x61\xac\x65\x8d\xd4\xd1\x07\x1d\x3c\x90\x55\xad\xa2\x90\x5c\xa1\xd0\xa1" + + "\xe6\xd9\x30\x89\x69\xd1\xad\xeb\xf0\x04\x1c\x7b\x8d\x1f\x5e\x1b\x74\x69\x0c\xcf\x52\x5b\x72\xa3\x36\x0d\xed\x67" + + "\xfa\xd5\x14\xea\x57\x66\xbb\xe1\x30\x5d\x98\x15\xa3\x04\x36\xc1\xaa\xaf\x34\xb3\x26\xc3\x09\xa6\x77\xe8\x89\x0f" + + "\xa3\xa0\x81\xbb\xb9\xcf\x69\x13\x7a\x00\xcb\xc0\xa8\x4e\xbe\xae\x95\x60\xe6\x9b\x2e\x87\x7a\xad\x4c\xf8\x78\xf3" + + "\x56\x1b\xb9\x67\x05\x1a\x85\xed\x8c\xf2\x71\xf2\x46\x3c\x6f\x73\x77\x25\xfd\x3a\x2c\x47\xb9\xa9\xce\x28\x85\xcf" + + "\x9a\x08\x9c\x2d\x95\x59\x9e\x55\xc2\x79\xb4\x67\x85\xc9\x1d\xbf\x1e\x86\x20\x8b\x51\x55\xc0\xf7\x5d\x52\xf6\x45" + + "\x39\xd2\xb9\x80\xee\xec\xe5\xff\x3d\xe4\xae\xc9\x45\xc4\xc7\x0e\x3d\x93\xc0\xd4\x35\x76\xe4\x22\x12\x2a\x01\xcd" + + "\xc4\xc9\xf3\xd6\x20\x26\x96\xe0\xbb\x5a\xf2\x27\x8d\xf5\xaa\x65\x3f\x4b\x65\xf2\x7b\x6a\x8e\xd4\xc5\x09\x01\x35" + + "\x4c\xae\x79\x63\x33\x68\xa4\xaf\x8e\xa6\x34\x97\x90\xa0\xfe\xb2\x20\x59\xf2\x25\x59\x1a\x6b\x61\x2b\x1c\x14\xe8" + + "\x31\xe7\xf8\x77\x38\x16\x65\x57\x46\xac\x2c\x03\x01\xd9\xe5\xcd\xdd\x74\x7e\x72\x9a\xb5\xa5\xc7\x85\x75\xc0\xff" + + "\x22\x54\xa7\x62\x15\xed\x4d\xe6\x81\x16\x10\xed\x37\xb6\x7d\x30\xb9\x26\xb5\x5d\x8b\xb1\x42\x9b\x4a\xa8\x5d\x17" + + "\x65\x1f\x19\xc1\x34\x98\x5a\x7c\x0c\x09\x12\x9c\xb7\x21\xa7\x3c\x19\xa4\xeb\xda\x2d\x31\x2a\x6a\x45\xdd\xfb\x5c" + + "\x66\xd3\xf7\xb8\x6b\xa9\xea\x36\xdd\xeb\xa6\xeb\xf5\x3e\xc3\x40\x2f\xa4\x72\xe9\x12\x98\xc0\x8a\x45\x75\xda\x92" + + "\x83\x13\xfc\x34\xea\xf6\xac\x58\xd0\x67\x34\x7f\xd1\x07\x70\x35\x49\x37\x25\x4c\xaf\xe6\x83\xe4\x2b\x26\x51\x65" + + "\x63\x3f\x95\x03\x67\x06\xb9\xa5\xa5\x5b\xe8\xf3\xd1\x69\x67\x02\x20\x9d\x33\xb6\xb4\xeb\x09\x84\xdc\x1a\xd7\x5c" + + "\xae\xf6\xfa\x18\xf3\x69\x9f\xae\x57\xe2\x8d\x1b\x78\xb3\x42\xea\xb8\x2d\xc0\x90\x21\x9f\xab\xf4\x87\x23\xef\x46" + + "\x58\xc9\x07\x31\x67\x90\xda\xa3\xd5\x42\x29\xee\xb9\x04\xfc\x91\xe1\xd3\xd0\xd7\x34\x52\xa3\x87\x85\x74\xf7\x8f" + + "\x20\xaa\x1b\xfd\xe5\x8c\x1e\xc1\xc4\x33\xdd\xab\x88\x23\x38\xd4\x8e\x75\xdf\x5a\x2a\x07\x62\xb2\x71\x8c\x44\x0b" + + "\xc8\xf7\x42\xfb\xc1\x60\x6d\x0c\x7b\xee\xfa\x0f\x0e\x4c\x6d\x71\x93\x2e\x4e\x1b\x22\x41\x42\x1a\xa8\x89\x72\x8c" + + "\x26\xc6\x70\x9f\x2e\xb2\x2a\xb1\x17\x46\x3c\xa0\x12\x7a\xd7\xd3\x90\xcf\x2d\xf9\x2e\xb8\x8b\xc7\xf4\x64\x41\x46" + + "\x7e\x66\x6e\x4d\xf1\x79\x7d\x33\x1b\x4f\xde\x4c\x79\x14\x3d\xe9\xb8\xfa\x14\x66\xe3\xd7\xe3\xd9\x78\x7a\x39\xbe" + + "\xdd\xdf\x68\x9d\xd0\x18\x7b\x9a\x80\xfa\x66\x0a\x57\xe3\xb7\xe3\xf9\x18\x2e\x2f\x6e\x2f\x2f\xae\xc6\xf4\xe4\xee" + + "\x1d\xcd\x75\xcd\x13\x6e\x02\x93\x92\xd2\xb7\x40\x85\x3e\xd2\x18\xce\xcb\x2e\xc9\x79\xea\x8f\x3b\xc9\x0f\x42\xa9" + + "\xc3\x22\x70\xe9\x37\x81\x78\x4a\x41\xf3\xff\x16\x95\x1a\x3d\x62\x63\xea\x1a\x7d\x03\xf9\xa7\xb6\x68\xdd\xe0\x70" + + "\xcf\x49\x77\x72\xea\x6f\xeb\x5c\xb7\x47\xcf\x9c\xfe\xf4\xf9\x1f\x02\xff\x1b\x00\x00\xff\xff\x66\x75\x1e\xc1\xab" + + "\x1c\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( diff --git a/internal/database/actions.go b/internal/database/actions.go index adc1ce2..2b3090e 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -122,6 +122,7 @@ func CreateMeasurement(sess sqlbuilder.Database, m Measurement, i string) (*Meas // XXX Do we want to have this be part of something else? m.StartTime = time.Now().UTC() + m.TestKeys = "" // XXX insert also the URL and stuff //m.Input = i diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go new file mode 100644 index 0000000..1712562 --- /dev/null +++ b/internal/database/actions_test.go @@ -0,0 +1,56 @@ +package database + +import ( + "database/sql" + "io/ioutil" + "os" + "testing" + "time" +) + +func TestMeasurementWorkflow(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + tmpdir, err := ioutil.TempDir("", "oonitest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + sess, err := Connect(tmpfile.Name()) + if err != nil { + t.Error(err) + } + result, err := CreateResult(sess, tmpdir, Result{ + TestGroupName: "websites", + StartTime: time.Now().UTC(), + }) + if err != nil { + t.Fatal(err) + } + + msmtTemplate := Measurement{ + ReportID: sql.NullString{String: "", Valid: false}, + TestName: "antani", + ResultID: result.ID, + ReportFilePath: tmpdir, + } + m1, err := CreateMeasurement(sess, msmtTemplate, "") + if err != nil { + t.Fatal(err) + } + + var m2 Measurement + err = sess.Collection("measurements").Find("id", m1.ID).One(&m2) + if err != nil { + t.Fatal(err) + } + if m2.ResultID != m1.ResultID { + t.Error("result_id mismatch") + } + +} diff --git a/internal/database/database_test.go b/internal/database/database_test.go index 0641d7d..355471c 100644 --- a/internal/database/database_test.go +++ b/internal/database/database_test.go @@ -13,6 +13,8 @@ func TestConnect(t *testing.T) { if err != nil { t.Error(err) } + defer os.Remove(tmpfile.Name()) + sess, err := Connect(tmpfile.Name()) if err != nil { t.Error(err) @@ -27,5 +29,4 @@ func TestConnect(t *testing.T) { log.Fatal("missing tables") } - defer os.Remove(tmpfile.Name()) } diff --git a/internal/database/models.go b/internal/database/models.go index 06a4a53..246b4c7 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -1,6 +1,7 @@ package database import ( + "database/sql" "os" "path/filepath" "time" @@ -28,25 +29,26 @@ type URL struct { // Measurement model type Measurement struct { - ID int64 `db:"id"` - TestName string `db:"test_name"` - StartTime time.Time `db:"start_time"` - Runtime float64 `db:"runtime"` // Fractional number of seconds - NetworkID int64 `db:"network_id"` // Used to include a Network - IsDone bool `db:"is_done"` - IsUploaded bool `db:"is_uploaded"` - IsFailed string `db:"is_failed"` - FailureMsg string `db:"failure_msg"` - IsUploadFailed bool `db:"is_upload_failed"` - UploadFailureMsg string `db:"upload_failure_msg"` - IsRerun bool `db:"is_rerun"` - ReportID string `db:"report_id"` - URLID string `db:"url_id"` // Used to reference URL - MeasurementID int64 `db:"measurement_id"` - IsAnomaly bool `db:"is_anomaly"` - TestKeys struct{} `db:"test_keys"` - ResultID int64 `db:"result_id"` - ReportFilePath string `db:"report_file_path"` + ID int64 `db:"id"` + TestName string `db:"test_name"` + StartTime time.Time `db:"start_time"` + Runtime float64 `db:"runtime"` // Fractional number of seconds + NetworkID int64 `db:"network_id"` // Used to include a Network + IsDone bool `db:"is_done"` + IsUploaded bool `db:"is_uploaded"` + IsFailed string `db:"is_failed"` + FailureMsg sql.NullString `db:"failure_msg,omitempty"` + IsUploadFailed bool `db:"is_upload_failed"` + UploadFailureMsg sql.NullString `db:"upload_failure_msg,omitempty"` + IsRerun bool `db:"is_rerun"` + ReportID sql.NullString `db:"report_id,omitempty"` + URLID string `db:"url_id"` // Used to reference URL + MeasurementID sql.NullInt64 `db:"measurement_id,omitempty"` + IsAnomaly sql.NullBool `db:"is_anomaly,omitempty"` + // FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462 + TestKeys string `db:"test_keys"` + ResultID int64 `db:"result_id"` + ReportFilePath string `db:"report_file_path"` } // Result model @@ -84,7 +86,7 @@ func (m *Measurement) SetGeoIPInfo() error { // Failed writes the error string to the measurement func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error { - m.FailureMsg = failure + m.FailureMsg = sql.NullString{String: failure, Valid: true} err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") @@ -107,7 +109,7 @@ func (m *Measurement) Done(sess sqlbuilder.Database) error { // UploadFailed writes the error string for the upload failure to the measurement func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) error { - m.UploadFailureMsg = failure + m.UploadFailureMsg = sql.NullString{String: failure, Valid: true} m.IsUploaded = false err := sess.Collection("measurements").Find("id", m.ID).Update(m) From b75cee5e845e22b3660ff65f82ee39654ac026da Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Sep 2018 16:13:04 +0200 Subject: [PATCH 08/65] Progress on schema and fixing unittests --- Makefile | 4 + data/migrations/1_create_msmt_results.sql | 21 +-- internal/bindata/bindata.go | 193 +++++++++++----------- internal/cli/run/run.go | 21 ++- internal/database/models.go | 9 +- nettests/nettests.go | 5 +- 6 files changed, 130 insertions(+), 123 deletions(-) diff --git a/Makefile b/Makefile index 4acf750..bf058c6 100644 --- a/Makefile +++ b/Makefile @@ -19,4 +19,8 @@ bindata: -nometadata \ -o internal/bindata/bindata.go -pkg bindata \ data/...; + +test-internal: + @$(GO) test -v ./internal/... + .PHONY: bindata diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 2341e09..440fc74 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -34,12 +34,12 @@ CREATE TABLE `urls` ( -- or distinguishing between wifi and mobile. CREATE TABLE `networks` ( `id` INTEGER PRIMARY KEY AUTOINCREMENT, - `network_name` VARCHAR(255), -- String name representing the network_name which by default is populated based + `network_name` VARCHAR(255) NOT NULL, -- String name representing the network_name which by default is populated based -- on the ASN. -- We use a separate key to reference the rows in -- this tables, because we may wish to "enrich" -- this with more data in the future. - `network_type` VARCHAR(16), -- One of wifi, mobile + `network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. -- The longest ip is an ipv6 address like: @@ -59,7 +59,7 @@ CREATE TABLE `results` ( -- That is to say: `SUM(runtime) FROM measurements` will always be <= -- `runtime FROM results` (most times <) `start_time` DATETIME NOT NULL, - `runtime` REAL NOT NULL, + `runtime` REAL, -- Used to indicate if the user has seen this result `is_viewed` TINYINT(1) NOT NULL, @@ -70,7 +70,10 @@ CREATE TABLE `results` ( `data_usage_down` INTEGER NOT NULL, -- It's probably reasonable to set the maximum length to 260 as this is the -- maximum length of file paths on windows. - `measurement_dir` VARCHAR(260) NOT NULL + `measurement_dir` VARCHAR(260) NOT NULL, + + `network_id` INTEGER NOT NULL, + FOREIGN KEY(`network_id`) REFERENCES `networks` (`id`) ); CREATE TABLE `measurements` ( @@ -87,13 +90,6 @@ CREATE TABLE `measurements` ( `start_time` DATETIME NOT NULL, `runtime` REAL NOT NULL, - -- For the purpose of populating the probe information in the results - -- views, you should pick the first measurement in the JOIN sorted by - -- start_time. - -- You don't have the guarantee that every (ip, asn, country, network_name) - -- is the same in a "measurement set" associated to a "result". - `network_id` INTEGER NOT NULL, - -- Note for golang: we used to have state be one of `done` and `active`, so -- this is equivalent to done being true or false. -- `state` TEXT, @@ -163,7 +159,6 @@ CREATE TABLE `measurements` ( FOREIGN KEY (`result_id`) REFERENCES `results`(`id`) ON DELETE CASCADE ON UPDATE CASCADE, -- If we delete a result we also want -- all the measurements to be deleted as well. - FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`), - FOREIGN KEY(`network_id`) REFERENCES `networks` (`id`) + FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`) ); -- +migrate StatementEnd diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 8c087c5..d636e72 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -130,105 +130,100 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x6f\xdb\xc6\xb2\xfe\xee\x5f\x31\x30\x8a\xd6\xc6\x95\xe4\x24" + - "\x37\x0d\xee\x75\x5b\x14\xae\xad\xe4\xa8\x8d\xe5\x40\x96\x4f\x13\x1c\x1c\x88\x2b\x72\x28\x6d\xbd\xdc\x65\xf6\x45" + - "\x8a\xce\xaf\x3f\x98\xd9\x25\x45\xca\x4e\xe2\x00\xcd\x07\x47\x22\x77\x67\xe7\xf5\x99\x67\x56\xc3\x21\xfc\x4f\x25" + - "\x57\x56\x78\x84\x2b\xb3\xd5\x47\xdd\x07\xb7\x5e\x78\xac\x50\xfb\xdf\x70\x25\xf5\xd1\xd1\xd5\xec\xe6\x1d\xcc\x2f" + - "\x7e\x7b\x3b\x86\xcc\xa2\x0b\xca\xbb\xec\xa7\xde\xd3\x0a\x85\x0b\x96\xf7\x1c\xbe\x0a\x56\x1d\x3e\xd2\xe8\xb7\xc6" + - "\xde\xd3\xe3\xc7\xcf\x1d\xeb\xa2\xff\xe6\xae\xfe\xa2\x82\x97\xb3\xf1\xc5\x7c\xdc\x3b\x11\x4e\x8e\x00\x32\x59\x64" + - "\x30\x99\xce\xc7\x6f\xc6\x33\x78\x37\x9b\x5c\x5f\xcc\x3e\xc0\x1f\xe3\x0f\x70\x71\x37\xbf\x99\x4c\x2f\x67\xe3\xeb" + - "\xf1\x74\x3e\xa0\x95\xc1\xaa\x0c\xfe\x79\x31\xbb\xfc\xc7\xc5\xec\xe4\xc5\x8f\x3f\x9e\xc2\xf4\x66\x0e\xd3\xbb\xb7" + - "\x6f\x07\x30\x1c\xc2\xfb\xf7\xef\x41\x3a\xf0\x6b\xe9\x40\x19\xbd\x02\xd4\x26\xac\xd6\xbf\xd2\xd6\x5c\x78\x5c\x19" + - "\xbb\x5b\xe4\xa6\xc0\xbd\x90\x43\x11\xf3\x35\x42\x2e\xbd\xfc\x0f\x6a\x25\x96\xd0\xec\x02\xda\x05\xa5\xb1\xe0\xd7" + - "\x78\x04\x4f\xfb\x37\x1c\x82\x93\x1e\x47\xf0\x27\x42\x70\x48\x5b\xc1\x79\x2b\xf5\x0a\xa6\x37\xd3\x31\x78\x03\x05" + - "\x6a\xe3\xbf\x45\xa0\x36\x70\xaf\xcd\x56\xf7\x35\x1b\x1d\xb1\x89\x26\x68\xff\xc0\xc2\x17\x7b\x0b\x1b\x03\xfd\xd6" + - "\x80\x42\xef\xd1\x42\xda\x13\xed\xdb\xae\x65\xbe\x66\xf7\x3d\x4d\xa3\xe1\x10\xee\x66\x6f\x61\x89\xe4\x6c\x07\xde" + - "\x1c\x9d\xc6\x64\xf9\x13\x21\xb7\x48\x49\x20\xc0\x61\x2d\x38\x1f\xbc\x58\xaa\xe8\xc3\x26\xb5\xf8\xcb\x0b\xb0\x28" + - "\x9c\xd1\xee\x9c\x76\x3e\x1f\xc1\x6b\x63\xc1\x99\x0a\xc1\x94\xec\xb2\x8d\xc4\xad\x83\xed\x1a\x2d\x82\x46\x2c\xf8" + - "\xa1\x37\x5e\x28\xd0\xa1\x5a\xa2\xa5\x85\x29\xb7\x8b\x56\xf6\x80\xa4\x49\xff\x83\x83\x95\x21\x8f\x7b\x03\x4b\x84" + - "\x2a\xe4\x6b\xa8\x8c\x45\xc0\xb2\x94\xb9\x44\xed\xe9\xcd\x5f\xc1\x79\x50\xc6\xdc\x87\x9a\xa5\xb3\x57\x48\xac\x35" + - "\x5b\x07\x52\x47\x9f\x0c\x87\xd1\x86\x11\x7d\x7a\x31\x82\x93\xca\x38\x0f\xb2\xaa\x8d\xf5\x42\xfb\x53\x32\x7b\x2b" + - "\xa2\x44\xb1\x31\xb2\x80\x22\xd4\x4a\xe6\xc2\x93\x02\x02\x96\x41\xe7\x6b\x92\x2a\x75\x69\x6c\x25\xbc\x34\x24\x59" + - "\x78\x56\xb5\xaf\x68\x6e\xaa\x8a\xde\x1a\x70\xb8\x41\x4b\xb6\x36\x4e\x23\x05\x83\x43\x4b\x5b\x8c\x66\x65\xc6\x9f" + - "\x44\x55\x2b\x3c\x4f\xbe\xaf\xc4\x0e\xb6\xd2\xad\x59\x91\xa2\xa0\xff\xb8\x26\x62\x04\x68\xbf\x32\x79\x3c\xbe\xb4" + - "\xa6\x6a\x1c\x5d\x5b\xb3\xc4\xf8\x84\xbe\xbe\x79\x77\x4b\xf2\x8c\x65\x19\x2e\xd4\x64\x27\x87\x4c\x28\x65\xb6\xac" + - "\x6b\xa3\x8a\x37\x70\x9c\x1b\x6b\x31\xf7\xc7\x20\xa0\x92\x2e\x57\xc2\x39\x59\x4a\x2c\xa0\x83\x3b\x49\x60\x21\x1d" + - "\xf9\x24\x48\xb7\x26\x31\x4b\xf4\x5b\x44\x0d\x5b\x59\x4a\x10\xba\x80\xca\x2c\x25\xf9\xb9\x0f\x19\x2d\x22\x7d\x2b" + - "\x6c\xa4\x8d\x0b\x2d\x2a\xec\xe3\x07\xd7\xfc\x6d\xac\x49\x7a\x0b\x16\x6b\x8b\x0e\xb5\x6f\xcc\xeb\xee\x4d\x05\xb2" + - "\xdc\x41\x81\xa5\x08\xca\x53\x08\x6a\x53\x07\x25\x3c\x16\xb0\x14\x0e\x8b\xaf\x55\x0e\x39\x40\xb3\xe4\x8b\xdb\xe9" + - "\xe8\x09\xab\x13\x78\x74\x0a\xe9\x1e\x77\xe4\x70\x8b\x25\x5a\xd4\x79\x8c\x68\xca\xd4\x27\x08\xdc\xa7\x82\x1b\xc0" + - "\x12\x73\x41\xe2\xb7\xfd\xac\x39\x46\x6d\x65\xbe\x3e\x7e\xaa\xb8\xad\xf4\xa9\xae\x0a\xe1\x45\xac\x18\x84\x32\xf8" + - "\x60\x71\xd4\x0d\x81\xdf\xd5\x9d\x10\x3c\x7f\x15\x23\x70\xa3\xb9\xda\x29\xfe\x83\x14\x7c\x42\x34\xc8\x64\xbd\x5f" + - "\xfc\xf2\x59\x17\xab\x63\xe0\x8c\x45\x47\xae\x89\x11\x6c\x83\x17\x73\xdb\x94\x20\x34\xc8\x7a\xf3\x92\x72\x4e\xd6" + - "\x9b\x57\x94\xc9\x16\x9d\x7b\x8a\xdf\xe7\x5c\x26\x7a\x85\x54\xe3\x35\x45\x3a\x0a\x6b\x85\x80\x92\xf7\x78\xfe\x04" + - "\x49\xcf\x9e\x3d\x7b\x76\xfe\xf5\x3f\x83\x27\x88\x8a\x09\x28\x1d\xfc\xef\xff\x43\xbe\x16\x96\x2d\xc9\x84\xd3\x5c" + - "\x0a\x27\x2f\x3b\x1e\xfa\x3b\x3a\x02\xe3\x79\xbf\x08\x1b\x6a\xc1\x35\xf8\x2d\x55\x98\x9c\x2a\x1d\xe4\x42\x13\xc4" + - "\x99\x18\xf4\xe3\x2d\x2e\xa9\x4d\xba\xe3\x01\x1c\xcb\x8a\xfe\xd6\x68\x19\x20\x75\x8e\xf4\xb5\x92\x45\xa1\x70\x69" + - "\x3e\x1d\xc7\xb8\x65\x1e\x9d\x5f\xac\xac\x09\xf5\x41\x49\x3f\x7f\xd5\x77\x40\xbf\x80\x0a\x59\x72\xc5\x78\x70\x5e" + - "\x58\xbf\xf0\xb2\x42\x86\x1b\x1b\x34\x7d\xee\x55\x43\x0b\xe4\xca\x19\x58\x8b\x0d\x36\xe2\x38\xc1\xbd\x69\x50\x8d" + - "\x13\xdd\x6c\xd0\xae\x51\x14\x64\x0f\x37\xbe\x08\xf8\x16\x19\x32\xe9\x08\xe3\xd7\x68\xa1\x14\xb9\x37\xd6\x45\xd0" + - "\x4f\xf2\x56\x06\xa4\x66\x84\x46\x20\xc3\x46\x7b\x5f\x09\x06\x18\xea\x01\x62\x77\x0e\xd9\xed\xdd\xf5\x49\x52\xf5" + - "\x14\x5e\xcf\x6e\xae\xa1\xc7\xe8\x60\x2b\x95\x02\xa1\xb6\x62\xe7\xc8\xbf\x3f\xff\xd2\x48\xca\xd2\xae\xb8\x69\x1f" + - "\x41\xee\x5f\xf4\xc2\xc1\xcf\xa7\xd1\xb5\x7b\xcf\x64\x70\x75\x31\x1f\xcf\x27\xd7\xe3\x03\x97\x36\xd2\x32\x98\x8d" + - "\x2f\xde\x76\x5e\x36\xc7\xdd\x39\xe4\x9e\x23\x75\x41\xcd\x0f\x41\x96\xfb\x4e\xb1\x16\x0e\x1c\x81\x3d\xe3\x46\xd4" + - "\x25\x65\x92\x5b\x50\xab\xc7\x22\x83\xf9\x64\xfa\x81\xf2\xf9\xf9\xe9\x23\xe2\x39\x87\xa8\x1c\xa1\x54\x62\x45\x52" + - "\x1f\x3d\x2d\x8a\xa6\x85\x05\x67\x1a\xf7\xcb\x3c\x58\x4a\x00\xb5\xa3\x98\x6b\xa9\x57\xa3\xf6\x6c\x5a\xf5\x99\x93" + - "\x79\x09\xc5\x7d\x11\x9c\x58\xe1\x22\xd4\xfb\x9c\xff\xfc\xaa\xc2\x6c\xf5\xe7\xd6\x0d\x87\x30\x21\x6e\x42\x2d\x57" + - "\x2c\x49\x1d\xe6\x40\xb1\x3f\x53\xcf\xf7\x6c\x43\x25\x3e\xc9\x2a\x54\xa0\x50\xaf\x3c\x03\xf3\x8b\x57\xcf\x40\x24" + - "\x8a\xcb\x54\xb7\xcd\xcb\x83\xb5\xa6\x84\x52\x2a\x84\x5a\xf8\x35\xf1\x04\xd8\x4a\x5d\x98\x6d\x82\xbe\xee\x2c\xb0" + - "\x28\xa4\xed\x80\xc3\xab\x0e\xcc\x3e\x52\xff\xfd\x94\xfb\x9b\x40\xe0\xbc\x79\x55\x8a\x1c\x97\xc6\xdc\x2f\x2a\x74" + - "\x0e\xf5\x0a\x6d\xf3\xc6\xa3\xc2\x95\x15\xd5\x51\x8b\x84\xc2\x3b\x51\xd7\xcd\xf7\xb5\xf7\xf5\x82\x6a\x10\xed\xa2" + - "\x94\xa8\x8a\x45\x25\xb4\xe4\xd6\x2c\x8d\xee\xad\x92\x7a\x23\x94\x2c\x16\x16\x3f\x06\x42\x12\x25\x75\xa7\xba\xdd" + - "\xba\xf9\xac\x0b\xdf\xc1\x9b\x3e\xd2\xbc\x7a\xf9\x20\x41\xfe\x8e\xd2\x79\x1d\x27\x0c\xa8\x83\xad\x8d\x63\x7c\x4c" + - "\xfc\xa2\xe1\x23\x91\xa4\x75\x19\x64\x6a\xb6\xa9\xac\x1b\x49\xcc\x9a\x07\xb0\x33\x01\xdc\xda\x04\x55\x40\x2d\xf3" + - "\xfb\xd8\x96\xa5\x75\xbe\x8b\x1d\x8d\x88\xdf\x6f\x26\x53\x70\xc6\x32\x99\xd9\x35\x92\xf6\x76\xb5\xd0\xf4\xc1\x04" + - "\xaa\xaa\x1f\x3c\x23\x23\xef\x5d\x05\x61\x85\xf6\x88\x0c\x6e\x40\xa4\x75\x07\x27\xb2\x1e\x80\x70\x7a\xd0\x74\x95" + - "\x41\x8f\x4f\x9d\x36\xf2\x62\x26\x83\x23\x8e\x25\x35\x08\x38\xee\x6a\xe7\x90\x48\xa5\x73\x26\x97\xcc\xb3\x08\x95" + - "\xe1\x38\xda\xdb\xb4\x84\x46\x6c\x37\x13\x1f\xba\x77\x6a\x7c\x9c\x40\x56\x46\x09\xbd\x3a\x27\xa0\x6f\xf0\x83\x2d" + - "\x71\x34\xb3\x76\x9a\x53\x16\x51\x81\x10\x3c\x13\xb9\x97\x1b\xcc\x06\xe0\xcc\x51\x97\xfb\x48\x07\xf8\x31\xc8\x8d" + - "\x50\x69\x9a\x60\xbc\x59\x22\x47\xcc\x06\x86\x9e\x52\x28\xb7\x77\x5f\xc6\xc7\x64\x30\x1f\xbf\x4f\x55\xf1\x04\x00" + - "\x4a\x9d\x3a\x02\x45\xab\xb0\x80\x02\x23\xee\x15\x20\xdd\x22\xd4\xca\x88\x02\x0b\x86\xc6\x01\x48\xed\x7c\x6a\x4b" + - "\x3c\xe2\x04\x27\xf5\x6a\xef\xf4\xb4\x7c\x51\x0a\xa9\xb0\x18\xc4\x30\x08\xdf\x90\x41\x6d\x52\x7c\x5b\xa9\x8c\x49" + - "\x9d\xc8\x14\xa1\x2d\x1c\x0e\x8a\x43\xef\x7b\xa0\xda\xec\x7c\x22\xa4\x1f\xca\x8f\x8a\x31\xd9\x0d\x9a\xa3\xd0\xf6" + - "\x91\x94\xd4\x14\x2a\xea\xea\x96\x97\x35\xf2\x2c\x0e\x69\x83\xf4\x7b\x4d\xa2\xa8\x2f\x01\x3c\xad\x08\x16\x17\x95" + - "\x5b\x1d\x0c\x09\x47\x07\xf6\x3c\x41\x58\x67\xe1\x97\x64\x52\x1f\x70\x0f\x5b\x18\x47\x81\x93\xab\x16\xd6\xcb\x3c" + - "\x28\x61\x7b\x8e\xa1\x2e\xba\xa4\x2e\x9a\x2c\x15\xba\xd8\xe7\x24\x5a\x2c\x4d\x62\x26\x77\x13\x46\x5a\x2f\xee\x31" + - "\x65\x2b\x71\x0d\x91\xc7\x09\xd7\x1b\x40\xc9\xcc\x64\x2d\x0b\x04\xe9\xdb\xe9\x6f\xef\x49\xee\xa2\x84\x26\x3c\x09" + - "\xc6\xbe\xc4\xc5\xad\x50\x38\x4f\xa3\x5c\x3b\x55\x8a\xa5\x54\xd2\xa7\xe1\xa4\x17\x81\x74\x39\x53\x18\xca\x2d\xa6" + - "\x54\x0d\xbf\x4a\x59\xdc\x19\x66\x4c\x82\x33\x16\xd0\x31\xfa\xd7\x36\x0a\x16\x6d\xd0\xdf\x90\x52\x0e\xed\x06\xed" + - "\xd0\x91\x8d\x91\x93\x2d\x64\x01\x16\x7d\xb0\x9a\xa1\x2e\x0d\xfd\x4a\x21\xf1\xb3\x11\xfc\xb6\xeb\x97\xca\x7e\xd3" + - "\xf7\x20\x75\x1d\x7c\x04\x56\xf2\xec\xc7\x40\xbe\x60\xeb\x6b\x49\xca\x97\xe8\xd3\x25\x4a\x57\xf9\xd6\x0d\xe3\x4f" + - "\xed\xc7\x37\xe3\x39\x37\x24\x77\x7e\x76\x26\x6a\x39\x32\x46\xcb\x91\x34\xf4\xf9\x6c\xf3\xfc\xac\xdb\x69\x7f\xe5" + - "\x53\x7f\xf9\x6e\x32\x7d\x77\x37\xff\xbe\x55\xe7\x97\xef\x66\xe3\x77\x37\xb3\xf9\x62\x72\xb5\x97\xef\xad\xc8\x7d" + - "\x07\xe8\xa5\xc7\x6a\x3f\xd5\x27\x02\xff\xaf\x7f\x67\xa0\xa4\xf3\x4d\x51\xe9\xa8\x77\xdb\x88\x7b\x04\x81\xaf\xdd" + - "\xbc\x81\x55\xa2\x25\xbf\xdf\xde\x4c\xe3\xa5\x41\xdf\x48\x1a\x42\x3b\xf4\x15\x5d\x1c\x2c\x36\x42\x05\x74\x70\x92" + - "\xb5\x7a\x67\x03\xc8\xd8\xa2\xec\x14\x84\xe5\x8a\x2e\x83\xda\x7b\x4f\xb4\xa4\xa6\x23\x9c\x8b\x82\x12\x5f\x28\x8b" + - "\xa2\xd8\xc5\x02\xa8\xad\xc9\x89\x2b\xb4\x61\xac\x65\x8d\xd4\xd1\x07\x1d\x3c\x90\x55\xad\xa2\x90\x5c\xa1\xd0\xa1" + - "\xe6\xd9\x30\x89\x69\xd1\xad\xeb\xf0\x04\x1c\x7b\x8d\x1f\x5e\x1b\x74\x69\x0c\xcf\x52\x5b\x72\xa3\x36\x0d\xed\x67" + - "\xfa\xd5\x14\xea\x57\x66\xbb\xe1\x30\x5d\x98\x15\xa3\x04\x36\xc1\xaa\xaf\x34\xb3\x26\xc3\x09\xa6\x77\xe8\x89\x0f" + - "\xa3\xa0\x81\xbb\xb9\xcf\x69\x13\x7a\x00\xcb\xc0\xa8\x4e\xbe\xae\x95\x60\xe6\x9b\x2e\x87\x7a\xad\x4c\xf8\x78\xf3" + - "\x56\x1b\xb9\x67\x05\x1a\x85\xed\x8c\xf2\x71\xf2\x46\x3c\x6f\x73\x77\x25\xfd\x3a\x2c\x47\xb9\xa9\xce\x28\x85\xcf" + - "\x9a\x08\x9c\x2d\x95\x59\x9e\x55\xc2\x79\xb4\x67\x85\xc9\x1d\xbf\x1e\x86\x20\x8b\x51\x55\xc0\xf7\x5d\x52\xf6\x45" + - "\x39\xd2\xb9\x80\xee\xec\xe5\xff\x3d\xe4\xae\xc9\x45\xc4\xc7\x0e\x3d\x93\xc0\xd4\x35\x76\xe4\x22\x12\x2a\x01\xcd" + - "\xc4\xc9\xf3\xd6\x20\x26\x96\xe0\xbb\x5a\xf2\x27\x8d\xf5\xaa\x65\x3f\x4b\x65\xf2\x7b\x6a\x8e\xd4\xc5\x09\x01\x35" + - "\x4c\xae\x79\x63\x33\x68\xa4\xaf\x8e\xa6\x34\x97\x90\xa0\xfe\xb2\x20\x59\xf2\x25\x59\x1a\x6b\x61\x2b\x1c\x14\xe8" + - "\x31\xe7\xf8\x77\x38\x16\x65\x57\x46\xac\x2c\x03\x01\xd9\xe5\xcd\xdd\x74\x7e\x72\x9a\xb5\xa5\xc7\x85\x75\xc0\xff" + - "\x22\x54\xa7\x62\x15\xed\x4d\xe6\x81\x16\x10\xed\x37\xb6\x7d\x30\xb9\x26\xb5\x5d\x8b\xb1\x42\x9b\x4a\xa8\x5d\x17" + - "\x65\x1f\x19\xc1\x34\x98\x5a\x7c\x0c\x09\x12\x9c\xb7\x21\xa7\x3c\x19\xa4\xeb\xda\x2d\x31\x2a\x6a\x45\xdd\xfb\x5c" + - "\x66\xd3\xf7\xb8\x6b\xa9\xea\x36\xdd\xeb\xa6\xeb\xf5\x3e\xc3\x40\x2f\xa4\x72\xe9\x12\x98\xc0\x8a\x45\x75\xda\x92" + - "\x83\x13\xfc\x34\xea\xf6\xac\x58\xd0\x67\x34\x7f\xd1\x07\x70\x35\x49\x37\x25\x4c\xaf\xe6\x83\xe4\x2b\x26\x51\x65" + - "\x63\x3f\x95\x03\x67\x06\xb9\xa5\xa5\x5b\xe8\xf3\xd1\x69\x67\x02\x20\x9d\x33\xb6\xb4\xeb\x09\x84\xdc\x1a\xd7\x5c" + - "\xae\xf6\xfa\x18\xf3\x69\x9f\xae\x57\xe2\x8d\x1b\x78\xb3\x42\xea\xb8\x2d\xc0\x90\x21\x9f\xab\xf4\x87\x23\xef\x46" + - "\x58\xc9\x07\x31\x67\x90\xda\xa3\xd5\x42\x29\xee\xb9\x04\xfc\x91\xe1\xd3\xd0\xd7\x34\x52\xa3\x87\x85\x74\xf7\x8f" + - "\x20\xaa\x1b\xfd\xe5\x8c\x1e\xc1\xc4\x33\xdd\xab\x88\x23\x38\xd4\x8e\x75\xdf\x5a\x2a\x07\x62\xb2\x71\x8c\x44\x0b" + - "\xc8\xf7\x42\xfb\xc1\x60\x6d\x0c\x7b\xee\xfa\x0f\x0e\x4c\x6d\x71\x93\x2e\x4e\x1b\x22\x41\x42\x1a\xa8\x89\x72\x8c" + - "\x26\xc6\x70\x9f\x2e\xb2\x2a\xb1\x17\x46\x3c\xa0\x12\x7a\xd7\xd3\x90\xcf\x2d\xf9\x2e\xb8\x8b\xc7\xf4\x64\x41\x46" + - "\x7e\x66\x6e\x4d\xf1\x79\x7d\x33\x1b\x4f\xde\x4c\x79\x14\x3d\xe9\xb8\xfa\x14\x66\xe3\xd7\xe3\xd9\x78\x7a\x39\xbe" + - "\xdd\xdf\x68\x9d\xd0\x18\x7b\x9a\x80\xfa\x66\x0a\x57\xe3\xb7\xe3\xf9\x18\x2e\x2f\x6e\x2f\x2f\xae\xc6\xf4\xe4\xee" + - "\x1d\xcd\x75\xcd\x13\x6e\x02\x93\x92\xd2\xb7\x40\x85\x3e\xd2\x18\xce\xcb\x2e\xc9\x79\xea\x8f\x3b\xc9\x0f\x42\xa9" + - "\xc3\x22\x70\xe9\x37\x81\x78\x4a\x41\xf3\xff\x16\x95\x1a\x3d\x62\x63\xea\x1a\x7d\x03\xf9\xa7\xb6\x68\xdd\xe0\x70" + - "\xcf\x49\x77\x72\xea\x6f\xeb\x5c\xb7\x47\xcf\x9c\xfe\xf4\xf9\x1f\x02\xff\x1b\x00\x00\xff\xff\x66\x75\x1e\xc1\xab" + - "\x1c\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\xff\x6f\xdb\x38\xb2\xff\x3d\x7f\xc5\x20\x58\xec\x4b\xf0\x6c\x27" + + "\xed\xeb\x16\xef\x72\xbb\x58\x64\x13\xb5\xe7\xdd\xc6\x29\x1c\xe7\xb6\xc5\xe1\x60\xd1\xd2\xc8\xe6\x86\x22\x55\x92" + + "\xb2\xea\xfb\xeb\x0f\x33\xa4\x64\xc9\x4d\xb3\x29\xb0\xfd\x21\xb5\x25\x72\x38\x5f\x3f\xf3\x19\x7a\x3c\x86\xff\x2d" + + "\xe5\xda\x0a\x8f\x70\x6d\x1a\x7d\xd4\x7f\x70\xe7\x85\xc7\x12\xb5\xff\x05\xd7\x52\x1f\x1d\x5d\xcf\x6f\xdf\xc3\xe2" + + "\xf2\x97\x77\x09\xa4\x16\x5d\xad\xbc\x4b\xff\x3e\x78\x5a\xa2\x70\xb5\xe5\x3d\x87\xaf\x6a\xab\x0e\x1f\x69\xf4\x8d" + + "\xb1\x0f\xf4\xf8\xf1\x73\x13\x9d\x0f\xdf\xdc\x57\x4f\x2a\x78\x35\x4f\x2e\x17\xc9\xe0\x44\x38\x39\x02\x48\x65\x9e" + + "\xc2\x74\xb6\x48\xde\x26\x73\x78\x3f\x9f\xde\x5c\xce\x3f\xc2\x6f\xc9\x47\xb8\xbc\x5f\xdc\x4e\x67\x57\xf3\xe4\x26" + + "\x99\x2d\x46\xb4\xb2\xb6\x2a\x85\x7f\x5e\xce\xaf\xfe\x71\x39\x3f\x79\xf9\xc3\x0f\xa7\x30\xbb\x5d\xc0\xec\xfe\xdd" + + "\xbb\x11\x8c\xc7\xf0\xe1\xc3\x07\x90\x0e\xfc\x46\x3a\x50\x46\xaf\x01\xb5\xa9\xd7\x9b\x9f\x69\x6b\x26\x3c\xae\x8d" + + "\xdd\x2d\x33\x93\xe3\x5e\xc8\xa1\x88\xc5\x06\x21\x93\x5e\xfe\x07\xb5\x12\x2b\x68\x77\x01\xed\x82\xc2\x58\xf0\x1b" + + "\x3c\x82\xe7\xfd\x1b\x8f\xc1\x49\x8f\x13\xf8\x1d\xa1\x76\x48\x5b\xc1\x79\x2b\xf5\x1a\x66\xb7\xb3\x04\xbc\x81\x1c" + + "\xb5\xf1\xdf\x22\x50\x1b\x78\xd0\xa6\xd1\x43\xcd\x26\x47\x6c\xa2\xa9\xb5\xff\xc2\xc2\x97\x7b\x0b\x5b\x03\x7d\x63" + + "\x40\xa1\xf7\x68\x21\xee\x09\xf6\x35\x1b\x99\x6d\xd8\x7d\xcf\xd3\x68\x3c\x86\xfb\xf9\x3b\x58\x21\x39\xdb\x81\x37" + + "\x47\xa7\x21\x59\x7e\x47\xc8\x2c\x52\x12\x08\x70\x58\x09\xce\x07\x2f\x56\x2a\xf8\xb0\x4d\x2d\xfe\xf2\x12\x2c\x0a" + + "\x67\xb4\xbb\xa0\x9d\x2f\x26\xf0\xc6\x58\x70\xa6\x44\x30\x05\xbb\x6c\x2b\xb1\x71\xd0\x6c\xd0\x22\x68\xc4\x9c\x1f" + + "\x7a\xe3\x85\x02\x5d\x97\x2b\xb4\xb4\x30\xe6\x76\xde\xc9\x1e\x91\x34\xe9\xff\xc7\xc1\xda\x90\xc7\xbd\x81\x15\x42" + + "\x59\x67\x1b\x28\x8d\x45\xc0\xa2\x90\x99\x44\xed\xe9\xcd\x1f\xb5\xf3\xa0\x8c\x79\xa8\x2b\x96\xce\x5e\x21\xb1\xd6" + + "\x34\x0e\xa4\x0e\x3e\x19\x8f\x83\x0d\x13\xfa\xf4\x72\x02\x27\xa5\x71\x1e\x64\x59\x19\xeb\x85\xf6\xa7\x64\x76\x23" + + "\x82\x44\xb1\x35\x32\x87\xbc\xae\x94\xcc\x84\x27\x05\x04\xac\x6a\x9d\x6d\x48\xaa\xd4\x85\xb1\xa5\xf0\xd2\x90\x64" + + "\xe1\x59\xd5\xa1\xa2\x99\x29\x4b\x7a\x6b\xc0\xe1\x16\x2d\xd9\xda\x3a\x8d\x14\xac\x1d\x5a\xda\x62\x34\x2b\x93\x7c" + + "\x16\x65\xa5\xf0\x22\xfa\xbe\x14\x3b\x68\xa4\xdb\xb0\x22\x79\x4e\xff\x71\x4d\x84\x08\xd0\x7e\x65\xb2\x70\x7c\x61" + + "\x4d\xd9\x3a\xba\xb2\x66\x85\xe1\x09\x7d\x7d\xfb\xfe\x8e\xe4\x19\xcb\x32\x5c\x5d\x91\x9d\x1c\x32\xa1\x94\x69\x58" + + "\xd7\x56\x15\x6f\xe0\x38\x33\xd6\x62\xe6\x8f\x41\x40\x29\x5d\xa6\x84\x73\xb2\x90\x98\x43\x0f\x77\xa2\xc0\x5c\x3a" + + "\xf2\x49\x2d\xdd\x86\xc4\xac\xd0\x37\x88\x1a\x1a\x59\x48\x10\x3a\x87\xd2\xac\x24\xf9\x79\x08\x19\x1d\x22\x7d\x2b" + + "\x6c\xc4\x8d\x4b\x2d\x4a\x7c\x0a\x3f\xee\x42\x71\xd2\x32\xb0\x58\x59\x74\xa8\x7d\x6b\x67\x5f\x48\xac\x94\xd5\x0e" + + "\x72\x2c\x44\xad\x3c\xc5\xa2\x32\x55\xad\x84\xc7\x1c\x56\xc2\x61\xfe\x67\x25\x44\x9e\xd0\x2c\xf9\xf2\x6e\x36\x79" + + "\xc6\xea\x88\x22\xbd\x8a\x7a\xc0\x1d\x79\xde\x62\x81\x16\x75\x16\x42\x1b\x53\xf6\x19\x02\xf7\x39\xe1\x46\xb0\xc2" + + "\x4c\x90\xf8\x66\x98\x3e\xc7\xa8\xad\xcc\x36\xc7\xcf\x15\xd7\x48\x1f\x0b\x2c\x17\x5e\x84\xd2\x41\x28\x6a\x5f\x5b" + + "\x9c\xf4\x63\xe1\x77\x55\x2f\x16\x2f\x5e\x1f\x84\xe2\x56\x73\xfd\x53\x46\x8c\x62\x3a\x10\xc6\x41\x2a\xab\xfd\xae" + + "\x57\xe7\xfd\x5d\x21\x82\xc6\xa2\x23\x1f\x85\x50\x76\x51\x0c\xd9\x6e\x0a\x10\x1a\x64\xb5\x7d\x45\x59\x28\xab\xed" + + "\x6b\xca\x6d\x8b\xce\x3d\x27\x00\x0b\x2e\x1c\xbd\x46\xaa\xfa\x8a\x42\x1e\x84\x75\x42\x40\xc9\x07\xbc\x78\x86\xa4" + + "\xf3\xf3\xf3\xf3\x8b\x3f\xff\x33\x7a\x86\xa8\x90\x89\xd2\xc1\xff\xfd\x0d\xb2\x8d\xb0\x6c\x49\x2a\x9c\xe6\xe2\x38" + + "\x79\xd5\xf3\xd0\x5f\xd1\x23\x18\xe1\x87\x65\xd9\x92\x0d\xae\xca\x6f\xa9\xcb\xe8\x54\xe9\x20\x13\x9a\x40\xcf\x84" + + "\xa0\x1f\x37\xb8\xa2\xc6\xe9\x8e\x47\x70\x2c\x4b\xfa\x5b\xa1\x65\xc8\xd4\x19\xd2\xd7\x52\xe6\xb9\xc2\x95\xf9\x7c" + + "\x1c\xe2\x96\x7a\x74\x7e\xb9\xb6\xa6\xae\x0e\x8a\x7c\x90\x58\xed\x99\x5d\x25\xe5\xb2\xe0\xd2\xf1\xe0\xbc\xb0\x7e" + + "\xe9\x65\x89\x0c\x40\xb6\xd6\xf4\x79\x50\x16\x1d\xb4\x2b\x67\x60\x23\xb6\xd8\x8a\xe3\x4c\xf7\xa6\xc5\x39\xce\x78" + + "\xb3\x45\xbb\x41\x91\x93\x3d\xdc\x0a\x43\x0b\xb0\xc8\x20\x4a\x47\x18\xbf\x41\x0b\x85\xc8\xbc\xb1\x2e\xb4\x81\x28" + + "\x6f\x6d\x40\x6a\xc6\x6c\x04\x32\x6c\xb2\xf7\x95\x60\xa4\xa1\xae\x20\x76\x17\x90\xde\xdd\xdf\x9c\x44\x55\x4f\xe1" + + "\xcd\xfc\xf6\x06\x06\x1c\x0f\x1a\xa9\x14\x08\xd5\x88\x9d\x23\xff\xfe\xf8\x53\x2b\x29\x8d\xbb\xc2\xa6\x7d\x04\xb9" + + "\xa3\xd1\x0b\x07\x3f\x9e\x06\xd7\xee\x3d\x93\xc2\xf5\xe5\x22\x59\x4c\x6f\x92\x03\x97\xb6\xd2\x52\x98\x27\x97\xef" + + "\x7a\x2f\xdb\xe3\xee\x1d\x72\x17\x92\x3a\xa7\x76\x88\x20\x8b\x7d\xef\xd8\x08\x07\x8e\xe0\x9f\x01\x24\xe8\x12\x33" + + "\xc9\x2d\xa9\xf9\x63\x9e\xc2\x62\x3a\xfb\x48\xf9\xfc\xe2\xf4\x11\xf1\x9c\x43\x54\x8e\x50\x28\xb1\x26\xa9\x8f\x9e" + + "\x16\x44\xd3\xc2\x9c\x33\x8d\x3b\x68\x56\x5b\x4a\x00\xb5\xa3\x98\x6b\xa9\xd7\x93\xee\x6c\x5a\xf5\x95\x93\x79\x09" + + "\xc5\x7d\x59\x3b\xb1\xc6\x65\x5d\xed\x73\xfe\xeb\xab\x72\xd3\xe8\xaf\xad\x1b\x8f\x61\x4a\x6c\x85\x9a\xb0\x58\x91" + + "\x3a\xcc\x8a\x42\xc7\x26\x16\xe0\xd9\x86\x52\x7c\x96\x65\x5d\x82\x42\xbd\xf6\x8c\xd0\x2f\x5f\x9f\x83\x88\xa4\x97" + + "\xc9\x6f\x97\x97\x07\x6b\x4d\x01\x85\x54\x08\x95\xf0\x1b\x62\x0e\xd0\x48\x9d\x9b\x26\x42\x5f\x7f\x3a\x58\xe6\xd2" + + "\xf6\xc0\xe1\xf5\xf9\x17\x4e\xef\x60\xbc\x5f\xec\x43\x83\xde\xdc\xce\x93\xe9\xdb\x19\x95\xfe\x49\x7f\xf9\x29\xcc" + + "\x93\x37\xc9\x3c\x99\x5d\x25\x77\x83\x9e\x4e\xc0\x71\xfa\x08\xbe\x0c\x53\xfa\x2f\x02\x99\x8b\xf6\x55\x21\x32\x5c" + + "\x19\xf3\xb0\x2c\xd1\x39\xd4\x6b\xb4\xed\x1b\x8f\x0a\xd7\x56\x94\x47\x1d\xd2\x0a\xef\x44\x55\xb5\xdf\x37\xde\x57" + + "\x4b\xaa\x71\xb4\xcb\x42\xa2\xca\x97\xa5\xd0\x92\x39\x80\x34\x7a\xb0\x4a\xea\xad\x50\x32\x5f\x5a\xfc\x54\x13\x52" + + "\x29\xa9\x7b\xe8\xe1\x36\xed\x67\x9d\xfb\x1e\x9e\x0d\x91\xec\xf5\xab\x2f\x12\xf0\xaf\x28\xcd\x99\xf1\x81\x94\xaf" + + "\x8d\x12\x7a\x7d\x41\x48\xd7\x16\x10\x81\x1c\x21\xa3\xc7\x1e\x3a\xa7\xa1\x2c\x08\xc2\x52\x91\x79\xb9\xc5\x74\x04" + + "\xce\x1c\xf5\x59\x80\x74\x80\x9f\x6a\xb9\x15\x2a\x12\x6c\x2e\xb8\x15\x32\x97\xb2\x35\xd7\x5e\x21\x94\xc3\x0e\xda" + + "\x52\x3e\x26\x85\x45\xf2\x21\x86\xed\x19\x15\x18\x5b\x55\xa8\x94\x4e\x61\x01\x39\x86\xc2\xcf\x41\xba\x65\x5d\x29" + + "\x23\x72\xcc\x19\x1b\x46\x20\xb5\xf3\x11\x97\x99\xf5\xd7\x4e\xea\x75\x2b\xad\x5b\xbe\x2c\x84\x54\x98\x8f\x42\x45" + + "\x09\xdf\xd2\x22\x6d\x7c\x38\xa4\x93\xca\x45\xb9\xcf\x50\xc8\xeb\x2e\xb2\xd4\x2b\xa8\x70\xfd\x00\x55\xda\x9d\xcf" + + "\xc4\xb4\x43\xf9\x41\x31\xa6\x7d\xb5\xe6\x28\x74\x40\xea\x36\xa6\x56\x39\x87\x8a\xda\x9a\xe5\x65\xad\x3c\x8b\x63" + + "\xda\x20\xfd\x5e\x93\x20\xea\x29\x84\xa3\x15\xb5\xc5\x65\xe9\xd6\x43\xde\xdc\xc2\xc0\xa1\xc3\x9e\x12\xd6\x5b\xf8" + + "\x94\x4c\x02\x42\xf7\x25\x86\x73\x14\x38\xb9\x2a\x61\xbd\xcc\x6a\x25\xec\xc0\x31\xd4\x46\x56\xd4\x46\xa2\xa5\x42" + + "\xe7\xfb\x9c\x44\x8b\x85\x89\xad\xf9\x7e\xca\x50\xe0\xc5\x03\xc6\x6c\xa5\x66\x2b\xb2\x30\xf4\x79\x03\x28\xb9\x35" + + "\x6f\x64\x8e\x20\x7d\x37\x10\xed\x3d\xc9\x6d\x84\x3a\x13\x0f\x47\x01\x98\xb7\x68\x77\xa0\x50\x38\x4f\xd3\x4d\x37" + + "\x68\x89\x95\x54\xd2\x47\x9a\x3e\x88\x40\xbc\xaf\xc8\x0d\xe5\x16\x73\x8a\x96\x60\xc4\x2c\xee\xd1\x7a\x13\xfb\x17" + + "\x0b\xe8\x19\xfd\x73\x17\x05\x8b\xb6\xd6\xdf\x90\x52\x0e\xed\x16\xed\xd8\x91\x8d\x81\x94\x2c\x65\x0e\x16\x7d\x6d" + + "\x35\x4d\x30\xbb\x38\x07\x2b\x85\x44\x50\x26\xf0\xcb\x6e\x58\x2a\xfb\x4d\xdf\x83\xd4\x55\xed\x47\xb0\x33\x35\x7b" + + "\xf6\x53\x4d\xbe\x60\xeb\x2b\x49\xca\x17\xe8\xe3\xbd\x42\x5f\xf9\xce\x0d\xc9\xe7\xee\xe3\xdb\x64\xc1\x88\xe9\x2e" + + "\xce\xce\x44\x25\x27\xc6\x68\x39\x91\x86\x3e\x9f\x6d\x5f\x9c\xf5\x5b\xc1\xcf\x7c\xea\x4f\xdf\x4d\x67\xef\xef\x17" + + "\xdf\x77\xea\xfc\xf4\xdd\x3c\x79\x7f\x3b\x5f\x2c\xa7\xd7\x7b\xf9\xde\x8a\x2c\x84\xa9\x90\x96\xd8\xbb\xc7\x72\x3f" + + "\xe8\x46\x06\xfb\xaf\x7f\xa7\xa0\xa4\xf3\x6d\x51\xe9\xa0\x77\xd7\x29\x06\x1d\x92\x6f\xa2\xbc\x81\x75\xec\xcb\xbf" + + "\xde\xdd\xce\xc2\x1c\x3d\x34\x92\xc6\xb1\x1e\x7f\x43\x17\x98\xf5\x56\xa8\x1a\x1d\x9c\xa4\x9d\xde\xe9\x08\x52\xb6" + + "\x28\x3d\x05\x61\xb9\xa2\x8b\x5a\xed\xbd\x27\xba\xae\xde\x13\xce\x45\x41\x89\x2f\x94\x45\x91\xef\x42\x01\x54\xd6" + + "\x64\xd4\xcc\xba\x30\x56\xb2\x42\x6a\x39\xa3\x1e\x1e\xc8\xb2\x52\x41\x48\xa6\x50\xe8\xba\xe2\xe1\x28\x8a\xe9\xd0" + + "\xad\xef\xf0\x08\x1c\x7b\x8d\x87\xd5\x7b\xd8\x67\x79\x98\x68\xc8\x8d\xda\xb4\xbc\x97\xf9\x47\x5b\xa8\x7f\x32\xdc" + + "\x8c\xc7\xf1\x0e\x29\x9f\x44\xb0\xa9\xad\xfa\x0a\xdf\x38\xcc\x70\x82\xe9\x1d\x7a\x22\x84\x28\x68\xf4\x6c\xaf\x38" + + "\xba\x84\x1e\xc1\xaa\x66\x54\x27\x5f\x57\x4a\x30\xf5\x8b\xf7\x25\x83\x56\x26\x7c\xb8\x8c\xaa\x8c\xd4\xbe\x9d\x67" + + "\x35\x0a\xdb\x1b\x6a\xc3\xe8\x89\x78\xd1\xe5\xee\x5a\xfa\x4d\xbd\x9a\x64\xa6\x3c\xa3\x14\x3e\x6b\x23\x70\xb6\x52" + + "\x66\x75\x56\x0a\xe7\xd1\x9e\xe5\x26\x73\xfc\x7a\x5c\xd7\x32\x9f\x94\x39\x7c\xdf\x67\x0d\x4f\xca\x91\xce\xd5\xe8" + + "\xce\x5e\xfd\xff\x97\xe4\x2d\xba\x88\x08\xc3\xa1\x67\x22\x98\xba\xd6\x8e\x4c\x38\x76\x8d\x80\x76\xe4\xe2\x81\x63" + + "\x14\x12\x4b\xf0\xf5\x25\xf9\x93\xe6\x5a\xb5\x6b\x65\xad\x94\xc9\x1e\xa8\x39\x52\x17\x27\x04\xd4\x30\xbd\xe1\x8d" + + "\x2d\xd3\x8e\x5f\x1d\x8d\x29\x2e\x22\x41\xf5\xb4\x20\x59\xf0\xbd\x51\x9c\xeb\xa0\x11\x0e\x72\xf4\x98\x71\xfc\xe3" + + "\xfa\x8f\x11\x60\xd2\x5f\x6f\xa7\xb3\x14\x04\xa4\x57\xb7\xf7\xb3\xc5\xc9\x69\xda\x95\x1e\x17\x56\x6b\x5e\x9c\x6b" + + "\x02\x54\xc7\x62\x15\xdd\xe5\xde\x81\x16\x10\xec\x37\xb6\x7b\x30\xbd\x21\xb5\x5d\x87\xb1\x42\x9b\x52\xa8\x5d\x1f" + + "\x65\x1f\x99\x41\x34\x98\x4a\x7c\xaa\x23\x24\x38\x6f\xeb\x8c\xf2\x64\x14\x6f\x30\x1b\x62\x54\xd4\x8a\xfa\x57\x9c" + + "\x4c\xf7\x1e\x70\xe7\x3a\xae\x19\xaf\x3a\xe3\x8d\xf3\x90\x61\xa0\x17\x52\xb9\x78\x2f\x4a\x60\xc5\xa2\x7a\x6d\xc9" + + "\xc1\x09\x7e\x9e\xf4\x7b\x56\x28\xe8\x33\x1a\x40\xe8\x03\xb8\x8a\xa4\x9b\x02\x66\xd7\x8b\x51\xf4\x15\x93\xa8\xa2" + + "\xb5\x9f\xca\x81\x33\x83\xdc\xd2\xd1\x2d\xf4\xd9\xe4\xb4\x47\x51\x49\xe7\x34\x58\xfa\x58\x35\x22\x64\xd6\xb8\xf6" + + "\xe2\x71\xd0\xd0\x28\x84\xc1\xf6\xc6\xc4\x4b\x28\xf0\x66\x8d\xd4\x7a\x3b\xa4\x21\x8b\xbe\x56\xf2\x5f\x0e\x7f\x5b" + + "\x61\x25\x1f\xc4\xe4\x41\x6a\x8f\x56\x0b\xa5\xb8\xf9\x52\x07\x78\x08\x60\x28\xc2\xdc\xc4\x53\xba\x1e\xe7\xd2\x3d" + + "\x3c\x02\xad\x6e\xf2\x87\x33\x7a\x02\x53\xcf\xbc\xaf\x24\xb2\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\x44\x69\xc3\x40\x85" + + "\x16\x90\x6f\x48\x56\x5d\x6a\x6f\x8c\x61\x17\xde\xfc\xc6\x11\xaa\x2c\x6e\xe3\x5d\x62\xcb\x28\x48\x48\x8b\x39\x41" + + "\x8e\xd1\x44\x1d\x1e\xe2\x95\x4e\x29\xf6\xc2\x88\x10\x94\x42\xef\x06\x1a\xf2\xb9\x05\xdf\x93\xf6\x81\x99\x9e\x2c" + + "\xc9\xc8\xa7\x27\xb8\xde\x78\xc6\x6d\xa8\x75\xf5\x70\x3c\x6b\x6f\x06\xe2\x70\x16\x90\xf9\x76\x06\xd7\xc9\xbb\x64" + + "\x91\xc0\xd5\xe5\xdd\xd5\xe5\x75\x42\x4f\xee\xdf\xd3\x04\xd2\x3e\xe1\x6e\x30\x2d\x28\x8f\x73\x54\xe8\x03\x9f\xe1" + + "\x04\xed\xb3\x9d\xe7\xfe\xf0\x11\xfd\x20\x94\x3a\xac\x06\x17\xef\xcb\xc3\x29\x39\x4d\xc2\x0d\x2a\x35\x79\xc4\xc6" + + "\xd8\x3e\x86\x06\xf2\xcf\x50\xfb\xd1\xf3\xab\x3f\x74\xfd\x37\x00\x00\xff\xff\x11\xda\x64\x3c\x8b\x1b\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 7c8a9aa..3d7df37 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -55,12 +55,23 @@ func init() { return err } - result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ - Name: *nettestGroup, - StartTime: time.Now().UTC(), - Country: ctx.Location.CountryCode, + network := database.Network{ + ASN: ctx.Location.ASN, + CountryCode: ctx.Location.CountryCode, NetworkName: ctx.Location.NetworkName, - ASN: fmt.Sprintf("%d", ctx.Location.ASN), + IP: ctx.Location.IP, + } + newID, err := ctx.DB.Collection("networks").Insert(network) + if err != nil { + log.WithError(err).Error("Failed to create the network row") + return nil + } + network.ID = newID.(int64) + + result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ + TestGroupName: *nettestGroup, + StartTime: time.Now().UTC(), + NetworkID: network.ID, }) if err != nil { log.Errorf("DB result error: %s", err) diff --git a/internal/database/models.go b/internal/database/models.go index 246b4c7..ffb641a 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -16,10 +16,11 @@ type Network struct { ID int64 `db:"id"` NetworkName string `db:"network_name"` IP string `db:"ip"` - ASN int `db:"asn"` + ASN uint `db:"asn"` CountryCode string `db:"country_code"` } +// URL represents URLs from the testing lists type URL struct { ID int64 `db:"id"` URL int64 `db:"url"` @@ -32,8 +33,7 @@ type Measurement struct { ID int64 `db:"id"` TestName string `db:"test_name"` StartTime time.Time `db:"start_time"` - Runtime float64 `db:"runtime"` // Fractional number of seconds - NetworkID int64 `db:"network_id"` // Used to include a Network + Runtime float64 `db:"runtime"` // Fractional number of seconds IsDone bool `db:"is_done"` IsUploaded bool `db:"is_uploaded"` IsFailed string `db:"is_failed"` @@ -56,7 +56,8 @@ type Result struct { ID int64 `db:"id"` TestGroupName string `db:"test_group_name"` StartTime time.Time `db:"start_time"` - Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds + NetworkID int64 `db:"network_id"` // Used to include a Network + Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds IsViewed bool `db:"is_viewed"` IsDone bool `db:"is_done"` DataUsageUp int64 `db:"data_usage_up"` diff --git a/nettests/nettests.go b/nettests/nettests.go index 66d720e..61915e1 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -1,6 +1,7 @@ package nettests import ( + "database/sql" "encoding/json" "fmt" "os" @@ -58,7 +59,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { c.msmts = make(map[int64]*database.Measurement) msmtTemplate := database.Measurement{ - ReportID: "", + ReportID: sql.NullString{String: "", Valid: false}, TestName: nt.Name, ResultID: c.res.ID, ReportFilePath: c.msmtPath, @@ -156,7 +157,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.report_created", func(e mk.Event) { log.Debugf("%s", e.Key) - msmtTemplate.ReportID = e.Value.ReportID + msmtTemplate.ReportID = sql.NullString{String: e.Value.ReportID, Valid: true} }) nt.On("status.geoip_lookup", func(e mk.Event) { From e6a23de1bae000d0558478eb10745b52ec632c4c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Sep 2018 16:15:06 +0200 Subject: [PATCH 09/65] Hello world travis.yml --- .travis.yml | 11 +++++++++++ 1 file changed, 11 insertions(+) create mode 100644 .travis.yml diff --git a/.travis.yml b/.travis.yml new file mode 100644 index 0000000..2476a9e --- /dev/null +++ b/.travis.yml @@ -0,0 +1,11 @@ +language: go +go: + - 1.9.x + - 1.x +install: + - go get -u github.com/golang/dep/... + - dep ensure + - make update-mk-libs + +script: + - make test-internal From 5ed67ee42251db247fb89bc651069ae958e68993 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Sep 2018 16:17:00 +0200 Subject: [PATCH 10/65] Update bindata --- internal/bindata/bindata.go | 127 ++++++++++++++++++------------------ 1 file changed, 64 insertions(+), 63 deletions(-) diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index d636e72..7be2ccb 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -132,7 +132,7 @@ func bindataDataDefaultconfigjson() (*asset, error) { var _bindataDataMigrations1createmsmtresultssql = []byte( "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\xff\x6f\xdb\x38\xb2\xff\x3d\x7f\xc5\x20\x58\xec\x4b\xf0\x6c\x27" + "\xed\xeb\x16\xef\x72\xbb\x58\x64\x13\xb5\xe7\xdd\xc6\x29\x1c\xe7\xb6\xc5\xe1\x60\xd1\xd2\xc8\xe6\x86\x22\x55\x92" + - "\xb2\xea\xfb\xeb\x0f\x33\xa4\x64\xc9\x4d\xb3\x29\xb0\xfd\x21\xb5\x25\x72\x38\x5f\x3f\xf3\x19\x7a\x3c\x86\xff\x2d" + + "\xb2\xea\xfb\xeb\x0f\x33\xa4\x64\xc9\x4d\xb3\x29\xb0\xfd\x21\xb5\x25\x72\x38\x5f\x3f\x9f\x19\x7a\x3c\x86\xff\x2d" + "\xe5\xda\x0a\x8f\x70\x6d\x1a\x7d\xd4\x7f\x70\xe7\x85\xc7\x12\xb5\xff\x05\xd7\x52\x1f\x1d\x5d\xcf\x6f\xdf\xc3\xe2" + "\xf2\x97\x77\x09\xa4\x16\x5d\xad\xbc\x4b\xff\x3e\x78\x5a\xa2\x70\xb5\xe5\x3d\x87\xaf\x6a\xab\x0e\x1f\x69\xf4\x8d" + "\xb1\x0f\xf4\xf8\xf1\x73\x13\x9d\x0f\xdf\xdc\x57\x4f\x2a\x78\x35\x4f\x2e\x17\xc9\xe0\x44\x38\x39\x02\x48\x65\x9e" + @@ -162,68 +162,69 @@ var _bindataDataMigrations1createmsmtresultssql = []byte( "\x57\xe7\xfd\x5d\x21\x82\xc6\xa2\x23\x1f\x85\x50\x76\x51\x0c\xd9\x6e\x0a\x10\x1a\x64\xb5\x7d\x45\x59\x28\xab\xed" + "\x6b\xca\x6d\x8b\xce\x3d\x27\x00\x0b\x2e\x1c\xbd\x46\xaa\xfa\x8a\x42\x1e\x84\x75\x42\x40\xc9\x07\xbc\x78\x86\xa4" + "\xf3\xf3\xf3\xf3\x8b\x3f\xff\x33\x7a\x86\xa8\x90\x89\xd2\xc1\xff\xfd\x0d\xb2\x8d\xb0\x6c\x49\x2a\x9c\xe6\xe2\x38" + - "\x79\xd5\xf3\xd0\x5f\xd1\x23\x18\xe1\x87\x65\xd9\x92\x0d\xae\xca\x6f\xa9\xcb\xe8\x54\xe9\x20\x13\x9a\x40\xcf\x84" + - "\xa0\x1f\x37\xb8\xa2\xc6\xe9\x8e\x47\x70\x2c\x4b\xfa\x5b\xa1\x65\xc8\xd4\x19\xd2\xd7\x52\xe6\xb9\xc2\x95\xf9\x7c" + - "\x1c\xe2\x96\x7a\x74\x7e\xb9\xb6\xa6\xae\x0e\x8a\x7c\x90\x58\xed\x99\x5d\x25\xe5\xb2\xe0\xd2\xf1\xe0\xbc\xb0\x7e" + - "\xe9\x65\x89\x0c\x40\xb6\xd6\xf4\x79\x50\x16\x1d\xb4\x2b\x67\x60\x23\xb6\xd8\x8a\xe3\x4c\xf7\xa6\xc5\x39\xce\x78" + - "\xb3\x45\xbb\x41\x91\x93\x3d\xdc\x0a\x43\x0b\xb0\xc8\x20\x4a\x47\x18\xbf\x41\x0b\x85\xc8\xbc\xb1\x2e\xb4\x81\x28" + - "\x6f\x6d\x40\x6a\xc6\x6c\x04\x32\x6c\xb2\xf7\x95\x60\xa4\xa1\xae\x20\x76\x17\x90\xde\xdd\xdf\x9c\x44\x55\x4f\xe1" + - "\xcd\xfc\xf6\x06\x06\x1c\x0f\x1a\xa9\x14\x08\xd5\x88\x9d\x23\xff\xfe\xf8\x53\x2b\x29\x8d\xbb\xc2\xa6\x7d\x04\xb9" + - "\xa3\xd1\x0b\x07\x3f\x9e\x06\xd7\xee\x3d\x93\xc2\xf5\xe5\x22\x59\x4c\x6f\x92\x03\x97\xb6\xd2\x52\x98\x27\x97\xef" + - "\x7a\x2f\xdb\xe3\xee\x1d\x72\x17\x92\x3a\xa7\x76\x88\x20\x8b\x7d\xef\xd8\x08\x07\x8e\xe0\x9f\x01\x24\xe8\x12\x33" + - "\xc9\x2d\xa9\xf9\x63\x9e\xc2\x62\x3a\xfb\x48\xf9\xfc\xe2\xf4\x11\xf1\x9c\x43\x54\x8e\x50\x28\xb1\x26\xa9\x8f\x9e" + - "\x16\x44\xd3\xc2\x9c\x33\x8d\x3b\x68\x56\x5b\x4a\x00\xb5\xa3\x98\x6b\xa9\xd7\x93\xee\x6c\x5a\xf5\x95\x93\x79\x09" + - "\xc5\x7d\x59\x3b\xb1\xc6\x65\x5d\xed\x73\xfe\xeb\xab\x72\xd3\xe8\xaf\xad\x1b\x8f\x61\x4a\x6c\x85\x9a\xb0\x58\x91" + - "\x3a\xcc\x8a\x42\xc7\x26\x16\xe0\xd9\x86\x52\x7c\x96\x65\x5d\x82\x42\xbd\xf6\x8c\xd0\x2f\x5f\x9f\x83\x88\xa4\x97" + - "\xc9\x6f\x97\x97\x07\x6b\x4d\x01\x85\x54\x08\x95\xf0\x1b\x62\x0e\xd0\x48\x9d\x9b\x26\x42\x5f\x7f\x3a\x58\xe6\xd2" + - "\xf6\xc0\xe1\xf5\xf9\x17\x4e\xef\x60\xbc\x5f\xec\x43\x83\xde\xdc\xce\x93\xe9\xdb\x19\x95\xfe\x49\x7f\xf9\x29\xcc" + - "\x93\x37\xc9\x3c\x99\x5d\x25\x77\x83\x9e\x4e\xc0\x71\xfa\x08\xbe\x0c\x53\xfa\x2f\x02\x99\x8b\xf6\x55\x21\x32\x5c" + - "\x19\xf3\xb0\x2c\xd1\x39\xd4\x6b\xb4\xed\x1b\x8f\x0a\xd7\x56\x94\x47\x1d\xd2\x0a\xef\x44\x55\xb5\xdf\x37\xde\x57" + - "\x4b\xaa\x71\xb4\xcb\x42\xa2\xca\x97\xa5\xd0\x92\x39\x80\x34\x7a\xb0\x4a\xea\xad\x50\x32\x5f\x5a\xfc\x54\x13\x52" + - "\x29\xa9\x7b\xe8\xe1\x36\xed\x67\x9d\xfb\x1e\x9e\x0d\x91\xec\xf5\xab\x2f\x12\xf0\xaf\x28\xcd\x99\xf1\x81\x94\xaf" + - "\x8d\x12\x7a\x7d\x41\x48\xd7\x16\x10\x81\x1c\x21\xa3\xc7\x1e\x3a\xa7\xa1\x2c\x08\xc2\x52\x91\x79\xb9\xc5\x74\x04" + - "\xce\x1c\xf5\x59\x80\x74\x80\x9f\x6a\xb9\x15\x2a\x12\x6c\x2e\xb8\x15\x32\x97\xb2\x35\xd7\x5e\x21\x94\xc3\x0e\xda" + - "\x52\x3e\x26\x85\x45\xf2\x21\x86\xed\x19\x15\x18\x5b\x55\xa8\x94\x4e\x61\x01\x39\x86\xc2\xcf\x41\xba\x65\x5d\x29" + - "\x23\x72\xcc\x19\x1b\x46\x20\xb5\xf3\x11\x97\x99\xf5\xd7\x4e\xea\x75\x2b\xad\x5b\xbe\x2c\x84\x54\x98\x8f\x42\x45" + - "\x09\xdf\xd2\x22\x6d\x7c\x38\xa4\x93\xca\x45\xb9\xcf\x50\xc8\xeb\x2e\xb2\xd4\x2b\xa8\x70\xfd\x00\x55\xda\x9d\xcf" + - "\xc4\xb4\x43\xf9\x41\x31\xa6\x7d\xb5\xe6\x28\x74\x40\xea\x36\xa6\x56\x39\x87\x8a\xda\x9a\xe5\x65\xad\x3c\x8b\x63" + - "\xda\x20\xfd\x5e\x93\x20\xea\x29\x84\xa3\x15\xb5\xc5\x65\xe9\xd6\x43\xde\xdc\xc2\xc0\xa1\xc3\x9e\x12\xd6\x5b\xf8" + - "\x94\x4c\x02\x42\xf7\x25\x86\x73\x14\x38\xb9\x2a\x61\xbd\xcc\x6a\x25\xec\xc0\x31\xd4\x46\x56\xd4\x46\xa2\xa5\x42" + - "\xe7\xfb\x9c\x44\x8b\x85\x89\xad\xf9\x7e\xca\x50\xe0\xc5\x03\xc6\x6c\xa5\x66\x2b\xb2\x30\xf4\x79\x03\x28\xb9\x35" + - "\x6f\x64\x8e\x20\x7d\x37\x10\xed\x3d\xc9\x6d\x84\x3a\x13\x0f\x47\x01\x98\xb7\x68\x77\xa0\x50\x38\x4f\xd3\x4d\x37" + - "\x68\x89\x95\x54\xd2\x47\x9a\x3e\x88\x40\xbc\xaf\xc8\x0d\xe5\x16\x73\x8a\x96\x60\xc4\x2c\xee\xd1\x7a\x13\xfb\x17" + - "\x0b\xe8\x19\xfd\x73\x17\x05\x8b\xb6\xd6\xdf\x90\x52\x0e\xed\x16\xed\xd8\x91\x8d\x81\x94\x2c\x65\x0e\x16\x7d\x6d" + - "\x35\x4d\x30\xbb\x38\x07\x2b\x85\x44\x50\x26\xf0\xcb\x6e\x58\x2a\xfb\x4d\xdf\x83\xd4\x55\xed\x47\xb0\x33\x35\x7b" + - "\xf6\x53\x4d\xbe\x60\xeb\x2b\x49\xca\x17\xe8\xe3\xbd\x42\x5f\xf9\xce\x0d\xc9\xe7\xee\xe3\xdb\x64\xc1\x88\xe9\x2e" + - "\xce\xce\x44\x25\x27\xc6\x68\x39\x91\x86\x3e\x9f\x6d\x5f\x9c\xf5\x5b\xc1\xcf\x7c\xea\x4f\xdf\x4d\x67\xef\xef\x17" + - "\xdf\x77\xea\xfc\xf4\xdd\x3c\x79\x7f\x3b\x5f\x2c\xa7\xd7\x7b\xf9\xde\x8a\x2c\x84\xa9\x90\x96\xd8\xbb\xc7\x72\x3f" + - "\xe8\x46\x06\xfb\xaf\x7f\xa7\xa0\xa4\xf3\x6d\x51\xe9\xa0\x77\xd7\x29\x06\x1d\x92\x6f\xa2\xbc\x81\x75\xec\xcb\xbf" + - "\xde\xdd\xce\xc2\x1c\x3d\x34\x92\xc6\xb1\x1e\x7f\x43\x17\x98\xf5\x56\xa8\x1a\x1d\x9c\xa4\x9d\xde\xe9\x08\x52\xb6" + - "\x28\x3d\x05\x61\xb9\xa2\x8b\x5a\xed\xbd\x27\xba\xae\xde\x13\xce\x45\x41\x89\x2f\x94\x45\x91\xef\x42\x01\x54\xd6" + - "\x64\xd4\xcc\xba\x30\x56\xb2\x42\x6a\x39\xa3\x1e\x1e\xc8\xb2\x52\x41\x48\xa6\x50\xe8\xba\xe2\xe1\x28\x8a\xe9\xd0" + - "\xad\xef\xf0\x08\x1c\x7b\x8d\x87\xd5\x7b\xd8\x67\x79\x98\x68\xc8\x8d\xda\xb4\xbc\x97\xf9\x47\x5b\xa8\x7f\x32\xdc" + - "\x8c\xc7\xf1\x0e\x29\x9f\x44\xb0\xa9\xad\xfa\x0a\xdf\x38\xcc\x70\x82\xe9\x1d\x7a\x22\x84\x28\x68\xf4\x6c\xaf\x38" + - "\xba\x84\x1e\xc1\xaa\x66\x54\x27\x5f\x57\x4a\x30\xf5\x8b\xf7\x25\x83\x56\x26\x7c\xb8\x8c\xaa\x8c\xd4\xbe\x9d\x67" + - "\x35\x0a\xdb\x1b\x6a\xc3\xe8\x89\x78\xd1\xe5\xee\x5a\xfa\x4d\xbd\x9a\x64\xa6\x3c\xa3\x14\x3e\x6b\x23\x70\xb6\x52" + - "\x66\x75\x56\x0a\xe7\xd1\x9e\xe5\x26\x73\xfc\x7a\x5c\xd7\x32\x9f\x94\x39\x7c\xdf\x67\x0d\x4f\xca\x91\xce\xd5\xe8" + - "\xce\x5e\xfd\xff\x97\xe4\x2d\xba\x88\x08\xc3\xa1\x67\x22\x98\xba\xd6\x8e\x4c\x38\x76\x8d\x80\x76\xe4\xe2\x81\x63" + - "\x14\x12\x4b\xf0\xf5\x25\xf9\x93\xe6\x5a\xb5\x6b\x65\xad\x94\xc9\x1e\xa8\x39\x52\x17\x27\x04\xd4\x30\xbd\xe1\x8d" + - "\x2d\xd3\x8e\x5f\x1d\x8d\x29\x2e\x22\x41\xf5\xb4\x20\x59\xf0\xbd\x51\x9c\xeb\xa0\x11\x0e\x72\xf4\x98\x71\xfc\xe3" + - "\xfa\x8f\x11\x60\xd2\x5f\x6f\xa7\xb3\x14\x04\xa4\x57\xb7\xf7\xb3\xc5\xc9\x69\xda\x95\x1e\x17\x56\x6b\x5e\x9c\x6b" + - "\x02\x54\xc7\x62\x15\xdd\xe5\xde\x81\x16\x10\xec\x37\xb6\x7b\x30\xbd\x21\xb5\x5d\x87\xb1\x42\x9b\x52\xa8\x5d\x1f" + - "\x65\x1f\x99\x41\x34\x98\x4a\x7c\xaa\x23\x24\x38\x6f\xeb\x8c\xf2\x64\x14\x6f\x30\x1b\x62\x54\xd4\x8a\xfa\x57\x9c" + - "\x4c\xf7\x1e\x70\xe7\x3a\xae\x19\xaf\x3a\xe3\x8d\xf3\x90\x61\xa0\x17\x52\xb9\x78\x2f\x4a\x60\xc5\xa2\x7a\x6d\xc9" + - "\xc1\x09\x7e\x9e\xf4\x7b\x56\x28\xe8\x33\x1a\x40\xe8\x03\xb8\x8a\xa4\x9b\x02\x66\xd7\x8b\x51\xf4\x15\x93\xa8\xa2" + - "\xb5\x9f\xca\x81\x33\x83\xdc\xd2\xd1\x2d\xf4\xd9\xe4\xb4\x47\x51\x49\xe7\x34\x58\xfa\x58\x35\x22\x64\xd6\xb8\xf6" + - "\xe2\x71\xd0\xd0\x28\x84\xc1\xf6\xc6\xc4\x4b\x28\xf0\x66\x8d\xd4\x7a\x3b\xa4\x21\x8b\xbe\x56\xf2\x5f\x0e\x7f\x5b" + - "\x61\x25\x1f\xc4\xe4\x41\x6a\x8f\x56\x0b\xa5\xb8\xf9\x52\x07\x78\x08\x60\x28\xc2\xdc\xc4\x53\xba\x1e\xe7\xd2\x3d" + - "\x3c\x02\xad\x6e\xf2\x87\x33\x7a\x02\x53\xcf\xbc\xaf\x24\xb2\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\x44\x69\xc3\x40\x85" + - "\x16\x90\x6f\x48\x56\x5d\x6a\x6f\x8c\x61\x17\xde\xfc\xc6\x11\xaa\x2c\x6e\xe3\x5d\x62\xcb\x28\x48\x48\x8b\x39\x41" + - "\x8e\xd1\x44\x1d\x1e\xe2\x95\x4e\x29\xf6\xc2\x88\x10\x94\x42\xef\x06\x1a\xf2\xb9\x05\xdf\x93\xf6\x81\x99\x9e\x2c" + - "\xc9\xc8\xa7\x27\xb8\xde\x78\xc6\x6d\xa8\x75\xf5\x70\x3c\x6b\x6f\x06\xe2\x70\x16\x90\xf9\x76\x06\xd7\xc9\xbb\x64" + - "\x91\xc0\xd5\xe5\xdd\xd5\xe5\x75\x42\x4f\xee\xdf\xd3\x04\xd2\x3e\xe1\x6e\x30\x2d\x28\x8f\x73\x54\xe8\x03\x9f\xe1" + - "\x04\xed\xb3\x9d\xe7\xfe\xf0\x11\xfd\x20\x94\x3a\xac\x06\x17\xef\xcb\xc3\x29\x39\x4d\xc2\x0d\x2a\x35\x79\xc4\xc6" + - "\xd8\x3e\x86\x06\xf2\xcf\x50\xfb\xd1\xf3\xab\x3f\x74\xfd\x37\x00\x00\xff\xff\x11\xda\x64\x3c\x8b\x1b\x00\x00") + "\x79\xd5\xf3\xd0\x5f\xc1\x11\x8c\xf0\xc3\xb2\x6c\x9b\x0d\xae\xca\x6f\xa9\xcb\xe8\x54\xe9\x20\x13\x9a\x40\xcf\x84" + + "\xa0\x1f\x37\xb8\x22\xe2\x74\xc7\x23\x38\x96\x25\xfd\xad\xd0\x32\x64\xea\x0c\xe9\x6b\x29\xf3\x5c\xe1\xca\x7c\x3e" + + "\x0e\x71\x4b\x3d\x3a\xbf\x5c\x5b\x53\x57\x07\x45\x3e\x48\xac\xf6\xcc\xae\x92\x72\x59\x70\xe9\x78\x70\x5e\x58\xbf" + + "\xf4\xb2\x44\x06\x20\x5b\x6b\xfa\x3c\x28\x8b\x0e\xda\x95\x33\xb0\x11\x5b\x6c\xc5\x71\xa6\x7b\xd3\xe2\x1c\x67\xbc" + + "\xd9\xa2\xdd\xa0\xc8\xc9\x1e\xa6\xc2\x40\x01\x16\x19\x44\xe9\x08\xe3\x37\x68\xa1\x10\x99\x37\xd6\x05\x1a\x88\xf2" + + "\xd6\x06\xa4\x66\xcc\x46\x20\xc3\x26\x7b\x5f\x09\x46\x1a\x62\x05\xb1\xbb\x80\xf4\xee\xfe\xe6\x24\xaa\x7a\x0a\x6f" + + "\xe6\xb7\x37\x30\xe8\xf1\xa0\x91\x4a\x81\x50\x8d\xd8\x39\xf2\xef\x8f\x3f\xb5\x92\xd2\xb8\x2b\x6c\xda\x47\x90\x19" + + "\x8d\x5e\x38\xf8\xf1\x34\xb8\x76\xef\x99\x14\xae\x2f\x17\xc9\x62\x7a\x93\x1c\xb8\xb4\x95\x96\xc2\x3c\xb9\x7c\x37" + + "\x3a\x6a\x4f\xb9\x77\xc8\xe4\x23\x75\x4e\x2c\x88\x20\x8b\x3d\x65\x6c\x84\x03\x47\xa8\xcf\xb8\x11\x54\x88\x09\xe4" + + "\x96\xc4\xf9\x98\xa7\xb0\x98\xce\x3e\x52\x1a\xbf\xe8\x47\x71\x90\x3a\x54\x85\x50\x28\xb1\x26\xa9\x8f\x9e\x16\x44" + + "\xd3\xc2\x9c\x13\x8c\x89\x33\xab\x2d\xc5\x5d\xed\x28\xd4\x5a\xea\xf5\xa4\x3b\x9b\x56\x7d\xe5\x64\x5e\x42\xe1\x5e" + + "\xd6\x4e\xac\x71\x59\x57\xfb\x54\xff\xfa\xaa\xdc\x34\xfa\x6b\xeb\xc6\x63\x98\x52\x93\x42\xdc\x2b\x56\xa4\x0e\x37" + + "\x43\x81\xa8\x89\xfc\x3d\xdb\x50\x8a\xcf\xb2\xac\x4b\x50\xa8\xd7\x9e\x81\xf9\xe5\xeb\x73\x10\xb1\xd7\xe5\x9e\xb7" + + "\x4b\xc7\x83\xb5\xa6\x80\x42\x2a\x84\x4a\xf8\x0d\x35\x0c\xd0\x48\x9d\x9b\x26\x22\x5e\x7f\x28\x58\xe6\xd2\xf6\x30" + + "\xe1\xf5\xf9\x17\x4e\xef\xd0\xbb\x5f\xe3\x43\x83\xde\xdc\xce\x93\xe9\xdb\x19\x55\xfc\x49\x7f\xf9\x29\xcc\x93\x37" + + "\xc9\x3c\x99\x5d\x25\x77\x03\x2a\x27\xbc\x38\x7d\x04\x56\x86\x99\xfc\x17\x61\xcb\x45\xfb\xaa\x10\x19\xae\x8c\x79" + + "\x58\x96\xe8\x1c\xea\x35\xda\xf6\x8d\x47\x85\x6b\x2b\xca\xa3\x0e\x60\x85\x77\xa2\xaa\xda\xef\x1b\xef\xab\x25\x95" + + "\x36\xda\x65\x21\x51\xe5\xcb\x52\x68\xc9\xd4\x2f\x8d\x1e\xac\x92\x7a\x2b\x94\xcc\x97\x16\x3f\xd5\x04\x50\x4a\xea" + + "\x1e\x68\xb8\x4d\xfb\x59\xe7\xbe\x07\x63\x43\x00\x7b\xfd\xea\x8b\x04\xfc\xe6\x8a\x7c\xa4\x76\x66\xc6\x87\x5e\x7c" + + "\x6d\x94\xd0\xeb\x0b\x02\xb8\xb6\x80\x08\xdb\x08\x10\x3d\xf6\x40\x39\x0d\x65\x41\xc8\x95\x8a\xcc\xcb\x2d\xa6\x23" + + "\x70\xe6\xa8\x4f\xfe\xd2\x01\x7e\xaa\xe5\x56\xa8\xd8\x57\x73\xc1\xad\x90\x5b\x28\x5b\x73\xed\x15\x42\x39\xec\x10" + + "\x2d\xe5\x63\x52\x58\x24\x1f\x62\xd8\x9e\x51\x81\x91\xa1\x42\xa5\x74\x0a\x0b\xc8\x31\x14\x7e\x0e\xd2\x2d\xeb\x4a" + + "\x19\x91\x63\xce\xd8\x30\x02\xa9\x9d\x8f\x70\xcc\xcd\x7e\xed\xa4\x5e\xb7\xd2\xba\xe5\xcb\x42\x48\x85\xf9\x28\x54" + + "\x94\xf0\x6d\x37\xa4\x8d\x0f\x87\x74\x52\xb9\x28\xf7\x19\x0a\x79\xdd\x45\x96\x28\x82\x0a\xd7\x0f\x50\xa5\xdd\xf9" + + "\x4c\x4c\x3b\x94\x1f\x14\xe3\x6e\xaf\xd6\x1c\x85\x0e\x48\xdd\xc6\xd4\x2a\xe7\x50\x11\x9b\x59\x5e\xd6\xca\xb3\x38" + + "\xa6\x0d\xd2\xef\x35\x09\xa2\x9e\x42\x38\x5a\x51\x5b\x5c\x96\x6e\x3d\x6c\x97\x5b\x18\x38\x74\xd8\x53\xc2\x7a\x0b" + + "\x9f\x92\x49\x40\xe8\xbe\xc4\x70\x8e\x02\x27\x57\x25\xac\x97\x59\xad\x84\x1d\x38\x86\x68\x64\x45\x34\x12\x2d\x15" + + "\x3a\xdf\xe7\x24\x5a\x2c\x4c\x64\xe4\xfb\x29\x43\x81\x17\x0f\x18\xb3\x95\x38\x56\x64\x61\xd6\xf3\x06\x50\x32\x23" + + "\x6f\x64\x8e\x20\x7d\x37\x07\xed\x3d\xc9\x34\x42\xcc\xc4\x33\x51\x00\xe6\x2d\xda\x1d\x28\x14\xce\xd3\x50\xd3\xcd" + + "\x57\x62\x25\x95\xf4\xb1\x3b\x1f\x44\x20\x5e\x53\xe4\x86\x72\x8b\x5b\x89\xb6\xaf\x88\x59\xdc\xeb\xe6\x4d\xe4\x2f" + + "\x16\xd0\x33\xfa\xe7\x2e\x0a\x16\x6d\xad\xbf\x21\xa5\x1c\xda\x2d\xda\xb1\x23\x1b\x43\x2f\xb2\x94\x39\x58\xf4\xb5" + + "\xd5\x34\xb8\xec\xe2\xf8\xab\x14\x52\x5f\x32\x81\x5f\x76\xc3\x52\xd9\x6f\xfa\x1e\xa4\xae\x6a\x3f\x82\x9d\xa9\xd9" + + "\xb3\x9f\x6a\xf2\x05\x5b\x5f\x49\x52\xbe\x40\x1f\xaf\x13\xfa\xca\x77\x6e\x48\x3e\x77\x1f\xdf\x26\x0b\x46\x4c\x77" + + "\x71\x76\x26\x2a\x39\x31\x46\xcb\x89\x34\xf4\xf9\x6c\xfb\xe2\xac\x4f\x05\x3f\xf3\xa9\x3f\x7d\x37\x9d\xbd\xbf\x5f" + + "\x7c\xdf\xa9\xf3\xd3\x77\xf3\xe4\xfd\xed\x7c\xb1\x9c\x5e\xef\xe5\x7b\x2b\xb2\x10\xa6\x42\x5a\x6a\xda\x3d\x96\xfb" + + "\xf9\x36\x36\xae\xff\xfa\x77\x0a\x4a\x3a\xdf\x16\x95\x0e\x7a\x77\x4c\x31\x60\x48\xbe\x80\xf2\x06\xd6\x91\x97\x7f" + + "\xbd\xbb\x9d\x85\xf1\x79\x68\x24\x4d\x61\xbd\xb6\x0d\x5d\x68\xa8\xb7\x42\xd5\xe8\xe0\x24\xed\xf4\x4e\x47\x90\xb2" + + "\x45\xe9\x29\x08\xcb\x15\x5d\xd4\x6a\xef\x3d\xd1\xb1\x7a\x4f\x38\x17\x05\x25\xbe\x50\x16\x45\xbe\x0b\x05\x50\x59" + + "\x93\x11\x99\x75\x61\xac\x64\x85\x44\x39\xa3\x1e\x1e\xc8\xb2\x52\x41\x48\xa6\x50\xe8\xba\xe2\x99\x28\x8a\xe9\xd0" + + "\xad\xef\xf0\x08\x1c\x7b\x8d\x87\xd5\x7b\xc8\xb3\x3c\x43\x34\xe4\x46\x6d\xda\x76\x97\xfb\x8f\xb6\x50\xff\x64\xa6" + + "\x19\x8f\xe3\xd5\x51\x3e\x89\x60\x53\x5b\xf5\x95\x7e\xe3\x30\xc3\x09\xa6\x77\xe8\xa9\x21\x44\x41\x13\x67\x7b\xb3" + + "\xd1\x25\xf4\x08\x56\x35\xa3\x3a\xf9\xba\x52\x82\x5b\xbf\x78\x4d\x32\xa0\x32\xe1\xc3\x1d\x54\x65\xa4\xf6\xed\x18" + + "\xab\x51\xd8\xde\x2c\x1b\x26\x4e\xc4\x8b\x2e\x77\xd7\xd2\x6f\xea\xd5\x24\x33\xe5\x19\xa5\xf0\x59\x1b\x81\xb3\x95" + + "\x32\xab\xb3\x52\x38\x8f\xf6\x2c\x37\x99\xe3\xd7\xe3\xba\x96\xf9\xa4\xcc\xe1\xfb\x7e\xd7\xf0\xa4\x1c\xe9\x5c\x8d" + + "\xee\xec\xd5\xff\x7f\xd9\xbc\x45\x17\x51\xc3\x70\xe8\x99\x08\xa6\xae\xb5\x23\x13\x8e\x5d\x23\xa0\x9d\xb4\x78\xce" + + "\x18\x85\xc4\x12\x7c\x6b\x49\xfe\xa4\x71\x56\xed\x5a\x59\x2b\x65\xb2\x07\x22\x47\x62\x71\x42\x40\x0d\xd3\x1b\xde" + + "\xd8\x76\xda\xf1\xab\xa3\xe9\xc4\x45\x24\xa8\x9e\x16\x24\x0b\xbe\x2e\x8a\xe3\x1c\x34\xc2\x41\x8e\x1e\x33\x8e\x7f" + + "\x5c\xff\x31\x02\x4c\xfa\xeb\xed\x74\x96\x82\x80\xf4\xea\xf6\x7e\xb6\x38\x39\x4d\xbb\xd2\xe3\xc2\x6a\xcd\x8b\xe3" + + "\x4c\x80\xea\x58\xac\xa2\xbb\xd3\x3b\xd0\x02\x82\xfd\xc6\x76\x0f\xa6\x37\xa4\xb6\xeb\x30\x56\x68\x53\x0a\xb5\xeb" + + "\xa3\xec\x23\x33\x88\x06\x53\x89\x4f\x75\x84\x04\xe7\x6d\x9d\x51\x9e\x8c\xe2\xc5\x65\x43\x1d\x15\x51\x51\xff\x66" + + "\x93\xdb\xbd\x07\xdc\xb9\xae\xd7\x8c\x37\x9c\xf1\xa2\x79\xd8\x61\xa0\x17\x52\xb9\x78\x1d\x4a\x60\xc5\xa2\x7a\xb4" + + "\xe4\xe0\x04\x3f\x4f\xfa\x9c\x15\x0a\xfa\x8c\x06\x10\xfa\x00\xae\x22\xe9\xa6\x80\xd9\xf5\x62\x14\x7d\xc5\x4d\x54" + + "\xd1\xda\x4f\xe5\xc0\x99\x41\x6e\xe9\xda\x2d\xf4\xd9\xe4\xb4\xd7\xa2\x92\xce\x69\xb0\xf4\xb1\x6a\x44\xc8\xac\x71" + + "\xed\x7d\xe3\x80\xd0\x28\x84\xc1\xf6\xc6\xc4\xbb\x27\xf0\x66\x8d\x44\xbd\x1d\xd2\x90\x45\x5f\x2b\xf9\x2f\x87\xbf" + + "\xad\xb0\x92\x0f\xe2\xe6\x41\x6a\x8f\x56\x0b\xa5\x98\x7c\x89\x01\x1e\x02\x18\x8a\x30\x37\xf1\x70\xae\xc7\xb9\x74" + + "\x0f\x8f\x40\xab\x9b\xfc\xe1\x8c\x9e\xc0\xd4\x73\xdf\x57\x52\xb3\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\xd4\xd2\x86\x81" + + "\x0a\x2d\x20\x5f\x8c\xac\xba\xd4\xde\x18\xc3\x2e\xbc\xf9\x8d\x23\x54\x59\xdc\xc6\x2b\xc4\xb6\xa3\x20\x21\x2d\xe6" + + "\x04\x39\x46\x53\xeb\xf0\x10\x6f\x72\x4a\xb1\x17\x46\x0d\x41\x29\xf4\x6e\xa0\x21\x9f\x5b\xf0\xf5\x68\x1f\x98\xe9" + + "\xc9\x92\x8c\x7c\x7a\x82\xeb\x8d\x67\x4c\x43\xad\xab\x87\xe3\x59\x7b\x21\x10\x87\xb3\x80\xcc\xb7\x33\xb8\x4e\xde" + + "\x25\x8b\x04\xae\x2e\xef\xae\x2e\xaf\x13\x7a\x72\xff\x9e\x26\x90\xf6\x09\xb3\xc1\xb4\xa0\x3c\xce\x51\xa1\x0f\xfd" + + "\x0c\x27\x68\xbf\xdb\x79\xee\xef\x1d\xd1\x0f\x42\xa9\xc3\x6a\x70\xf1\x9a\x3c\x9c\x92\xd3\x24\xdc\xa0\x52\x93\x47" + + "\x6c\x8c\xf4\x31\x34\x90\x7f\x7d\xda\x8f\x9e\x5f\xfd\x7d\xeb\xbf\x01\x00\x00\xff\xff\x90\x8a\x5a\xbc\x82\x1b\x00" + + "\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( From 71ed0e969f303dcc017cf6eeeb335faf532408f7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 6 Sep 2018 16:20:10 +0200 Subject: [PATCH 11/65] Add Readme.md entry on testing --- Readme.md | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/Readme.md b/Readme.md index ec60e7a..85659fb 100644 --- a/Readme.md +++ b/Readme.md @@ -86,3 +86,9 @@ It should now be possible to build ooni by running: ``` make build ``` + +To run internal tests do: + +``` +make test-internal +``` From 35bd334cfca530d9bb977a0ae6b613850cde872d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 12:55:27 +0200 Subject: [PATCH 12/65] Update the measurement, network and url creation to the new schema --- internal/cli/run/run.go | 25 ++-------- internal/database/actions.go | 69 ++++++++++++++++++--------- internal/database/actions_test.go | 20 ++++---- internal/database/models.go | 4 +- nettests/nettests.go | 41 ++++++++++------ nettests/websites/web_connectivity.go | 59 +++++++++++++++++++---- 6 files changed, 138 insertions(+), 80 deletions(-) diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 3d7df37..88785c3 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -3,9 +3,7 @@ package run import ( "errors" "fmt" - "path/filepath" "strings" - "time" "github.com/alecthomas/kingpin" "github.com/apex/log" @@ -14,7 +12,6 @@ import ( "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests/groups" - "github.com/ooni/probe-cli/utils" ) func init() { @@ -55,24 +52,13 @@ func init() { return err } - network := database.Network{ - ASN: ctx.Location.ASN, - CountryCode: ctx.Location.CountryCode, - NetworkName: ctx.Location.NetworkName, - IP: ctx.Location.IP, - } - newID, err := ctx.DB.Collection("networks").Insert(network) + network, err := database.CreateNetwork(ctx.DB, ctx.Location) if err != nil { log.WithError(err).Error("Failed to create the network row") return nil } - network.ID = newID.(int64) - result, err := database.CreateResult(ctx.DB, ctx.Home, database.Result{ - TestGroupName: *nettestGroup, - StartTime: time.Now().UTC(), - NetworkID: network.ID, - }) + result, err := database.CreateResult(ctx.DB, ctx.Home, *nettestGroup, network.ID) if err != nil { log.Errorf("DB result error: %s", err) return err @@ -80,16 +66,13 @@ func init() { for _, nt := range group.Nettests { log.Debugf("Running test %T", nt) - msmtPath := filepath.Join(ctx.TempDir, - fmt.Sprintf("msmt-%T-%s.jsonl", nt, - time.Now().UTC().Format(utils.ResultTimestamp))) - - ctl := nettests.NewController(nt, ctx, result, msmtPath) + ctl := nettests.NewController(nt, ctx, result) if err = nt.Run(ctl); err != nil { log.WithError(err).Errorf("Failed to run %s", group.Label) return err } } + if err = result.Finished(ctx.DB, group.Summary); err != nil { return err } diff --git a/internal/database/actions.go b/internal/database/actions.go index 2b3090e..851ad4e 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -1,6 +1,7 @@ package database import ( + "database/sql" "time" "github.com/apex/log" @@ -117,41 +118,65 @@ func ListResults(db sqlbuilder.Database) ([]*Result, []*Result, error) { // CreateMeasurement writes the measurement to the database a returns a pointer // to the Measurement -func CreateMeasurement(sess sqlbuilder.Database, m Measurement, i string) (*Measurement, error) { - col := sess.Collection("measurements") +func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, resultID int64, reportFilePath string, urlID sql.NullInt64) (*Measurement, error) { + msmt := Measurement{ + ReportID: reportID, + TestName: testName, + ResultID: resultID, + ReportFilePath: reportFilePath, + URLID: urlID, + // XXX Do we want to have this be part of something else? + StartTime: time.Now().UTC(), + TestKeys: "", + } - // XXX Do we want to have this be part of something else? - m.StartTime = time.Now().UTC() - m.TestKeys = "" - - // XXX insert also the URL and stuff - //m.Input = i - //m.State = "active" - - newID, err := col.Insert(m) + newID, err := sess.Collection("measurements").Insert(msmt) if err != nil { return nil, errors.Wrap(err, "creating measurement") } - m.ID = newID.(int64) - return &m, nil + msmt.ID = newID.(int64) + return &msmt, nil } // CreateResult writes the Result to the database a returns a pointer // to the Result -func CreateResult(sess sqlbuilder.Database, homePath string, r Result) (*Result, error) { - log.Debugf("Creating result %v", r) +func CreateResult(sess sqlbuilder.Database, homePath string, testGroupName string, networkID int64) (*Result, error) { + startTime := time.Now().UTC() - col := sess.Collection("results") - - p, err := utils.MakeResultsDir(homePath, r.TestGroupName, r.StartTime) + p, err := utils.MakeResultsDir(homePath, testGroupName, startTime) if err != nil { return nil, err } - r.MeasurementDir = p - newID, err := col.Insert(r) + + result := Result{ + TestGroupName: testGroupName, + StartTime: startTime, + NetworkID: networkID, + } + result.MeasurementDir = p + log.Debugf("Creating result %v", result) + + newID, err := sess.Collection("results").Insert(result) if err != nil { return nil, errors.Wrap(err, "creating result") } - r.ID = newID.(int64) - return &r, nil + result.ID = newID.(int64) + return &result, nil +} + +// CreateNetwork will create a new network in the network table +func CreateNetwork(sess sqlbuilder.Database, location *utils.LocationInfo) (*Network, error) { + network := Network{ + ASN: location.ASN, + CountryCode: location.CountryCode, + NetworkName: location.NetworkName, + IP: location.IP, + } + newID, err := sess.Collection("networks").Insert(network) + if err != nil { + return nil, err + } + + network.ID = newID.(int64) + return &network, nil } diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 1712562..6cb1ec9 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -5,7 +5,6 @@ import ( "io/ioutil" "os" "testing" - "time" ) func TestMeasurementWorkflow(t *testing.T) { @@ -25,21 +24,18 @@ func TestMeasurementWorkflow(t *testing.T) { if err != nil { t.Error(err) } - result, err := CreateResult(sess, tmpdir, Result{ - TestGroupName: "websites", - StartTime: time.Now().UTC(), - }) + result, err := CreateResult(sess, tmpdir, "websites", 0) if err != nil { t.Fatal(err) } - msmtTemplate := Measurement{ - ReportID: sql.NullString{String: "", Valid: false}, - TestName: "antani", - ResultID: result.ID, - ReportFilePath: tmpdir, - } - m1, err := CreateMeasurement(sess, msmtTemplate, "") + reportID := sql.NullString{String: "", Valid: false} + testName := "antani" + resultID := result.ID + reportFilePath := tmpdir + urlID := sql.NullInt64{Int64: 0, Valid: false} + + m1, err := CreateMeasurement(sess, reportID, testName, resultID, reportFilePath, urlID) if err != nil { t.Fatal(err) } diff --git a/internal/database/models.go b/internal/database/models.go index ffb641a..8cfe1e8 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -23,7 +23,7 @@ type Network struct { // URL represents URLs from the testing lists type URL struct { ID int64 `db:"id"` - URL int64 `db:"url"` + URL string `db:"url"` CategoryCode string `db:"category_code"` CountryCode string `db:"country_code"` } @@ -42,7 +42,7 @@ type Measurement struct { UploadFailureMsg sql.NullString `db:"upload_failure_msg,omitempty"` IsRerun bool `db:"is_rerun"` ReportID sql.NullString `db:"report_id,omitempty"` - URLID string `db:"url_id"` // Used to reference URL + URLID sql.NullInt64 `db:"url_id,omitempty"` // Used to reference URL MeasurementID sql.NullInt64 `db:"measurement_id,omitempty"` IsAnomaly sql.NullBool `db:"is_anomaly,omitempty"` // FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462 diff --git a/nettests/nettests.go b/nettests/nettests.go index 61915e1..398b60f 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -6,6 +6,7 @@ import ( "fmt" "os" "path/filepath" + "time" "github.com/apex/log" "github.com/fatih/color" @@ -24,7 +25,10 @@ type Nettest interface { } // NewController creates a nettest controller -func NewController(nt Nettest, ctx *ooni.Context, res *database.Result, msmtPath string) *Controller { +func NewController(nt Nettest, ctx *ooni.Context, res *database.Result) *Controller { + msmtPath := filepath.Join(ctx.TempDir, + fmt.Sprintf("msmt-%T-%s.jsonl", nt, + time.Now().UTC().Format(utils.ResultTimestamp))) return &Controller{ Ctx: ctx, nt: nt, @@ -36,11 +40,12 @@ func NewController(nt Nettest, ctx *ooni.Context, res *database.Result, msmtPath // Controller is passed to the run method of every Nettest // each nettest instance has one controller type Controller struct { - Ctx *ooni.Context - res *database.Result - nt Nettest - msmts map[int64]*database.Measurement - msmtPath string // XXX maybe we can drop this and just use a temporary file + Ctx *ooni.Context + res *database.Result + nt Nettest + msmts map[int64]*database.Measurement + msmtPath string // XXX maybe we can drop this and just use a temporary file + inputIdxMap map[int64]int64 // Used to map mk idx to database id } func getCaBundlePath() string { @@ -51,6 +56,11 @@ func getCaBundlePath() string { return "/etc/ssl/cert.pem" } +func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error { + c.inputIdxMap = inputIdxMap + return nil +} + // Init should be called once to initialise the nettest func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("Init: %v", nt) @@ -58,12 +68,11 @@ func (c *Controller) Init(nt *mk.Nettest) error { c.msmts = make(map[int64]*database.Measurement) - msmtTemplate := database.Measurement{ - ReportID: sql.NullString{String: "", Valid: false}, - TestName: nt.Name, - ResultID: c.res.ID, - ReportFilePath: c.msmtPath, - } + // These values are shared by every measurement + reportID := sql.NullString{String: "", Valid: false} + testName := nt.Name + resultID := c.res.ID + reportFilePath := c.msmtPath // This is to workaround homedirs having UTF-8 characters in them. // See: https://github.com/measurement-kit/measurement-kit/issues/1635 @@ -157,7 +166,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.report_created", func(e mk.Event) { log.Debugf("%s", e.Key) - msmtTemplate.ReportID = sql.NullString{String: e.Value.ReportID, Valid: true} + reportID = sql.NullString{String: e.Value.ReportID, Valid: true} }) nt.On("status.geoip_lookup", func(e mk.Event) { @@ -175,7 +184,11 @@ func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf(color.RedString(e.Key)) idx := e.Value.Idx - msmt, err := database.CreateMeasurement(c.Ctx.DB, msmtTemplate, e.Value.Input) + urlID := sql.NullInt64{Int64: 0, Valid: false} + if c.inputIdxMap != nil { + urlID = sql.NullInt64{Int64: c.inputIdxMap[idx], Valid: true} + } + msmt, err := database.CreateMeasurement(c.Ctx.DB, reportID, testName, resultID, reportFilePath, urlID) if err != nil { log.WithError(err).Error("Failed to create measurement") return diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index d149207..b4db6d4 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -6,7 +6,9 @@ import ( "io/ioutil" "net/http" + "github.com/apex/log" "github.com/measurement-kit/go-measurement-kit" + "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/nettests" "github.com/pkg/errors" ) @@ -14,6 +16,7 @@ import ( // URLInfo contains the URL and the citizenlab category code for that URL type URLInfo struct { URL string `json:"url"` + CountryCode string `json:"country_code"` CategoryCode string `json:"category_code"` } @@ -24,10 +27,11 @@ type URLResponse struct { const orchestrateBaseURL = "https://events.proteus.test.ooni.io" -func lookupURLs(ctl *nettests.Controller) ([]string, error) { +func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { var ( - parsed = new(URLResponse) - urls []string + parsed = new(URLResponse) + urls []string + urlIDMap map[int64]int64 ) // XXX pass in the configuration for category codes reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", @@ -36,22 +40,58 @@ func lookupURLs(ctl *nettests.Controller) ([]string, error) { resp, err := http.Get(reqURL) if err != nil { - return urls, errors.Wrap(err, "failed to perform request") + return urls, urlIDMap, errors.Wrap(err, "failed to perform request") } body, err := ioutil.ReadAll(resp.Body) if err != nil { - return urls, errors.Wrap(err, "failed to read response body") + return urls, urlIDMap, errors.Wrap(err, "failed to read response body") } err = json.Unmarshal([]byte(body), &parsed) if err != nil { - return urls, errors.Wrap(err, "failed to parse json") + return urls, urlIDMap, errors.Wrap(err, "failed to parse json") } - for _, url := range parsed.Results { + for idx, url := range parsed.Results { + var urlID int64 + + res, err := ctl.Ctx.DB.Update("urls").Set( + "url", url.URL, + "category_code", url.CategoryCode, + "country_code", url.CountryCode, + ).Where("url = ? AND country_code = ?", url.URL, url.CountryCode).Exec() + + if err != nil { + log.Error("Failed to write to the URL table") + } else { + affected, err := res.RowsAffected() + + if err != nil { + log.Error("Failed to get affected row count") + } else if affected == 0 { + newID, err := ctl.Ctx.DB.Collection("urls").Insert( + database.URL{ + URL: url.URL, + CategoryCode: url.CategoryCode, + CountryCode: url.CountryCode, + }) + if err != nil { + log.Error("Failed to insert into the URLs table") + } + urlID = newID.(int64) + } else { + lastID, err := res.LastInsertId() + if err != nil { + log.Error("failed to get URL ID") + } + urlID = lastID + } + } + + urlIDMap[int64(idx)] = urlID urls = append(urls, url.URL) } - return urls, nil + return urls, urlIDMap, nil } // WebConnectivity test implementation @@ -63,10 +103,11 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error { nt := mk.NewNettest("WebConnectivity") ctl.Init(nt) - urls, err := lookupURLs(ctl) + urls, urlIDMap, err := lookupURLs(ctl) if err != nil { return err } + ctl.SetInputIdxMap(urlIDMap) nt.Options.Inputs = urls return nt.Run() From 1516a7ea741ba3166d3825bc18a3d387a53b0de9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 12:56:00 +0200 Subject: [PATCH 13/65] Fix bug in constraint --- data/migrations/1_create_msmt_results.sql | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 440fc74..2b44ec9 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -125,7 +125,7 @@ CREATE TABLE `measurements` ( `report_id` VARCHAR(255), -- This can be NULL when no report file has been -- created. - `url_id` INTEGER NOT NULL, + `url_id` INTEGER, -- This is not yet a feature of the collector, but we are planning to add -- this at some point in the near future. From ecc8216ceb11b5dfea867e9f6dfa4da3238b76f4 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 12:56:06 +0200 Subject: [PATCH 14/65] Update bindata --- internal/bindata/bindata.go | 44 ++++++++++++++++++------------------- 1 file changed, 22 insertions(+), 22 deletions(-) diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 7be2ccb..7734bde 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -202,28 +202,28 @@ var _bindataDataMigrations1createmsmtresultssql = []byte( "\x45\xe9\x29\x08\xcb\x15\x5d\xd4\x6a\xef\x3d\xd1\xb1\x7a\x4f\x38\x17\x05\x25\xbe\x50\x16\x45\xbe\x0b\x05\x50\x59" + "\x93\x11\x99\x75\x61\xac\x64\x85\x44\x39\xa3\x1e\x1e\xc8\xb2\x52\x41\x48\xa6\x50\xe8\xba\xe2\x99\x28\x8a\xe9\xd0" + "\xad\xef\xf0\x08\x1c\x7b\x8d\x87\xd5\x7b\xc8\xb3\x3c\x43\x34\xe4\x46\x6d\xda\x76\x97\xfb\x8f\xb6\x50\xff\x64\xa6" + - "\x19\x8f\xe3\xd5\x51\x3e\x89\x60\x53\x5b\xf5\x95\x7e\xe3\x30\xc3\x09\xa6\x77\xe8\xa9\x21\x44\x41\x13\x67\x7b\xb3" + - "\xd1\x25\xf4\x08\x56\x35\xa3\x3a\xf9\xba\x52\x82\x5b\xbf\x78\x4d\x32\xa0\x32\xe1\xc3\x1d\x54\x65\xa4\xf6\xed\x18" + - "\xab\x51\xd8\xde\x2c\x1b\x26\x4e\xc4\x8b\x2e\x77\xd7\xd2\x6f\xea\xd5\x24\x33\xe5\x19\xa5\xf0\x59\x1b\x81\xb3\x95" + - "\x32\xab\xb3\x52\x38\x8f\xf6\x2c\x37\x99\xe3\xd7\xe3\xba\x96\xf9\xa4\xcc\xe1\xfb\x7e\xd7\xf0\xa4\x1c\xe9\x5c\x8d" + - "\xee\xec\xd5\xff\x7f\xd9\xbc\x45\x17\x51\xc3\x70\xe8\x99\x08\xa6\xae\xb5\x23\x13\x8e\x5d\x23\xa0\x9d\xb4\x78\xce" + - "\x18\x85\xc4\x12\x7c\x6b\x49\xfe\xa4\x71\x56\xed\x5a\x59\x2b\x65\xb2\x07\x22\x47\x62\x71\x42\x40\x0d\xd3\x1b\xde" + - "\xd8\x76\xda\xf1\xab\xa3\xe9\xc4\x45\x24\xa8\x9e\x16\x24\x0b\xbe\x2e\x8a\xe3\x1c\x34\xc2\x41\x8e\x1e\x33\x8e\x7f" + - "\x5c\xff\x31\x02\x4c\xfa\xeb\xed\x74\x96\x82\x80\xf4\xea\xf6\x7e\xb6\x38\x39\x4d\xbb\xd2\xe3\xc2\x6a\xcd\x8b\xe3" + - "\x4c\x80\xea\x58\xac\xa2\xbb\xd3\x3b\xd0\x02\x82\xfd\xc6\x76\x0f\xa6\x37\xa4\xb6\xeb\x30\x56\x68\x53\x0a\xb5\xeb" + - "\xa3\xec\x23\x33\x88\x06\x53\x89\x4f\x75\x84\x04\xe7\x6d\x9d\x51\x9e\x8c\xe2\xc5\x65\x43\x1d\x15\x51\x51\xff\x66" + - "\x93\xdb\xbd\x07\xdc\xb9\xae\xd7\x8c\x37\x9c\xf1\xa2\x79\xd8\x61\xa0\x17\x52\xb9\x78\x1d\x4a\x60\xc5\xa2\x7a\xb4" + - "\xe4\xe0\x04\x3f\x4f\xfa\x9c\x15\x0a\xfa\x8c\x06\x10\xfa\x00\xae\x22\xe9\xa6\x80\xd9\xf5\x62\x14\x7d\xc5\x4d\x54" + - "\xd1\xda\x4f\xe5\xc0\x99\x41\x6e\xe9\xda\x2d\xf4\xd9\xe4\xb4\xd7\xa2\x92\xce\x69\xb0\xf4\xb1\x6a\x44\xc8\xac\x71" + - "\xed\x7d\xe3\x80\xd0\x28\x84\xc1\xf6\xc6\xc4\xbb\x27\xf0\x66\x8d\x44\xbd\x1d\xd2\x90\x45\x5f\x2b\xf9\x2f\x87\xbf" + - "\xad\xb0\x92\x0f\xe2\xe6\x41\x6a\x8f\x56\x0b\xa5\x98\x7c\x89\x01\x1e\x02\x18\x8a\x30\x37\xf1\x70\xae\xc7\xb9\x74" + - "\x0f\x8f\x40\xab\x9b\xfc\xe1\x8c\x9e\xc0\xd4\x73\xdf\x57\x52\xb3\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\xd4\xd2\x86\x81" + - "\x0a\x2d\x20\x5f\x8c\xac\xba\xd4\xde\x18\xc3\x2e\xbc\xf9\x8d\x23\x54\x59\xdc\xc6\x2b\xc4\xb6\xa3\x20\x21\x2d\xe6" + - "\x04\x39\x46\x53\xeb\xf0\x10\x6f\x72\x4a\xb1\x17\x46\x0d\x41\x29\xf4\x6e\xa0\x21\x9f\x5b\xf0\xf5\x68\x1f\x98\xe9" + - "\xc9\x92\x8c\x7c\x7a\x82\xeb\x8d\x67\x4c\x43\xad\xab\x87\xe3\x59\x7b\x21\x10\x87\xb3\x80\xcc\xb7\x33\xb8\x4e\xde" + - "\x25\x8b\x04\xae\x2e\xef\xae\x2e\xaf\x13\x7a\x72\xff\x9e\x26\x90\xf6\x09\xb3\xc1\xb4\xa0\x3c\xce\x51\xa1\x0f\xfd" + - "\x0c\x27\x68\xbf\xdb\x79\xee\xef\x1d\xd1\x0f\x42\xa9\xc3\x6a\x70\xf1\x9a\x3c\x9c\x92\xd3\x24\xdc\xa0\x52\x93\x47" + - "\x6c\x8c\xf4\x31\x34\x90\x7f\x7d\xda\x8f\x9e\x5f\xfd\x7d\xeb\xbf\x01\x00\x00\xff\xff\x90\x8a\x5a\xbc\x82\x1b\x00" + + "\x19\x8f\xe3\xd5\x51\x3e\x89\x60\x53\x5b\xd5\xef\x37\xbe\x4c\x6c\x42\xe7\x1d\x7a\xea\x03\x51\xd0\xa0\xd9\x5e\x68" + + "\x74\x79\x3c\x82\x55\xcd\x60\x4e\x2e\xae\x94\xe0\x8e\x2f\xde\x8e\x0c\x18\x4c\xf8\x70\xf5\x54\x19\xa9\x7d\x3b\xbd" + + "\x6a\x14\xb6\x37\xc2\x86\x41\x13\xf1\xa2\x4b\xd9\xb5\xf4\x9b\x7a\x35\xc9\x4c\x79\x46\x99\x7b\xd6\x3a\xfe\x6c\xa5" + + "\xcc\xea\xac\x14\xce\xa3\x3d\xcb\x4d\xe6\xf8\xf5\xb8\xae\x65\x3e\x29\x73\xf8\xbe\xdf\x2c\x3c\x29\x47\x3a\x57\xa3" + + "\x3b\x7b\xf5\xff\x5f\xf6\x6c\xd1\x33\xd4\x27\x1c\x7a\x26\x62\xa8\x6b\xed\xc8\x84\x63\xd7\x08\x68\x07\x2c\x1e\x2f" + + "\x46\x21\x9f\x04\x5f\x56\x92\x3f\x69\x8a\x55\xbb\x56\xd6\x4a\x99\xec\x81\x38\x91\xc8\x9b\x80\x4f\xc3\xf4\x86\x37" + + "\xb6\x0d\x76\xfc\xea\x68\x28\x71\x11\x00\xaa\xa7\x05\xc9\x82\x6f\x89\xe2\x14\x07\x8d\x70\x90\xa3\xc7\x8c\xc3\x1e" + + "\xd7\x7f\x8c\xb8\x92\xfe\x7a\x3b\x9d\xa5\x20\x20\xbd\xba\xbd\x9f\x2d\x4e\x4e\xd3\xae\xe2\xb8\x9e\x5a\xf3\xe2\x14" + + "\x13\x10\x3a\xd6\xa8\xe8\xae\xf2\x0e\xb4\x80\x60\xbf\xb1\xdd\x83\xe9\x0d\xa9\xed\x3a\x68\x15\xda\x94\x42\xed\xfa" + + "\xe0\xfa\xc8\xe8\xa1\xc1\x54\xe2\x53\x1d\x91\xc0\x79\x5b\x67\x94\x27\xa3\x78\x5f\xd9\x50\x23\x45\x0c\xd4\xbf\xd0" + + "\xe4\x2e\xef\x01\x77\xae\x6b\x31\xe3\xc5\x66\xbc\x5f\x1e\x36\x16\xe8\x85\x54\x2e\xde\x82\x12\x46\xb1\xa8\x1e\x1b" + + "\x39\x38\xc1\xcf\x93\x3e\x55\x85\x3a\x3e\xa3\xb9\x83\x3e\x80\xab\x48\xba\x29\x60\x76\xbd\x18\x45\x5f\x71\xef\x54" + + "\xb4\xf6\x53\x39\x70\x66\x90\x5b\xba\x2e\x0b\x7d\x36\x39\xed\x75\xa6\xa4\x73\x1a\x2c\x7d\x8c\x66\x10\x32\x6b\x5c" + + "\x7b\xcd\x38\xe0\x31\x0a\x61\xb0\xbd\x31\xf1\xca\x09\xbc\x59\x23\x31\x6e\x07\x30\x64\xd1\x57\x26\x8b\x47\x66\xbe" + + "\xad\xb0\x92\x0f\xe2\x9e\x41\x6a\x8f\x56\x0b\xa5\x98\x73\x09\xf8\x1f\x02\x06\x8a\x30\x2e\xf1\x4c\xae\xc7\xb9\x74" + + "\x0f\x8f\x20\xaa\x9b\xfc\xe1\x8c\x9e\xc0\xd4\x73\xbb\x57\x52\x8f\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\xd4\xc9\x86\x39" + + "\x0a\x2d\x20\xdf\x87\xac\xba\xd4\xde\x18\xc3\x2e\xbc\xf9\x8d\x23\x54\x59\xdc\xc6\x9b\xc3\xb6\x91\x20\x21\x2d\xe6" + + "\x04\x39\x46\x53\xc7\xf0\x10\x2f\x70\x4a\xb1\x17\x46\x7d\x40\x29\xf4\x6e\xa0\x21\x9f\x5b\xf0\xad\x68\x1f\x8f\xe9" + + "\xc9\x92\x8c\x7c\x7a\x70\xeb\x4d\x65\xcc\x3e\xad\xab\x87\x53\x59\x7b\x0f\x10\x67\xb2\x00\xc8\xb7\x33\xb8\x4e\xde" + + "\x25\x8b\x04\xae\x2e\xef\xae\x2e\xaf\x13\x7a\x72\xff\x9e\x06\x8f\xf6\x09\x93\xc0\xb4\xa0\x3c\xce\x51\xa1\x0f\x6d" + + "\x0c\x27\x68\xbf\xc9\x79\xee\xcf\x1c\xd1\x0f\x42\xa9\xc3\x6a\x70\xf1\x76\x3c\x9c\x92\xd3\x00\xdc\xa0\x52\x93\x47" + + "\x6c\x8c\xac\x31\x34\x90\x7f\x74\xda\x4f\x9c\x5f\xfd\x59\xeb\xbf\x01\x00\x00\xff\xff\xbd\xb9\x3d\xa5\x79\x1b\x00" + "\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { From a6da3799a78680721ae7b8fc35f22ac63612243a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 14:00:53 +0200 Subject: [PATCH 15/65] Fix travis maybe --- .travis.yml | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 2476a9e..60d6d34 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,11 +1,12 @@ language: go go: - - 1.9.x - - 1.x +- 1.9.x +- 1.x + install: - - go get -u github.com/golang/dep/... - - dep ensure - - make update-mk-libs +- go get -u github.com/golang/dep/... +- dep ensure +- make update-mk-libs script: - - make test-internal +- make test-internal From dabbdc15d281bcd824edee370e5f86a9b021c32e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 14:06:08 +0200 Subject: [PATCH 16/65] Fix bug in the test creation workflow --- internal/database/actions.go | 2 ++ internal/database/actions_test.go | 15 ++++++++++++++- internal/database/models.go | 1 + 3 files changed, 17 insertions(+), 1 deletion(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index 851ad4e..4235dd2 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -170,6 +170,8 @@ func CreateNetwork(sess sqlbuilder.Database, location *utils.LocationInfo) (*Net ASN: location.ASN, CountryCode: location.CountryCode, NetworkName: location.NetworkName, + // On desktop we consider it to always be wifi + NetworkType: "wifi", IP: location.IP, } newID, err := sess.Collection("networks").Insert(network) diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 6cb1ec9..261e2e9 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -5,6 +5,8 @@ import ( "io/ioutil" "os" "testing" + + "github.com/ooni/probe-cli/utils" ) func TestMeasurementWorkflow(t *testing.T) { @@ -24,7 +26,18 @@ func TestMeasurementWorkflow(t *testing.T) { if err != nil { t.Error(err) } - result, err := CreateResult(sess, tmpdir, "websites", 0) + + location := utils.LocationInfo{ + ASN: 0, + CountryCode: "IT", + NetworkName: "Unknown", + } + network, err := CreateNetwork(sess, &location) + if err != nil { + t.Fatal(err) + } + + result, err := CreateResult(sess, tmpdir, "websites", network.ID) if err != nil { t.Fatal(err) } diff --git a/internal/database/models.go b/internal/database/models.go index 8cfe1e8..69b679d 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -15,6 +15,7 @@ import ( type Network struct { ID int64 `db:"id"` NetworkName string `db:"network_name"` + NetworkType string `db:"network_type"` IP string `db:"ip"` ASN uint `db:"asn"` CountryCode string `db:"country_code"` From af6fb616d8804c00d01471d0e7d04da9ed4a097d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 14:09:42 +0200 Subject: [PATCH 17/65] Specify osx as the travis os --- .travis.yml | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.travis.yml b/.travis.yml index 60d6d34..7e30f98 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,3 +1,5 @@ +# Due to MK dependencies only macOS development is supported +os: osx language: go go: - 1.9.x From c2ea0c2a630d82d51fd248c52dcc7a7ada9ab011 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 15:16:20 +0200 Subject: [PATCH 18/65] Fix List* workflows and add unittests for them --- .travis.yml | 2 +- Makefile | 5 ++ internal/database/actions.go | 129 +++++++++--------------------- internal/database/actions_test.go | 23 +++++- internal/database/models.go | 23 +++++- 5 files changed, 83 insertions(+), 99 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7e30f98..6b1cb71 100644 --- a/.travis.yml +++ b/.travis.yml @@ -6,7 +6,7 @@ go: - 1.x install: -- go get -u github.com/golang/dep/... +- make install-dev-deps - dep ensure - make update-mk-libs diff --git a/Makefile b/Makefile index bf058c6..9b3dd13 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,10 @@ GO ?= go +install-dev-deps: + @$(GO) get -u github.com/golang/dep/... + @$(GO) get golang.org/x/tools/cmd/cover + @$(GO) get github.com/mattn/goveralls + build: @echo "Building dist/ooni" @$(GO) build -i -o dist/ooni cmd/ooni/main.go diff --git a/internal/database/actions.go b/internal/database/actions.go index 4235dd2..6f7dc77 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -7,111 +7,54 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/utils" "github.com/pkg/errors" + db "upper.io/db.v3" "upper.io/db.v3/lib/sqlbuilder" ) // ListMeasurements given a result ID -func ListMeasurements(db sqlbuilder.Database, resultID int64) ([]*Measurement, error) { - measurements := []*Measurement{} +func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementURLNetwork, error) { + measurements := []MeasurementURLNetwork{} - /* - FIXME - rows, err := db.Query(`SELECT id, name, - start_time, runtime, - country, - asn, - summary, - input - FROM measurements - WHERE result_id = ? - ORDER BY start_time;`, resultID) - if err != nil { - return measurements, errors.Wrap(err, "failed to get measurement list") - } - - for rows.Next() { - msmt := Measurement{} - err = rows.Scan(&msmt.ID, &msmt.Name, - &msmt.StartTime, &msmt.Runtime, - &msmt.CountryCode, - &msmt.ASN, - &msmt.Summary, &msmt.Input, - //&result.DataUsageUp, &result.DataUsageDown) - ) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - measurements = append(measurements, &msmt) - } - */ + req := sess.Select( + "networks.id as network_id", + "results.id as result_id", + "urls.id as url_id", + db.Raw("networks.*"), + db.Raw("urls.*"), + db.Raw("measurements.*"), + ).From("results"). + Join("measurements").On("results.id = measurements.result_id"). + Join("networks").On("results.network_id = networks.id"). + LeftJoin("urls").On("urls.id = measurements.url_id"). + OrderBy("measurements.start_time"). + Where("results.id = ?", resultID) + if err := req.All(&measurements); err != nil { + log.Errorf("failed to run query %s: %v", req.String(), err) + return measurements, err + } return measurements, nil } // ListResults return the list of results -func ListResults(db sqlbuilder.Database) ([]*Result, []*Result, error) { - doneResults := []*Result{} - incompleteResults := []*Result{} +func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) { + doneResults := []ResultNetwork{} + incompleteResults := []ResultNetwork{} - /* - FIXME - rows, err := db.Query(`SELECT id, name, - start_time, runtime, - network_name, country, - asn, - summary, done - FROM results - WHERE done = 1 - ORDER BY start_time;`) - if err != nil { - return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") - } - for rows.Next() { - result := Result{} - err = rows.Scan(&result.ID, &result.Name, - &result.StartTime, &result.Runtime, - &result.NetworkName, &result.Country, - &result.ASN, - &result.Summary, &result.Done, - //&result.DataUsageUp, &result.DataUsageDown) - ) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - doneResults = append(doneResults, &result) - } - */ + req := sess.Select( + "networks.id AS network_id", + db.Raw("results.*"), + db.Raw("networks.*"), + ).From("results"). + Join("networks").On("results.network_id = networks.id"). + OrderBy("results.start_time") - /* - FIXME - rows, err := db.Query(`SELECT - id, name, - start_time, - network_name, country, - asn - FROM results - WHERE done != 1 - ORDER BY start_time;`) - if err != nil { - return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") - } - */ - - /* - for rows.Next() { - result := Result{Done: false} - err = rows.Scan(&result.ID, &result.Name, &result.StartTime, - &result.NetworkName, &result.Country, - &result.ASN) - if err != nil { - log.WithError(err).Error("failed to fetch a row") - continue - } - incompleteResults = append(incompleteResults, &result) - } - */ + if err := req.Where("is_done = true").All(&doneResults); err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } + if err := req.Where("is_done = false").All(&incompleteResults); err != nil { + return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") + } return doneResults, incompleteResults, nil } diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 261e2e9..8c7e770 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -6,6 +6,7 @@ import ( "os" "testing" + "github.com/apex/log" "github.com/ooni/probe-cli/utils" ) @@ -14,7 +15,8 @@ func TestMeasurementWorkflow(t *testing.T) { if err != nil { t.Fatal(err) } - defer os.Remove(tmpfile.Name()) + log.Infof("%s", tmpfile.Name()) + //defer os.Remove(tmpfile.Name()) tmpdir, err := ioutil.TempDir("", "oonitest") if err != nil { @@ -62,4 +64,23 @@ func TestMeasurementWorkflow(t *testing.T) { t.Error("result_id mismatch") } + done, incomplete, err := ListResults(sess) + if err != nil { + t.Fatal(err) + } + + if len(incomplete) != 1 { + t.Error("there should be 1 incomplete measurement") + } + if len(done) != 0 { + t.Error("there should be 0 done measurements") + } + + msmts, err := ListMeasurements(sess, resultID) + if err != nil { + t.Fatal(err) + } + if msmts[0].Network.NetworkType != "wifi" { + t.Error("network_type should be wifi") + } } diff --git a/internal/database/models.go b/internal/database/models.go index 69b679d..1327f63 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -11,6 +11,21 @@ import ( "upper.io/db.v3/lib/sqlbuilder" ) +// ResultNetwork is used to represent the structure made from the JOIN +// between the results and networks tables. +type ResultNetwork struct { + Result `db:",inline"` + Network `db:",inline"` +} + +// MeasurementURLNetwork is used for the JOIN between Measurement and URL +type MeasurementURLNetwork struct { + Measurement `db:",inline"` + Network `db:",inline"` + NetworkID int64 `db:"network_id"` + URL `db:",inline"` +} + // Network represents a network tested by the user type Network struct { ID int64 `db:"id"` @@ -23,10 +38,10 @@ type Network struct { // URL represents URLs from the testing lists type URL struct { - ID int64 `db:"id"` - URL string `db:"url"` - CategoryCode string `db:"category_code"` - CountryCode string `db:"country_code"` + ID sql.NullInt64 `db:"id"` + URL sql.NullString `db:"url"` + CategoryCode sql.NullString `db:"category_code"` + CountryCode sql.NullString `db:"country_code"` } // Measurement model From 6f0defc6728be872ba51d0c1cf9dc3334b7877fb Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 15:23:29 +0200 Subject: [PATCH 19/65] Refactor URL table row creation into the actions file --- internal/database/actions.go | 46 +++++++++++++++++++++++++++ nettests/websites/web_connectivity.go | 35 ++------------------ 2 files changed, 48 insertions(+), 33 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index 6f7dc77..6b9c93f 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -125,3 +125,49 @@ func CreateNetwork(sess sqlbuilder.Database, location *utils.LocationInfo) (*Net network.ID = newID.(int64) return &network, nil } + +// CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't +// exists, otherwise it will update the category code of the one already in +// there. +func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string, countryCode string) (int64, error) { + var urlID int64 + + res, err := sess.Update("urls").Set( + "url", url, + "category_code", categoryCode, + "country_code", countryCode, + ).Where("url = ? AND country_code = ?", url, countryCode).Exec() + + if err != nil { + log.Error("Failed to write to the URL table") + return 0, err + } + affected, err := res.RowsAffected() + + if err != nil { + log.Error("Failed to get affected row count") + return 0, err + } + if affected == 0 { + newID, err := sess.Collection("urls").Insert( + URL{ + URL: sql.NullString{String: url, Valid: true}, + CategoryCode: sql.NullString{String: categoryCode, Valid: true}, + CountryCode: sql.NullString{String: countryCode, Valid: true}, + }) + if err != nil { + log.Error("Failed to insert into the URLs table") + return 0, err + } + urlID = newID.(int64) + } else { + lastID, err := res.LastInsertId() + if err != nil { + log.Error("failed to get URL ID") + return 0, err + } + urlID = lastID + } + + return urlID, nil +} diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index b4db6d4..32a7bf0 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -53,41 +53,10 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { } for idx, url := range parsed.Results { - var urlID int64 - - res, err := ctl.Ctx.DB.Update("urls").Set( - "url", url.URL, - "category_code", url.CategoryCode, - "country_code", url.CountryCode, - ).Where("url = ? AND country_code = ?", url.URL, url.CountryCode).Exec() - + urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode) if err != nil { - log.Error("Failed to write to the URL table") - } else { - affected, err := res.RowsAffected() - - if err != nil { - log.Error("Failed to get affected row count") - } else if affected == 0 { - newID, err := ctl.Ctx.DB.Collection("urls").Insert( - database.URL{ - URL: url.URL, - CategoryCode: url.CategoryCode, - CountryCode: url.CountryCode, - }) - if err != nil { - log.Error("Failed to insert into the URLs table") - } - urlID = newID.(int64) - } else { - lastID, err := res.LastInsertId() - if err != nil { - log.Error("failed to get URL ID") - } - urlID = lastID - } + log.Error("failed to add to the URL table") } - urlIDMap[int64(idx)] = urlID urls = append(urls, url.URL) } From ca505dfff324ea51e736d975ea52b140b0b68a5f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 15:29:16 +0200 Subject: [PATCH 20/65] Write a unittest for the URL creation function --- internal/database/actions_test.go | 39 +++++++++++++++++++++++++++---- 1 file changed, 35 insertions(+), 4 deletions(-) diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 8c7e770..0da42c5 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -6,7 +6,6 @@ import ( "os" "testing" - "github.com/apex/log" "github.com/ooni/probe-cli/utils" ) @@ -15,8 +14,7 @@ func TestMeasurementWorkflow(t *testing.T) { if err != nil { t.Fatal(err) } - log.Infof("%s", tmpfile.Name()) - //defer os.Remove(tmpfile.Name()) + defer os.Remove(tmpfile.Name()) tmpdir, err := ioutil.TempDir("", "oonitest") if err != nil { @@ -26,7 +24,7 @@ func TestMeasurementWorkflow(t *testing.T) { sess, err := Connect(tmpfile.Name()) if err != nil { - t.Error(err) + t.Fatal(err) } location := utils.LocationInfo{ @@ -84,3 +82,36 @@ func TestMeasurementWorkflow(t *testing.T) { t.Error("network_type should be wifi") } } + +func TestURLCreation(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) + + tmpdir, err := ioutil.TempDir("", "oonitest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + sess, err := Connect(tmpfile.Name()) + if err != nil { + t.Fatal(err) + } + + newID1, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX") + if err != nil { + t.Fatal(err) + } + + newID2, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX") + if err != nil { + t.Fatal(err) + } + + if newID2 != newID1 { + t.Error("inserting the same URL with different category code should produce the same result") + } +} From b727aba85441b7e21452937140a7121d3e25386c Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Fri, 7 Sep 2018 15:39:22 +0200 Subject: [PATCH 21/65] Update result listing --- internal/cli/list/list.go | 25 ++++++++++++------------- 1 file changed, 12 insertions(+), 13 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 73bc901..d06693b 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -42,15 +42,15 @@ func init() { } for idx, result := range incompleteResults { output.ResultItem(output.ResultItemData{ - ID: result.ID, + ID: result.Result.ID, Index: idx, TotalCount: len(incompleteResults), Name: result.TestGroupName, StartTime: result.StartTime, - NetworkName: "FIXME", //result.NetworkName, - Country: "FIXME", //result.Country, - ASN: "FIXME", //result.ASN, - Summary: "{}", //result.Summary, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + Summary: "{}", //result.Summary, Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, @@ -58,26 +58,25 @@ func init() { } resultSummary := output.ResultSummaryData{} - netCount := make(map[string]int) + netCount := make(map[uint]int) output.SectionTitle("Results") for idx, result := range doneResults { output.ResultItem(output.ResultItemData{ - ID: result.ID, + ID: result.Result.ID, Index: idx, TotalCount: len(doneResults), Name: result.TestGroupName, StartTime: result.StartTime, - NetworkName: "FIXME", //result.NetworkName, - Country: "FIXME", //result.Country, - ASN: "FIXME", //result.ASN, - Summary: "{}", //result.Summary, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + Summary: "{}", //result.Summary, Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, }) resultSummary.TotalTests++ - // FIXME - // netCount[result.ASN]++ + netCount[result.Network.ASN]++ resultSummary.TotalDataUsageUp += result.DataUsageUp resultSummary.TotalDataUsageDown += result.DataUsageDown } From b29071f37b887160269745d15f7e5b4e0e8698f6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 12:41:28 +0200 Subject: [PATCH 22/65] Replace summary with test_keys --- data/migrations/1_create_msmt_results.sql | 2 +- internal/cli/list/list.go | 51 ++++--- internal/cli/run/run.go | 2 +- internal/database/actions.go | 46 ++++++ internal/database/actions_test.go | 36 ++++- internal/database/models.go | 22 +-- internal/log/handlers/cli/results.go | 60 ++++---- internal/output/output.go | 58 +++---- nettests/groups/groups.go | 142 ------------------ nettests/im/facebook_messenger.go | 18 +-- nettests/im/telegram.go | 20 +-- nettests/im/whatsapp.go | 20 +-- .../http_header_field_manipulation.go | 14 +- .../middlebox/http_invalid_request_line.go | 14 +- nettests/nettests.go | 25 ++- nettests/performance/dash.go | 24 +-- nettests/performance/ndt.go | 31 ++-- nettests/summary/summary.go | 44 ------ nettests/websites/web_connectivity.go | 26 ++-- 19 files changed, 274 insertions(+), 381 deletions(-) delete mode 100644 nettests/summary/summary.go diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 2b44ec9..17d1c17 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -41,7 +41,7 @@ CREATE TABLE `networks` ( -- this with more data in the future. `network_type` VARCHAR(16) NOT NULL, -- One of wifi, mobile - `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. + `ip` VARCHAR(40) NOT NULL, -- Stores a string representation of an ipv4 or ipv6 address. -- The longest ip is an ipv6 address like: -- 0000:0000:0000:0000:0000:0000:0000:0000, -- which is 39 chars. diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index d06693b..66be3e4 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -41,19 +41,22 @@ func init() { output.SectionTitle("Incomplete results") } for idx, result := range incompleteResults { + output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, - Index: idx, - TotalCount: len(incompleteResults), - Name: result.TestGroupName, - StartTime: result.StartTime, - NetworkName: result.Network.NetworkName, - Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), - Summary: "{}", //result.Summary, - Done: result.IsDone, - DataUsageUp: result.DataUsageUp, - DataUsageDown: result.DataUsageDown, + ID: result.Result.ID, + Index: idx, + TotalCount: len(incompleteResults), + Name: result.TestGroupName, + StartTime: result.StartTime, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + MeasurementCount: 0, + MeasurementAnomalyCount: 0, + TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now + Done: result.IsDone, + DataUsageUp: result.DataUsageUp, + DataUsageDown: result.DataUsageDown, }) } @@ -61,16 +64,22 @@ func init() { netCount := make(map[uint]int) output.SectionTitle("Results") for idx, result := range doneResults { + totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID) + if err != nil { + log.WithError(err).Error("failed to list measurement counts") + } output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, - Index: idx, - TotalCount: len(doneResults), - Name: result.TestGroupName, - StartTime: result.StartTime, - NetworkName: result.Network.NetworkName, - Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), - Summary: "{}", //result.Summary, + ID: result.Result.ID, + Index: idx, + TotalCount: len(doneResults), + Name: result.TestGroupName, + StartTime: result.StartTime, + NetworkName: result.Network.NetworkName, + Country: result.Network.CountryCode, + ASN: fmt.Sprintf("AS%d", result.Network.ASN), + TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now + MeasurementCount: totalCount, + MeasurementAnomalyCount: anmlyCount, Done: result.IsDone, DataUsageUp: result.DataUsageUp, DataUsageDown: result.DataUsageDown, diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 88785c3..47333ff 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -73,7 +73,7 @@ func init() { } } - if err = result.Finished(ctx.DB, group.Summary); err != nil { + if err = result.Finished(ctx.DB); err != nil { return err } return nil diff --git a/internal/database/actions.go b/internal/database/actions.go index 6b9c93f..f17229d 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "encoding/json" "time" "github.com/apex/log" @@ -36,6 +37,33 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR return measurements, nil } +// GetMeasurementCounts returns the number of anomalous and total measurement for a given result +func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) { + var ( + totalCount uint64 + anmlyCount uint64 + err error + ) + col := sess.Collection("measurements") + + // XXX these two queries can be done with a single query + totalCount, err = col.Find("result_id", resultID). + Count() + if err != nil { + log.WithError(err).Error("failed to get total count") + return totalCount, anmlyCount, err + } + + anmlyCount, err = col.Find("result_id", resultID). + And(db.Cond{"is_anomaly": true}).Count() + if err != nil { + log.WithError(err).Error("failed to get anmly count") + return totalCount, anmlyCount, err + } + + return totalCount, anmlyCount, err +} + // ListResults return the list of results func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, error) { doneResults := []ResultNetwork{} @@ -168,6 +196,24 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string } urlID = lastID } + log.Debugf("returning url %d", urlID) return urlID, nil } + +// AddTestKeys writes the summary to the measurement +func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error { + tkBytes, err := json.Marshal(tk) + if err != nil { + log.WithError(err).Error("failed to serialize summary") + } + isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly + msmt.TestKeys = string(tkBytes) + msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true} + + err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt) + if err != nil { + return errors.Wrap(err, "updating measurement") + } + return nil +} diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 0da42c5..a060b40 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -83,18 +83,48 @@ func TestMeasurementWorkflow(t *testing.T) { } } -func TestURLCreation(t *testing.T) { +func TestNetworkCreate(t *testing.T) { tmpfile, err := ioutil.TempFile("", "dbtest") if err != nil { t.Fatal(err) } defer os.Remove(tmpfile.Name()) - tmpdir, err := ioutil.TempDir("", "oonitest") + sess, err := Connect(tmpfile.Name()) if err != nil { t.Fatal(err) } - defer os.RemoveAll(tmpdir) + + l1 := utils.LocationInfo{ + ASN: 2, + CountryCode: "IT", + NetworkName: "Antaninet", + } + + l2 := utils.LocationInfo{ + ASN: 3, + CountryCode: "IT", + NetworkName: "Fufnet", + } + + _, err = CreateNetwork(sess, &l1) + if err != nil { + t.Fatal(err) + } + + _, err = CreateNetwork(sess, &l2) + if err != nil { + t.Fatal(err) + } + +} + +func TestURLCreation(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + defer os.Remove(tmpfile.Name()) sess, err := Connect(tmpfile.Name()) if err != nil { diff --git a/internal/database/models.go b/internal/database/models.go index 1327f63..a593f24 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -6,7 +6,6 @@ import ( "path/filepath" "time" - "github.com/ooni/probe-cli/nettests/summary" "github.com/pkg/errors" "upper.io/db.v3/lib/sqlbuilder" ) @@ -28,7 +27,7 @@ type MeasurementURLNetwork struct { // Network represents a network tested by the user type Network struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` NetworkName string `db:"network_name"` NetworkType string `db:"network_type"` IP string `db:"ip"` @@ -38,7 +37,7 @@ type Network struct { // URL represents URLs from the testing lists type URL struct { - ID sql.NullInt64 `db:"id"` + ID sql.NullInt64 `db:"id,omitempty"` URL sql.NullString `db:"url"` CategoryCode sql.NullString `db:"category_code"` CountryCode sql.NullString `db:"country_code"` @@ -46,7 +45,7 @@ type URL struct { // Measurement model type Measurement struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` TestName string `db:"test_name"` StartTime time.Time `db:"start_time"` Runtime float64 `db:"runtime"` // Fractional number of seconds @@ -69,7 +68,7 @@ type Measurement struct { // Result model type Result struct { - ID int64 `db:"id"` + ID int64 `db:"id,omitempty"` TestGroupName string `db:"test_group_name"` StartTime time.Time `db:"start_time"` NetworkID int64 `db:"network_id"` // Used to include a Network @@ -82,7 +81,7 @@ type Result struct { } // Finished marks the result as done and sets the runtime -func (r *Result) Finished(sess sqlbuilder.Database, makeSummary summary.ResultSummaryFunc) error { +func (r *Result) Finished(sess sqlbuilder.Database) error { if r.IsDone == true || r.Runtime != 0 { return errors.New("Result is already finished") } @@ -147,17 +146,6 @@ func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error { return nil } -// WriteSummary writes the summary to the measurement -func (m *Measurement) WriteSummary(sess sqlbuilder.Database, summary string) error { - // XXX remove m.Summary = summary - - err := sess.Collection("measurements").Find("id", m.ID).Update(m) - if err != nil { - return errors.Wrap(err, "updating measurement") - } - return nil -} - // AddToResult adds a measurement to a result func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) error { var err error diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 630145b..d22398a 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -9,7 +9,6 @@ import ( "github.com/apex/log" "github.com/ooni/probe-cli/internal/util" - "github.com/ooni/probe-cli/nettests/summary" ) func formatSpeed(speed int64) string { @@ -24,55 +23,51 @@ func formatSpeed(speed int64) string { return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) } -var summarizers = map[string]func(string) []string{ - "websites": func(ss string) []string { - var summary summary.WebsitesSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } +// PerformanceTestKeys is the result summary for a performance test +type PerformanceTestKeys struct { + Upload int64 `json:"upload"` + Download int64 `json:"download"` + Ping float64 `json:"ping"` + Bitrate int64 `json:"median_bitrate"` +} + +var summarizers = map[string]func(uint64, uint64, string) []string{ + "websites": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("%d tested", summary.Tested), - fmt.Sprintf("%d blocked", summary.Blocked), + fmt.Sprintf("%d tested", totalCount), + fmt.Sprintf("%d blocked", anomalyCount), "", } }, - "performance": func(ss string) []string { - var summary summary.PerformanceSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { + "performance": func(totalCount uint64, anomalyCount uint64, ss string) []string { + var tk PerformanceTestKeys + if err := json.Unmarshal([]byte(ss), &tk); err != nil { return nil } return []string{ - fmt.Sprintf("Download: %s", formatSpeed(summary.Download)), - fmt.Sprintf("Upload: %s", formatSpeed(summary.Upload)), - fmt.Sprintf("Ping: %.2fms", summary.Ping), + fmt.Sprintf("Download: %s", formatSpeed(tk.Download)), + fmt.Sprintf("Upload: %s", formatSpeed(tk.Upload)), + fmt.Sprintf("Ping: %.2fms", tk.Ping), } }, - "im": func(ss string) []string { - var summary summary.IMSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } + "im": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("%d tested", summary.Tested), - fmt.Sprintf("%d blocked", summary.Blocked), + fmt.Sprintf("%d tested", totalCount), + fmt.Sprintf("%d blocked", anomalyCount), "", } }, - "middlebox": func(ss string) []string { - var summary summary.MiddleboxSummary - if err := json.Unmarshal([]byte(ss), &summary); err != nil { - return nil - } + "middlebox": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ - fmt.Sprintf("Detected: %v", summary.Detected), + fmt.Sprintf("Detected: %v", anomalyCount > 0), "", "", } }, } -func makeSummary(name string, ss string) []string { - return summarizers[name](ss) +func makeSummary(name string, totalCount uint64, anomalyCount uint64, ss string) []string { + return summarizers[name](totalCount, anomalyCount, ss) } func logResultItem(w io.Writer, f log.Fields) error { @@ -98,7 +93,10 @@ func logResultItem(w io.Writer, f log.Fields) error { fmt.Fprintf(w, "┃ "+firstRow+" ┃\n") fmt.Fprintf(w, "┡"+strings.Repeat("━", colWidth*2+2)+"┩\n") - summary := makeSummary(name, f.Get("summary").(string)) + summary := makeSummary(name, + f.Get("measurement_count").(uint64), + f.Get("measurement_anomaly_count").(uint64), + f.Get("test_keys").(string)) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", util.RightPad(name, colWidth), diff --git a/internal/output/output.go b/internal/output/output.go index 2758fd8..7d6bf77 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -21,38 +21,42 @@ func Progress(key string, perc float64, msg string) { // ResultItemData is the metadata about a result type ResultItemData struct { - ID int64 - Name string - StartTime time.Time - Summary string - Runtime float64 - Country string - NetworkName string - ASN string - Done bool - DataUsageDown int64 - DataUsageUp int64 - Index int - TotalCount int + ID int64 + Name string + StartTime time.Time + TestKeys string + MeasurementCount uint64 + MeasurementAnomalyCount uint64 + Runtime float64 + Country string + NetworkName string + ASN string + Done bool + DataUsageDown int64 + DataUsageUp int64 + Index int + TotalCount int } // ResultItem logs a progress type event func ResultItem(result ResultItemData) { log.WithFields(log.Fields{ - "type": "result_item", - "id": result.ID, - "name": result.Name, - "start_time": result.StartTime, - "summary": result.Summary, - "country": result.Country, - "network_name": result.NetworkName, - "asn": result.ASN, - "runtime": result.Runtime, - "done": result.Done, - "data_usage_down": result.DataUsageDown, - "data_usage_up": result.DataUsageUp, - "index": result.Index, - "total_count": result.TotalCount, + "type": "result_item", + "id": result.ID, + "name": result.Name, + "start_time": result.StartTime, + "test_keys": result.TestKeys, + "measurement_count": result.MeasurementCount, + "measurement_anomaly_count": result.MeasurementAnomalyCount, + "country": result.Country, + "network_name": result.NetworkName, + "asn": result.ASN, + "runtime": result.Runtime, + "done": result.Done, + "data_usage_down": result.DataUsageDown, + "data_usage_up": result.DataUsageUp, + "index": result.Index, + "total_count": result.TotalCount, }).Info("result item") } diff --git a/nettests/groups/groups.go b/nettests/groups/groups.go index 8b68dac..e209c55 100644 --- a/nettests/groups/groups.go +++ b/nettests/groups/groups.go @@ -1,14 +1,10 @@ package groups import ( - "encoding/json" - - "github.com/apex/log" "github.com/ooni/probe-cli/nettests" "github.com/ooni/probe-cli/nettests/im" "github.com/ooni/probe-cli/nettests/middlebox" "github.com/ooni/probe-cli/nettests/performance" - "github.com/ooni/probe-cli/nettests/summary" "github.com/ooni/probe-cli/nettests/websites" ) @@ -16,7 +12,6 @@ import ( type NettestGroup struct { Label string Nettests []nettests.Nettest - Summary summary.ResultSummaryFunc } // NettestGroups that can be run by the user @@ -26,35 +21,6 @@ var NettestGroups = map[string]NettestGroup{ Nettests: []nettests.Nettest{ websites.WebConnectivity{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"WebConnectivity"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - // XXX to generate this I need to create the summary map as a list - var summary summary.WebsitesSummary - summary.Tested = 0 - summary.Blocked = 0 - for _, msmtSummaryStr := range m["WebConnectivity"] { - var wcSummary websites.WebConnectivitySummary - - err := json.Unmarshal([]byte(msmtSummaryStr), &wcSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal WebConnectivity summary") - return "", err - } - if wcSummary.Blocked { - summary.Blocked++ - } - summary.Tested++ - } - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "performance": NettestGroup{ Label: "Performance", @@ -62,38 +28,6 @@ var NettestGroups = map[string]NettestGroup{ performance.Dash{}, performance.NDT{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"Dash", "Ndt"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - var ( - err error - ndtSummary performance.NDTSummary - dashSummary performance.DashSummary - summary summary.PerformanceSummary - ) - err = json.Unmarshal([]byte(m["Dash"][0]), &dashSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal Dash summary") - return "", err - } - err = json.Unmarshal([]byte(m["Ndt"][0]), &ndtSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal NDT summary") - return "", err - } - summary.Bitrate = dashSummary.Bitrate - summary.Download = ndtSummary.Download - summary.Upload = ndtSummary.Upload - summary.Ping = ndtSummary.AvgRTT - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "middlebox": NettestGroup{ Label: "Middleboxes", @@ -101,35 +35,6 @@ var NettestGroups = map[string]NettestGroup{ middlebox.HTTPInvalidRequestLine{}, middlebox.HTTPHeaderFieldManipulation{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"HttpInvalidRequestLine", "HttpHeaderFieldManipulation"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - - var ( - err error - hhfmSummary middlebox.HTTPHeaderFieldManipulationSummary - hirlSummary middlebox.HTTPInvalidRequestLineSummary - summary summary.MiddleboxSummary - ) - err = json.Unmarshal([]byte(m["HttpHeaderFieldManipulation"][0]), &hhfmSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal hhfm summary") - return "", err - } - err = json.Unmarshal([]byte(m["HttpInvalidRequestLine"][0]), &hirlSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal hirl summary") - return "", err - } - summary.Detected = hirlSummary.Tampering == true || hhfmSummary.Tampering == true - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, "im": NettestGroup{ Label: "Instant Messaging", @@ -138,52 +43,5 @@ var NettestGroups = map[string]NettestGroup{ im.Telegram{}, im.WhatsApp{}, }, - Summary: func(m summary.SummaryMap) (string, error) { - if err := summary.CheckRequiredKeys([]string{"Whatsapp", "Telegram", "FacebookMessenger"}, m); err != nil { - log.WithError(err).Error("missing keys") - return "", err - } - var ( - err error - waSummary im.WhatsAppSummary - tgSummary im.TelegramSummary - fbSummary im.FacebookMessengerSummary - summary summary.IMSummary - ) - err = json.Unmarshal([]byte(m["Whatsapp"][0]), &waSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal whatsapp summary") - return "", err - } - err = json.Unmarshal([]byte(m["Telegram"][0]), &tgSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal telegram summary") - return "", err - } - err = json.Unmarshal([]byte(m["FacebookMessenger"][0]), &fbSummary) - if err != nil { - log.WithError(err).Error("failed to unmarshal facebook summary") - return "", err - } - // XXX it could actually be that some are not tested when the - // configuration is changed. - summary.Tested = 3 - summary.Blocked = 0 - if fbSummary.Blocked == true { - summary.Blocked++ - } - if tgSummary.Blocked == true { - summary.Blocked++ - } - if waSummary.Blocked == true { - summary.Blocked++ - } - - summaryBytes, err := json.Marshal(summary) - if err != nil { - return "", err - } - return string(summaryBytes), nil - }, }, } diff --git a/nettests/im/facebook_messenger.go b/nettests/im/facebook_messenger.go index d6d97b2..4f7d716 100644 --- a/nettests/im/facebook_messenger.go +++ b/nettests/im/facebook_messenger.go @@ -16,15 +16,15 @@ func (h FacebookMessenger) Run(ctl *nettests.Controller) error { return mknt.Run() } -// FacebookMessengerSummary for the test -type FacebookMessengerSummary struct { - DNSBlocking bool - TCPBlocking bool - Blocked bool +// FacebookMessengerTestKeys for the test +type FacebookMessengerTestKeys struct { + DNSBlocking bool `json:"facebook_dns_blocking"` + TCPBlocking bool `json:"facebook_tcp_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h FacebookMessenger) GetTestKeys(tk map[string]interface{}) interface{} { var ( dnsBlocking bool tcpBlocking bool @@ -41,10 +41,10 @@ func (h FacebookMessenger) Summary(tk map[string]interface{}) interface{} { tcpBlocking = tk["facebook_tcp_blocking"].(bool) } - return FacebookMessengerSummary{ + return FacebookMessengerTestKeys{ DNSBlocking: dnsBlocking, TCPBlocking: tcpBlocking, - Blocked: dnsBlocking || tcpBlocking, + IsAnomaly: dnsBlocking || tcpBlocking, } } diff --git a/nettests/im/telegram.go b/nettests/im/telegram.go index 6c2984b..0ef4943 100644 --- a/nettests/im/telegram.go +++ b/nettests/im/telegram.go @@ -16,16 +16,16 @@ func (h Telegram) Run(ctl *nettests.Controller) error { return mknt.Run() } -// TelegramSummary for the test -type TelegramSummary struct { - HTTPBlocking bool - TCPBlocking bool - WebBlocking bool - Blocked bool +// TelegramTestKeys for the test +type TelegramTestKeys struct { + HTTPBlocking bool `json:"telegram_http_blocking"` + TCPBlocking bool `json:"telegram_tcp_blocking"` + WebBlocking bool `json:"telegram_web_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h Telegram) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h Telegram) GetTestKeys(tk map[string]interface{}) interface{} { var ( tcpBlocking bool httpBlocking bool @@ -48,11 +48,11 @@ func (h Telegram) Summary(tk map[string]interface{}) interface{} { webBlocking = tk["telegram_web_status"].(string) == "blocked" } - return TelegramSummary{ + return TelegramTestKeys{ TCPBlocking: tcpBlocking, HTTPBlocking: httpBlocking, WebBlocking: webBlocking, - Blocked: webBlocking || httpBlocking || tcpBlocking, + IsAnomaly: webBlocking || httpBlocking || tcpBlocking, } } diff --git a/nettests/im/whatsapp.go b/nettests/im/whatsapp.go index 36a18c3..19b2199 100644 --- a/nettests/im/whatsapp.go +++ b/nettests/im/whatsapp.go @@ -16,16 +16,16 @@ func (h WhatsApp) Run(ctl *nettests.Controller) error { return mknt.Run() } -// WhatsAppSummary for the test -type WhatsAppSummary struct { - RegistrationServerBlocking bool - WebBlocking bool - EndpointsBlocking bool - Blocked bool +// WhatsAppTestKeys for the test +type WhatsAppTestKeys struct { + RegistrationServerBlocking bool `json:"registration_server_blocking"` + WebBlocking bool `json:"whatsapp_web_blocking"` + EndpointsBlocking bool `json:"whatsapp_endpoints_blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h WhatsApp) GetTestKeys(tk map[string]interface{}) interface{} { var ( webBlocking bool registrationBlocking bool @@ -46,11 +46,11 @@ func (h WhatsApp) Summary(tk map[string]interface{}) interface{} { webBlocking = computeBlocking("whatsapp_web_status") endpointsBlocking = computeBlocking("whatsapp_endpoints_status") - return WhatsAppSummary{ + return WhatsAppTestKeys{ RegistrationServerBlocking: registrationBlocking, WebBlocking: webBlocking, EndpointsBlocking: endpointsBlocking, - Blocked: registrationBlocking || webBlocking || endpointsBlocking, + IsAnomaly: registrationBlocking || webBlocking || endpointsBlocking, } } diff --git a/nettests/middlebox/http_header_field_manipulation.go b/nettests/middlebox/http_header_field_manipulation.go index 822909d..f5df162 100644 --- a/nettests/middlebox/http_header_field_manipulation.go +++ b/nettests/middlebox/http_header_field_manipulation.go @@ -16,13 +16,13 @@ func (h HTTPHeaderFieldManipulation) Run(ctl *nettests.Controller) error { return mknt.Run() } -// HTTPHeaderFieldManipulationSummary for the test -type HTTPHeaderFieldManipulationSummary struct { - Tampering bool +// HTTPHeaderFieldManipulationTestKeys for the test +type HTTPHeaderFieldManipulationTestKeys struct { + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys returns a projection of the tests keys needed for the views +func (h HTTPHeaderFieldManipulation) GetTestKeys(tk map[string]interface{}) interface{} { tampering := false for _, v := range tk["tampering"].(map[string]interface{}) { t, ok := v.(bool) @@ -32,8 +32,8 @@ func (h HTTPHeaderFieldManipulation) Summary(tk map[string]interface{}) interfac } } - return HTTPHeaderFieldManipulationSummary{ - Tampering: tampering, + return HTTPHeaderFieldManipulationTestKeys{ + IsAnomaly: tampering, } } diff --git a/nettests/middlebox/http_invalid_request_line.go b/nettests/middlebox/http_invalid_request_line.go index 947e64e..b815c94 100644 --- a/nettests/middlebox/http_invalid_request_line.go +++ b/nettests/middlebox/http_invalid_request_line.go @@ -16,17 +16,17 @@ func (h HTTPInvalidRequestLine) Run(ctl *nettests.Controller) error { return mknt.Run() } -// HTTPInvalidRequestLineSummary for the test -type HTTPInvalidRequestLineSummary struct { - Tampering bool +// HTTPInvalidRequestLineTestKeys for the test +type HTTPInvalidRequestLineTestKeys struct { + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (h HTTPInvalidRequestLine) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (h HTTPInvalidRequestLine) GetTestKeys(tk map[string]interface{}) interface{} { tampering := tk["tampering"].(bool) - return HTTPInvalidRequestLineSummary{ - Tampering: tampering, + return HTTPInvalidRequestLineTestKeys{ + IsAnomaly: tampering, } } diff --git a/nettests/nettests.go b/nettests/nettests.go index 398b60f..5f4e242 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -20,7 +20,7 @@ import ( // Nettest interface. Every Nettest should implement this. type Nettest interface { Run(*Controller) error - Summary(map[string]interface{}) interface{} + GetTestKeys(map[string]interface{}) interface{} LogSummary(string) error } @@ -119,7 +119,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { IncludeIP: c.Ctx.Config.Sharing.IncludeIP, IncludeASN: c.Ctx.Config.Sharing.IncludeASN, IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, - LogLevel: "INFO", + LogLevel: "DEBUG", ProbeCC: c.Ctx.Location.CountryCode, ProbeASN: fmt.Sprintf("AS%d", c.Ctx.Location.ASN), @@ -139,18 +139,16 @@ func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("GeoIPCountryPath: %s", nt.Options.GeoIPCountryPath) nt.On("log", func(e mk.Event) { - log.Debugf(color.RedString(e.Key)) - level := e.Value.LogLevel msg := e.Value.Message switch level { case "ERROR": - log.Error(msg) + log.Errorf("%v: %s", color.RedString("mklog"), msg) case "INFO": - log.Info(msg) + log.Infof("%v: %s", color.BlueString("mklog"), msg) default: - log.Debug(msg) + log.Debugf("%v: %s", color.WhiteString("mklog"), msg) } }) @@ -285,6 +283,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { } }) + log.Debugf("Registered all the handlers") return nil } @@ -307,13 +306,13 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) { var entry Entry json.Unmarshal([]byte(jsonStr), &entry) - summary := c.nt.Summary(entry.TestKeys) - summaryBytes, err := json.Marshal(summary) - if err != nil { - log.WithError(err).Error("failed to serialize summary") - } + tk := c.nt.GetTestKeys(entry.TestKeys) + log.Debugf("Fetching: %s %v", idx, c.msmts[idx]) - c.msmts[idx].WriteSummary(c.Ctx.DB, string(summaryBytes)) + err := database.AddTestKeys(c.Ctx.DB, c.msmts[idx], tk) + if err != nil { + log.WithError(err).Error("failed to add test keys to summary") + } } // MKStart is the interface for the mk.Nettest Start() function diff --git a/nettests/performance/dash.go b/nettests/performance/dash.go index cced30c..3d63809 100644 --- a/nettests/performance/dash.go +++ b/nettests/performance/dash.go @@ -16,22 +16,24 @@ func (d Dash) Run(ctl *nettests.Controller) error { return dash.Run() } -// DashSummary for the test +// DashTestKeys for the test // TODO: process 'receiver_data' to provide an array of performance for a chart. -type DashSummary struct { - Latency float64 - Bitrate int64 - Delay float64 +type DashTestKeys struct { + Latency float64 `json:"connect_latency"` + Bitrate int64 `json:"median_bitrate"` + Delay float64 `json:"min_playout_delay"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (d Dash) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (d Dash) GetTestKeys(tk map[string]interface{}) interface{} { simple := tk["simple"].(map[string]interface{}) - return DashSummary{ - Latency: simple["connect_latency"].(float64), - Bitrate: int64(simple["median_bitrate"].(float64)), - Delay: simple["min_playout_delay"].(float64), + return DashTestKeys{ + IsAnomaly: false, + Latency: simple["connect_latency"].(float64), + Bitrate: int64(simple["median_bitrate"].(float64)), + Delay: simple["min_playout_delay"].(float64), } } diff --git a/nettests/performance/ndt.go b/nettests/performance/ndt.go index c9e8098..590b779 100644 --- a/nettests/performance/ndt.go +++ b/nettests/performance/ndt.go @@ -16,26 +16,27 @@ func (n NDT) Run(ctl *nettests.Controller) error { return nt.Run() } -// NDTSummary for the test -type NDTSummary struct { - Upload int64 - Download int64 - Ping int64 - MaxRTT float64 - AvgRTT float64 - MinRTT float64 - MSS int64 - OutOfOrder int64 - PacketLoss float64 - Timeouts int64 +// NDTTestKeys for the test +type NDTTestKeys struct { + Upload int64 `json:"upload"` + Download int64 `json:"download"` + Ping int64 `json:"ping"` + MaxRTT float64 `json:"max_rtt"` + AvgRTT float64 `json:"avg_rtt"` + MinRTT float64 `json:"min_rtt"` + MSS int64 `json:"mss"` + OutOfOrder int64 `json:"out_of_order"` + PacketLoss float64 `json:"packet_loss"` + Timeouts int64 `json:"timeouts"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (n NDT) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (n NDT) GetTestKeys(tk map[string]interface{}) interface{} { simple := tk["simple"].(map[string]interface{}) advanced := tk["advanced"].(map[string]interface{}) - return NDTSummary{ + return NDTTestKeys{ Upload: int64(simple["upload"].(float64)), Download: int64(simple["download"].(float64)), Ping: int64(simple["ping"].(float64)), diff --git a/nettests/summary/summary.go b/nettests/summary/summary.go deleted file mode 100644 index f225927..0000000 --- a/nettests/summary/summary.go +++ /dev/null @@ -1,44 +0,0 @@ -package summary - -import "fmt" - -// ResultSummaryFunc is the function used to generate result summaries -type ResultSummaryFunc func(SummaryMap) (string, error) - -// SummaryMap contains a mapping from test name to serialized summary for it -type SummaryMap map[string][]string - -// PerformanceSummary is the result summary for a performance test -type PerformanceSummary struct { - Upload int64 - Download int64 - Ping float64 - Bitrate int64 -} - -// MiddleboxSummary is the summary for the middlebox tests -type MiddleboxSummary struct { - Detected bool -} - -// IMSummary is the summary for the im tests -type IMSummary struct { - Tested uint - Blocked uint -} - -// WebsitesSummary is the summary for the websites test -type WebsitesSummary struct { - Tested uint - Blocked uint -} - -func CheckRequiredKeys(rk []string, m SummaryMap) error { - for _, key := range rk { - if _, ok := m[key]; ok { - continue - } - return fmt.Errorf("missing SummaryMap key '%s'", key) - } - return nil -} diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 32a7bf0..6427bd4 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -29,10 +29,11 @@ const orchestrateBaseURL = "https://events.proteus.test.ooni.io" func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { var ( - parsed = new(URLResponse) - urls []string - urlIDMap map[int64]int64 + parsed = new(URLResponse) + urls []string ) + urlIDMap := make(map[int64]int64) + log.Debug("Looking up URLs") // XXX pass in the configuration for category codes reqURL := fmt.Sprintf("%s/api/v1/urls?probe_cc=%s", orchestrateBaseURL, @@ -53,6 +54,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { } for idx, url := range parsed.Results { + log.Debugf("Going over URL %d", idx) urlID, err := database.CreateOrUpdateURL(ctl.Ctx.DB, url.URL, url.CategoryCode, url.CountryCode) if err != nil { log.Error("failed to add to the URL table") @@ -82,15 +84,15 @@ func (n WebConnectivity) Run(ctl *nettests.Controller) error { return nt.Run() } -// WebConnectivitySummary for the test -type WebConnectivitySummary struct { - Accessible bool - Blocking string - Blocked bool +// WebConnectivityTestKeys for the test +type WebConnectivityTestKeys struct { + Accessible bool `json:"accessible"` + Blocking string `json:"blocking"` + IsAnomaly bool `json:"-"` } -// Summary generates a summary for a test run -func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { +// GetTestKeys generates a summary for a test run +func (n WebConnectivity) GetTestKeys(tk map[string]interface{}) interface{} { var ( blocked bool blocking string @@ -117,10 +119,10 @@ func (n WebConnectivity) Summary(tk map[string]interface{}) interface{} { accessible = tk["accessible"].(bool) } - return WebConnectivitySummary{ + return WebConnectivityTestKeys{ Accessible: accessible, Blocking: blocking, - Blocked: blocked, + IsAnomaly: blocked, } } From 73fafacc2c36d5fd782017aa21bc7a3bffff969d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 12:53:43 +0200 Subject: [PATCH 23/65] Zap unused method --- internal/database/models.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/database/models.go b/internal/database/models.go index a593f24..fd23e5d 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -95,11 +95,6 @@ func (r *Result) Finished(sess sqlbuilder.Database) error { return nil } -// SetGeoIPInfo for the Measurement -func (m *Measurement) SetGeoIPInfo() error { - return nil -} - // Failed writes the error string to the measurement func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error { m.FailureMsg = sql.NullString{String: failure, Valid: true} From 898332064b375f4d44a77c8dc3e62a698c4dae2b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 15:03:52 +0200 Subject: [PATCH 24/65] Add result testkeys projection --- internal/cli/list/list.go | 7 +++++-- internal/database/actions.go | 18 ++++++++++++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 66be3e4..c2f1e74 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -41,7 +41,6 @@ func init() { output.SectionTitle("Incomplete results") } for idx, result := range incompleteResults { - output.ResultItem(output.ResultItemData{ ID: result.Result.ID, Index: idx, @@ -68,6 +67,10 @@ func init() { if err != nil { log.WithError(err).Error("failed to list measurement counts") } + testKeys, err := database.GetResultTestKeys(ctx.DB, result.Result.ID) + if err != nil { + log.WithError(err).Error("failed to get testKeys") + } output.ResultItem(output.ResultItemData{ ID: result.Result.ID, Index: idx, @@ -77,7 +80,7 @@ func init() { NetworkName: result.Network.NetworkName, Country: result.Network.CountryCode, ASN: fmt.Sprintf("AS%d", result.Network.ASN), - TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now + TestKeys: testKeys, MeasurementCount: totalCount, MeasurementAnomalyCount: anmlyCount, Done: result.IsDone, diff --git a/internal/database/actions.go b/internal/database/actions.go index f17229d..a002663 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -37,6 +37,24 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR return measurements, nil } +// GetResultTestKeys returns a list of TestKeys for a given measurements +func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) { + res := sess.Collection("measurements").Find("result_id", resultID) + defer res.Close() + + var msmt Measurement + for res.Next(&msmt) { + if msmt.TestName == "web_connectivity" { + break + } + // We only really care about the NDT TestKeys + if msmt.TestName == "ndt" { + return msmt.TestKeys, nil + } + } + return "{}", nil +} + // GetMeasurementCounts returns the number of anomalous and total measurement for a given result func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uint64, error) { var ( From 5055eaa88ad62feadab09a823dd46e4042228eda Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 15:15:29 +0200 Subject: [PATCH 25/65] Add info and reset commands --- cmd/ooni/main.go | 1 + internal/cli/info/info.go | 14 ++++++++++++-- internal/cli/reset/reset.go | 30 ++++++++++++++++++++++++++++++ 3 files changed, 43 insertions(+), 2 deletions(-) create mode 100644 internal/cli/reset/reset.go diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index 2b56450..748fb31 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -9,6 +9,7 @@ import ( _ "github.com/ooni/probe-cli/internal/cli/list" _ "github.com/ooni/probe-cli/internal/cli/nettest" _ "github.com/ooni/probe-cli/internal/cli/onboard" + _ "github.com/ooni/probe-cli/internal/cli/reset" _ "github.com/ooni/probe-cli/internal/cli/run" _ "github.com/ooni/probe-cli/internal/cli/show" _ "github.com/ooni/probe-cli/internal/cli/upload" diff --git a/internal/cli/info/info.go b/internal/cli/info/info.go index 5743dff..13f4834 100644 --- a/internal/cli/info/info.go +++ b/internal/cli/info/info.go @@ -10,8 +10,18 @@ func init() { cmd := root.Command("info", "Display information about OONI Probe") cmd.Action(func(_ *kingpin.ParseContext) error { - log.Info("Info") - log.Error("this function is not implemented") + ctx, err := root.Init() + if err != nil { + log.Errorf("%s", err) + return err + } + log.WithFields(log.Fields{ + "path": ctx.Home, + }).Info("Home") + log.WithFields(log.Fields{ + "path": ctx.TempDir, + }).Info("TempDir") + return nil }) } diff --git a/internal/cli/reset/reset.go b/internal/cli/reset/reset.go new file mode 100644 index 0000000..9a07ed8 --- /dev/null +++ b/internal/cli/reset/reset.go @@ -0,0 +1,30 @@ +package clean + +import ( + "os" + + "github.com/alecthomas/kingpin" + "github.com/apex/log" + "github.com/ooni/probe-cli/internal/cli/root" +) + +func init() { + cmd := root.Command("reset", "Cleanup an old or experimental installation") + force := cmd.Flag("force", "Force deleting the OONI Home").Bool() + + cmd.Action(func(_ *kingpin.ParseContext) error { + ctx, err := root.Init() + if err != nil { + log.Errorf("%s", err) + return err + } + if *force == true { + os.RemoveAll(ctx.Home) + log.Infof("Deleted %s", ctx.Home) + } else { + log.Infof("Run with --force to delete %s", ctx.Home) + } + + return nil + }) +} From 21e3a08b0e86beb9245b28e904758c118dc1e9cd Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 16:28:54 +0200 Subject: [PATCH 26/65] vendor strcase from: https://github.com/iancoleman/strcase/commit/3605ed457bf7f8caa1371b4fafadadc026673479 --- utils/strcase/LICENSE | 22 ++++++ utils/strcase/README.md | 23 ++++++ utils/strcase/camel.go | 75 ++++++++++++++++++ utils/strcase/camel_test.go | 68 +++++++++++++++++ utils/strcase/numbers.go | 38 ++++++++++ utils/strcase/snake.go | 94 +++++++++++++++++++++++ utils/strcase/snake_test.go | 147 ++++++++++++++++++++++++++++++++++++ 7 files changed, 467 insertions(+) create mode 100644 utils/strcase/LICENSE create mode 100644 utils/strcase/README.md create mode 100644 utils/strcase/camel.go create mode 100644 utils/strcase/camel_test.go create mode 100644 utils/strcase/numbers.go create mode 100644 utils/strcase/snake.go create mode 100644 utils/strcase/snake_test.go diff --git a/utils/strcase/LICENSE b/utils/strcase/LICENSE new file mode 100644 index 0000000..3e87ff7 --- /dev/null +++ b/utils/strcase/LICENSE @@ -0,0 +1,22 @@ +The MIT License (MIT) + +Copyright (c) 2015 Ian Coleman +Copyright (c) 2018 Ma_124, + +Permission is hereby granted, free of charge, to any person obtaining a copy +of this software and associated documentation files (the "Software"), to deal +in the Software without restriction, including without limitation the rights +to use, copy, modify, merge, publish, distribute, sublicense, and/or sell +copies of the Software, and to permit persons to whom the Software is +furnished to do so, Subject to the following conditions: + +The above copyright notice and this permission notice shall be included in all +copies or Substantial portions of the Software. + +THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR +IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, +FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE +AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER +LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, +OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE +SOFTWARE. diff --git a/utils/strcase/README.md b/utils/strcase/README.md new file mode 100644 index 0000000..978b46d --- /dev/null +++ b/utils/strcase/README.md @@ -0,0 +1,23 @@ +# strcase +[![Godoc Reference](https://godoc.org/github.com/iancoleman/strcase?status.svg)](http://godoc.org/github.com/iancoleman/strcase) +[![Build Status](https://travis-ci.org/iancoleman/strcase.svg)](https://travis-ci.org/iancoleman/strcase) +[![Coverage](http://gocover.io/_badge/github.com/iancoleman/strcase?0)](http://gocover.io/github.com/iancoleman/strcase) + +strcase is a go package for converting string case to [snake case](https://en.wikipedia.org/wiki/Snake_case) or [camel case](https://en.wikipedia.org/wiki/CamelCase). + +## Example + +```go +s := "AnyKind of_string" +``` + +| Function | Result | +|-----------------------------------|----------------------| +| `ToSnake(s)` | `any_kind_of_string` | +| `ToScreamingSnake(s)` | `ANY_KIND_OF_STRING` | +| `ToKebab(s)` | `any-kind-of-string` | +| `ToScreamingKebab(s)` | `ANY-KIND-OF-STRING` | +| `ToDelimited(s, '.')` | `any.kind.of.string` | +| `ToScreamingDelimited(s, '.')` | `ANY.KIND.OF.STRING` | +| `ToCamel(s)` | `AnyKindOfString` | +| `ToLowerCamel(s)` | `anyKindOfString` | diff --git a/utils/strcase/camel.go b/utils/strcase/camel.go new file mode 100644 index 0000000..7c2e2b7 --- /dev/null +++ b/utils/strcase/camel.go @@ -0,0 +1,75 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcase + +import ( + "strings" +) + +// Converts a string to CamelCase +func toCamelInitCase(s string, initCase bool) string { + s = addWordBoundariesToNumbers(s) + s = strings.Trim(s, " ") + n := "" + capNext := initCase + for _, v := range s { + if v >= 'A' && v <= 'Z' { + n += string(v) + } + if v >= '0' && v <= '9' { + n += string(v) + } + if v >= 'a' && v <= 'z' { + if capNext { + n += strings.ToUpper(string(v)) + } else { + n += string(v) + } + } + if v == '_' || v == ' ' || v == '-' { + capNext = true + } else { + capNext = false + } + } + return n +} + +// Converts a string to CamelCase +func ToCamel(s string) string { + return toCamelInitCase(s, true) +} + +// Converts a string to lowerCamelCase +func ToLowerCamel(s string) string { + if s == "" { + return s + } + if r := rune(s[0]); r >= 'A' && r <= 'Z' { + s = strings.ToLower(string(r)) + s[1:] + } + return toCamelInitCase(s, false) +} diff --git a/utils/strcase/camel_test.go b/utils/strcase/camel_test.go new file mode 100644 index 0000000..b62b9c4 --- /dev/null +++ b/utils/strcase/camel_test.go @@ -0,0 +1,68 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcase + +import ( + "testing" +) + +func TestToCamel(t *testing.T) { + cases := [][]string{ + []string{"test_case", "TestCase"}, + []string{"test", "Test"}, + []string{"TestCase", "TestCase"}, + []string{" test case ", "TestCase"}, + []string{"", ""}, + []string{"many_many_words", "ManyManyWords"}, + []string{"AnyKind of_string", "AnyKindOfString"}, + []string{"odd-fix", "OddFix"}, + []string{"numbers2And55with000", "Numbers2And55With000"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToCamel(in) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} + +func TestToLowerCamel(t *testing.T) { + cases := [][]string{ + []string{"foo-bar", "fooBar"}, + []string{"TestCase", "testCase"}, + []string{"", ""}, + []string{"AnyKind of_string", "anyKindOfString"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToLowerCamel(in) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} diff --git a/utils/strcase/numbers.go b/utils/strcase/numbers.go new file mode 100644 index 0000000..fdf07cb --- /dev/null +++ b/utils/strcase/numbers.go @@ -0,0 +1,38 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcase + +import ( + "regexp" +) + +var numberSequence = regexp.MustCompile(`([a-zA-Z])(\d+)([a-zA-Z]?)`) +var numberReplacement = []byte(`$1 $2 $3`) + +func addWordBoundariesToNumbers(s string) string { + b := []byte(s) + b = numberSequence.ReplaceAll(b, numberReplacement) + return string(b) +} diff --git a/utils/strcase/snake.go b/utils/strcase/snake.go new file mode 100644 index 0000000..1d2f520 --- /dev/null +++ b/utils/strcase/snake.go @@ -0,0 +1,94 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +// Package strcase converts strings to snake_case or CamelCase +package strcase + +import ( + "strings" +) + +// Converts a string to snake_case +func ToSnake(s string) string { + return ToDelimited(s, '_') +} + +// Converts a string to SCREAMING_SNAKE_CASE +func ToScreamingSnake(s string) string { + return ToScreamingDelimited(s, '_', true) +} + +// Converts a string to kebab-case +func ToKebab(s string) string { + return ToDelimited(s, '-') +} + +// Converts a string to SCREAMING-KEBAB-CASE +func ToScreamingKebab(s string) string { + return ToScreamingDelimited(s, '-', true) +} + +// Converts a string to delimited.snake.case (in this case `del = '.'`) +func ToDelimited(s string, del uint8) string { + return ToScreamingDelimited(s, del, false) +} + +// Converts a string to SCREAMING.DELIMITED.SNAKE.CASE (in this case `del = '.'; screaming = true`) or delimited.snake.case (in this case `del = '.'; screaming = false`) +func ToScreamingDelimited(s string, del uint8, screaming bool) string { + s = addWordBoundariesToNumbers(s) + s = strings.Trim(s, " ") + n := "" + for i, v := range s { + // treat acronyms as words, eg for JSONData -> JSON is a whole word + nextCaseIsChanged := false + if i+1 < len(s) { + next := s[i+1] + if (v >= 'A' && v <= 'Z' && next >= 'a' && next <= 'z') || (v >= 'a' && v <= 'z' && next >= 'A' && next <= 'Z') { + nextCaseIsChanged = true + } + } + + if i > 0 && n[len(n)-1] != del && nextCaseIsChanged { + // add underscore if next letter case type is changed + if v >= 'A' && v <= 'Z' { + n += string(del) + string(v) + } else if v >= 'a' && v <= 'z' { + n += string(v) + string(del) + } + } else if v == ' ' || v == '_' || v == '-' { + // replace spaces/underscores with delimiters + n += string(del) + } else { + n = n + string(v) + } + } + + if screaming { + n = strings.ToUpper(n) + } else { + n = strings.ToLower(n) + } + return n +} diff --git a/utils/strcase/snake_test.go b/utils/strcase/snake_test.go new file mode 100644 index 0000000..9c0f9d2 --- /dev/null +++ b/utils/strcase/snake_test.go @@ -0,0 +1,147 @@ +/* + * The MIT License (MIT) + * + * Copyright (c) 2015 Ian Coleman + * Copyright (c) 2018 Ma_124, + * + * Permission is hereby granted, free of charge, to any person obtaining a copy + * of this software and associated documentation files (the "Software"), to deal + * in the Software without restriction, including without limitation the rights + * to use, copy, modify, merge, publish, distribute, sublicense, and/or sell + * copies of the Software, and to permit persons to whom the Software is + * furnished to do so, Subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in all + * copies or Substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE + * AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, + * OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE + * SOFTWARE. + */ + +package strcase + +import ( + "testing" +) + +func TestToSnake(t *testing.T) { + cases := [][]string{ + []string{"testCase", "test_case"}, + []string{"TestCase", "test_case"}, + []string{"Test Case", "test_case"}, + []string{" Test Case", "test_case"}, + []string{"Test Case ", "test_case"}, + []string{" Test Case ", "test_case"}, + []string{"test", "test"}, + []string{"test_case", "test_case"}, + []string{"Test", "test"}, + []string{"", ""}, + []string{"ManyManyWords", "many_many_words"}, + []string{"manyManyWords", "many_many_words"}, + []string{"AnyKind of_string", "any_kind_of_string"}, + []string{"numbers2and55with000", "numbers_2_and_55_with_000"}, + []string{"JSONData", "json_data"}, + []string{"userID", "user_id"}, + []string{"AAAbbb", "aa_abbb"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToSnake(in) + if result != out { + t.Error("'" + in + "'('" + result + "' != '" + out + "')") + } + } +} + +func TestToDelimited(t *testing.T) { + cases := [][]string{ + []string{"testCase", "test@case"}, + []string{"TestCase", "test@case"}, + []string{"Test Case", "test@case"}, + []string{" Test Case", "test@case"}, + []string{"Test Case ", "test@case"}, + []string{" Test Case ", "test@case"}, + []string{"test", "test"}, + []string{"test_case", "test@case"}, + []string{"Test", "test"}, + []string{"", ""}, + []string{"ManyManyWords", "many@many@words"}, + []string{"manyManyWords", "many@many@words"}, + []string{"AnyKind of_string", "any@kind@of@string"}, + []string{"numbers2and55with000", "numbers@2@and@55@with@000"}, + []string{"JSONData", "json@data"}, + []string{"userID", "user@id"}, + []string{"AAAbbb", "aa@abbb"}, + []string{"test-case", "test@case"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToDelimited(in, '@') + if result != out { + t.Error("'" + in + "' ('" + result + "' != '" + out + "')") + } + } +} + +func TestToScreamingSnake(t *testing.T) { + cases := [][]string{ + []string{"testCase", "TEST_CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingSnake(in) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} + +func TestToKebab(t *testing.T) { + cases := [][]string{ + []string{"testCase", "test-case"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToKebab(in) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} + +func TestToScreamingKebab(t *testing.T) { + cases := [][]string{ + []string{"testCase", "TEST-CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingKebab(in) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} + +func TestToScreamingDelimited(t *testing.T) { + cases := [][]string{ + []string{"testCase", "TEST.CASE"}, + } + for _, i := range cases { + in := i[0] + out := i[1] + result := ToScreamingDelimited(in, '.', true) + if result != out { + t.Error("'" + result + "' != '" + out + "'") + } + } +} From e0c0acffebdd89ee54a93be7181caab895a289b6 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 16:29:14 +0200 Subject: [PATCH 27/65] Make error reporting more robust to panic --- cmd/ooni/main.go | 11 ++--------- internal/cli/app/app.go | 8 ++++++-- internal/database/actions.go | 18 ++++++++++++++++-- nettests/nettests.go | 8 ++++++-- 4 files changed, 30 insertions(+), 15 deletions(-) diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index 748fb31..d6567bc 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -2,7 +2,6 @@ package main import ( // commands - "github.com/apex/log" _ "github.com/ooni/probe-cli/internal/cli/geoip" _ "github.com/ooni/probe-cli/internal/cli/info" @@ -14,17 +13,11 @@ import ( _ "github.com/ooni/probe-cli/internal/cli/show" _ "github.com/ooni/probe-cli/internal/cli/upload" _ "github.com/ooni/probe-cli/internal/cli/version" + "github.com/ooni/probe-cli/internal/crashreport" "github.com/ooni/probe-cli/internal/cli/app" - "github.com/ooni/probe-cli/internal/crashreport" ) func main() { - crashreport.CapturePanicAndWait(func() { - err := app.Run() - if err == nil { - return - } - log.WithError(err).Fatal("main exit") - }, nil) + crashreport.CapturePanicAndWait(app.Run, nil) } diff --git a/internal/cli/app/app.go b/internal/cli/app/app.go index 539874c..c1b4cfb 100644 --- a/internal/cli/app/app.go +++ b/internal/cli/app/app.go @@ -3,13 +3,17 @@ package app import ( "os" + "github.com/apex/log" ooni "github.com/ooni/probe-cli" "github.com/ooni/probe-cli/internal/cli/root" ) // Run the app. This is the main app entry point -func Run() error { +func Run() { root.Cmd.Version(ooni.Version) _, err := root.Cmd.Parse(os.Args[1:]) - return err + if err != nil { + log.WithError(err).Error("failed to parse arguments") + } + return } diff --git a/internal/database/actions.go b/internal/database/actions.go index a002663..0a997ac 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "encoding/json" + "reflect" "time" "github.com/apex/log" @@ -221,16 +222,29 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string // AddTestKeys writes the summary to the measurement func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) error { + var ( + isAnomaly bool + isAnomalyValid bool + ) tkBytes, err := json.Marshal(tk) if err != nil { log.WithError(err).Error("failed to serialize summary") } - isAnomaly := tk.(struct{ IsAnomaly bool }).IsAnomaly + + // This is necessary so that we can extract from the the opaque testKeys just + // the IsAnomaly field of bool type. + // Maybe generics are not so bad after-all, heh golang? + isAnomalyValue := reflect.ValueOf(tk).FieldByName("IsAnomaly") + if isAnomalyValue.IsValid() == true && isAnomalyValue.Kind() == reflect.Bool { + isAnomaly = isAnomalyValue.Bool() + isAnomalyValid = true + } msmt.TestKeys = string(tkBytes) - msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: true} + msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid} err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt) if err != nil { + log.WithError(err).Error("failed to update measurement") return errors.Wrap(err, "updating measurement") } return nil diff --git a/nettests/nettests.go b/nettests/nettests.go index 5f4e242..ce9b720 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -12,9 +12,11 @@ import ( "github.com/fatih/color" "github.com/measurement-kit/go-measurement-kit" ooni "github.com/ooni/probe-cli" + "github.com/ooni/probe-cli/internal/crashreport" "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/output" "github.com/ooni/probe-cli/utils" + "github.com/ooni/probe-cli/utils/strcase" ) // Nettest interface. Every Nettest should implement this. @@ -70,7 +72,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { // These values are shared by every measurement reportID := sql.NullString{String: "", Valid: false} - testName := nt.Name + testName := strcase.ToSnake(nt.Name) resultID := c.res.ID reportFilePath := c.msmtPath @@ -270,7 +272,9 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("measurement", func(e mk.Event) { log.Debugf("status.end") - c.OnEntry(e.Value.Idx, e.Value.JSONStr) + crashreport.CapturePanicAndWait(func() { + c.OnEntry(e.Value.Idx, e.Value.JSONStr) + }, nil) }) nt.On("status.end", func(e mk.Event) { From e5c508a94c8540e4abaa14a3684f844d860b0470 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 16:56:32 +0200 Subject: [PATCH 28/65] Fix bug in result listing --- internal/cli/list/list.go | 8 ++++---- internal/database/actions.go | 4 +++- internal/database/models.go | 6 ++++-- internal/log/handlers/cli/cli.go | 10 ++++++---- 4 files changed, 17 insertions(+), 11 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index c2f1e74..13ef534 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -42,7 +42,7 @@ func init() { } for idx, result := range incompleteResults { output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, + ID: result.ResultID, Index: idx, TotalCount: len(incompleteResults), Name: result.TestGroupName, @@ -63,16 +63,16 @@ func init() { netCount := make(map[uint]int) output.SectionTitle("Results") for idx, result := range doneResults { - totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID) + totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.ResultID) if err != nil { log.WithError(err).Error("failed to list measurement counts") } - testKeys, err := database.GetResultTestKeys(ctx.DB, result.Result.ID) + testKeys, err := database.GetResultTestKeys(ctx.DB, result.ResultID) if err != nil { log.WithError(err).Error("failed to get testKeys") } output.ResultItem(output.ResultItemData{ - ID: result.Result.ID, + ID: result.ResultID, Index: idx, TotalCount: len(doneResults), Name: result.TestGroupName, diff --git a/internal/database/actions.go b/internal/database/actions.go index 0a997ac..6cfd1f9 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -80,6 +80,7 @@ func GetMeasurementCounts(sess sqlbuilder.Database, resultID int64) (uint64, uin return totalCount, anmlyCount, err } + log.Debugf("counts: %d, %d, %d", resultID, totalCount, anmlyCount) return totalCount, anmlyCount, err } @@ -90,8 +91,9 @@ func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, er req := sess.Select( "networks.id AS network_id", - db.Raw("results.*"), + "results.id AS result_id", db.Raw("networks.*"), + db.Raw("results.*"), ).From("results"). Join("networks").On("results.network_id = networks.id"). OrderBy("results.start_time") diff --git a/internal/database/models.go b/internal/database/models.go index fd23e5d..83b2226 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -13,8 +13,10 @@ import ( // ResultNetwork is used to represent the structure made from the JOIN // between the results and networks tables. type ResultNetwork struct { - Result `db:",inline"` - Network `db:",inline"` + Result `db:",inline"` + ResultID int64 `db:"result_id"` + Network `db:",inline"` + NetworkID int64 `db:"network_id"` } // MeasurementURLNetwork is used for the JOIN between Measurement and URL diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index ca716d5..0aa718d 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -105,11 +105,13 @@ func logTable(w io.Writer, f log.Fields) error { func (h *Handler) TypedLog(t string, e *log.Entry) error { switch t { case "progress": - var err error - s := fmt.Sprintf("%.2f%%: %-25s", e.Fields.Get("percentage").(float64)*100, e.Message) - fmt.Fprintf(h.Writer, s) + perc := e.Fields.Get("percentage").(float64) * 100 + s := fmt.Sprintf(" %s\n %-25s", + bold.Sprintf("%.2f%%", perc), + bold.Sprint(e.Message)) + fmt.Fprint(h.Writer, s) fmt.Fprintln(h.Writer) - return err + return nil case "table": return logTable(h.Writer, e.Fields) case "result_item": From 8657c3b8c62fa3e2c3557438949c354e64f0b101 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 17:09:00 +0200 Subject: [PATCH 29/65] Remove rbgterm from Gopkg.toml --- Gopkg.toml | 4 ---- 1 file changed, 4 deletions(-) diff --git a/Gopkg.toml b/Gopkg.toml index 1a01b9c..b14984a 100644 --- a/Gopkg.toml +++ b/Gopkg.toml @@ -30,10 +30,6 @@ required = ["github.com/shuLhan/go-bindata/go-bindata"] name = "github.com/alecthomas/kingpin" version = "2.2.6" -[[constraint]] - branch = "master" - name = "github.com/aybabtme/rgbterm" - [prune] go-tests = true unused-packages = true From aa4e333987d4e5328a1481c191df159571b27048 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 17:09:13 +0200 Subject: [PATCH 30/65] Update mk --- Gopkg.lock | 13 ++----------- 1 file changed, 2 insertions(+), 11 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 26e99af..979b6c9 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -46,15 +46,6 @@ packages = ["."] revision = "ed7bcb39ff10f39ab08e317ce16df282845852fa" -[[projects]] - branch = "master" - name = "github.com/jmoiron/sqlx" - packages = [ - ".", - "reflectx" - ] - revision = "2aeb6a910c2b94f2d5eb53d9895d80e27264ec41" - [[projects]] name = "github.com/mattn/go-colorable" packages = ["."] @@ -77,7 +68,7 @@ branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "4fe2e61c300930aedc10713557b6e05f29631fc0" + revision = "b44943bb7da37f768d30315a2227dac86feac546" [[projects]] branch = "master" @@ -166,6 +157,6 @@ [solve-meta] analyzer-name = "dep" analyzer-version = 1 - inputs-digest = "bb552d1e6530dab8cdd5cc7a6c60bf2b9afbe77e6ea20a31b6d60fc44ad05e26" + inputs-digest = "45f2fee1cf7e2abceb2500715c0f158ad17d51b52f8200cb1bb51c369b15402f" solver-name = "gps-cdcl" solver-version = 1 From 53791076c9f34d9d906d6a49b481729802410703 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 17:37:47 +0200 Subject: [PATCH 31/65] Disable input list randomization --- nettests/nettests.go | 1 + 1 file changed, 1 insertion(+) diff --git a/nettests/nettests.go b/nettests/nettests.go index ce9b720..899acfd 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -129,6 +129,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { DisableReportFile: false, DisableCollector: false, + RandomizeInput: false, // It's important to disable input randomization to ensure the URLs are written in sync to the DB SoftwareName: "ooniprobe", SoftwareVersion: ooni.Version, From 1637fdbf93e1f92e2c59d797e9cbaac7fde4976d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 10 Sep 2018 18:03:32 +0200 Subject: [PATCH 32/65] Structured output of measurements --- internal/cli/list/list.go | 4 ++-- internal/database/actions.go | 1 + internal/database/models.go | 1 + internal/log/handlers/cli/cli.go | 4 ++-- internal/output/output.go | 27 +++++++++++++++++++++++++++ nettests/nettests.go | 7 ------- 6 files changed, 33 insertions(+), 11 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 13ef534..fdf93ff 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -27,8 +27,8 @@ func init() { log.WithError(err).Error("failed to list measurements") return err } - for idx, msmt := range measurements { - fmt.Printf("%d: %v\n", idx, msmt) + for _, msmt := range measurements { + output.MeasurementItem(msmt) } } else { doneResults, incompleteResults, err := database.ListResults(ctx.DB) diff --git a/internal/database/actions.go b/internal/database/actions.go index 6cfd1f9..2fe12c6 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -18,6 +18,7 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR measurements := []MeasurementURLNetwork{} req := sess.Select( + "measurements.id as msmt_tbl_id", "networks.id as network_id", "results.id as result_id", "urls.id as url_id", diff --git a/internal/database/models.go b/internal/database/models.go index 83b2226..60c510b 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -22,6 +22,7 @@ type ResultNetwork struct { // MeasurementURLNetwork is used for the JOIN between Measurement and URL type MeasurementURLNetwork struct { Measurement `db:",inline"` + MsmtTblID int64 `db:"msmt_tbl_id"` Network `db:",inline"` NetworkID int64 `db:"network_id"` URL `db:",inline"` diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index 0aa718d..8e4c2ec 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -136,10 +136,10 @@ func (h *Handler) DefaultLog(e *log.Entry) error { if name == "source" { continue } - s += fmt.Sprintf(" %s=%s", color.Sprint(name), e.Fields.Get(name)) + s += fmt.Sprintf(" %s=%v", color.Sprint(name), e.Fields.Get(name)) } - fmt.Fprintf(h.Writer, s) + fmt.Fprint(h.Writer, s) fmt.Fprintln(h.Writer) return nil diff --git a/internal/output/output.go b/internal/output/output.go index 7d6bf77..0e6a214 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -7,6 +7,7 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/util" ) @@ -19,6 +20,32 @@ func Progress(key string, perc float64, msg string) { }).Info(msg) } +// MeasurementItem logs a progress type event +func MeasurementItem(msmt database.MeasurementURLNetwork) { + log.WithFields(log.Fields{ + "type": "measurement_item", + "id": msmt.MsmtTblID, + "test_name": msmt.TestName, + "start_time": msmt.StartTime, + "test_keys": msmt.TestKeys, + "probe_cc": msmt.Network.CountryCode, + "network_name": msmt.Network.NetworkName, + "asn": msmt.Network.ASN, + "runtime": msmt.Runtime, + "url": msmt.URL.URL, + "url_category_code": msmt.URL.CategoryCode, + "url_country_code": msmt.URL.CountryCode, + "is_anomaly": msmt.IsAnomaly, + "is_uploaded": msmt.IsUploaded, + "is_upload_failed": msmt.IsUploadFailed, + "upload_failure_msg": msmt.UploadFailureMsg.String, + "is_failed": msmt.IsFailed, + "failure_msg": msmt.FailureMsg.String, + "is_done": msmt.IsDone, + "report_file_path": msmt.ReportFilePath, + }).Info("measurement") +} + // ResultItemData is the metadata about a result type ResultItemData struct { ID int64 diff --git a/nettests/nettests.go b/nettests/nettests.go index 899acfd..bb0a176 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -172,13 +172,6 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.geoip_lookup", func(e mk.Event) { log.Debugf(color.RedString(e.Key)) - - /* FIXME - Put this into the network table - msmtTemplate.ASN = e.Value.ProbeASN - msmtTemplate.IP = e.Value.ProbeIP - msmtTemplate.CountryCode = e.Value.ProbeCC - */ }) nt.On("status.measurement_start", func(e mk.Event) { From aabe55cc51b54f48c2c93a0abf5c649fa67e3208 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 12:39:47 +0200 Subject: [PATCH 33/65] Update Makefile target for downloading libs --- .travis.yml | 4 ++-- Gopkg.lock | 2 +- Makefile | 9 ++++++--- 3 files changed, 9 insertions(+), 6 deletions(-) diff --git a/.travis.yml b/.travis.yml index 158e4d0..1b86ea9 100644 --- a/.travis.yml +++ b/.travis.yml @@ -7,7 +7,7 @@ go: install: - make install-dev-deps - dep ensure -- make update-mk-libs +- make download-mk-libs-macos script: - make test-internal -- make build \ No newline at end of file +- make build diff --git a/Gopkg.lock b/Gopkg.lock index 979b6c9..2d6eb69 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -68,7 +68,7 @@ branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "b44943bb7da37f768d30315a2227dac86feac546" + revision = "33cbd62d4a73b68af069f70a797c2250332c059a" [[projects]] branch = "master" diff --git a/Makefile b/Makefile index 9b3dd13..25e9fa1 100644 --- a/Makefile +++ b/Makefile @@ -14,10 +14,13 @@ build-windows: @echo "Building dist/ooni.exe" CC=x86_64-w64-mingw32-gcc GOOS=windows GOARCH=amd64 CGO_ENABLED=1 go build -o dist/ooni.exe -x cmd/ooni/main.go -update-mk-libs: +download-mk-libs: @echo "updating mk-libs" - @cd vendor/github.com/measurement-kit/go-measurement-kit && curl -L -o master.zip https://github.com/measurement-kit/golang-prebuilt/archive/master.zip && unzip master.zip && mv golang-prebuilt-master libs && rm master.zip # This is a hack to workaround: https://github.com/golang/dep/issues/1240 -.PHONY: update-mk-libs + @cd vendor/github.com/measurement-kit/go-measurement-kit && ./download-libs.sh +download-mk-libs-macos: + @echo "updating mk-libs" + @cd vendor/github.com/measurement-kit/go-measurement-kit && ./download-libs.sh macos +.PHONY: update-mk-libs-macos bindata: @$(GO) run vendor/github.com/shuLhan/go-bindata/go-bindata/*.go \ From 66ffbde27013471dd8e4eab0470b74d1fe8409d7 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 12:40:39 +0200 Subject: [PATCH 34/65] Update readme entry --- Readme.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Readme.md b/Readme.md index 85659fb..18e809c 100644 --- a/Readme.md +++ b/Readme.md @@ -32,7 +32,7 @@ On macOS you can build a windows and macOS ooni binary. This can be done by running: ``` -make update-mk-libs +make download-mk-libs ``` This will download the prebuilt measurement-kit binaries. From 31ccb1c18176665b35fbfc09a136ffa00dac556f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 15:19:08 +0200 Subject: [PATCH 35/65] Improvements to the list output --- internal/database/actions.go | 2 ++ internal/database/models.go | 5 ++++- internal/output/output.go | 15 ++++++++------- 3 files changed, 14 insertions(+), 8 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index 2fe12c6..88334b5 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -118,6 +118,8 @@ func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testNa ResultID: resultID, ReportFilePath: reportFilePath, URLID: urlID, + IsFailed: false, + IsDone: false, // XXX Do we want to have this be part of something else? StartTime: time.Now().UTC(), TestKeys: "", diff --git a/internal/database/models.go b/internal/database/models.go index 60c510b..6d188aa 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -25,6 +25,8 @@ type MeasurementURLNetwork struct { MsmtTblID int64 `db:"msmt_tbl_id"` Network `db:",inline"` NetworkID int64 `db:"network_id"` + Result `db:",inline"` + ResultID int64 `db:"result_id"` URL `db:",inline"` } @@ -54,7 +56,7 @@ type Measurement struct { Runtime float64 `db:"runtime"` // Fractional number of seconds IsDone bool `db:"is_done"` IsUploaded bool `db:"is_uploaded"` - IsFailed string `db:"is_failed"` + IsFailed bool `db:"is_failed"` FailureMsg sql.NullString `db:"failure_msg,omitempty"` IsUploadFailed bool `db:"is_upload_failed"` UploadFailureMsg sql.NullString `db:"upload_failure_msg,omitempty"` @@ -101,6 +103,7 @@ func (r *Result) Finished(sess sqlbuilder.Database) error { // Failed writes the error string to the measurement func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error { m.FailureMsg = sql.NullString{String: failure, Valid: true} + m.IsFailed = true err := sess.Collection("measurements").Find("id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") diff --git a/internal/output/output.go b/internal/output/output.go index 0e6a214..2c92698 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -26,22 +26,23 @@ func MeasurementItem(msmt database.MeasurementURLNetwork) { "type": "measurement_item", "id": msmt.MsmtTblID, "test_name": msmt.TestName, - "start_time": msmt.StartTime, + "test_group_name": msmt.Result.TestGroupName, + "start_time": msmt.Measurement.StartTime, "test_keys": msmt.TestKeys, "probe_cc": msmt.Network.CountryCode, "network_name": msmt.Network.NetworkName, "asn": msmt.Network.ASN, - "runtime": msmt.Runtime, - "url": msmt.URL.URL, - "url_category_code": msmt.URL.CategoryCode, - "url_country_code": msmt.URL.CountryCode, - "is_anomaly": msmt.IsAnomaly, + "runtime": msmt.Measurement.Runtime, + "url": msmt.URL.URL.String, + "url_category_code": msmt.URL.CategoryCode.String, + "url_country_code": msmt.URL.CountryCode.String, + "is_anomaly": msmt.IsAnomaly.Bool, "is_uploaded": msmt.IsUploaded, "is_upload_failed": msmt.IsUploadFailed, "upload_failure_msg": msmt.UploadFailureMsg.String, "is_failed": msmt.IsFailed, "failure_msg": msmt.FailureMsg.String, - "is_done": msmt.IsDone, + "is_done": msmt.Measurement.IsDone, "report_file_path": msmt.ReportFilePath, }).Info("measurement") } From 8da12a9c07401d7a5af7d2d81f28d18e32def024 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 15:40:42 +0200 Subject: [PATCH 36/65] Add measurement summary to list command --- internal/cli/list/list.go | 18 ++++++++++++++++++ internal/database/actions.go | 6 ++++++ internal/database/models.go | 20 +++++++++++++------- internal/output/output.go | 23 ++++++++++++++++++++--- 4 files changed, 57 insertions(+), 10 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index fdf93ff..304da18 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -27,9 +27,27 @@ func init() { log.WithError(err).Error("failed to list measurements") return err } + + msmtSummary := output.MeasurementSummaryData{ + TotalCount: 0, + AnomalyCount: 0, + DataUsageUp: 0, + DataUsageDown: 0, + } for _, msmt := range measurements { + // FIXME this logic should be adjusted for test groups that have many + // measurements in them + if msmtSummary.DataUsageUp == 0 { + msmtSummary.DataUsageUp = msmt.DataUsageUp + msmtSummary.DataUsageDown = msmt.DataUsageDown + } + if msmt.IsAnomaly.Bool == true { + msmtSummary.AnomalyCount++ + } + msmtSummary.TotalCount++ output.MeasurementItem(msmt) } + output.MeasurementSummary(msmtSummary) } else { doneResults, incompleteResults, err := database.ListResults(ctx.DB) if err != nil { diff --git a/internal/database/actions.go b/internal/database/actions.go index 88334b5..7b5ff68 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -19,8 +19,14 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR req := sess.Select( "measurements.id as msmt_tbl_id", + "measurements.is_done as measurement_is_done", + "measurements.start_time as measurement_start_time", + "measurements.runtime as measurement_runtime", "networks.id as network_id", "results.id as result_id", + "results.start_time as result_start_time", + "results.is_done as result_is_done", + "results.runtime as result_runtime", "urls.id as url_id", db.Raw("networks.*"), db.Raw("urls.*"), diff --git a/internal/database/models.go b/internal/database/models.go index 6d188aa..9fb103d 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -21,13 +21,19 @@ type ResultNetwork struct { // MeasurementURLNetwork is used for the JOIN between Measurement and URL type MeasurementURLNetwork struct { - Measurement `db:",inline"` - MsmtTblID int64 `db:"msmt_tbl_id"` - Network `db:",inline"` - NetworkID int64 `db:"network_id"` - Result `db:",inline"` - ResultID int64 `db:"result_id"` - URL `db:",inline"` + Measurement `db:",inline"` + MeasurementStartTime time.Time `db:"measurement_start_time"` + MeasurementIsDone bool `db:"measurement_is_done"` + MeasurementRuntime float64 `db:"measurement_runtime"` + MsmtTblID int64 `db:"msmt_tbl_id"` + Network `db:",inline"` + NetworkID int64 `db:"network_id"` + Result `db:",inline"` + ResultID int64 `db:"result_id"` + ResultRuntime float64 `db:"result_runtime"` + ResultStartTime time.Time `db:"result_start_time"` + ResultIsDone bool `db:"result_is_done"` + URL `db:",inline"` } // Network represents a network tested by the user diff --git a/internal/output/output.go b/internal/output/output.go index 2c92698..466b57a 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -20,6 +20,23 @@ func Progress(key string, perc float64, msg string) { }).Info(msg) } +type MeasurementSummaryData struct { + TotalCount int64 + AnomalyCount int64 + DataUsageUp int64 + DataUsageDown int64 +} + +func MeasurementSummary(msmt MeasurementSummaryData) { + log.WithFields(log.Fields{ + "type": "measurement_summary", + "total_count": msmt.TotalCount, + "anomaly_count": msmt.AnomalyCount, + "data_usage_down": msmt.DataUsageDown, + "data_usage_up": msmt.DataUsageUp, + }).Info("measurement summary") +} + // MeasurementItem logs a progress type event func MeasurementItem(msmt database.MeasurementURLNetwork) { log.WithFields(log.Fields{ @@ -27,12 +44,12 @@ func MeasurementItem(msmt database.MeasurementURLNetwork) { "id": msmt.MsmtTblID, "test_name": msmt.TestName, "test_group_name": msmt.Result.TestGroupName, - "start_time": msmt.Measurement.StartTime, + "start_time": msmt.MeasurementStartTime, "test_keys": msmt.TestKeys, "probe_cc": msmt.Network.CountryCode, "network_name": msmt.Network.NetworkName, "asn": msmt.Network.ASN, - "runtime": msmt.Measurement.Runtime, + "runtime": msmt.MeasurementRuntime, "url": msmt.URL.URL.String, "url_category_code": msmt.URL.CategoryCode.String, "url_country_code": msmt.URL.CountryCode.String, @@ -42,7 +59,7 @@ func MeasurementItem(msmt database.MeasurementURLNetwork) { "upload_failure_msg": msmt.UploadFailureMsg.String, "is_failed": msmt.IsFailed, "failure_msg": msmt.FailureMsg.String, - "is_done": msmt.Measurement.IsDone, + "is_done": msmt.MeasurementIsDone, "report_file_path": msmt.ReportFilePath, }).Info("measurement") } From d02ed117f97f1fef6e872be132b9a0ad13b61cca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 16:36:09 +0200 Subject: [PATCH 37/65] Include total_runtime in the summary of measurements --- internal/cli/list/list.go | 4 ++++ internal/database/actions.go | 1 + internal/output/output.go | 2 ++ 3 files changed, 7 insertions(+) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 304da18..371ff90 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -33,8 +33,12 @@ func init() { AnomalyCount: 0, DataUsageUp: 0, DataUsageDown: 0, + TotalRuntime: 0, } for _, msmt := range measurements { + if msmtSummary.TotalRuntime == 0 { + msmtSummary.TotalRuntime = msmt.ResultRuntime + } // FIXME this logic should be adjusted for test groups that have many // measurements in them if msmtSummary.DataUsageUp == 0 { diff --git a/internal/database/actions.go b/internal/database/actions.go index 7b5ff68..ddfa07e 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -27,6 +27,7 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR "results.start_time as result_start_time", "results.is_done as result_is_done", "results.runtime as result_runtime", + "results.test_group_name as test_group_name", "urls.id as url_id", db.Raw("networks.*"), db.Raw("urls.*"), diff --git a/internal/output/output.go b/internal/output/output.go index 466b57a..fac6faf 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -21,6 +21,7 @@ func Progress(key string, perc float64, msg string) { } type MeasurementSummaryData struct { + TotalRuntime float64 TotalCount int64 AnomalyCount int64 DataUsageUp int64 @@ -30,6 +31,7 @@ type MeasurementSummaryData struct { func MeasurementSummary(msmt MeasurementSummaryData) { log.WithFields(log.Fields{ "type": "measurement_summary", + "total_runtime": msmt.TotalRuntime, "total_count": msmt.TotalCount, "anomaly_count": msmt.AnomalyCount, "data_usage_down": msmt.DataUsageDown, From 0ee0c2812973145ce02f39257dfd17922884b3b5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 18:06:15 +0200 Subject: [PATCH 38/65] Resolve country code ambigious selection --- internal/cli/list/list.go | 6 ++-- internal/database/actions.go | 2 ++ internal/database/models.go | 21 ++++++++----- internal/log/handlers/cli/results.go | 2 +- internal/output/output.go | 46 ++++++++++++++-------------- 5 files changed, 41 insertions(+), 36 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 371ff90..9d4e0a8 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -1,8 +1,6 @@ package list import ( - "fmt" - "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" @@ -71,7 +69,7 @@ func init() { StartTime: result.StartTime, NetworkName: result.Network.NetworkName, Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), + ASN: result.Network.ASN, MeasurementCount: 0, MeasurementAnomalyCount: 0, TestKeys: "{}", // FIXME this used to be Summary we probably need to use a list now @@ -101,7 +99,7 @@ func init() { StartTime: result.StartTime, NetworkName: result.Network.NetworkName, Country: result.Network.CountryCode, - ASN: fmt.Sprintf("AS%d", result.Network.ASN), + ASN: result.Network.ASN, TestKeys: testKeys, MeasurementCount: totalCount, MeasurementAnomalyCount: anmlyCount, diff --git a/internal/database/actions.go b/internal/database/actions.go index ddfa07e..ba36145 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -23,12 +23,14 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR "measurements.start_time as measurement_start_time", "measurements.runtime as measurement_runtime", "networks.id as network_id", + "networks.country_code as network_country_code", "results.id as result_id", "results.start_time as result_start_time", "results.is_done as result_is_done", "results.runtime as result_runtime", "results.test_group_name as test_group_name", "urls.id as url_id", + "urls.country_code as url_country_code", db.Raw("networks.*"), db.Raw("urls.*"), db.Raw("measurements.*"), diff --git a/internal/database/models.go b/internal/database/models.go index 9fb103d..fbbb982 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -26,14 +26,19 @@ type MeasurementURLNetwork struct { MeasurementIsDone bool `db:"measurement_is_done"` MeasurementRuntime float64 `db:"measurement_runtime"` MsmtTblID int64 `db:"msmt_tbl_id"` - Network `db:",inline"` - NetworkID int64 `db:"network_id"` - Result `db:",inline"` - ResultID int64 `db:"result_id"` - ResultRuntime float64 `db:"result_runtime"` - ResultStartTime time.Time `db:"result_start_time"` - ResultIsDone bool `db:"result_is_done"` - URL `db:",inline"` + + Network `db:",inline"` + NetworkID int64 `db:"network_id"` + NetworkCountryCode string `db:"network_country_code"` + + Result `db:",inline"` + ResultID int64 `db:"result_id"` + ResultRuntime float64 `db:"result_runtime"` + ResultStartTime time.Time `db:"result_start_time"` + ResultIsDone bool `db:"result_is_done"` + + URL `db:",inline"` + URLCountryCode sql.NullString `db:"url_country_code"` } // Network represents a network tested by the user diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index d22398a..887117a 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -77,7 +77,7 @@ func logResultItem(w io.Writer, f log.Fields) error { name := f.Get("name").(string) startTime := f.Get("start_time").(time.Time) networkName := f.Get("network_name").(string) - asn := fmt.Sprintf("AS %s", f.Get("asn").(string)) + asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("country").(string)) //runtime := f.Get("runtime").(float64) //dataUsageUp := f.Get("dataUsageUp").(int64) //dataUsageDown := f.Get("dataUsageDown").(int64) diff --git a/internal/output/output.go b/internal/output/output.go index fac6faf..bbd139d 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -42,27 +42,27 @@ func MeasurementSummary(msmt MeasurementSummaryData) { // MeasurementItem logs a progress type event func MeasurementItem(msmt database.MeasurementURLNetwork) { log.WithFields(log.Fields{ - "type": "measurement_item", - "id": msmt.MsmtTblID, - "test_name": msmt.TestName, - "test_group_name": msmt.Result.TestGroupName, - "start_time": msmt.MeasurementStartTime, - "test_keys": msmt.TestKeys, - "probe_cc": msmt.Network.CountryCode, - "network_name": msmt.Network.NetworkName, - "asn": msmt.Network.ASN, - "runtime": msmt.MeasurementRuntime, - "url": msmt.URL.URL.String, - "url_category_code": msmt.URL.CategoryCode.String, - "url_country_code": msmt.URL.CountryCode.String, - "is_anomaly": msmt.IsAnomaly.Bool, - "is_uploaded": msmt.IsUploaded, - "is_upload_failed": msmt.IsUploadFailed, - "upload_failure_msg": msmt.UploadFailureMsg.String, - "is_failed": msmt.IsFailed, - "failure_msg": msmt.FailureMsg.String, - "is_done": msmt.MeasurementIsDone, - "report_file_path": msmt.ReportFilePath, + "type": "measurement_item", + "id": msmt.MsmtTblID, + "test_name": msmt.TestName, + "test_group_name": msmt.Result.TestGroupName, + "start_time": msmt.MeasurementStartTime, + "test_keys": msmt.TestKeys, + "network_country_code": msmt.NetworkCountryCode, + "network_name": msmt.Network.NetworkName, + "asn": msmt.Network.ASN, + "runtime": msmt.MeasurementRuntime, + "url": msmt.URL.URL.String, + "url_category_code": msmt.URL.CategoryCode.String, + "url_country_code": msmt.URLCountryCode.String, + "is_anomaly": msmt.IsAnomaly.Bool, + "is_uploaded": msmt.IsUploaded, + "is_upload_failed": msmt.IsUploadFailed, + "upload_failure_msg": msmt.UploadFailureMsg.String, + "is_failed": msmt.IsFailed, + "failure_msg": msmt.FailureMsg.String, + "is_done": msmt.MeasurementIsDone, + "report_file_path": msmt.ReportFilePath, }).Info("measurement") } @@ -77,7 +77,7 @@ type ResultItemData struct { Runtime float64 Country string NetworkName string - ASN string + ASN uint Done bool DataUsageDown int64 DataUsageUp int64 @@ -95,7 +95,7 @@ func ResultItem(result ResultItemData) { "test_keys": result.TestKeys, "measurement_count": result.MeasurementCount, "measurement_anomaly_count": result.MeasurementAnomalyCount, - "country": result.Country, + "network_country_code": result.Country, "network_name": result.NetworkName, "asn": result.ASN, "runtime": result.Runtime, From 044748c2ce4b83a5c3352e95d27fff3bcb84f317 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 18:16:14 +0200 Subject: [PATCH 39/65] Fix exposing the country code --- internal/log/handlers/cli/results.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 887117a..9c3462f 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -77,7 +77,7 @@ func logResultItem(w io.Writer, f log.Fields) error { name := f.Get("name").(string) startTime := f.Get("start_time").(time.Time) networkName := f.Get("network_name").(string) - asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("country").(string)) + asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string)) //runtime := f.Get("runtime").(float64) //dataUsageUp := f.Get("dataUsageUp").(int64) //dataUsageDown := f.Get("dataUsageDown").(int64) From 2081c25b7344865ecfb2976cc695d95c67342693 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 11 Sep 2018 18:41:15 +0200 Subject: [PATCH 40/65] Expose the median_bitrate in the list view --- internal/database/actions.go | 21 ++++++++++++++++----- internal/database/actions_test.go | 20 ++++++++++++++++++++ internal/database/models.go | 8 ++++++++ internal/log/handlers/cli/results.go | 15 ++++----------- 4 files changed, 48 insertions(+), 16 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index ba36145..a0b3f89 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -53,17 +53,28 @@ func GetResultTestKeys(sess sqlbuilder.Database, resultID int64) (string, error) res := sess.Collection("measurements").Find("result_id", resultID) defer res.Close() - var msmt Measurement + var ( + msmt Measurement + tk PerformanceTestKeys + ) for res.Next(&msmt) { if msmt.TestName == "web_connectivity" { break } - // We only really care about the NDT TestKeys - if msmt.TestName == "ndt" { - return msmt.TestKeys, nil + // We only really care about performance keys + if msmt.TestName == "ndt" || msmt.TestName == "dash" { + if err := json.Unmarshal([]byte(msmt.TestKeys), &tk); err != nil { + log.WithError(err).Error("failed to parse testKeys") + return "{}", err + } } } - return "{}", nil + b, err := json.Marshal(tk) + if err != nil { + log.WithError(err).Error("failed to serialize testKeys") + return "{}", err + } + return string(b), nil } // GetMeasurementCounts returns the number of anomalous and total measurement for a given result diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index a060b40..504f175 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -2,6 +2,7 @@ package database import ( "database/sql" + "encoding/json" "io/ioutil" "os" "testing" @@ -145,3 +146,22 @@ func TestURLCreation(t *testing.T) { t.Error("inserting the same URL with different category code should produce the same result") } } + +func TestPerformanceTestKeys(t *testing.T) { + var tk PerformanceTestKeys + + ndtS := "{\"download\":100.0,\"upload\":20.0,\"ping\":2.2}" + dashS := "{\"median_bitrate\":102.0}" + if err := json.Unmarshal([]byte(ndtS), &tk); err != nil { + t.Fatal("failed to parse ndtS") + } + if err := json.Unmarshal([]byte(dashS), &tk); err != nil { + t.Fatal("failed to parse dashS") + } + if tk.Bitrate != 102.0 { + t.Fatalf("error Bitrate %f", tk.Bitrate) + } + if tk.Download != 100.0 { + t.Fatalf("error Download %f", tk.Download) + } +} diff --git a/internal/database/models.go b/internal/database/models.go index fbbb982..dfb9e54 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -96,6 +96,14 @@ type Result struct { MeasurementDir string `db:"measurement_dir"` } +// PerformanceTestKeys is the result summary for a performance test +type PerformanceTestKeys struct { + Upload float64 `json:"upload"` + Download float64 `json:"download"` + Ping float64 `json:"ping"` + Bitrate float64 `json:"median_bitrate"` +} + // Finished marks the result as done and sets the runtime func (r *Result) Finished(sess sqlbuilder.Database) error { if r.IsDone == true || r.Runtime != 0 { diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 9c3462f..8d9a2a1 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -8,12 +8,13 @@ import ( "time" "github.com/apex/log" + "github.com/ooni/probe-cli/internal/database" "github.com/ooni/probe-cli/internal/util" ) -func formatSpeed(speed int64) string { +func formatSpeed(speed float64) string { if speed < 1000 { - return fmt.Sprintf("%d Kbit/s", speed) + return fmt.Sprintf("%.2f Kbit/s", speed) } else if speed < 1000*1000 { return fmt.Sprintf("%.2f Mbit/s", float32(speed)/1000) } else if speed < 1000*1000*1000 { @@ -23,14 +24,6 @@ func formatSpeed(speed int64) string { return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) } -// PerformanceTestKeys is the result summary for a performance test -type PerformanceTestKeys struct { - Upload int64 `json:"upload"` - Download int64 `json:"download"` - Ping float64 `json:"ping"` - Bitrate int64 `json:"median_bitrate"` -} - var summarizers = map[string]func(uint64, uint64, string) []string{ "websites": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ @@ -40,7 +33,7 @@ var summarizers = map[string]func(uint64, uint64, string) []string{ } }, "performance": func(totalCount uint64, anomalyCount uint64, ss string) []string { - var tk PerformanceTestKeys + var tk database.PerformanceTestKeys if err := json.Unmarshal([]byte(ss), &tk); err != nil { return nil } From 54af7170d4cb7ce8c0eb3398c882048d59b404ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 12 Sep 2018 13:42:16 +0200 Subject: [PATCH 41/65] Add support for data_usage event --- data/migrations/1_create_msmt_results.sql | 4 ++-- internal/cli/list/list.go | 4 ++-- internal/database/models.go | 4 ++-- internal/log/handlers/cli/results.go | 6 +++--- internal/output/output.go | 12 +++++------ nettests/nettests.go | 26 ++++++++++++++++++++++- 6 files changed, 40 insertions(+), 16 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 17d1c17..5ff8b53 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -66,8 +66,8 @@ CREATE TABLE `results` ( -- This is a flag used to indicate if the result is done or is currently running. `is_done` TINYINT(1) NOT NULL, - `data_usage_up` INTEGER NOT NULL, - `data_usage_down` INTEGER NOT NULL, + `data_usage_up` REAL NOT NULL, + `data_usage_down` REAL NOT NULL, -- It's probably reasonable to set the maximum length to 260 as this is the -- maximum length of file paths on windows. `measurement_dir` VARCHAR(260) NOT NULL, diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 9d4e0a8..28ebd92 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -29,8 +29,8 @@ func init() { msmtSummary := output.MeasurementSummaryData{ TotalCount: 0, AnomalyCount: 0, - DataUsageUp: 0, - DataUsageDown: 0, + DataUsageUp: 0.0, + DataUsageDown: 0.0, TotalRuntime: 0, } for _, msmt := range measurements { diff --git a/internal/database/models.go b/internal/database/models.go index dfb9e54..87d95ef 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -91,8 +91,8 @@ type Result struct { Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds IsViewed bool `db:"is_viewed"` IsDone bool `db:"is_done"` - DataUsageUp int64 `db:"data_usage_up"` - DataUsageDown int64 `db:"data_usage_down"` + DataUsageUp float64 `db:"data_usage_up"` + DataUsageDown float64 `db:"data_usage_down"` MeasurementDir string `db:"measurement_dir"` } diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 8d9a2a1..4f1a795 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -113,8 +113,8 @@ func logResultSummary(w io.Writer, f log.Fields) error { networks := f.Get("total_networks").(int64) tests := f.Get("total_tests").(int64) - dataUp := f.Get("total_data_usage_up").(int64) - dataDown := f.Get("total_data_usage_down").(int64) + dataUp := f.Get("total_data_usage_up").(float64) + dataDown := f.Get("total_data_usage_down").(float64) if tests == 0 { fmt.Fprintf(w, "No results\n") fmt.Fprintf(w, "Try running:\n") @@ -125,7 +125,7 @@ func logResultSummary(w io.Writer, f log.Fields) error { fmt.Fprintf(w, " │ %s │ %s │ %s │\n", util.RightPad(fmt.Sprintf("%d tests", tests), 12), util.RightPad(fmt.Sprintf("%d nets", networks), 12), - util.RightPad(fmt.Sprintf("%d ⬆ %d ⬇", dataUp, dataDown), 12)) + util.RightPad(fmt.Sprintf("%.0f ⬆ %.0f ⬇", dataUp, dataDown), 12)) fmt.Fprintf(w, " └──────────────┴──────────────┴──────────────┘\n") return nil diff --git a/internal/output/output.go b/internal/output/output.go index bbd139d..8a1328a 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -24,8 +24,8 @@ type MeasurementSummaryData struct { TotalRuntime float64 TotalCount int64 AnomalyCount int64 - DataUsageUp int64 - DataUsageDown int64 + DataUsageUp float64 + DataUsageDown float64 } func MeasurementSummary(msmt MeasurementSummaryData) { @@ -79,8 +79,8 @@ type ResultItemData struct { NetworkName string ASN uint Done bool - DataUsageDown int64 - DataUsageUp int64 + DataUsageDown float64 + DataUsageUp float64 Index int TotalCount int } @@ -109,8 +109,8 @@ func ResultItem(result ResultItemData) { type ResultSummaryData struct { TotalTests int64 - TotalDataUsageUp int64 - TotalDataUsageDown int64 + TotalDataUsageUp float64 + TotalDataUsageDown float64 TotalNetworks int64 } diff --git a/nettests/nettests.go b/nettests/nettests.go index bb0a176..1f5cd58 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -63,6 +63,12 @@ func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error { return nil } +type StatusEnd struct { + DownloadedKB float64 `json:"download_kb"` + UploadedKB float64 `json:"uploaded_kb"` + Failure string `json:"failure"` +} + // Init should be called once to initialise the nettest func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("Init: %v", nt) @@ -273,12 +279,27 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.end", func(e mk.Event) { log.Debugf("status.end") + for idx, msmt := range c.msmts { log.Debugf("adding msmt#%d to result", idx) if err := msmt.AddToResult(c.Ctx.DB, c.res); err != nil { log.WithError(err).Error("failed to add to result") } } + + var endMsg StatusEnd + err := json.Unmarshal([]byte(e.Value.JSONStr), &endMsg) + if err != nil { + log.WithError(err).Errorf("failed to extract status.end message %s", e.Value.JSONStr) + return + } + + if endMsg.Failure != "" { + log.Errorf("Failure in status.end: %s", endMsg.Failure) + } + + c.res.DataUsageDown += endMsg.DownloadedKB + c.res.DataUsageDown += endMsg.UploadedKB }) log.Debugf("Registered all the handlers") @@ -303,7 +324,10 @@ func (c *Controller) OnEntry(idx int64, jsonStr string) { log.Debugf("OnEntry") var entry Entry - json.Unmarshal([]byte(jsonStr), &entry) + if err := json.Unmarshal([]byte(jsonStr), &entry); err != nil { + log.WithError(err).Error("failed to parse onEntry") + return + } tk := c.nt.GetTestKeys(entry.TestKeys) log.Debugf("Fetching: %s %v", idx, c.msmts[idx]) From 4880d31120492c01041617034208b2cfbd31e852 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 12 Sep 2018 13:48:18 +0200 Subject: [PATCH 42/65] Fix data usage count --- Gopkg.lock | 2 +- nettests/nettests.go | 21 ++++----------------- 2 files changed, 5 insertions(+), 18 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index 2d6eb69..c9e1d9b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -68,7 +68,7 @@ branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "33cbd62d4a73b68af069f70a797c2250332c059a" + revision = "4b5013c619f6f0293fd67b6892a7f90066d3c18d" [[projects]] branch = "master" diff --git a/nettests/nettests.go b/nettests/nettests.go index 1f5cd58..89ec036 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -63,12 +63,6 @@ func (c *Controller) SetInputIdxMap(inputIdxMap map[int64]int64) error { return nil } -type StatusEnd struct { - DownloadedKB float64 `json:"download_kb"` - UploadedKB float64 `json:"uploaded_kb"` - Failure string `json:"failure"` -} - // Init should be called once to initialise the nettest func (c *Controller) Init(nt *mk.Nettest) error { log.Debugf("Init: %v", nt) @@ -287,19 +281,12 @@ func (c *Controller) Init(nt *mk.Nettest) error { } } - var endMsg StatusEnd - err := json.Unmarshal([]byte(e.Value.JSONStr), &endMsg) - if err != nil { - log.WithError(err).Errorf("failed to extract status.end message %s", e.Value.JSONStr) - return + if e.Value.Failure != "" { + log.Errorf("Failure in status.end: %s", e.Value.Failure) } - if endMsg.Failure != "" { - log.Errorf("Failure in status.end: %s", endMsg.Failure) - } - - c.res.DataUsageDown += endMsg.DownloadedKB - c.res.DataUsageDown += endMsg.UploadedKB + c.res.DataUsageDown += e.Value.DownloadedKB + c.res.DataUsageDown += e.Value.UploadedKB }) log.Debugf("Registered all the handlers") From 2b1fb662fbe2a891544cac71ca8d93a105ad8181 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 12 Sep 2018 14:03:07 +0200 Subject: [PATCH 43/65] Fix showing data usage indicators --- internal/log/handlers/cli/results.go | 20 +++++++++++++++----- internal/util/util.go | 6 +++++- nettests/nettests.go | 2 +- 3 files changed, 21 insertions(+), 7 deletions(-) diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 4f1a795..4ed7365 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -24,6 +24,18 @@ func formatSpeed(speed float64) string { return fmt.Sprintf("%.2f Tbit/s", float32(speed)/(1000*1000*1000)) } +func formatSize(size float64) string { + if size < 1024 { + return fmt.Sprintf("%.1fK", size) + } else if size < 1024*1024 { + return fmt.Sprintf("%.1fM", size/1024.0) + } else if size < 1024*1024*1024 { + return fmt.Sprintf("%.1fG", size/(1024.0*1024.0)) + } + // WTF, you crazy? + return fmt.Sprintf("%.1fT", size/(1024*1024*1024)) +} + var summarizers = map[string]func(uint64, uint64, string) []string{ "websites": func(totalCount uint64, anomalyCount uint64, ss string) []string { return []string{ @@ -102,9 +114,7 @@ func logResultItem(w io.Writer, f log.Fields) error { util.RightPad(summary[2], colWidth))) if index == totalCount-1 { - fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────┬") - fmt.Fprintf(w, strings.Repeat("─", colWidth*2-44)) - fmt.Fprintf(w, "┘\n") + fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────────┬┘\n") } return nil } @@ -125,8 +135,8 @@ func logResultSummary(w io.Writer, f log.Fields) error { fmt.Fprintf(w, " │ %s │ %s │ %s │\n", util.RightPad(fmt.Sprintf("%d tests", tests), 12), util.RightPad(fmt.Sprintf("%d nets", networks), 12), - util.RightPad(fmt.Sprintf("%.0f ⬆ %.0f ⬇", dataUp, dataDown), 12)) - fmt.Fprintf(w, " └──────────────┴──────────────┴──────────────┘\n") + util.RightPad(fmt.Sprintf("⬆ %s ⬇ %s", formatSize(dataUp), formatSize(dataDown)), 16)) + fmt.Fprintf(w, " └──────────────┴──────────────┴──────────────────┘\n") return nil } diff --git a/internal/util/util.go b/internal/util/util.go index c47d301..7d64ccf 100644 --- a/internal/util/util.go +++ b/internal/util/util.go @@ -38,7 +38,11 @@ func EscapeAwareRuneCountInString(s string) int { } func RightPad(str string, length int) string { - return str + strings.Repeat(" ", length-EscapeAwareRuneCountInString(str)) + c := length - EscapeAwareRuneCountInString(str) + if c < 0 { + c = 0 + } + return str + strings.Repeat(" ", c) } // WrapString wraps the given string within lim width in characters. diff --git a/nettests/nettests.go b/nettests/nettests.go index 89ec036..8da672c 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -286,7 +286,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { } c.res.DataUsageDown += e.Value.DownloadedKB - c.res.DataUsageDown += e.Value.UploadedKB + c.res.DataUsageUp += e.Value.UploadedKB }) log.Debugf("Registered all the handlers") From b1ae8bc13e36233c9fa50122baa481fa8e3ba99a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 12 Sep 2018 15:41:54 +0200 Subject: [PATCH 44/65] Show data usage in summary of measurements --- internal/cli/list/list.go | 4 ++-- internal/database/actions.go | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 28ebd92..4a8bbf4 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -34,11 +34,11 @@ func init() { TotalRuntime: 0, } for _, msmt := range measurements { + // We assume that since these are summary level information the first + // item will contain the information necessary. if msmtSummary.TotalRuntime == 0 { msmtSummary.TotalRuntime = msmt.ResultRuntime } - // FIXME this logic should be adjusted for test groups that have many - // measurements in them if msmtSummary.DataUsageUp == 0 { msmtSummary.DataUsageUp = msmt.DataUsageUp msmtSummary.DataUsageDown = msmt.DataUsageDown diff --git a/internal/database/actions.go b/internal/database/actions.go index a0b3f89..84699ed 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -34,6 +34,7 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR db.Raw("networks.*"), db.Raw("urls.*"), db.Raw("measurements.*"), + db.Raw("results.*"), ).From("results"). Join("measurements").On("results.id = measurements.result_id"). Join("networks").On("results.network_id = networks.id"). From 7f5df077823409408d66a455e53d3e22979f4f23 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Wed, 12 Sep 2018 18:47:57 +0200 Subject: [PATCH 45/65] Fix a very annoying logic bug in URL insertion Write test case for it --- internal/database/actions.go | 64 +++++++++++++-------------- internal/database/actions_test.go | 31 ++++++++++++- nettests/nettests.go | 2 +- nettests/websites/web_connectivity.go | 1 + 4 files changed, 63 insertions(+), 35 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index 84699ed..bb0ea74 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -202,48 +202,48 @@ func CreateNetwork(sess sqlbuilder.Database, location *utils.LocationInfo) (*Net // CreateOrUpdateURL will create a new URL entry to the urls table if it doesn't // exists, otherwise it will update the category code of the one already in // there. -func CreateOrUpdateURL(sess sqlbuilder.Database, url string, categoryCode string, countryCode string) (int64, error) { - var urlID int64 - - res, err := sess.Update("urls").Set( - "url", url, - "category_code", categoryCode, - "country_code", countryCode, - ).Where("url = ? AND country_code = ?", url, countryCode).Exec() +func CreateOrUpdateURL(sess sqlbuilder.Database, urlStr string, categoryCode string, countryCode string) (int64, error) { + var url URL + tx, err := sess.NewTx(nil) if err != nil { - log.Error("Failed to write to the URL table") + log.WithError(err).Error("failed to create transaction") return 0, err } - affected, err := res.RowsAffected() + res := tx.Collection("urls").Find( + db.Cond{"url": urlStr, "country_code": countryCode}, + ) + err = res.One(&url) - if err != nil { - log.Error("Failed to get affected row count") - return 0, err - } - if affected == 0 { - newID, err := sess.Collection("urls").Insert( - URL{ - URL: sql.NullString{String: url, Valid: true}, - CategoryCode: sql.NullString{String: categoryCode, Valid: true}, - CountryCode: sql.NullString{String: countryCode, Valid: true}, - }) - if err != nil { + if err == db.ErrNoMoreRows { + url = URL{ + URL: sql.NullString{String: urlStr, Valid: true}, + CategoryCode: sql.NullString{String: categoryCode, Valid: true}, + CountryCode: sql.NullString{String: countryCode, Valid: true}, + } + newID, insErr := tx.Collection("urls").Insert(url) + if insErr != nil { log.Error("Failed to insert into the URLs table") - return 0, err + return 0, insErr } - urlID = newID.(int64) + url.ID = sql.NullInt64{Int64: newID.(int64), Valid: true} + } else if err != nil { + log.WithError(err).Error("Failed to get single result") + return 0, err } else { - lastID, err := res.LastInsertId() - if err != nil { - log.Error("failed to get URL ID") - return 0, err - } - urlID = lastID + url.CategoryCode = sql.NullString{String: categoryCode, Valid: true} + res.Update(url) } - log.Debugf("returning url %d", urlID) - return urlID, nil + err = tx.Commit() + if err != nil { + log.WithError(err).Error("Failed to write to the URL table") + return 0, err + } + + log.Debugf("returning url %d", url.ID.Int64) + + return url.ID.Int64, nil } // AddTestKeys writes the summary to the measurement diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 504f175..608aa3d 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -132,12 +132,27 @@ func TestURLCreation(t *testing.T) { t.Fatal(err) } - newID1, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX") + newID1, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX") if err != nil { t.Fatal(err) } - newID2, err := CreateOrUpdateURL(sess, "https://google.com", "GMB", "XX") + newID2, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX") + if err != nil { + t.Fatal(err) + } + + newID3, err := CreateOrUpdateURL(sess, "https://facebook.com", "GRP", "XX") + if err != nil { + t.Fatal(err) + } + + newID4, err := CreateOrUpdateURL(sess, "https://facebook.com", "GMP", "XX") + if err != nil { + t.Fatal(err) + } + + newID5, err := CreateOrUpdateURL(sess, "https://google.com", "SRCH", "XX") if err != nil { t.Fatal(err) } @@ -145,6 +160,18 @@ func TestURLCreation(t *testing.T) { if newID2 != newID1 { t.Error("inserting the same URL with different category code should produce the same result") } + + if newID3 == newID1 { + t.Error("inserting different URL should produce different ids") + } + + if newID4 != newID3 { + t.Error("inserting the same URL with different category code should produce the same result") + } + + if newID5 != newID1 { + t.Error("the ID of google should still be the same") + } } func TestPerformanceTestKeys(t *testing.T) { diff --git a/nettests/nettests.go b/nettests/nettests.go index 8da672c..c01b290 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -176,12 +176,12 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.measurement_start", func(e mk.Event) { log.Debugf(color.RedString(e.Key)) - idx := e.Value.Idx urlID := sql.NullInt64{Int64: 0, Valid: false} if c.inputIdxMap != nil { urlID = sql.NullInt64{Int64: c.inputIdxMap[idx], Valid: true} } + log.Debugf("👁 %d %s %d", idx, e.Value.Input, urlID.Int64) msmt, err := database.CreateMeasurement(c.Ctx.DB, reportID, testName, resultID, reportFilePath, urlID) if err != nil { log.WithError(err).Error("Failed to create measurement") diff --git a/nettests/websites/web_connectivity.go b/nettests/websites/web_connectivity.go index 6427bd4..4121207 100644 --- a/nettests/websites/web_connectivity.go +++ b/nettests/websites/web_connectivity.go @@ -59,6 +59,7 @@ func lookupURLs(ctl *nettests.Controller) ([]string, map[int64]int64, error) { if err != nil { log.Error("failed to add to the URL table") } + log.Debugf("Mapped URL %s to idx %d and urlID %d", url.URL, idx, urlID) urlIDMap[int64(idx)] = urlID urls = append(urls, url.URL) } From 251f136b53a9f084d158b3bb00f4333c3bdef714 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Sep 2018 11:02:49 +0200 Subject: [PATCH 46/65] Move include_country to the sharing options --- config/parser_test.go | 2 +- config/settings.go | 10 +- config/testdata/valid-config.json | 2 +- data/default-config.json | 2 +- internal/bindata/bindata.go | 229 +++++++++++++++--------------- 5 files changed, 122 insertions(+), 123 deletions(-) diff --git a/config/parser_test.go b/config/parser_test.go index 010dbe6..e26a569 100644 --- a/config/parser_test.go +++ b/config/parser_test.go @@ -13,7 +13,7 @@ func TestParseConfig(t *testing.T) { if len(config.NettestGroups.Middlebox.EnabledTests) < 0 { t.Error("at least one middlebox test should be enabled") } - if config.Advanced.IncludeCountry == false { + if config.Sharing.IncludeCountry == false { t.Error("country should be included") } } diff --git a/config/settings.go b/config/settings.go index c8ddf06..49ca276 100644 --- a/config/settings.go +++ b/config/settings.go @@ -110,15 +110,15 @@ type Notifications struct { // Sharing settings type Sharing struct { - IncludeIP bool `json:"include_ip"` - IncludeASN bool `json:"include_asn"` - IncludeGPS bool `json:"include_gps"` - UploadResults bool `json:"upload_results"` + IncludeIP bool `json:"include_ip"` + IncludeASN bool `json:"include_asn"` + IncludeCountry bool `json:"include_country"` + IncludeGPS bool `json:"include_gps"` + UploadResults bool `json:"upload_results"` } // Advanced settings type Advanced struct { - IncludeCountry bool `json:"include_country"` UseDomainFronting bool `json:"use_domain_fronting"` SendCrashReports bool `json:"send_crash_reports"` } diff --git a/config/testdata/valid-config.json b/config/testdata/valid-config.json index c6c3875..a3fecb9 100644 --- a/config/testdata/valid-config.json +++ b/config/testdata/valid-config.json @@ -5,6 +5,7 @@ "auto_update": true, "sharing": { "include_ip": false, + "include_country": true, "include_asn": true, "include_gps": true, "upload_results": true @@ -56,7 +57,6 @@ } }, "advanced": { - "include_country": true, "use_domain_fronting": false, "send_crash_reports": true } diff --git a/data/default-config.json b/data/default-config.json index cd72e53..6f6c042 100644 --- a/data/default-config.json +++ b/data/default-config.json @@ -7,6 +7,7 @@ "sharing": { "include_ip": false, "include_asn": true, + "include_country": true, "include_gps": true, "upload_results": true }, @@ -57,7 +58,6 @@ } }, "advanced": { - "include_country": true, "use_domain_fronting": false, "send_crash_reports": true } diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index 7734bde..b699aae 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -80,26 +80,26 @@ func (fi bindataFileInfo) Sys() interface{} { } var _bindataDataDefaultconfigjson = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\x41\x6f\xdb\x3c\x0c\x86\xef\xf9\x15\x82\xce\x75\x53\xe0\xbb\xe5" + - "\xf8\xdd\x76\xd8\x3a\x60\xbb\x15\x85\x20\x5b\xb4\x4d\x4c\x26\x35\x91\x4e\x16\x0c\xfd\xef\x83\xdc\x24\x56\xda\xae" + - "\xeb\xd1\xef\x4b\x53\xe2\x43\x52\xbf\x37\xc6\x58\x67\x77\xc6\x7e\x1f\x51\x0c\x8a\x39\xf2\x9c\xcd\xfd\xfd\x97\x4f" + - "\xe6\x6b\xe6\x16\x4c\xc7\xd4\xe3\x60\x7a\x8c\x70\x6b\xbe\x01\x98\x51\x35\xc9\x6e\xbb\x65\x26\xbc\x45\xde\x8e\x10" + - "\xd3\x36\x95\xd8\xa6\x8b\x68\x7a\xce\xa6\x48\xf6\x66\x49\xbd\x87\x2c\xc8\x64\x77\xe6\xee\x59\x40\xea\x39\x4f\x10" + - "\x5c\xc7\x24\x40\x6a\x77\xa6\xf7\x51\xe0\xe4\x8a\x6b\x41\xbd\xdd\x19\xcd\xf3\xb3\xe6\x67\x65\x37\xa7\xe0\x15\x6a" + - "\x59\x46\x9f\x91\x06\xbb\x33\xa5\x06\x63\x2c\x52\x17\xe7\x00\x0e\x53\x9d\xb2\x32\xbc\x50\x95\xa0\x32\x86\x24\xd7" + - "\xc6\x9c\x22\xfb\xe0\x32\xc8\x1c\xf5\xec\x6d\x8c\x79\x5a\x4e\x26\x56\xec\xb1\xf3\x8a\x4c\xb2\x9e\x0f\xe4\xdb\x08" + - "\xe1\x3a\xd3\x12\x7b\x74\x4c\x4e\x41\xd4\x75\x3c\xa5\x08\xfa\x0c\xe4\xcd\x30\x82\x83\x9c\xef\x7f\x39\xb1\x20\x98" + - "\xbc\x42\x58\xb2\x5c\x55\xbd\x9e\x5a\x97\x7c\x52\x97\xf0\x92\xee\x61\x91\x8d\xb1\x07\x68\x9b\x8e\x89\xa0\x53\xdc" + - "\xa3\x1e\xed\xcd\xd9\xe9\x7d\x07\x2d\xf3\x8f\x66\x02\x11\xa0\x01\xf2\xea\x1d\x46\xaf\xe2\x53\x5a\x15\x85\x08\x43" + - "\xf6\xd3\xaa\x04\x2f\xe3\xfa\x45\x41\xd7\x8f\x32\x31\x0d\xd2\xde\x47\x0c\x4d\x86\x9f\x33\x88\x36\x11\x09\x5e\x84" + - "\x8c\xe0\x03\xe4\xa6\x47\x88\xa1\x99\x3c\x61\x9a\xe3\x42\xd9\x2e\x61\x8f\xa7\xe2\x26\x26\x1d\xe3\xd1\xf9\x18\xf9" + - "\xe0\xa9\x2b\x63\x61\xff\xbb\xbb\xfb\xfc\xbf\xbd\x10\x5b\x68\x0b\x68\x81\x55\xf5\xe8\x00\xad\xa0\xc2\xaa\x54\xac" + - "\x3a\xaf\x30\x70\xc6\xc5\x7d\x78\x5c\xec\xa7\xcb\xa4\x88\x7a\x52\x57\xd8\xf8\xa1\x6e\xc0\x3b\xb0\xdf\x87\xfa\x16" + - "\xd6\x1a\xec\x49\xba\xbe\x47\x82\x5c\xb6\xe7\x54\xf4\x47\x6e\x50\x1a\x71\x4e\x55\x77\xc7\x09\xe4\x3d\xe4\x82\xae" + - "\x4c\x97\x7d\xc3\x73\x89\xb3\xbe\x0e\x28\x8d\xfe\xeb\xdf\x95\x79\xfd\xfb\x55\x19\x13\x86\x10\xa1\xe5\x5f\x1f\x2c" + - "\xe2\xdf\x03\xf4\xc1\x11\xba\xf0\x5c\x57\x2b\xec\x0b\xcd\xf0\xfa\x1d\xe9\x78\x26\xcd\xc7\x17\x2f\x83\x80\x0b\x3c" + - "\x79\x24\xd7\x67\xa6\xd3\x2e\xd6\xab\x27\x40\xc1\x75\xb9\x70\xc8\x50\x10\xd4\xef\xc7\xe6\x69\xf3\x27\x00\x00\xff" + - "\xff\x42\x02\xc0\xed\x72\x05\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\x8c\x94\x41\x6f\xdb\x3e\x0c\xc5\xef\xf9\x14\x82\xce\x75\x53\xe0\x7f\xcb" + + "\xf1\x7f\xdb\x61\xeb\x80\xed\x56\x14\x82\x6c\xd1\x36\x31\x99\xd4\x44\x3a\x59\x30\xf4\xbb\x0f\x72\x93\x58\xd9\xb2" + + "\xae\x47\xbf\x47\xd3\x7a\x3f\x52\xfe\xb9\x31\xc6\x3a\xbb\x33\xf6\xeb\x88\x62\x50\xcc\x91\xe7\x6c\x1e\x1f\x3f\x7d" + + "\x30\x9f\x33\xb7\x60\x3a\xa6\x1e\x07\xd3\x63\x84\x7b\xf3\x05\xc0\x8c\xaa\x49\x76\xdb\x2d\x33\xe1\x3d\xf2\x76\x84" + + "\x98\xb6\xa9\xd4\x36\x5d\x44\xd3\x73\x36\x45\xb2\x77\x4b\xeb\x3d\x64\x41\x26\xbb\x33\x0f\xaf\x02\x52\xcf\x79\x82" + + "\xe0\x3a\x26\x01\x52\xbb\x33\xbd\x8f\x02\x27\x57\x5c\x0b\xea\xed\xce\x68\x9e\x5f\x35\x3f\x2b\xbb\x39\x05\xaf\x50" + + "\xcb\x32\xfa\x8c\x34\xd8\x9d\x29\x19\x8c\xb1\x48\x5d\x9c\x03\x38\x4c\x75\xcb\xca\xf0\x42\x55\x83\xca\xe8\x78\x26" + + "\xcd\xc7\xdb\xe6\x90\xe4\xda\x98\x53\x64\x1f\x5c\x06\x99\xa3\x9e\xbd\x8d\x31\x2f\xcb\xb1\x88\x15\x7b\xec\xbc\x22" + + "\x93\xac\x87\x03\xf2\x6d\x84\x70\xdd\x69\xa9\x3d\x3a\x26\xa7\x20\xea\x3a\x9e\x52\x04\x7d\xa5\x75\xb3\x8c\xe0\x20" + + "\xe7\x70\x97\x2f\x16\x3e\x93\x57\x08\x4b\x97\x2b\x24\xeb\x57\x6b\x1e\x27\x75\x29\x2f\xed\x9e\x16\xd9\x18\x7b\x80" + + "\xb6\xe9\x98\x08\x3a\xc5\x3d\xea\xd1\xde\x9d\x9d\xde\x77\xd0\x32\x7f\x6b\x26\x10\x01\x1a\x20\xaf\xde\x61\xf4\x2a" + + "\x3e\xa5\x55\x51\x88\x30\x64\x3f\xad\x4a\xf0\x32\xae\x4f\x14\x74\x7d\x28\xeb\xd4\x20\xed\x7d\xc4\xd0\x64\xf8\x3e" + + "\x83\x68\x13\x91\xe0\xb7\x92\x11\x7c\x80\xdc\xf4\x08\x31\x34\x93\x27\x4c\x73\x5c\x28\xdb\xa5\xec\xf9\x14\x6e\x62" + + "\xd2\x31\x1e\x9d\x8f\x91\x0f\x9e\xba\xb2\x33\xf6\xbf\x87\x87\x8f\xff\xdb\x0b\xb1\x85\xb6\x80\x16\x58\xd5\x8c\x0e" + + "\xd0\x0a\x2a\xac\x4a\xc5\xaa\xf3\x0a\x03\x67\x5c\xdc\xa7\xe7\xc5\x7e\xb9\x6c\x8a\xa8\x27\x75\x85\x8d\x1f\xea\x01" + + "\xbc\x01\xfb\x6d\xa8\xb7\xb0\xd6\x60\x4f\xd2\xf5\x39\x12\xe4\x72\xb5\x4e\xa1\xdf\x73\x82\x32\x88\x73\xab\x7a\x3a" + + "\x4e\x20\xef\x21\x17\x74\x65\xbb\xec\x0d\xcf\x25\xce\xfa\x67\x41\x19\xf4\x5f\xdf\xae\xcc\xeb\xd7\xaf\x62\x4c\x18" + + "\x42\x84\x96\x7f\xbc\x33\xc4\xbf\x17\xe8\x9d\x2b\x74\xe1\xb9\x5e\xad\xb0\x2f\x34\xc3\xba\x23\xb3\x80\x0b\x3c\x79" + + "\x24\xd7\x67\xa6\xd3\x75\xab\x6f\x97\x00\x05\xd7\xe5\x12\x35\x43\x49\x59\xff\x22\x36\x2f\x9b\x5f\x01\x00\x00\xff" + + "\xff\x96\xe8\x5a\xf8\x72\x05\x00\x00") func bindataDataDefaultconfigjsonBytes() ([]byte, error) { return bindataRead( @@ -130,101 +130,100 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\xff\x6f\xdb\x38\xb2\xff\x3d\x7f\xc5\x20\x58\xec\x4b\xf0\x6c\x27" + - "\xed\xeb\x16\xef\x72\xbb\x58\x64\x13\xb5\xe7\xdd\xc6\x29\x1c\xe7\xb6\xc5\xe1\x60\xd1\xd2\xc8\xe6\x86\x22\x55\x92" + - "\xb2\xea\xfb\xeb\x0f\x33\xa4\x64\xc9\x4d\xb3\x29\xb0\xfd\x21\xb5\x25\x72\x38\x5f\x3f\x9f\x19\x7a\x3c\x86\xff\x2d" + - "\xe5\xda\x0a\x8f\x70\x6d\x1a\x7d\xd4\x7f\x70\xe7\x85\xc7\x12\xb5\xff\x05\xd7\x52\x1f\x1d\x5d\xcf\x6f\xdf\xc3\xe2" + - "\xf2\x97\x77\x09\xa4\x16\x5d\xad\xbc\x4b\xff\x3e\x78\x5a\xa2\x70\xb5\xe5\x3d\x87\xaf\x6a\xab\x0e\x1f\x69\xf4\x8d" + - "\xb1\x0f\xf4\xf8\xf1\x73\x13\x9d\x0f\xdf\xdc\x57\x4f\x2a\x78\x35\x4f\x2e\x17\xc9\xe0\x44\x38\x39\x02\x48\x65\x9e" + - "\xc2\x74\xb6\x48\xde\x26\x73\x78\x3f\x9f\xde\x5c\xce\x3f\xc2\x6f\xc9\x47\xb8\xbc\x5f\xdc\x4e\x67\x57\xf3\xe4\x26" + - "\x99\x2d\x46\xb4\xb2\xb6\x2a\x85\x7f\x5e\xce\xaf\xfe\x71\x39\x3f\x79\xf9\xc3\x0f\xa7\x30\xbb\x5d\xc0\xec\xfe\xdd" + - "\xbb\x11\x8c\xc7\xf0\xe1\xc3\x07\x90\x0e\xfc\x46\x3a\x50\x46\xaf\x01\xb5\xa9\xd7\x9b\x9f\x69\x6b\x26\x3c\xae\x8d" + - "\xdd\x2d\x33\x93\xe3\x5e\xc8\xa1\x88\xc5\x06\x21\x93\x5e\xfe\x07\xb5\x12\x2b\x68\x77\x01\xed\x82\xc2\x58\xf0\x1b" + - "\x3c\x82\xe7\xfd\x1b\x8f\xc1\x49\x8f\x13\xf8\x1d\xa1\x76\x48\x5b\xc1\x79\x2b\xf5\x1a\x66\xb7\xb3\x04\xbc\x81\x1c" + - "\xb5\xf1\xdf\x22\x50\x1b\x78\xd0\xa6\xd1\x43\xcd\x26\x47\x6c\xa2\xa9\xb5\xff\xc2\xc2\x97\x7b\x0b\x5b\x03\x7d\x63" + - "\x40\xa1\xf7\x68\x21\xee\x09\xf6\x35\x1b\x99\x6d\xd8\x7d\xcf\xd3\x68\x3c\x86\xfb\xf9\x3b\x58\x21\x39\xdb\x81\x37" + - "\x47\xa7\x21\x59\x7e\x47\xc8\x2c\x52\x12\x08\x70\x58\x09\xce\x07\x2f\x56\x2a\xf8\xb0\x4d\x2d\xfe\xf2\x12\x2c\x0a" + - "\x67\xb4\xbb\xa0\x9d\x2f\x26\xf0\xc6\x58\x70\xa6\x44\x30\x05\xbb\x6c\x2b\xb1\x71\xd0\x6c\xd0\x22\x68\xc4\x9c\x1f" + - "\x7a\xe3\x85\x02\x5d\x97\x2b\xb4\xb4\x30\xe6\x76\xde\xc9\x1e\x91\x34\xe9\xff\xc7\xc1\xda\x90\xc7\xbd\x81\x15\x42" + - "\x59\x67\x1b\x28\x8d\x45\xc0\xa2\x90\x99\x44\xed\xe9\xcd\x1f\xb5\xf3\xa0\x8c\x79\xa8\x2b\x96\xce\x5e\x21\xb1\xd6" + - "\x34\x0e\xa4\x0e\x3e\x19\x8f\x83\x0d\x13\xfa\xf4\x72\x02\x27\xa5\x71\x1e\x64\x59\x19\xeb\x85\xf6\xa7\x64\x76\x23" + - "\x82\x44\xb1\x35\x32\x87\xbc\xae\x94\xcc\x84\x27\x05\x04\xac\x6a\x9d\x6d\x48\xaa\xd4\x85\xb1\xa5\xf0\xd2\x90\x64" + - "\xe1\x59\xd5\xa1\xa2\x99\x29\x4b\x7a\x6b\xc0\xe1\x16\x2d\xd9\xda\x3a\x8d\x14\xac\x1d\x5a\xda\x62\x34\x2b\x93\x7c" + - "\x16\x65\xa5\xf0\x22\xfa\xbe\x14\x3b\x68\xa4\xdb\xb0\x22\x79\x4e\xff\x71\x4d\x84\x08\xd0\x7e\x65\xb2\x70\x7c\x61" + - "\x4d\xd9\x3a\xba\xb2\x66\x85\xe1\x09\x7d\x7d\xfb\xfe\x8e\xe4\x19\xcb\x32\x5c\x5d\x91\x9d\x1c\x32\xa1\x94\x69\x58" + - "\xd7\x56\x15\x6f\xe0\x38\x33\xd6\x62\xe6\x8f\x41\x40\x29\x5d\xa6\x84\x73\xb2\x90\x98\x43\x0f\x77\xa2\xc0\x5c\x3a" + - "\xf2\x49\x2d\xdd\x86\xc4\xac\xd0\x37\x88\x1a\x1a\x59\x48\x10\x3a\x87\xd2\xac\x24\xf9\x79\x08\x19\x1d\x22\x7d\x2b" + - "\x6c\xc4\x8d\x4b\x2d\x4a\x7c\x0a\x3f\xee\x42\x71\xd2\x32\xb0\x58\x59\x74\xa8\x7d\x6b\x67\x5f\x48\xac\x94\xd5\x0e" + - "\x72\x2c\x44\xad\x3c\xc5\xa2\x32\x55\xad\x84\xc7\x1c\x56\xc2\x61\xfe\x67\x25\x44\x9e\xd0\x2c\xf9\xf2\x6e\x36\x79" + - "\xc6\xea\x88\x22\xbd\x8a\x7a\xc0\x1d\x79\xde\x62\x81\x16\x75\x16\x42\x1b\x53\xf6\x19\x02\xf7\x39\xe1\x46\xb0\xc2" + - "\x4c\x90\xf8\x66\x98\x3e\xc7\xa8\xad\xcc\x36\xc7\xcf\x15\xd7\x48\x1f\x0b\x2c\x17\x5e\x84\xd2\x41\x28\x6a\x5f\x5b" + - "\x9c\xf4\x63\xe1\x77\x55\x2f\x16\x2f\x5e\x1f\x84\xe2\x56\x73\xfd\x53\x46\x8c\x62\x3a\x10\xc6\x41\x2a\xab\xfd\xae" + - "\x57\xe7\xfd\x5d\x21\x82\xc6\xa2\x23\x1f\x85\x50\x76\x51\x0c\xd9\x6e\x0a\x10\x1a\x64\xb5\x7d\x45\x59\x28\xab\xed" + - "\x6b\xca\x6d\x8b\xce\x3d\x27\x00\x0b\x2e\x1c\xbd\x46\xaa\xfa\x8a\x42\x1e\x84\x75\x42\x40\xc9\x07\xbc\x78\x86\xa4" + - "\xf3\xf3\xf3\xf3\x8b\x3f\xff\x33\x7a\x86\xa8\x90\x89\xd2\xc1\xff\xfd\x0d\xb2\x8d\xb0\x6c\x49\x2a\x9c\xe6\xe2\x38" + - "\x79\xd5\xf3\xd0\x5f\xc1\x11\x8c\xf0\xc3\xb2\x6c\x9b\x0d\xae\xca\x6f\xa9\xcb\xe8\x54\xe9\x20\x13\x9a\x40\xcf\x84" + - "\xa0\x1f\x37\xb8\x22\xe2\x74\xc7\x23\x38\x96\x25\xfd\xad\xd0\x32\x64\xea\x0c\xe9\x6b\x29\xf3\x5c\xe1\xca\x7c\x3e" + - "\x0e\x71\x4b\x3d\x3a\xbf\x5c\x5b\x53\x57\x07\x45\x3e\x48\xac\xf6\xcc\xae\x92\x72\x59\x70\xe9\x78\x70\x5e\x58\xbf" + - "\xf4\xb2\x44\x06\x20\x5b\x6b\xfa\x3c\x28\x8b\x0e\xda\x95\x33\xb0\x11\x5b\x6c\xc5\x71\xa6\x7b\xd3\xe2\x1c\x67\xbc" + - "\xd9\xa2\xdd\xa0\xc8\xc9\x1e\xa6\xc2\x40\x01\x16\x19\x44\xe9\x08\xe3\x37\x68\xa1\x10\x99\x37\xd6\x05\x1a\x88\xf2" + - "\xd6\x06\xa4\x66\xcc\x46\x20\xc3\x26\x7b\x5f\x09\x46\x1a\x62\x05\xb1\xbb\x80\xf4\xee\xfe\xe6\x24\xaa\x7a\x0a\x6f" + - "\xe6\xb7\x37\x30\xe8\xf1\xa0\x91\x4a\x81\x50\x8d\xd8\x39\xf2\xef\x8f\x3f\xb5\x92\xd2\xb8\x2b\x6c\xda\x47\x90\x19" + - "\x8d\x5e\x38\xf8\xf1\x34\xb8\x76\xef\x99\x14\xae\x2f\x17\xc9\x62\x7a\x93\x1c\xb8\xb4\x95\x96\xc2\x3c\xb9\x7c\x37" + - "\x3a\x6a\x4f\xb9\x77\xc8\xe4\x23\x75\x4e\x2c\x88\x20\x8b\x3d\x65\x6c\x84\x03\x47\xa8\xcf\xb8\x11\x54\x88\x09\xe4" + - "\x96\xc4\xf9\x98\xa7\xb0\x98\xce\x3e\x52\x1a\xbf\xe8\x47\x71\x90\x3a\x54\x85\x50\x28\xb1\x26\xa9\x8f\x9e\x16\x44" + - "\xd3\xc2\x9c\x13\x8c\x89\x33\xab\x2d\xc5\x5d\xed\x28\xd4\x5a\xea\xf5\xa4\x3b\x9b\x56\x7d\xe5\x64\x5e\x42\xe1\x5e" + - "\xd6\x4e\xac\x71\x59\x57\xfb\x54\xff\xfa\xaa\xdc\x34\xfa\x6b\xeb\xc6\x63\x98\x52\x93\x42\xdc\x2b\x56\xa4\x0e\x37" + - "\x43\x81\xa8\x89\xfc\x3d\xdb\x50\x8a\xcf\xb2\xac\x4b\x50\xa8\xd7\x9e\x81\xf9\xe5\xeb\x73\x10\xb1\xd7\xe5\x9e\xb7" + - "\x4b\xc7\x83\xb5\xa6\x80\x42\x2a\x84\x4a\xf8\x0d\x35\x0c\xd0\x48\x9d\x9b\x26\x22\x5e\x7f\x28\x58\xe6\xd2\xf6\x30" + - "\xe1\xf5\xf9\x17\x4e\xef\xd0\xbb\x5f\xe3\x43\x83\xde\xdc\xce\x93\xe9\xdb\x19\x55\xfc\x49\x7f\xf9\x29\xcc\x93\x37" + - "\xc9\x3c\x99\x5d\x25\x77\x03\x2a\x27\xbc\x38\x7d\x04\x56\x86\x99\xfc\x17\x61\xcb\x45\xfb\xaa\x10\x19\xae\x8c\x79" + - "\x58\x96\xe8\x1c\xea\x35\xda\xf6\x8d\x47\x85\x6b\x2b\xca\xa3\x0e\x60\x85\x77\xa2\xaa\xda\xef\x1b\xef\xab\x25\x95" + - "\x36\xda\x65\x21\x51\xe5\xcb\x52\x68\xc9\xd4\x2f\x8d\x1e\xac\x92\x7a\x2b\x94\xcc\x97\x16\x3f\xd5\x04\x50\x4a\xea" + - "\x1e\x68\xb8\x4d\xfb\x59\xe7\xbe\x07\x63\x43\x00\x7b\xfd\xea\x8b\x04\xfc\xe6\x8a\x7c\xa4\x76\x66\xc6\x87\x5e\x7c" + - "\x6d\x94\xd0\xeb\x0b\x02\xb8\xb6\x80\x08\xdb\x08\x10\x3d\xf6\x40\x39\x0d\x65\x41\xc8\x95\x8a\xcc\xcb\x2d\xa6\x23" + - "\x70\xe6\xa8\x4f\xfe\xd2\x01\x7e\xaa\xe5\x56\xa8\xd8\x57\x73\xc1\xad\x90\x5b\x28\x5b\x73\xed\x15\x42\x39\xec\x10" + - "\x2d\xe5\x63\x52\x58\x24\x1f\x62\xd8\x9e\x51\x81\x91\xa1\x42\xa5\x74\x0a\x0b\xc8\x31\x14\x7e\x0e\xd2\x2d\xeb\x4a" + - "\x19\x91\x63\xce\xd8\x30\x02\xa9\x9d\x8f\x70\xcc\xcd\x7e\xed\xa4\x5e\xb7\xd2\xba\xe5\xcb\x42\x48\x85\xf9\x28\x54" + - "\x94\xf0\x6d\x37\xa4\x8d\x0f\x87\x74\x52\xb9\x28\xf7\x19\x0a\x79\xdd\x45\x96\x28\x82\x0a\xd7\x0f\x50\xa5\xdd\xf9" + - "\x4c\x4c\x3b\x94\x1f\x14\xe3\x6e\xaf\xd6\x1c\x85\x0e\x48\xdd\xc6\xd4\x2a\xe7\x50\x11\x9b\x59\x5e\xd6\xca\xb3\x38" + - "\xa6\x0d\xd2\xef\x35\x09\xa2\x9e\x42\x38\x5a\x51\x5b\x5c\x96\x6e\x3d\x6c\x97\x5b\x18\x38\x74\xd8\x53\xc2\x7a\x0b" + - "\x9f\x92\x49\x40\xe8\xbe\xc4\x70\x8e\x02\x27\x57\x25\xac\x97\x59\xad\x84\x1d\x38\x86\x68\x64\x45\x34\x12\x2d\x15" + - "\x3a\xdf\xe7\x24\x5a\x2c\x4c\x64\xe4\xfb\x29\x43\x81\x17\x0f\x18\xb3\x95\x38\x56\x64\x61\xd6\xf3\x06\x50\x32\x23" + - "\x6f\x64\x8e\x20\x7d\x37\x07\xed\x3d\xc9\x34\x42\xcc\xc4\x33\x51\x00\xe6\x2d\xda\x1d\x28\x14\xce\xd3\x50\xd3\xcd" + - "\x57\x62\x25\x95\xf4\xb1\x3b\x1f\x44\x20\x5e\x53\xe4\x86\x72\x8b\x5b\x89\xb6\xaf\x88\x59\xdc\xeb\xe6\x4d\xe4\x2f" + - "\x16\xd0\x33\xfa\xe7\x2e\x0a\x16\x6d\xad\xbf\x21\xa5\x1c\xda\x2d\xda\xb1\x23\x1b\x43\x2f\xb2\x94\x39\x58\xf4\xb5" + - "\xd5\x34\xb8\xec\xe2\xf8\xab\x14\x52\x5f\x32\x81\x5f\x76\xc3\x52\xd9\x6f\xfa\x1e\xa4\xae\x6a\x3f\x82\x9d\xa9\xd9" + - "\xb3\x9f\x6a\xf2\x05\x5b\x5f\x49\x52\xbe\x40\x1f\xaf\x13\xfa\xca\x77\x6e\x48\x3e\x77\x1f\xdf\x26\x0b\x46\x4c\x77" + - "\x71\x76\x26\x2a\x39\x31\x46\xcb\x89\x34\xf4\xf9\x6c\xfb\xe2\xac\x4f\x05\x3f\xf3\xa9\x3f\x7d\x37\x9d\xbd\xbf\x5f" + - "\x7c\xdf\xa9\xf3\xd3\x77\xf3\xe4\xfd\xed\x7c\xb1\x9c\x5e\xef\xe5\x7b\x2b\xb2\x10\xa6\x42\x5a\x6a\xda\x3d\x96\xfb" + - "\xf9\x36\x36\xae\xff\xfa\x77\x0a\x4a\x3a\xdf\x16\x95\x0e\x7a\x77\x4c\x31\x60\x48\xbe\x80\xf2\x06\xd6\x91\x97\x7f" + - "\xbd\xbb\x9d\x85\xf1\x79\x68\x24\x4d\x61\xbd\xb6\x0d\x5d\x68\xa8\xb7\x42\xd5\xe8\xe0\x24\xed\xf4\x4e\x47\x90\xb2" + - "\x45\xe9\x29\x08\xcb\x15\x5d\xd4\x6a\xef\x3d\xd1\xb1\x7a\x4f\x38\x17\x05\x25\xbe\x50\x16\x45\xbe\x0b\x05\x50\x59" + - "\x93\x11\x99\x75\x61\xac\x64\x85\x44\x39\xa3\x1e\x1e\xc8\xb2\x52\x41\x48\xa6\x50\xe8\xba\xe2\x99\x28\x8a\xe9\xd0" + - "\xad\xef\xf0\x08\x1c\x7b\x8d\x87\xd5\x7b\xc8\xb3\x3c\x43\x34\xe4\x46\x6d\xda\x76\x97\xfb\x8f\xb6\x50\xff\x64\xa6" + - "\x19\x8f\xe3\xd5\x51\x3e\x89\x60\x53\x5b\xd5\xef\x37\xbe\x4c\x6c\x42\xe7\x1d\x7a\xea\x03\x51\xd0\xa0\xd9\x5e\x68" + - "\x74\x79\x3c\x82\x55\xcd\x60\x4e\x2e\xae\x94\xe0\x8e\x2f\xde\x8e\x0c\x18\x4c\xf8\x70\xf5\x54\x19\xa9\x7d\x3b\xbd" + - "\x6a\x14\xb6\x37\xc2\x86\x41\x13\xf1\xa2\x4b\xd9\xb5\xf4\x9b\x7a\x35\xc9\x4c\x79\x46\x99\x7b\xd6\x3a\xfe\x6c\xa5" + - "\xcc\xea\xac\x14\xce\xa3\x3d\xcb\x4d\xe6\xf8\xf5\xb8\xae\x65\x3e\x29\x73\xf8\xbe\xdf\x2c\x3c\x29\x47\x3a\x57\xa3" + - "\x3b\x7b\xf5\xff\x5f\xf6\x6c\xd1\x33\xd4\x27\x1c\x7a\x26\x62\xa8\x6b\xed\xc8\x84\x63\xd7\x08\x68\x07\x2c\x1e\x2f" + - "\x46\x21\x9f\x04\x5f\x56\x92\x3f\x69\x8a\x55\xbb\x56\xd6\x4a\x99\xec\x81\x38\x91\xc8\x9b\x80\x4f\xc3\xf4\x86\x37" + - "\xb6\x0d\x76\xfc\xea\x68\x28\x71\x11\x00\xaa\xa7\x05\xc9\x82\x6f\x89\xe2\x14\x07\x8d\x70\x90\xa3\xc7\x8c\xc3\x1e" + - "\xd7\x7f\x8c\xb8\x92\xfe\x7a\x3b\x9d\xa5\x20\x20\xbd\xba\xbd\x9f\x2d\x4e\x4e\xd3\xae\xe2\xb8\x9e\x5a\xf3\xe2\x14" + - "\x13\x10\x3a\xd6\xa8\xe8\xae\xf2\x0e\xb4\x80\x60\xbf\xb1\xdd\x83\xe9\x0d\xa9\xed\x3a\x68\x15\xda\x94\x42\xed\xfa" + - "\xe0\xfa\xc8\xe8\xa1\xc1\x54\xe2\x53\x1d\x91\xc0\x79\x5b\x67\x94\x27\xa3\x78\x5f\xd9\x50\x23\x45\x0c\xd4\xbf\xd0" + - "\xe4\x2e\xef\x01\x77\xae\x6b\x31\xe3\xc5\x66\xbc\x5f\x1e\x36\x16\xe8\x85\x54\x2e\xde\x82\x12\x46\xb1\xa8\x1e\x1b" + - "\x39\x38\xc1\xcf\x93\x3e\x55\x85\x3a\x3e\xa3\xb9\x83\x3e\x80\xab\x48\xba\x29\x60\x76\xbd\x18\x45\x5f\x71\xef\x54" + - "\xb4\xf6\x53\x39\x70\x66\x90\x5b\xba\x2e\x0b\x7d\x36\x39\xed\x75\xa6\xa4\x73\x1a\x2c\x7d\x8c\x66\x10\x32\x6b\x5c" + - "\x7b\xcd\x38\xe0\x31\x0a\x61\xb0\xbd\x31\xf1\xca\x09\xbc\x59\x23\x31\x6e\x07\x30\x64\xd1\x57\x26\x8b\x47\x66\xbe" + - "\xad\xb0\x92\x0f\xe2\x9e\x41\x6a\x8f\x56\x0b\xa5\x98\x73\x09\xf8\x1f\x02\x06\x8a\x30\x2e\xf1\x4c\xae\xc7\xb9\x74" + - "\x0f\x8f\x20\xaa\x9b\xfc\xe1\x8c\x9e\xc0\xd4\x73\xbb\x57\x52\x8f\xe0\x50\x3b\xd6\xbd\xb1\x54\x17\xd4\xc9\x86\x39" + - "\x0a\x2d\x20\xdf\x87\xac\xba\xd4\xde\x18\xc3\x2e\xbc\xf9\x8d\x23\x54\x59\xdc\xc6\x9b\xc3\xb6\x91\x20\x21\x2d\xe6" + - "\x04\x39\x46\x53\xc7\xf0\x10\x2f\x70\x4a\xb1\x17\x46\x7d\x40\x29\xf4\x6e\xa0\x21\x9f\x5b\xf0\xad\x68\x1f\x8f\xe9" + - "\xc9\x92\x8c\x7c\x7a\x70\xeb\x4d\x65\xcc\x3e\xad\xab\x87\x53\x59\x7b\x0f\x10\x67\xb2\x00\xc8\xb7\x33\xb8\x4e\xde" + - "\x25\x8b\x04\xae\x2e\xef\xae\x2e\xaf\x13\x7a\x72\xff\x9e\x06\x8f\xf6\x09\x93\xc0\xb4\xa0\x3c\xce\x51\xa1\x0f\x6d" + - "\x0c\x27\x68\xbf\xc9\x79\xee\xcf\x1c\xd1\x0f\x42\xa9\xc3\x6a\x70\xf1\x76\x3c\x9c\x92\xd3\x00\xdc\xa0\x52\x93\x47" + - "\x6c\x8c\xac\x31\x34\x90\x7f\x74\xda\x4f\x9c\x5f\xfd\x59\xeb\xbf\x01\x00\x00\xff\xff\xbd\xb9\x3d\xa5\x79\x1b\x00" + - "\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x73\xdb\x36\x12\xfe\xee\x5f\xb1\xe3\xe9\xf4\xec\x39\x49\x76" + + "\x72\x69\xe6\xce\xd7\x4e\xc7\xb5\x99\x9c\xda\x58\xce\xc8\xf2\x35\x99\x9b\x1b\x11\x22\x97\x12\x6a\x10\x60\x00\x50" + + "\x8c\xee\xd7\xdf\xec\x02\xa4\x48\xc5\x75\x9d\x99\xe6\x83\x23\x91\xc0\x62\x5f\x9f\x67\x17\x1a\x8f\xe1\xaf\xa5\x5c" + + "\x5b\xe1\x11\xae\x4d\xa3\x8f\xfa\x0f\xee\xbc\xf0\x58\xa2\xf6\x3f\xe1\x5a\xea\xa3\xa3\xeb\xf9\xed\x7b\x58\x5c\xfe" + + "\xf4\x2e\x81\xd4\xa2\xab\x95\x77\xe9\x3f\x07\x4f\x4b\x14\xae\xb6\xbc\xe7\xf0\x55\x6d\xd5\xe1\x23\x8d\xbe\x31\xf6" + + "\x81\x1e\x3f\x7e\x6e\xa2\xf3\xe1\x9b\xfb\xea\x49\x05\xaf\xe6\xc9\xe5\x22\x19\x9c\x08\x27\x47\x00\xa9\xcc\x53\x98" + + "\xce\x16\xc9\xdb\x64\x0e\xef\xe7\xd3\x9b\xcb\xf9\x47\xf8\x25\xf9\x08\x97\xf7\x8b\xdb\xe9\xec\x6a\x9e\xdc\x24\xb3" + + "\xc5\x88\x56\xd6\x56\xa5\xf0\xef\xcb\xf9\xd5\xbf\x2e\xe7\x27\x2f\xbf\xfb\xee\x14\x66\xb7\x0b\x98\xdd\xbf\x7b\x37" + + "\x82\xf1\x18\x3e\x7c\xf8\x00\xd2\x81\xdf\x48\x07\xca\xe8\x35\xa0\x36\xf5\x7a\xf3\x23\x6d\xcd\x84\xc7\xb5\xb1\xbb" + + "\x65\x66\x72\xdc\x0b\x39\x14\xb1\xd8\x20\x64\xd2\xcb\xff\xa1\x56\x62\x05\xed\x2e\xa0\x5d\x50\x18\x0b\x7e\x83\x47" + + "\xf0\xbc\x7f\xe3\x31\x38\xe9\x71\x02\xbf\x22\xd4\x0e\x69\x2b\x38\x6f\xa5\x5e\xc3\xec\x76\x96\x80\x37\x90\xa3\x36" + + "\xfe\x6b\x04\x6a\x03\x0f\xda\x34\x7a\xa8\xd9\xe4\x88\x4d\x34\xb5\xf6\x5f\x58\xf8\x72\x6f\x61\x6b\xa0\x6f\x0c\x28" + + "\xf4\x1e\x2d\xc4\x3d\xc1\xbe\x66\x23\xb3\x0d\xbb\xef\x79\x1a\x8d\xc7\x70\x3f\x7f\x07\x2b\x24\x67\x3b\xf0\xe6\xe8" + + "\x34\x24\xcb\xaf\x08\x99\x45\x4a\x02\x01\x0e\x2b\xc1\xf9\xe0\xc5\x4a\x05\x1f\xb6\xa9\xc5\x5f\x5e\x82\x45\xe1\x8c" + + "\x76\x17\xb4\xf3\xc5\x04\xde\x18\x0b\xce\x94\x08\xa6\x60\x97\x6d\x25\x36\x0e\x9a\x0d\x5a\x04\x8d\x98\xf3\x43\x6f" + + "\xbc\x50\xa0\xeb\x72\x85\x96\x16\xc6\xdc\xce\x3b\xd9\x23\x92\x26\xfd\x5f\x1c\xac\x0d\x79\xdc\x1b\x58\x21\x94\x75" + + "\xb6\x81\xd2\x58\x04\x2c\x0a\x99\x49\xd4\x9e\xde\xfc\x56\x3b\x0f\xca\x98\x87\xba\x62\xe9\xec\x15\x12\x6b\x4d\xe3" + + "\x40\xea\xe0\x93\xf1\x38\xd8\x30\xa1\x4f\x2f\x27\x70\x52\x1a\xe7\x41\x96\x95\xb1\x5e\x68\x7f\x4a\x66\x37\x22\x48" + + "\x14\x5b\x23\x73\xc8\xeb\x4a\xc9\x4c\x78\x52\x40\xc0\xaa\xd6\xd9\x86\xa4\x4a\x5d\x18\x5b\x0a\x2f\x0d\x49\x16\x9e" + + "\x55\x1d\x2a\x9a\x99\xb2\xa4\xb7\x06\x1c\x6e\xd1\x92\xad\xad\xd3\x48\xc1\xda\xa1\xa5\x2d\x46\xb3\x32\xc9\x67\x51" + + "\x56\x0a\x2f\xa2\xef\x4b\xb1\x83\x46\xba\x0d\x2b\x92\xe7\xf4\x1f\xd7\x44\x88\x00\xed\x57\x26\x0b\xc7\x17\xd6\x94" + + "\xad\xa3\x2b\x6b\x56\x18\x9e\xd0\xd7\xb7\xef\xef\x48\x9e\xb1\x2c\xc3\xd5\x15\xd9\xc9\x21\x13\x4a\x99\x86\x75\x6d" + + "\x55\xf1\x06\x8e\x33\x63\x2d\x66\xfe\x18\x04\x94\xd2\x65\x4a\x38\x27\x0b\x89\x39\xf4\x70\x27\x0a\xcc\xa5\x23\x9f" + + "\xd4\xd2\x6d\x48\xcc\x0a\x7d\x83\xa8\xa1\x91\x85\x04\xa1\x73\x28\xcd\x4a\x92\x9f\x87\x90\xd1\x21\xd2\xd7\xc2\x46" + + "\xdc\xb8\xd4\xa2\xc4\xa7\xf0\xe3\x2e\x14\x27\x2d\x03\x8b\x95\x45\x87\xda\xb7\x76\xf6\x85\xc4\x4a\x59\xed\x20\xc7" + + "\x42\xd4\xca\x53\x2c\x2a\x53\xd5\x4a\x78\xcc\x61\x25\x1c\xe6\x7f\x54\x42\xe4\x09\xcd\x92\x2f\xef\x66\x93\x67\xac" + + "\x8e\x28\xd2\xab\xa8\x07\xdc\x91\xe7\x2d\x16\x68\x51\x67\x21\xb4\x31\x65\x9f\x21\x70\x9f\x13\x6e\x04\x2b\xcc\x04" + + "\x89\x6f\x86\xe9\x73\x8c\xda\xca\x6c\x73\xfc\x5c\x71\x8d\xf4\xb1\xc0\x72\xe1\x45\x28\x1d\x84\xa2\xf6\xb5\xc5\x49" + + "\x3f\x16\x7e\x57\xf5\x62\xf1\xe2\xf5\x41\x28\x6e\x35\xd7\x3f\x65\xc4\x28\xa6\x03\x63\x9c\xac\xf6\x9b\x5e\x9d\xf7" + + "\x37\x85\x00\x1a\x8b\x8e\x5c\x14\x22\xd9\x05\x31\x24\xbb\x29\x40\x68\x90\xd5\xf6\x15\x25\xa1\xac\xb6\xaf\x29\xb5" + + "\x2d\x3a\xf7\x1c\xff\x2f\xb8\x6e\xf4\x1a\xa9\xe8\x2b\x8a\x78\x10\xd6\x09\x01\x25\x1f\xf0\xe2\x19\x92\xce\xcf\xcf" + + "\xcf\x2f\xfe\xf8\xcf\xe8\x19\xa2\x42\x22\x4a\x07\x7f\xfb\x07\x64\x1b\x61\xd9\x92\x54\x38\xcd\xb5\x71\xf2\xaa\xe7" + + "\xa1\x3f\x83\x22\x18\xe0\x87\x55\xd9\xf6\x1a\x5c\x94\x5f\x53\x96\xd1\xa9\xd2\x41\x26\x34\x61\x9e\x09\x31\x3f\x6e" + + "\x70\x45\xbc\xe9\x8e\x47\x70\x2c\x4b\xfa\x5b\xa1\x65\xc4\xd4\x19\xd2\xd7\x52\xe6\xb9\xc2\x95\xf9\x7c\x1c\xe2\x96" + + "\x7a\x74\x7e\xb9\xb6\xa6\xae\x0e\x6a\x7c\x90\x57\xed\x99\x5d\x21\xe5\xb2\xe0\xca\xf1\xe0\xbc\xb0\x7e\xe9\x65\x89" + + "\x8c\x3f\xb6\xd6\xf4\x79\x50\x15\x1d\xb2\x2b\x67\x60\x23\xb6\xd8\x8a\xe3\x44\xf7\xa6\x85\x39\x4e\x78\xb3\x45\xbb" + + "\x41\x91\x93\x3d\xcc\x84\x81\x01\x2c\x32\x86\xd2\x11\xc6\x6f\xd0\x42\x21\x32\x6f\xac\x0b\x2c\x10\xe5\xad\x0d\x48" + + "\xcd\x90\x8d\x40\x86\x4d\xf6\xbe\x12\x0c\x34\x44\x0a\x62\x77\x01\xe9\xdd\xfd\xcd\x49\x54\xf5\x14\xde\xcc\x6f\x6f" + + "\x60\xd0\xe2\x41\x23\x95\x02\xa1\x1a\xb1\x73\xe4\xdf\xef\x7f\x68\x25\xa5\x71\x57\xd8\xb4\x8f\x20\x13\x1a\xbd\x70" + + "\xf0\xfd\x69\x70\xed\xde\x33\x29\x5c\x5f\x2e\x92\xc5\xf4\x26\x39\x70\x69\x2b\x2d\x85\x79\x72\xf9\x6e\x74\xd4\x9e" + + "\x72\xef\x90\xb9\x47\xea\x9c\x48\x10\x41\x16\x7b\xc6\xd8\x08\x07\x8e\x40\x9f\x61\x23\xa8\x10\x13\xc8\x2d\x89\xf2" + + "\x31\x4f\x61\x31\x9d\x7d\xa4\x34\x7e\xd1\x8f\xe2\x20\x75\xa8\x0a\xa1\x50\x62\x4d\x52\x1f\x3d\x2d\x88\xa6\x85\x39" + + "\x27\x18\xf3\x66\x56\x5b\x8a\xbb\xda\x51\xa8\xb5\xd4\xeb\x49\x77\x36\xad\xfa\x9d\x93\x79\x09\x85\x7b\x59\x3b\xb1" + + "\xc6\x65\x5d\x05\x93\x9f\x58\x92\x9b\x46\x3f\xba\x68\x3c\x86\x29\x75\x27\x44\xba\x62\x45\x8a\x70\x17\x14\x18\x9a" + + "\x58\xdf\xb3\xf6\xa5\xf8\x2c\xcb\xba\x04\x85\x7a\xed\x19\x91\x5f\xbe\x3e\x07\x11\x9b\x5c\x6e\x76\xbb\x44\x3c\x58" + + "\x6b\x0a\x28\xa4\x42\xa8\x84\xdf\x50\xa7\x00\x8d\xd4\xb9\x69\x22\xd6\xf5\xa7\x81\x65\x2e\x6d\x0f\x0d\x5e\x9f\x7f" + + "\xe1\xee\x0e\xb6\xfb\xd5\x3d\x34\xe8\xcd\xed\x3c\x99\xbe\x9d\x51\xad\x9f\xf4\x97\x9f\xc2\x3c\x79\x93\xcc\x93\xd9" + + "\x55\x72\x37\xe0\x70\x42\x8a\xd3\x47\x00\x65\x98\xc3\x7f\x12\xaa\x5c\xb4\xaf\x0a\x91\xe1\xca\x98\x87\x65\x89\xce" + + "\xa1\x5e\xa3\x6d\xdf\x78\x54\xb8\xb6\xa2\x3c\xea\xa0\x55\x78\x27\xaa\xaa\xfd\xbe\xf1\xbe\x5a\x52\x51\xa3\x5d\x16" + + "\x12\x55\xbe\x2c\x85\x96\xcc\xf9\xd2\xe8\xc1\x2a\xa9\xb7\x42\xc9\x7c\x69\xf1\x53\x4d\xd0\xa4\xa4\xee\xc1\x85\xdb" + + "\xb4\x9f\x75\xee\x7b\x00\x36\x84\xae\xd7\xaf\xbe\x48\xbd\xaf\xae\xc5\x47\xaa\x66\x66\x7c\x68\xc2\xd7\x46\x09\xbd" + + "\xbe\x20\x68\x6b\x4b\x87\x50\x8d\xa0\xd0\x63\x0f\x8e\xd3\x50\x10\x84\x59\xa9\xc8\xbc\xdc\x62\x3a\x02\x67\x8e\xfa" + + "\xac\x2f\x1d\xe0\xa7\x5a\x6e\x85\x8a\x0d\x35\x97\xda\x0a\xb9\x77\xb2\x35\x57\x5d\x21\x94\xc3\x0e\xcb\x52\x3e\x26" + + "\x85\x45\xf2\x21\x86\xed\x19\xb5\x17\xb9\x29\x54\x4a\xa7\xb0\x80\x1c\x43\xc9\xe7\x20\xdd\xb2\xae\x94\x11\x39\xe6" + + "\x8c\x0a\x23\x90\xda\xf9\x08\xc4\xdc\xe5\xd7\x4e\xea\x75\x2b\xad\x5b\xbe\x2c\x84\x54\x98\x8f\x42\x45\x09\xdf\xb6" + + "\x41\xda\xf8\x70\x48\x27\x95\x8b\x72\x9f\xa1\x90\xd7\x5d\x64\x89\x1c\xa8\x70\xfd\x00\x4f\xda\x9d\xcf\x44\xb3\x43" + + "\xf9\x41\x31\x6e\xf3\x6a\xcd\x51\xe8\x20\xd4\x6d\x4c\xad\x72\x0e\x15\xf1\x98\xe5\x65\xad\x3c\x8b\x63\xda\x20\xfd" + + "\x5e\x93\x20\xea\x29\x6c\xa3\x15\xb5\xc5\x65\xe9\xd6\xc3\x3e\xb9\x85\x81\x43\x87\x3d\x25\xac\xb7\xf0\x29\x99\x04" + + "\x84\xee\x4b\xf4\xe6\x28\x70\x72\x55\xc2\x7a\x99\xd5\x4a\xd8\x81\x63\x88\x40\x56\x44\x20\xd1\x52\xa1\xf3\x7d\x4e" + + "\xa2\xc5\xc2\x44\x2e\xbe\x9f\x32\x14\x78\xf1\x80\x31\x5b\x89\x5d\x45\x16\x86\x3c\x6f\x00\x25\x73\xf1\x46\xe6\x08" + + "\xd2\x77\x03\xd0\xde\x93\x4c\x20\xc4\x49\x3c\x0c\x05\x60\xde\xa2\xdd\x81\x42\xe1\x3c\x4d\x33\xdd\x60\x25\x56\x52" + + "\x49\x1f\xdb\xf2\x41\x04\xe2\xfd\x44\x6e\x28\xb7\xb8\x89\x68\x3b\x8a\x98\xc5\xbd\x36\xde\x44\xe6\x62\x01\x3d\xa3" + + "\x7f\xec\xa2\x60\xd1\xd6\xfa\x2b\x52\xca\xa1\xdd\xa2\x1d\x3b\xb2\x31\x74\x21\x4b\x99\x83\x45\x5f\x5b\x4d\x13\xcb" + + "\x2e\xce\xbd\x4a\x21\x75\x24\x13\xf8\x69\x37\x2c\x95\xfd\xa6\x6f\x41\xea\xaa\xf6\x23\xd8\x99\x9a\x3d\xfb\xa9\x26" + + "\x5f\xb0\xf5\x95\x24\xe5\x0b\xf4\xf1\x1e\xa1\xaf\x7c\xe7\x86\xe4\x73\xf7\xf1\x6d\xb2\x60\xc4\x74\x17\x67\x67\xa2" + + "\x92\x13\x63\xb4\x9c\x48\x43\x9f\xcf\xb6\x2f\xce\xfa\x54\xf0\x23\x9f\xfa\xc3\x37\xd3\xd9\xfb\xfb\xc5\xb7\x9d\x3a" + + "\x3f\x7c\x33\x4f\xde\xdf\xce\x17\xcb\xe9\xf5\x5e\xbe\xb7\x22\x0b\x61\x2a\xa4\xa5\x76\xdd\x63\xb9\x1f\x6c\x63\xcb" + + "\xfa\x9f\xff\xa6\xa0\xa4\xf3\x6d\x51\xe9\xa0\x77\xc7\x14\x03\x86\xe4\x9b\x27\x6f\x60\x1d\x79\xf9\xe7\xbb\xdb\x59" + + "\x98\x9b\x87\x46\xd2\xf8\xd5\x6b\xd8\xd0\x85\x56\x7a\x2b\x54\x8d\x0e\x4e\xd2\x4e\xef\x74\x04\x29\x5b\x94\x9e\x82" + + "\xb0\x5c\xd1\x45\xad\xf6\xde\x13\x1d\xab\xf7\x84\x73\x51\x50\xe2\x0b\x65\x51\xe4\xbb\x50\x00\x95\x35\x19\x91\x59" + + "\x17\xc6\x4a\x56\x48\x94\x33\xea\xe1\x81\x2c\x2b\x15\x84\x64\x0a\x85\xae\x2b\x9e\x86\xa2\x98\x0e\xdd\xfa\x0e\x8f" + + "\xc0\xb1\xd7\x78\x58\xbd\x87\x3c\xcb\xd3\x43\x43\x6e\xd4\xa6\x6d\x74\xb9\xff\x68\x0b\xf5\x0f\xa6\x99\xf1\x38\xde" + + "\x19\xe5\x93\x08\x36\xb5\x55\xfd\x7e\xe3\xcb\xc4\x26\x74\xde\xa1\xa7\x0e\x10\x05\x4d\x98\xed\x4d\x46\x97\xc7\x23" + + "\x58\xd5\x0c\xe6\xe4\xe2\x4a\x09\xee\xf5\xe2\xb5\xc8\x80\xc1\x84\x0f\x77\x4e\x95\x91\xda\xb7\x63\xab\x46\x61\x7b" + + "\xb3\x6b\x18\x31\x11\x2f\xba\x94\x5d\x4b\xbf\xa9\x57\x93\xcc\x94\x67\x94\xb9\x67\xad\xe3\xcf\x56\xca\xac\xce\x4a" + + "\xe1\x3c\xda\xb3\xdc\x64\x8e\x5f\x8f\xeb\x5a\xe6\x93\x32\x87\x6f\xfb\xcd\xc2\x93\x72\xa4\x73\x35\xba\xb3\x57\x7f" + + "\xff\xb2\x67\x8b\x9e\xa1\x3e\xe1\xd0\x33\x11\x43\x5d\x6b\x47\x26\x1c\xbb\x46\x40\x3b\x5a\xf1\x60\x31\x0a\xf9\x24" + + "\xf8\x96\x92\xfc\x49\xf3\xab\xda\xb5\xb2\x56\xca\x64\x0f\xc4\x89\x44\xde\x04\x7c\x1a\xa6\x37\xbc\xb1\x6d\xad\xe3" + + "\x57\x47\xe3\x88\x8b\x00\x50\x3d\x2d\x48\x16\x7c\x3d\x14\xe7\x37\x68\x84\x83\x1c\x3d\x66\x1c\xf6\xb8\xfe\x63\xc4" + + "\x95\xf4\xe7\xdb\xe9\x2c\x05\x01\xe9\xd5\xed\xfd\x6c\x71\x72\x9a\x76\x15\xc7\xf5\xd4\x9a\x17\xe7\x97\x80\xd0\xb1" + + "\x46\x45\x77\x87\x77\xa0\x05\x04\xfb\x8d\xed\x1e\x4c\x6f\x48\x6d\xd7\x41\xab\xd0\xa6\x14\x6a\xd7\x07\xd7\x47\x86" + + "\x0e\x0d\xa6\x12\x9f\xea\x88\x04\xce\xdb\x3a\xa3\x3c\x19\xc5\x8b\xca\x86\x1a\x29\x62\xa0\xfe\x4d\x26\x77\x79\x0f" + + "\xb8\x73\x5d\x8b\x19\x6f\x34\xe3\xc5\xf2\xb0\xb1\x40\x2f\xa4\x72\xf1\xfa\x93\x30\x8a\x45\xf5\xd8\xc8\xc1\x09\x7e" + + "\x9e\xf4\xa9\x2a\xd4\xf1\x19\x0d\x1d\xf4\x01\x5c\x45\xd2\x4d\x01\xb3\xeb\xc5\x28\xfa\x8a\x7b\xa7\xa2\xb5\x9f\xca" + + "\x81\x33\x83\xdc\xd2\x75\x59\xe8\xb3\xc9\x69\xaf\x33\x25\x9d\xd3\x60\xe9\x63\x34\x83\x90\x59\xe3\xda\xfb\xc5\x01" + + "\x8f\x51\x08\x83\xed\x8d\x89\x77\x4d\xe0\xcd\x1a\x89\x71\x3b\x80\x21\x8b\x7e\x67\xb2\x78\x64\xda\xdb\x0a\x2b\xf9" + + "\x20\xee\x19\xa4\xf6\x68\xb5\x50\x8a\x39\x97\x80\xff\x21\x60\xa0\x08\xe3\x12\x4f\xe3\x7a\x9c\x4b\xf7\xf0\x08\xa2" + + "\xba\xc9\x6f\xce\xe8\x09\x4c\x3d\xb7\x7b\x25\xf5\x08\x0e\xb5\x63\xdd\x1b\x4b\x75\x41\x9d\x6c\x98\xa3\xd0\x02\xf2" + + "\x4d\xc8\xaa\x4b\xed\x8d\x31\xec\xc2\x9b\x5f\x38\x42\x95\xc5\x6d\xbc\x32\x6c\x1b\x09\x12\xd2\x62\x4e\x90\x63\x34" + + "\x75\x0c\x0f\xf1\xea\xa6\x14\x7b\x61\xd4\x07\x94\x42\xef\x06\x1a\xf2\xb9\x05\x5f\x87\xf6\xf1\x98\x9e\x2c\xc9\xc8" + + "\xa7\x07\xb7\xde\x54\xc6\xec\xd3\xba\x7a\x38\x95\xb5\x37\x00\x71\x26\x0b\x80\x7c\x3b\x83\xeb\xe4\x5d\xb2\x48\xe0" + + "\xea\xf2\xee\xea\xf2\x3a\xa1\x27\xf7\xef\x69\xf0\x68\x9f\x30\x09\x4c\x0b\xca\xe3\x1c\x15\xfa\xd0\xc6\x70\x82\xf6" + + "\x9b\x9c\xe7\xfe\xbe\x11\xfd\x20\x94\x3a\xac\x06\x17\xaf\xc5\xc3\x29\x39\x0d\xc0\x0d\x2a\x35\x79\xc4\xc6\xc8\x1a" + + "\x43\x03\xf9\xd7\xa6\xfd\xc4\xf9\xbb\xbf\x67\xfd\x3f\x00\x00\xff\xff\xc9\xf1\x5a\xc3\x72\x1b\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( From 4ed94dfc5392143ac62a54dfd2f01bc759cb58c8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Sep 2018 14:54:56 +0200 Subject: [PATCH 47/65] Improve the presentation of the measurement listing from the CLI --- cmd/ooni/main.go | 7 +- internal/cli/list/list.go | 34 ++++--- internal/crashreport/crashreport.go | 25 ++++-- internal/log/handlers/cli/cli.go | 4 + internal/log/handlers/cli/measurements.go | 103 ++++++++++++++++++++++ internal/onboard/onboard.go | 2 +- internal/output/output.go | 25 ++++-- nettests/nettests.go | 2 +- 8 files changed, 177 insertions(+), 25 deletions(-) create mode 100644 internal/log/handlers/cli/measurements.go diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index d6567bc..0492259 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -3,6 +3,7 @@ package main import ( // commands + "github.com/apex/log" _ "github.com/ooni/probe-cli/internal/cli/geoip" _ "github.com/ooni/probe-cli/internal/cli/info" _ "github.com/ooni/probe-cli/internal/cli/list" @@ -19,5 +20,9 @@ import ( ) func main() { - crashreport.CapturePanicAndWait(app.Run, nil) + err, _ := crashreport.CapturePanic(app.Run, nil) + if err != nil { + log.WithError(err.(error)).Error("panic in app.Run") + crashreport.Wait() + } } diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 4a8bbf4..0534a87 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -27,27 +27,41 @@ func init() { } msmtSummary := output.MeasurementSummaryData{ - TotalCount: 0, - AnomalyCount: 0, - DataUsageUp: 0.0, - DataUsageDown: 0.0, - TotalRuntime: 0, + TotalCount: 0, + AnomalyCount: 0, + DataUsageUp: 0.0, + DataUsageDown: 0.0, + TotalRuntime: 0, + ASN: 0, + NetworkName: "", + NetworkCountryCode: "ZZ", } - for _, msmt := range measurements { + isFirst := true + isLast := false + for idx, msmt := range measurements { + if idx > 0 { + isFirst = false + } + if idx == len(measurements)-1 { + isLast = true + } + // We assume that since these are summary level information the first // item will contain the information necessary. - if msmtSummary.TotalRuntime == 0 { + if isFirst { msmtSummary.TotalRuntime = msmt.ResultRuntime - } - if msmtSummary.DataUsageUp == 0 { msmtSummary.DataUsageUp = msmt.DataUsageUp msmtSummary.DataUsageDown = msmt.DataUsageDown + msmtSummary.NetworkName = msmt.NetworkName + msmtSummary.NetworkCountryCode = msmt.NetworkCountryCode + msmtSummary.ASN = msmt.ASN + msmtSummary.StartTime = msmt.MeasurementStartTime } if msmt.IsAnomaly.Bool == true { msmtSummary.AnomalyCount++ } msmtSummary.TotalCount++ - output.MeasurementItem(msmt) + output.MeasurementItem(msmt, isFirst, isLast) } output.MeasurementSummary(msmtSummary) } else { diff --git a/internal/crashreport/crashreport.go b/internal/crashreport/crashreport.go index 21e2c1c..29801f2 100644 --- a/internal/crashreport/crashreport.go +++ b/internal/crashreport/crashreport.go @@ -1,6 +1,7 @@ package crashreport import ( + "github.com/apex/log" "github.com/getsentry/raven-go" ) @@ -8,13 +9,15 @@ import ( // crash reporting logic a no-op. var Disabled = false +var client *raven.Client + // CapturePanic is a wrapper around raven.CapturePanic that becomes a noop if // `Disabled` is set to true. func CapturePanic(f func(), tags map[string]string) (interface{}, string) { if Disabled == true { return nil, "" } - return raven.CapturePanic(f, tags) + return client.CapturePanic(f, tags) } // CapturePanicAndWait is a wrapper around raven.CapturePanicAndWait that becomes a noop if @@ -23,7 +26,7 @@ func CapturePanicAndWait(f func(), tags map[string]string) (interface{}, string) if Disabled == true { return nil, "" } - return raven.CapturePanicAndWait(f, tags) + return client.CapturePanicAndWait(f, tags) } // CaptureError is a wrapper around raven.CaptureError @@ -31,7 +34,7 @@ func CaptureError(err error, tags map[string]string) string { if Disabled == true { return "" } - return raven.CaptureError(err, tags) + return client.CaptureError(err, tags) } // CaptureErrorAndWait is a wrapper around raven.CaptureErrorAndWait @@ -39,9 +42,21 @@ func CaptureErrorAndWait(err error, tags map[string]string) string { if Disabled == true { return "" } - return raven.CaptureErrorAndWait(err, tags) + return client.CaptureErrorAndWait(err, tags) +} + +// Wait will block on sending messages to the sentry server +func Wait() { + if Disabled == false { + log.Info("sending exception backtrace") + client.Wait() + } } func init() { - raven.SetDSN("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954") + var err error + client, err = raven.NewClient("https://cb4510e090f64382ac371040c19b2258:8448daeebfa643c289ef398f8645980b@sentry.io/1234954", nil) + if err != nil { + log.WithError(err).Error("failed to create a raven client") + } } diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index 8e4c2ec..a167ad6 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -114,6 +114,10 @@ func (h *Handler) TypedLog(t string, e *log.Entry) error { return nil case "table": return logTable(h.Writer, e.Fields) + case "measurement_item": + return logMeasurementItem(h.Writer, e.Fields) + case "measurement_summary": + return logMeasurementSummary(h.Writer, e.Fields) case "result_item": return logResultItem(h.Writer, e.Fields) case "result_summary": diff --git a/internal/log/handlers/cli/measurements.go b/internal/log/handlers/cli/measurements.go new file mode 100644 index 0000000..e3c4f0c --- /dev/null +++ b/internal/log/handlers/cli/measurements.go @@ -0,0 +1,103 @@ +package cli + +import ( + "fmt" + "io" + "strings" + "time" + + "github.com/apex/log" + "github.com/ooni/probe-cli/internal/util" +) + +func statusIcon(ok bool) string { + if ok { + return "✓" + } + return "❌" +} +func logMeasurementItem(w io.Writer, f log.Fields) error { + colWidth := 24 + + rID := f.Get("id").(int64) + testName := f.Get("test_name").(string) + + // We currently don't use these fields in the view + //testGroupName := f.Get("test_group_name").(string) + //networkName := f.Get("network_name").(string) + //asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string)) + testKeys := f.Get("test_keys").(string) + + isAnomaly := f.Get("is_anomaly").(bool) + isFailed := f.Get("is_failed").(bool) + isUploaded := f.Get("is_uploaded").(bool) + url := f.Get("url").(string) + urlCategoryCode := f.Get("url_category_code").(string) + + isFirst := f.Get("is_first").(bool) + isLast := f.Get("is_last").(bool) + if isFirst { + fmt.Fprintf(w, "┏"+strings.Repeat("━", colWidth*2+2)+"┓\n") + } else { + fmt.Fprintf(w, "┢"+strings.Repeat("━", colWidth*2+2)+"┪\n") + } + + anomalyStr := fmt.Sprintf("ok: %s", statusIcon(!isAnomaly)) + uploadStr := fmt.Sprintf("uploaded: %s", statusIcon(isUploaded)) + failureStr := fmt.Sprintf("success: %s", statusIcon(!isFailed)) + + fmt.Fprintf(w, fmt.Sprintf("│ %s │\n", + util.RightPad( + fmt.Sprintf("#%d", rID), colWidth*2))) + + if url != "" { + fmt.Fprintf(w, fmt.Sprintf("│ %s │\n", + util.RightPad( + fmt.Sprintf("%s (%s)", url, urlCategoryCode), colWidth*2))) + } + + fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", + util.RightPad(testName, colWidth), + util.RightPad(anomalyStr, colWidth))) + + fmt.Fprintf(w, fmt.Sprintf("│ %s │\n", + util.RightPad(testKeys, colWidth*2))) + fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", + util.RightPad(failureStr, colWidth), + util.RightPad(uploadStr, colWidth))) + + if isLast { + fmt.Fprintf(w, "└┬────────────────────────────────────────────────┬┘\n") + } + return nil +} + +func logMeasurementSummary(w io.Writer, f log.Fields) error { + colWidth := 12 + + totalCount := f.Get("total_count").(int64) + anomalyCount := f.Get("anomaly_count").(int64) + totalRuntime := f.Get("total_runtime").(float64) + dataUp := f.Get("data_usage_up").(float64) + dataDown := f.Get("data_usage_down").(float64) + + startTime := f.Get("start_time").(time.Time) + + asn := f.Get("asn").(uint) + countryCode := f.Get("network_country_code").(string) + networkName := f.Get("network_name").(string) + + fmt.Fprintf(w, " │ %s │\n", + util.RightPad(startTime.Format(time.RFC822), (colWidth+3)*3), + ) + fmt.Fprintf(w, " │ %s │\n", + util.RightPad(fmt.Sprintf("AS%d, %s (%s)", asn, networkName, countryCode), (colWidth+3)*3), + ) + fmt.Fprintf(w, " │ %s %s %s │\n", + util.RightPad(fmt.Sprintf("%.2fs", totalRuntime), colWidth), + util.RightPad(fmt.Sprintf("%d/%d anmls", anomalyCount, totalCount), colWidth), + util.RightPad(fmt.Sprintf("⬆ %s ⬇ %s", formatSize(dataUp), formatSize(dataDown)), colWidth+4)) + fmt.Fprintf(w, " └────────────────────────────────────────────────┘\n") + + return nil +} diff --git a/internal/onboard/onboard.go b/internal/onboard/onboard.go index f34a9ff..2144d0d 100644 --- a/internal/onboard/onboard.go +++ b/internal/onboard/onboard.go @@ -126,7 +126,7 @@ func Onboarding(config *config.Config) error { config.Lock() config.InformedConsent = true - config.Advanced.IncludeCountry = settings.IncludeCountry + config.Sharing.IncludeCountry = settings.IncludeCountry config.Advanced.SendCrashReports = settings.SendCrashReports config.Sharing.IncludeIP = settings.IncludeIP config.Sharing.IncludeASN = settings.IncludeNetwork diff --git a/internal/output/output.go b/internal/output/output.go index 8a1328a..0391cd3 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -21,11 +21,15 @@ func Progress(key string, perc float64, msg string) { } type MeasurementSummaryData struct { - TotalRuntime float64 - TotalCount int64 - AnomalyCount int64 - DataUsageUp float64 - DataUsageDown float64 + TotalRuntime float64 + TotalCount int64 + AnomalyCount int64 + DataUsageUp float64 + DataUsageDown float64 + ASN uint + NetworkName string + NetworkCountryCode string + StartTime time.Time } func MeasurementSummary(msmt MeasurementSummaryData) { @@ -36,13 +40,20 @@ func MeasurementSummary(msmt MeasurementSummaryData) { "anomaly_count": msmt.AnomalyCount, "data_usage_down": msmt.DataUsageDown, "data_usage_up": msmt.DataUsageUp, + "asn": msmt.ASN, + "network_country_code": msmt.NetworkCountryCode, + "network_name": msmt.NetworkName, + "start_time": msmt.StartTime, }).Info("measurement summary") } // MeasurementItem logs a progress type event -func MeasurementItem(msmt database.MeasurementURLNetwork) { +func MeasurementItem(msmt database.MeasurementURLNetwork, isFirst bool, isLast bool) { log.WithFields(log.Fields{ - "type": "measurement_item", + "type": "measurement_item", + "is_first": isFirst, + "is_last": isLast, + "id": msmt.MsmtTblID, "test_name": msmt.TestName, "test_group_name": msmt.Result.TestGroupName, diff --git a/nettests/nettests.go b/nettests/nettests.go index c01b290..e02caaf 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -120,7 +120,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.Options = mk.NettestOptions{ IncludeIP: c.Ctx.Config.Sharing.IncludeIP, IncludeASN: c.Ctx.Config.Sharing.IncludeASN, - IncludeCountry: c.Ctx.Config.Advanced.IncludeCountry, + IncludeCountry: c.Ctx.Config.Sharing.IncludeCountry, LogLevel: "DEBUG", ProbeCC: c.Ctx.Location.CountryCode, From 867204adfb9d5e203fed1443dc7b3c270c1adaba Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Thu, 13 Sep 2018 15:59:29 +0200 Subject: [PATCH 48/65] Improve the testKeys logging --- internal/log/handlers/cli/measurements.go | 32 +++++++++++++++++++++-- 1 file changed, 30 insertions(+), 2 deletions(-) diff --git a/internal/log/handlers/cli/measurements.go b/internal/log/handlers/cli/measurements.go index e3c4f0c..85dca51 100644 --- a/internal/log/handlers/cli/measurements.go +++ b/internal/log/handlers/cli/measurements.go @@ -1,6 +1,8 @@ package cli import ( + "bytes" + "encoding/json" "fmt" "io" "strings" @@ -16,6 +18,28 @@ func statusIcon(ok bool) string { } return "❌" } + +func logTestKeys(w io.Writer, testKeys string) error { + colWidth := 24 + + var out bytes.Buffer + if err := json.Indent(&out, []byte(testKeys), "", " "); err != nil { + return err + } + + testKeysLines := strings.Split(string(out.Bytes()), "\n") + if len(testKeysLines) > 1 { + testKeysLines = testKeysLines[1 : len(testKeysLines)-1] + testKeysLines[0] = "{" + testKeysLines[0][1:] + testKeysLines[len(testKeysLines)-1] = testKeysLines[len(testKeysLines)-1] + "}" + } + for _, line := range testKeysLines { + fmt.Fprintf(w, fmt.Sprintf("│ %s │\n", + util.RightPad(line, colWidth*2))) + } + return nil +} + func logMeasurementItem(w io.Writer, f log.Fields) error { colWidth := 24 @@ -60,12 +84,16 @@ func logMeasurementItem(w io.Writer, f log.Fields) error { util.RightPad(testName, colWidth), util.RightPad(anomalyStr, colWidth))) - fmt.Fprintf(w, fmt.Sprintf("│ %s │\n", - util.RightPad(testKeys, colWidth*2))) fmt.Fprintf(w, fmt.Sprintf("│ %s %s│\n", util.RightPad(failureStr, colWidth), util.RightPad(uploadStr, colWidth))) + if testKeys != "" { + if err := logTestKeys(w, testKeys); err != nil { + return err + } + } + if isLast { fmt.Fprintf(w, "└┬────────────────────────────────────────────────┬┘\n") } From 5cae6b0b83da46bf19ddd45b936c7112e6d90499 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 11:51:54 +0200 Subject: [PATCH 49/65] Make it possible to disable uploading of results --- internal/cli/run/run.go | 6 ++++++ nettests/nettests.go | 2 +- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/internal/cli/run/run.go b/internal/cli/run/run.go index 47333ff..e88a483 100644 --- a/internal/cli/run/run.go +++ b/internal/cli/run/run.go @@ -26,6 +26,8 @@ func init() { fmt.Sprintf("the nettest group to run. Supported tests are: %s", strings.Join(nettestGroupNames, ", "))).Required().String() + noCollector := cmd.Flag("no-collector", "Disable uploading measurements to a collector").Bool() + cmd.Action(func(_ *kingpin.ParseContext) error { log.Infof("Starting %s", *nettestGroup) ctx, err := root.Init() @@ -39,6 +41,10 @@ func init() { return err } + if *noCollector == true { + ctx.Config.Sharing.UploadResults = false + } + group, ok := groups.NettestGroups[*nettestGroup] if !ok { log.Errorf("No test group named %s", *nettestGroup) diff --git a/nettests/nettests.go b/nettests/nettests.go index e02caaf..730ad0f 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -128,7 +128,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { ProbeIP: c.Ctx.Location.IP, DisableReportFile: false, - DisableCollector: false, + DisableCollector: !c.Ctx.Config.Sharing.UploadResults, RandomizeInput: false, // It's important to disable input randomization to ensure the URLs are written in sync to the DB SoftwareName: "ooniprobe", SoftwareVersion: ooni.Version, From bd6a5acf5b33083167bf578e49a0e1f223b81c32 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 11:52:10 +0200 Subject: [PATCH 50/65] Sent ooniprobe-desktop as the software name --- nettests/nettests.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/nettests/nettests.go b/nettests/nettests.go index 730ad0f..e7b8ed1 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -130,7 +130,7 @@ func (c *Controller) Init(nt *mk.Nettest) error { DisableReportFile: false, DisableCollector: !c.Ctx.Config.Sharing.UploadResults, RandomizeInput: false, // It's important to disable input randomization to ensure the URLs are written in sync to the DB - SoftwareName: "ooniprobe", + SoftwareName: "ooniprobe-desktop", SoftwareVersion: ooni.Version, OutputPath: msmtPath, From 4f6c8aa18b69987501e92250ed314cd8cd8632e5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 11:52:42 +0200 Subject: [PATCH 51/65] Basic ooni unittest --- ooni_test.go | 27 +++++++++++++++++++++++++++ 1 file changed, 27 insertions(+) create mode 100644 ooni_test.go diff --git a/ooni_test.go b/ooni_test.go new file mode 100644 index 0000000..0e2df77 --- /dev/null +++ b/ooni_test.go @@ -0,0 +1,27 @@ +package ooni + +import ( + "io/ioutil" + "os" + "path" + "testing" +) + +func TestInit(t *testing.T) { + ooniHome, err := ioutil.TempDir("", "oonihome") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(ooniHome) + + ctx := NewContext("", ooniHome) + if err := ctx.Init(); err != nil { + t.Error(err) + t.Fatal("failed to init the context") + } + + configPath := path.Join(ooniHome, "config.json") + if _, err := os.Stat(configPath); os.IsNotExist(err) { + t.Fatal("config file was not created") + } +} From 4738f7f32d54e4d258144f69b63cf0e976438617 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 13:00:00 +0200 Subject: [PATCH 52/65] Workaround to keep track of upload state --- nettests/nettests.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/nettests/nettests.go b/nettests/nettests.go index e7b8ed1..3b4455b 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -181,7 +181,6 @@ func (c *Controller) Init(nt *mk.Nettest) error { if c.inputIdxMap != nil { urlID = sql.NullInt64{Int64: c.inputIdxMap[idx], Valid: true} } - log.Debugf("👁 %d %s %d", idx, e.Value.Input, urlID.Int64) msmt, err := database.CreateMeasurement(c.Ctx.DB, reportID, testName, resultID, reportFilePath, urlID) if err != nil { log.WithError(err).Error("Failed to create measurement") @@ -250,8 +249,11 @@ func (c *Controller) Init(nt *mk.Nettest) error { nt.On("status.measurement_submission", func(e mk.Event) { log.Debugf(color.RedString(e.Key)) - if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil { - log.WithError(err).Error("failed to mark msmt as uploaded") + // XXX maybe this should change once MK is aligned with the spec + if c.Ctx.Config.Sharing.UploadResults == true { + if err := c.msmts[e.Value.Idx].UploadSucceeded(c.Ctx.DB); err != nil { + log.WithError(err).Error("failed to mark msmt as uploaded") + } } }) From 6c49bd694febcdd5e5b4a7fadc0f2c786943cd0b Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 13:04:48 +0200 Subject: [PATCH 53/65] Improve output of incomplete measurements Rename done to is_done --- internal/log/handlers/cli/results.go | 8 +++++++- internal/output/output.go | 2 +- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/internal/log/handlers/cli/results.go b/internal/log/handlers/cli/results.go index 4ed7365..2c367aa 100644 --- a/internal/log/handlers/cli/results.go +++ b/internal/log/handlers/cli/results.go @@ -80,6 +80,7 @@ func logResultItem(w io.Writer, f log.Fields) error { rID := f.Get("id").(int64) name := f.Get("name").(string) + isDone := f.Get("is_done").(bool) startTime := f.Get("start_time").(time.Time) networkName := f.Get("network_name").(string) asn := fmt.Sprintf("AS%d (%s)", f.Get("asn").(uint), f.Get("network_country_code").(string)) @@ -114,7 +115,12 @@ func logResultItem(w io.Writer, f log.Fields) error { util.RightPad(summary[2], colWidth))) if index == totalCount-1 { - fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────────┬┘\n") + if isDone == true { + fmt.Fprintf(w, "└┬──────────────┬──────────────┬──────────────────┬┘\n") + } else { + // We want the incomplete section to not have a footer + fmt.Fprintf(w, "└──────────────────────────────────────────────────┘\n") + } } return nil } diff --git a/internal/output/output.go b/internal/output/output.go index 0391cd3..0081bb2 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -110,7 +110,7 @@ func ResultItem(result ResultItemData) { "network_name": result.NetworkName, "asn": result.ASN, "runtime": result.Runtime, - "done": result.Done, + "is_done": result.Done, "data_usage_down": result.DataUsageDown, "data_usage_up": result.DataUsageUp, "index": result.Index, From d55cccc236cdbeadcadb71ae4657f005961ee08d Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 16:16:56 +0200 Subject: [PATCH 54/65] Exit with non-zero code when trying to do interactive onboarding and --batch is set --- internal/cli/app/app.go | 3 ++- internal/cli/onboard/onboard.go | 5 +++++ internal/cli/root/root.go | 3 +++ ooni.go | 4 ++++ 4 files changed, 14 insertions(+), 1 deletion(-) diff --git a/internal/cli/app/app.go b/internal/cli/app/app.go index c1b4cfb..0cb7a07 100644 --- a/internal/cli/app/app.go +++ b/internal/cli/app/app.go @@ -13,7 +13,8 @@ func Run() { root.Cmd.Version(ooni.Version) _, err := root.Cmd.Parse(os.Args[1:]) if err != nil { - log.WithError(err).Error("failed to parse arguments") + log.WithError(err).Error("failure in main command") + os.Exit(1) } return } diff --git a/internal/cli/onboard/onboard.go b/internal/cli/onboard/onboard.go index 108672a..ddbe97f 100644 --- a/internal/cli/onboard/onboard.go +++ b/internal/cli/onboard/onboard.go @@ -1,6 +1,8 @@ package onboard import ( + "errors" + "github.com/alecthomas/kingpin" "github.com/apex/log" "github.com/ooni/probe-cli/internal/cli/root" @@ -29,6 +31,9 @@ func init() { } return nil } + if ctx.IsBatch == true { + return errors.New("cannot do onboarding in batch mode") + } return onboard.Onboarding(ctx.Config) }) diff --git a/internal/cli/root/root.go b/internal/cli/root/root.go index c7749da..dd34dfb 100644 --- a/internal/cli/root/root.go +++ b/internal/cli/root/root.go @@ -48,6 +48,9 @@ func init() { if err != nil { return nil, err } + if *isBatch { + ctx.IsBatch = true + } return ctx, nil } diff --git a/ooni.go b/ooni.go index fdd740a..52d85bd 100644 --- a/ooni.go +++ b/ooni.go @@ -23,6 +23,7 @@ type Context struct { Config *config.Config DB sqlbuilder.Database Location *utils.LocationInfo + IsBatch bool Home string TempDir string @@ -60,6 +61,9 @@ func (c *Context) LocationLookup() error { // config option is set to false func (c *Context) MaybeOnboarding() error { if c.Config.InformedConsent == false { + if c.IsBatch == true { + return errors.New("cannot run onboarding in batch mode") + } if err := onboard.Onboarding(c.Config); err != nil { return errors.Wrap(err, "onboarding") } From 30c60414230be09460154b15ff55ed21767d9943 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 16:33:56 +0200 Subject: [PATCH 55/65] Remove unused nettest command --- cmd/ooni/main.go | 1 - internal/cli/nettest/nettest.go | 17 ----------------- internal/cli/reset/reset.go | 2 +- 3 files changed, 1 insertion(+), 19 deletions(-) delete mode 100644 internal/cli/nettest/nettest.go diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index 0492259..017382b 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -7,7 +7,6 @@ import ( _ "github.com/ooni/probe-cli/internal/cli/geoip" _ "github.com/ooni/probe-cli/internal/cli/info" _ "github.com/ooni/probe-cli/internal/cli/list" - _ "github.com/ooni/probe-cli/internal/cli/nettest" _ "github.com/ooni/probe-cli/internal/cli/onboard" _ "github.com/ooni/probe-cli/internal/cli/reset" _ "github.com/ooni/probe-cli/internal/cli/run" diff --git a/internal/cli/nettest/nettest.go b/internal/cli/nettest/nettest.go deleted file mode 100644 index 3517a66..0000000 --- a/internal/cli/nettest/nettest.go +++ /dev/null @@ -1,17 +0,0 @@ -package nettest - -import ( - "github.com/alecthomas/kingpin" - "github.com/apex/log" - "github.com/ooni/probe-cli/internal/cli/root" -) - -func init() { - cmd := root.Command("nettest", "Run a specific nettest") - - cmd.Action(func(_ *kingpin.ParseContext) error { - log.Info("Nettest") - log.Error("this function is not implemented") - return nil - }) -} diff --git a/internal/cli/reset/reset.go b/internal/cli/reset/reset.go index 9a07ed8..bc7562e 100644 --- a/internal/cli/reset/reset.go +++ b/internal/cli/reset/reset.go @@ -1,4 +1,4 @@ -package clean +package reset import ( "os" From 19f07208c344feb917e367a731b06b80c7eb8eea Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 16:38:35 +0200 Subject: [PATCH 56/65] Update to latest mk --- Gopkg.lock | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/Gopkg.lock b/Gopkg.lock index c9e1d9b..cc34202 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -68,7 +68,7 @@ branch = "master" name = "github.com/measurement-kit/go-measurement-kit" packages = ["."] - revision = "4b5013c619f6f0293fd67b6892a7f90066d3c18d" + revision = "18bd1b84e534aacc7292c22da5c900b6d7bff41b" [[projects]] branch = "master" From 1fe8eedfb343f173be1a6d010f727e0dbb98b754 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 16:38:41 +0200 Subject: [PATCH 57/65] Bump ooni version number --- ooni.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ooni.go b/ooni.go index 52d85bd..cb7d78a 100644 --- a/ooni.go +++ b/ooni.go @@ -16,7 +16,7 @@ import ( "upper.io/db.v3/lib/sqlbuilder" ) -const Version = "3.0.0-dev.0" +const Version = "3.0.0-dev.1" // Context for OONI Probe type Context struct { From 52bfe5df6ca954e7e90f7a4f148ddf59322d3d71 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 17:30:29 +0200 Subject: [PATCH 58/65] Add support for deleting measurements --- internal/database/actions.go | 29 +++++++++--- internal/database/actions_test.go | 74 +++++++++++++++++++++++++++++++ internal/database/database.go | 5 +++ 3 files changed, 102 insertions(+), 6 deletions(-) diff --git a/internal/database/actions.go b/internal/database/actions.go index bb0ea74..d712e16 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "encoding/json" + "os" "reflect" "time" @@ -112,24 +113,40 @@ func ListResults(sess sqlbuilder.Database) ([]ResultNetwork, []ResultNetwork, er incompleteResults := []ResultNetwork{} req := sess.Select( - "networks.id AS network_id", - "results.id AS result_id", db.Raw("networks.*"), db.Raw("results.*"), ).From("results"). - Join("networks").On("results.network_id = networks.id"). - OrderBy("results.start_time") + Join("networks").On("results.network_id = networks.network_id"). + OrderBy("results.result_start_time") - if err := req.Where("is_done = true").All(&doneResults); err != nil { + if err := req.Where("result_is_done = true").All(&doneResults); err != nil { return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") } - if err := req.Where("is_done = false").All(&incompleteResults); err != nil { + if err := req.Where("result_is_done = false").All(&incompleteResults); err != nil { return doneResults, incompleteResults, errors.Wrap(err, "failed to get result done list") } return doneResults, incompleteResults, nil } +// DeleteResult will delete a particular result and the relative measurement on +// disk. +func DeleteResult(sess sqlbuilder.Database, resultID int64) error { + var result Result + res := sess.Collection("results").Find("result_id", resultID) + if err := res.One(&result); err != nil { + log.WithError(err).Error("error in obtaining the result") + return err + } + if err := res.Delete(); err != nil { + log.WithError(err).Error("failed to delete the result directory") + return err + } + + os.RemoveAll(result.MeasurementDir) + return nil +} + // CreateMeasurement writes the measurement to the database a returns a pointer // to the Measurement func CreateMeasurement(sess sqlbuilder.Database, reportID sql.NullString, testName string, resultID int64, reportFilePath string, urlID sql.NullInt64) (*Measurement, error) { diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 608aa3d..152a6b8 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -84,6 +84,80 @@ func TestMeasurementWorkflow(t *testing.T) { } } +func TestDeleteResult(t *testing.T) { + tmpfile, err := ioutil.TempFile("", "dbtest") + if err != nil { + t.Fatal(err) + } + fmt.Printf("%s", tmpfile.Name()) + //defer os.Remove(tmpfile.Name()) + + tmpdir, err := ioutil.TempDir("", "oonitest") + if err != nil { + t.Fatal(err) + } + defer os.RemoveAll(tmpdir) + + sess, err := Connect(tmpfile.Name()) + if err != nil { + t.Fatal(err) + } + + location := utils.LocationInfo{ + ASN: 0, + CountryCode: "IT", + NetworkName: "Unknown", + } + network, err := CreateNetwork(sess, &location) + if err != nil { + t.Fatal(err) + } + + result, err := CreateResult(sess, tmpdir, "websites", network.ID) + if err != nil { + t.Fatal(err) + } + + reportID := sql.NullString{String: "", Valid: false} + testName := "antani" + resultID := result.ID + reportFilePath := tmpdir + urlID := sql.NullInt64{Int64: 0, Valid: false} + + m1, err := CreateMeasurement(sess, reportID, testName, resultID, reportFilePath, urlID) + if err != nil { + t.Fatal(err) + } + + var m2 Measurement + err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2) + if err != nil { + t.Fatal(err) + } + if m2.ResultID != m1.ResultID { + t.Error("result_id mismatch") + } + + err = DeleteResult(sess, resultID) + if err != nil { + t.Fatal(err) + } + totalResults, err := sess.Collection("results").Find().Count() + if err != nil { + t.Fatal(err) + } + totalMeasurements, err := sess.Collection("measurements").Find().Count() + if err != nil { + t.Fatal(err) + } + if totalResults != 0 { + t.Fatal("results should be zero") + } + if totalMeasurements != 0 { + t.Fatal("measurements should be zero") + } +} + func TestNetworkCreate(t *testing.T) { tmpfile, err := ioutil.TempFile("", "dbtest") if err != nil { diff --git a/internal/database/database.go b/internal/database/database.go index 8da63f7..577e022 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -30,8 +30,13 @@ func RunMigrations(db *sql.DB) error { func Connect(path string) (db sqlbuilder.Database, err error) { settings := sqlite.ConnectionURL{ Database: path, + Options: map[string]string{"_foreign_keys": "1"}, } sess, err := sqlite.Open(settings) + if err != nil { + log.WithError(err).Error("failed to open the DB") + return nil, err + } err = RunMigrations(sess.Driver().(*sql.DB)) if err != nil { From 18a89f4cbd1e116cee1c800a879d5eefd473b807 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 17:30:38 +0200 Subject: [PATCH 59/65] [schema-change] Prefix all columns that could cause conflicts --- data/migrations/1_create_msmt_results.sql | 58 ++++--- internal/bindata/bindata.go | 190 +++++++++++----------- internal/cli/app/app.go | 2 +- internal/cli/list/list.go | 14 +- internal/database/actions.go | 27 +-- internal/database/actions_test.go | 3 +- internal/database/models.go | 86 ++++------ internal/output/output.go | 12 +- 8 files changed, 185 insertions(+), 207 deletions(-) diff --git a/data/migrations/1_create_msmt_results.sql b/data/migrations/1_create_msmt_results.sql index 5ff8b53..a78a408 100644 --- a/data/migrations/1_create_msmt_results.sql +++ b/data/migrations/1_create_msmt_results.sql @@ -12,13 +12,13 @@ DROP TABLE `networks`; -- +migrate StatementBegin CREATE TABLE `urls` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `url_id` INTEGER PRIMARY KEY AUTOINCREMENT, `url` VARCHAR(255) NOT NULL, -- XXX is this long enough? `category_code` VARCHAR(5) NOT NULL, -- The citizenlab category code for the -- site. We use the string NONE to denote -- no known category code. - `country_code` VARCHAR(2) NOT NULL -- The two letter country code which this + `url_country_code` VARCHAR(2) NOT NULL -- The two letter country code which this -- URL belongs to ); @@ -33,7 +33,7 @@ CREATE TABLE `urls` ( -- or add support for allowing the user to "correct" a misclassified measurement -- or distinguishing between wifi and mobile. CREATE TABLE `networks` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `network_id` INTEGER PRIMARY KEY AUTOINCREMENT, `network_name` VARCHAR(255) NOT NULL, -- String name representing the network_name which by default is populated based -- on the ASN. -- We use a separate key to reference the rows in @@ -46,11 +46,11 @@ CREATE TABLE `networks` ( -- 0000:0000:0000:0000:0000:0000:0000:0000, -- which is 39 chars. `asn` INT(4) NOT NULL, - `country_code` VARCHAR(2) NOT NULL -- The two letter country code + `network_country_code` VARCHAR(2) NOT NULL -- The two letter country code ); CREATE TABLE `results` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `result_id` INTEGER PRIMARY KEY AUTOINCREMENT, -- This can be one of "websites", "im", "performance", "middlebox". `test_group_name` VARCHAR(16) NOT NULL, -- We use a different start_time and runtime, because we want to also have @@ -58,26 +58,28 @@ CREATE TABLE `results` ( -- go into the test. -- That is to say: `SUM(runtime) FROM measurements` will always be <= -- `runtime FROM results` (most times <) - `start_time` DATETIME NOT NULL, - `runtime` REAL, + `result_start_time` DATETIME NOT NULL, + `result_runtime` REAL, -- Used to indicate if the user has seen this result - `is_viewed` TINYINT(1) NOT NULL, + `result_is_viewed` TINYINT(1) NOT NULL, -- This is a flag used to indicate if the result is done or is currently running. - `is_done` TINYINT(1) NOT NULL, - `data_usage_up` REAL NOT NULL, - `data_usage_down` REAL NOT NULL, + `result_is_done` TINYINT(1) NOT NULL, + `result_data_usage_up` REAL NOT NULL, + `result_data_usage_down` REAL NOT NULL, -- It's probably reasonable to set the maximum length to 260 as this is the -- maximum length of file paths on windows. `measurement_dir` VARCHAR(260) NOT NULL, `network_id` INTEGER NOT NULL, - FOREIGN KEY(`network_id`) REFERENCES `networks` (`id`) + CONSTRAINT `fk_network_id` + FOREIGN KEY(`network_id`) + REFERENCES `networks`(`network_id`) ); CREATE TABLE `measurements` ( - `id` INTEGER PRIMARY KEY AUTOINCREMENT, + `measurement_id` INTEGER PRIMARY KEY AUTOINCREMENT, -- This can be one of: -- facebook_messenger -- telegram @@ -87,31 +89,31 @@ CREATE TABLE `measurements` ( -- dash -- ndt `test_name` VARCHAR(64) NOT NULL, - `start_time` DATETIME NOT NULL, - `runtime` REAL NOT NULL, + `measurement_start_time` DATETIME NOT NULL, + `measurement_runtime` REAL NOT NULL, -- Note for golang: we used to have state be one of `done` and `active`, so -- this is equivalent to done being true or false. -- `state` TEXT, - `is_done` TINYINT(1) NOT NULL, + `measurement_is_done` TINYINT(1) NOT NULL, -- The reason to have a dedicated is_uploaded flag, instead of just using -- is_upload_failed, is that we may not have uploaded the measurement due -- to a setting. - `is_uploaded` TINYINT(1) NOT NULL, + `measurement_is_uploaded` TINYINT(1) NOT NULL, -- This is the measurement failed to run and the user should be offerred to -- re-run it. - `is_failed` TINYINT(1) NOT NULL, - `failure_msg` VARCHAR(255), + `measurement_is_failed` TINYINT(1) NOT NULL, + `measurement_failure_msg` VARCHAR(255), - `is_upload_failed` TINYINT(1) NOT NULL, - `upload_failure_msg` VARCHAR(255), + `measurement_is_upload_failed` TINYINT(1) NOT NULL, + `measurement_upload_failure_msg` VARCHAR(255), -- Is used to indicate that this particular measurement has been re-run and -- therefore the UI can take this into account to either hide it from the -- result view or at the very least disable the ability to re-run it. -- XXX do we also want to have a reference to the re-run measurement? - `is_rerun` TINYINT(1) NOT NULL, + `measurement_is_rerun` TINYINT(1) NOT NULL, -- This is the server-side report_id returned by the collector. By using -- report_id & input, you can query the api to fetch this measurement. @@ -131,7 +133,7 @@ CREATE TABLE `measurements` ( -- this at some point in the near future. -- See: https://github.com/ooni/pipeline/blob/master/docs/ooni-uuid.md & -- https://github.com/ooni/pipeline/issues/48 - `measurement_id` INT(64), + `collector_measurement_id` INT(64), -- This indicates in the case of a websites test, that a site is likely -- blocked, or for an IM test if the IM tests says the app is likely @@ -156,9 +158,11 @@ CREATE TABLE `measurements` ( -- have many measurements per file. `report_file_path` VARCHAR(260) NOT NULL, - FOREIGN KEY (`result_id`) REFERENCES `results`(`id`) - ON DELETE CASCADE ON UPDATE CASCADE, -- If we delete a result we also want - -- all the measurements to be deleted as well. - FOREIGN KEY (`url_id`) REFERENCES `urls`(`id`) + CONSTRAINT `fk_result_id` + FOREIGN KEY (`result_id`) + REFERENCES `results`(`result_id`) + ON DELETE CASCADE, -- If we delete a result we also want + -- all the measurements to be deleted as well. + FOREIGN KEY (`url_id`) REFERENCES `urls`(`url_id`) ); -- +migrate StatementEnd diff --git a/internal/bindata/bindata.go b/internal/bindata/bindata.go index b699aae..390d000 100644 --- a/internal/bindata/bindata.go +++ b/internal/bindata/bindata.go @@ -130,100 +130,102 @@ func bindataDataDefaultconfigjson() (*asset, error) { } var _bindataDataMigrations1createmsmtresultssql = []byte( - "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xac\x59\x6d\x73\xdb\x36\x12\xfe\xee\x5f\xb1\xe3\xe9\xf4\xec\x39\x49\x76" + - "\x72\x69\xe6\xce\xd7\x4e\xc7\xb5\x99\x9c\xda\x58\xce\xc8\xf2\x35\x99\x9b\x1b\x11\x22\x97\x12\x6a\x10\x60\x00\x50" + - "\x8c\xee\xd7\xdf\xec\x02\xa4\x48\xc5\x75\x9d\x99\xe6\x83\x23\x91\xc0\x62\x5f\x9f\x67\x17\x1a\x8f\xe1\xaf\xa5\x5c" + - "\x5b\xe1\x11\xae\x4d\xa3\x8f\xfa\x0f\xee\xbc\xf0\x58\xa2\xf6\x3f\xe1\x5a\xea\xa3\xa3\xeb\xf9\xed\x7b\x58\x5c\xfe" + - "\xf4\x2e\x81\xd4\xa2\xab\x95\x77\xe9\x3f\x07\x4f\x4b\x14\xae\xb6\xbc\xe7\xf0\x55\x6d\xd5\xe1\x23\x8d\xbe\x31\xf6" + - "\x81\x1e\x3f\x7e\x6e\xa2\xf3\xe1\x9b\xfb\xea\x49\x05\xaf\xe6\xc9\xe5\x22\x19\x9c\x08\x27\x47\x00\xa9\xcc\x53\x98" + - "\xce\x16\xc9\xdb\x64\x0e\xef\xe7\xd3\x9b\xcb\xf9\x47\xf8\x25\xf9\x08\x97\xf7\x8b\xdb\xe9\xec\x6a\x9e\xdc\x24\xb3" + - "\xc5\x88\x56\xd6\x56\xa5\xf0\xef\xcb\xf9\xd5\xbf\x2e\xe7\x27\x2f\xbf\xfb\xee\x14\x66\xb7\x0b\x98\xdd\xbf\x7b\x37" + - "\x82\xf1\x18\x3e\x7c\xf8\x00\xd2\x81\xdf\x48\x07\xca\xe8\x35\xa0\x36\xf5\x7a\xf3\x23\x6d\xcd\x84\xc7\xb5\xb1\xbb" + - "\x65\x66\x72\xdc\x0b\x39\x14\xb1\xd8\x20\x64\xd2\xcb\xff\xa1\x56\x62\x05\xed\x2e\xa0\x5d\x50\x18\x0b\x7e\x83\x47" + - "\xf0\xbc\x7f\xe3\x31\x38\xe9\x71\x02\xbf\x22\xd4\x0e\x69\x2b\x38\x6f\xa5\x5e\xc3\xec\x76\x96\x80\x37\x90\xa3\x36" + - "\xfe\x6b\x04\x6a\x03\x0f\xda\x34\x7a\xa8\xd9\xe4\x88\x4d\x34\xb5\xf6\x5f\x58\xf8\x72\x6f\x61\x6b\xa0\x6f\x0c\x28" + - "\xf4\x1e\x2d\xc4\x3d\xc1\xbe\x66\x23\xb3\x0d\xbb\xef\x79\x1a\x8d\xc7\x70\x3f\x7f\x07\x2b\x24\x67\x3b\xf0\xe6\xe8" + - "\x34\x24\xcb\xaf\x08\x99\x45\x4a\x02\x01\x0e\x2b\xc1\xf9\xe0\xc5\x4a\x05\x1f\xb6\xa9\xc5\x5f\x5e\x82\x45\xe1\x8c" + - "\x76\x17\xb4\xf3\xc5\x04\xde\x18\x0b\xce\x94\x08\xa6\x60\x97\x6d\x25\x36\x0e\x9a\x0d\x5a\x04\x8d\x98\xf3\x43\x6f" + - "\xbc\x50\xa0\xeb\x72\x85\x96\x16\xc6\xdc\xce\x3b\xd9\x23\x92\x26\xfd\x5f\x1c\xac\x0d\x79\xdc\x1b\x58\x21\x94\x75" + - "\xb6\x81\xd2\x58\x04\x2c\x0a\x99\x49\xd4\x9e\xde\xfc\x56\x3b\x0f\xca\x98\x87\xba\x62\xe9\xec\x15\x12\x6b\x4d\xe3" + - "\x40\xea\xe0\x93\xf1\x38\xd8\x30\xa1\x4f\x2f\x27\x70\x52\x1a\xe7\x41\x96\x95\xb1\x5e\x68\x7f\x4a\x66\x37\x22\x48" + - "\x14\x5b\x23\x73\xc8\xeb\x4a\xc9\x4c\x78\x52\x40\xc0\xaa\xd6\xd9\x86\xa4\x4a\x5d\x18\x5b\x0a\x2f\x0d\x49\x16\x9e" + - "\x55\x1d\x2a\x9a\x99\xb2\xa4\xb7\x06\x1c\x6e\xd1\x92\xad\xad\xd3\x48\xc1\xda\xa1\xa5\x2d\x46\xb3\x32\xc9\x67\x51" + - "\x56\x0a\x2f\xa2\xef\x4b\xb1\x83\x46\xba\x0d\x2b\x92\xe7\xf4\x1f\xd7\x44\x88\x00\xed\x57\x26\x0b\xc7\x17\xd6\x94" + - "\xad\xa3\x2b\x6b\x56\x18\x9e\xd0\xd7\xb7\xef\xef\x48\x9e\xb1\x2c\xc3\xd5\x15\xd9\xc9\x21\x13\x4a\x99\x86\x75\x6d" + - "\x55\xf1\x06\x8e\x33\x63\x2d\x66\xfe\x18\x04\x94\xd2\x65\x4a\x38\x27\x0b\x89\x39\xf4\x70\x27\x0a\xcc\xa5\x23\x9f" + - "\xd4\xd2\x6d\x48\xcc\x0a\x7d\x83\xa8\xa1\x91\x85\x04\xa1\x73\x28\xcd\x4a\x92\x9f\x87\x90\xd1\x21\xd2\xd7\xc2\x46" + - "\xdc\xb8\xd4\xa2\xc4\xa7\xf0\xe3\x2e\x14\x27\x2d\x03\x8b\x95\x45\x87\xda\xb7\x76\xf6\x85\xc4\x4a\x59\xed\x20\xc7" + - "\x42\xd4\xca\x53\x2c\x2a\x53\xd5\x4a\x78\xcc\x61\x25\x1c\xe6\x7f\x54\x42\xe4\x09\xcd\x92\x2f\xef\x66\x93\x67\xac" + - "\x8e\x28\xd2\xab\xa8\x07\xdc\x91\xe7\x2d\x16\x68\x51\x67\x21\xb4\x31\x65\x9f\x21\x70\x9f\x13\x6e\x04\x2b\xcc\x04" + - "\x89\x6f\x86\xe9\x73\x8c\xda\xca\x6c\x73\xfc\x5c\x71\x8d\xf4\xb1\xc0\x72\xe1\x45\x28\x1d\x84\xa2\xf6\xb5\xc5\x49" + - "\x3f\x16\x7e\x57\xf5\x62\xf1\xe2\xf5\x41\x28\x6e\x35\xd7\x3f\x65\xc4\x28\xa6\x03\x63\x9c\xac\xf6\x9b\x5e\x9d\xf7" + - "\x37\x85\x00\x1a\x8b\x8e\x5c\x14\x22\xd9\x05\x31\x24\xbb\x29\x40\x68\x90\xd5\xf6\x15\x25\xa1\xac\xb6\xaf\x29\xb5" + - "\x2d\x3a\xf7\x1c\xff\x2f\xb8\x6e\xf4\x1a\xa9\xe8\x2b\x8a\x78\x10\xd6\x09\x01\x25\x1f\xf0\xe2\x19\x92\xce\xcf\xcf" + - "\xcf\x2f\xfe\xf8\xcf\xe8\x19\xa2\x42\x22\x4a\x07\x7f\xfb\x07\x64\x1b\x61\xd9\x92\x54\x38\xcd\xb5\x71\xf2\xaa\xe7" + - "\xa1\x3f\x83\x22\x18\xe0\x87\x55\xd9\xf6\x1a\x5c\x94\x5f\x53\x96\xd1\xa9\xd2\x41\x26\x34\x61\x9e\x09\x31\x3f\x6e" + - "\x70\x45\xbc\xe9\x8e\x47\x70\x2c\x4b\xfa\x5b\xa1\x65\xc4\xd4\x19\xd2\xd7\x52\xe6\xb9\xc2\x95\xf9\x7c\x1c\xe2\x96" + - "\x7a\x74\x7e\xb9\xb6\xa6\xae\x0e\x6a\x7c\x90\x57\xed\x99\x5d\x21\xe5\xb2\xe0\xca\xf1\xe0\xbc\xb0\x7e\xe9\x65\x89" + - "\x8c\x3f\xb6\xd6\xf4\x79\x50\x15\x1d\xb2\x2b\x67\x60\x23\xb6\xd8\x8a\xe3\x44\xf7\xa6\x85\x39\x4e\x78\xb3\x45\xbb" + - "\x41\x91\x93\x3d\xcc\x84\x81\x01\x2c\x32\x86\xd2\x11\xc6\x6f\xd0\x42\x21\x32\x6f\xac\x0b\x2c\x10\xe5\xad\x0d\x48" + - "\xcd\x90\x8d\x40\x86\x4d\xf6\xbe\x12\x0c\x34\x44\x0a\x62\x77\x01\xe9\xdd\xfd\xcd\x49\x54\xf5\x14\xde\xcc\x6f\x6f" + - "\x60\xd0\xe2\x41\x23\x95\x02\xa1\x1a\xb1\x73\xe4\xdf\xef\x7f\x68\x25\xa5\x71\x57\xd8\xb4\x8f\x20\x13\x1a\xbd\x70" + - "\xf0\xfd\x69\x70\xed\xde\x33\x29\x5c\x5f\x2e\x92\xc5\xf4\x26\x39\x70\x69\x2b\x2d\x85\x79\x72\xf9\x6e\x74\xd4\x9e" + - "\x72\xef\x90\xb9\x47\xea\x9c\x48\x10\x41\x16\x7b\xc6\xd8\x08\x07\x8e\x40\x9f\x61\x23\xa8\x10\x13\xc8\x2d\x89\xf2" + - "\x31\x4f\x61\x31\x9d\x7d\xa4\x34\x7e\xd1\x8f\xe2\x20\x75\xa8\x0a\xa1\x50\x62\x4d\x52\x1f\x3d\x2d\x88\xa6\x85\x39" + - "\x27\x18\xf3\x66\x56\x5b\x8a\xbb\xda\x51\xa8\xb5\xd4\xeb\x49\x77\x36\xad\xfa\x9d\x93\x79\x09\x85\x7b\x59\x3b\xb1" + - "\xc6\x65\x5d\x05\x93\x9f\x58\x92\x9b\x46\x3f\xba\x68\x3c\x86\x29\x75\x27\x44\xba\x62\x45\x8a\x70\x17\x14\x18\x9a" + - "\x58\xdf\xb3\xf6\xa5\xf8\x2c\xcb\xba\x04\x85\x7a\xed\x19\x91\x5f\xbe\x3e\x07\x11\x9b\x5c\x6e\x76\xbb\x44\x3c\x58" + - "\x6b\x0a\x28\xa4\x42\xa8\x84\xdf\x50\xa7\x00\x8d\xd4\xb9\x69\x22\xd6\xf5\xa7\x81\x65\x2e\x6d\x0f\x0d\x5e\x9f\x7f" + - "\xe1\xee\x0e\xb6\xfb\xd5\x3d\x34\xe8\xcd\xed\x3c\x99\xbe\x9d\x51\xad\x9f\xf4\x97\x9f\xc2\x3c\x79\x93\xcc\x93\xd9" + - "\x55\x72\x37\xe0\x70\x42\x8a\xd3\x47\x00\x65\x98\xc3\x7f\x12\xaa\x5c\xb4\xaf\x0a\x91\xe1\xca\x98\x87\x65\x89\xce" + - "\xa1\x5e\xa3\x6d\xdf\x78\x54\xb8\xb6\xa2\x3c\xea\xa0\x55\x78\x27\xaa\xaa\xfd\xbe\xf1\xbe\x5a\x52\x51\xa3\x5d\x16" + - "\x12\x55\xbe\x2c\x85\x96\xcc\xf9\xd2\xe8\xc1\x2a\xa9\xb7\x42\xc9\x7c\x69\xf1\x53\x4d\xd0\xa4\xa4\xee\xc1\x85\xdb" + - "\xb4\x9f\x75\xee\x7b\x00\x36\x84\xae\xd7\xaf\xbe\x48\xbd\xaf\xae\xc5\x47\xaa\x66\x66\x7c\x68\xc2\xd7\x46\x09\xbd" + - "\xbe\x20\x68\x6b\x4b\x87\x50\x8d\xa0\xd0\x63\x0f\x8e\xd3\x50\x10\x84\x59\xa9\xc8\xbc\xdc\x62\x3a\x02\x67\x8e\xfa" + - "\xac\x2f\x1d\xe0\xa7\x5a\x6e\x85\x8a\x0d\x35\x97\xda\x0a\xb9\x77\xb2\x35\x57\x5d\x21\x94\xc3\x0e\xcb\x52\x3e\x26" + - "\x85\x45\xf2\x21\x86\xed\x19\xb5\x17\xb9\x29\x54\x4a\xa7\xb0\x80\x1c\x43\xc9\xe7\x20\xdd\xb2\xae\x94\x11\x39\xe6" + - "\x8c\x0a\x23\x90\xda\xf9\x08\xc4\xdc\xe5\xd7\x4e\xea\x75\x2b\xad\x5b\xbe\x2c\x84\x54\x98\x8f\x42\x45\x09\xdf\xb6" + - "\x41\xda\xf8\x70\x48\x27\x95\x8b\x72\x9f\xa1\x90\xd7\x5d\x64\x89\x1c\xa8\x70\xfd\x00\x4f\xda\x9d\xcf\x44\xb3\x43" + - "\xf9\x41\x31\x6e\xf3\x6a\xcd\x51\xe8\x20\xd4\x6d\x4c\xad\x72\x0e\x15\xf1\x98\xe5\x65\xad\x3c\x8b\x63\xda\x20\xfd" + - "\x5e\x93\x20\xea\x29\x6c\xa3\x15\xb5\xc5\x65\xe9\xd6\xc3\x3e\xb9\x85\x81\x43\x87\x3d\x25\xac\xb7\xf0\x29\x99\x04" + - "\x84\xee\x4b\xf4\xe6\x28\x70\x72\x55\xc2\x7a\x99\xd5\x4a\xd8\x81\x63\x88\x40\x56\x44\x20\xd1\x52\xa1\xf3\x7d\x4e" + - "\xa2\xc5\xc2\x44\x2e\xbe\x9f\x32\x14\x78\xf1\x80\x31\x5b\x89\x5d\x45\x16\x86\x3c\x6f\x00\x25\x73\xf1\x46\xe6\x08" + - "\xd2\x77\x03\xd0\xde\x93\x4c\x20\xc4\x49\x3c\x0c\x05\x60\xde\xa2\xdd\x81\x42\xe1\x3c\x4d\x33\xdd\x60\x25\x56\x52" + - "\x49\x1f\xdb\xf2\x41\x04\xe2\xfd\x44\x6e\x28\xb7\xb8\x89\x68\x3b\x8a\x98\xc5\xbd\x36\xde\x44\xe6\x62\x01\x3d\xa3" + - "\x7f\xec\xa2\x60\xd1\xd6\xfa\x2b\x52\xca\xa1\xdd\xa2\x1d\x3b\xb2\x31\x74\x21\x4b\x99\x83\x45\x5f\x5b\x4d\x13\xcb" + - "\x2e\xce\xbd\x4a\x21\x75\x24\x13\xf8\x69\x37\x2c\x95\xfd\xa6\x6f\x41\xea\xaa\xf6\x23\xd8\x99\x9a\x3d\xfb\xa9\x26" + - "\x5f\xb0\xf5\x95\x24\xe5\x0b\xf4\xf1\x1e\xa1\xaf\x7c\xe7\x86\xe4\x73\xf7\xf1\x6d\xb2\x60\xc4\x74\x17\x67\x67\xa2" + - "\x92\x13\x63\xb4\x9c\x48\x43\x9f\xcf\xb6\x2f\xce\xfa\x54\xf0\x23\x9f\xfa\xc3\x37\xd3\xd9\xfb\xfb\xc5\xb7\x9d\x3a" + - "\x3f\x7c\x33\x4f\xde\xdf\xce\x17\xcb\xe9\xf5\x5e\xbe\xb7\x22\x0b\x61\x2a\xa4\xa5\x76\xdd\x63\xb9\x1f\x6c\x63\xcb" + - "\xfa\x9f\xff\xa6\xa0\xa4\xf3\x6d\x51\xe9\xa0\x77\xc7\x14\x03\x86\xe4\x9b\x27\x6f\x60\x1d\x79\xf9\xe7\xbb\xdb\x59" + - "\x98\x9b\x87\x46\xd2\xf8\xd5\x6b\xd8\xd0\x85\x56\x7a\x2b\x54\x8d\x0e\x4e\xd2\x4e\xef\x74\x04\x29\x5b\x94\x9e\x82" + - "\xb0\x5c\xd1\x45\xad\xf6\xde\x13\x1d\xab\xf7\x84\x73\x51\x50\xe2\x0b\x65\x51\xe4\xbb\x50\x00\x95\x35\x19\x91\x59" + - "\x17\xc6\x4a\x56\x48\x94\x33\xea\xe1\x81\x2c\x2b\x15\x84\x64\x0a\x85\xae\x2b\x9e\x86\xa2\x98\x0e\xdd\xfa\x0e\x8f" + - "\xc0\xb1\xd7\x78\x58\xbd\x87\x3c\xcb\xd3\x43\x43\x6e\xd4\xa6\x6d\x74\xb9\xff\x68\x0b\xf5\x0f\xa6\x99\xf1\x38\xde" + - "\x19\xe5\x93\x08\x36\xb5\x55\xfd\x7e\xe3\xcb\xc4\x26\x74\xde\xa1\xa7\x0e\x10\x05\x4d\x98\xed\x4d\x46\x97\xc7\x23" + - "\x58\xd5\x0c\xe6\xe4\xe2\x4a\x09\xee\xf5\xe2\xb5\xc8\x80\xc1\x84\x0f\x77\x4e\x95\x91\xda\xb7\x63\xab\x46\x61\x7b" + - "\xb3\x6b\x18\x31\x11\x2f\xba\x94\x5d\x4b\xbf\xa9\x57\x93\xcc\x94\x67\x94\xb9\x67\xad\xe3\xcf\x56\xca\xac\xce\x4a" + - "\xe1\x3c\xda\xb3\xdc\x64\x8e\x5f\x8f\xeb\x5a\xe6\x93\x32\x87\x6f\xfb\xcd\xc2\x93\x72\xa4\x73\x35\xba\xb3\x57\x7f" + - "\xff\xb2\x67\x8b\x9e\xa1\x3e\xe1\xd0\x33\x11\x43\x5d\x6b\x47\x26\x1c\xbb\x46\x40\x3b\x5a\xf1\x60\x31\x0a\xf9\x24" + - "\xf8\x96\x92\xfc\x49\xf3\xab\xda\xb5\xb2\x56\xca\x64\x0f\xc4\x89\x44\xde\x04\x7c\x1a\xa6\x37\xbc\xb1\x6d\xad\xe3" + - "\x57\x47\xe3\x88\x8b\x00\x50\x3d\x2d\x48\x16\x7c\x3d\x14\xe7\x37\x68\x84\x83\x1c\x3d\x66\x1c\xf6\xb8\xfe\x63\xc4" + - "\x95\xf4\xe7\xdb\xe9\x2c\x05\x01\xe9\xd5\xed\xfd\x6c\x71\x72\x9a\x76\x15\xc7\xf5\xd4\x9a\x17\xe7\x97\x80\xd0\xb1" + - "\x46\x45\x77\x87\x77\xa0\x05\x04\xfb\x8d\xed\x1e\x4c\x6f\x48\x6d\xd7\x41\xab\xd0\xa6\x14\x6a\xd7\x07\xd7\x47\x86" + - "\x0e\x0d\xa6\x12\x9f\xea\x88\x04\xce\xdb\x3a\xa3\x3c\x19\xc5\x8b\xca\x86\x1a\x29\x62\xa0\xfe\x4d\x26\x77\x79\x0f" + - "\xb8\x73\x5d\x8b\x19\x6f\x34\xe3\xc5\xf2\xb0\xb1\x40\x2f\xa4\x72\xf1\xfa\x93\x30\x8a\x45\xf5\xd8\xc8\xc1\x09\x7e" + - "\x9e\xf4\xa9\x2a\xd4\xf1\x19\x0d\x1d\xf4\x01\x5c\x45\xd2\x4d\x01\xb3\xeb\xc5\x28\xfa\x8a\x7b\xa7\xa2\xb5\x9f\xca" + - "\x81\x33\x83\xdc\xd2\x75\x59\xe8\xb3\xc9\x69\xaf\x33\x25\x9d\xd3\x60\xe9\x63\x34\x83\x90\x59\xe3\xda\xfb\xc5\x01" + - "\x8f\x51\x08\x83\xed\x8d\x89\x77\x4d\xe0\xcd\x1a\x89\x71\x3b\x80\x21\x8b\x7e\x67\xb2\x78\x64\xda\xdb\x0a\x2b\xf9" + - "\x20\xee\x19\xa4\xf6\x68\xb5\x50\x8a\x39\x97\x80\xff\x21\x60\xa0\x08\xe3\x12\x4f\xe3\x7a\x9c\x4b\xf7\xf0\x08\xa2" + - "\xba\xc9\x6f\xce\xe8\x09\x4c\x3d\xb7\x7b\x25\xf5\x08\x0e\xb5\x63\xdd\x1b\x4b\x75\x41\x9d\x6c\x98\xa3\xd0\x02\xf2" + - "\x4d\xc8\xaa\x4b\xed\x8d\x31\xec\xc2\x9b\x5f\x38\x42\x95\xc5\x6d\xbc\x32\x6c\x1b\x09\x12\xd2\x62\x4e\x90\x63\x34" + - "\x75\x0c\x0f\xf1\xea\xa6\x14\x7b\x61\xd4\x07\x94\x42\xef\x06\x1a\xf2\xb9\x05\x5f\x87\xf6\xf1\x98\x9e\x2c\xc9\xc8" + - "\xa7\x07\xb7\xde\x54\xc6\xec\xd3\xba\x7a\x38\x95\xb5\x37\x00\x71\x26\x0b\x80\x7c\x3b\x83\xeb\xe4\x5d\xb2\x48\xe0" + - "\xea\xf2\xee\xea\xf2\x3a\xa1\x27\xf7\xef\x69\xf0\x68\x9f\x30\x09\x4c\x0b\xca\xe3\x1c\x15\xfa\xd0\xc6\x70\x82\xf6" + - "\x9b\x9c\xe7\xfe\xbe\x11\xfd\x20\x94\x3a\xac\x06\x17\xaf\xc5\xc3\x29\x39\x0d\xc0\x0d\x2a\x35\x79\xc4\xc6\xc8\x1a" + - "\x43\x03\xf9\xd7\xa6\xfd\xc4\xf9\xbb\xbf\x67\xfd\x3f\x00\x00\xff\xff\xc9\xf1\x5a\xc3\x72\x1b\x00\x00") + "\x1f\x8b\x08\x00\x00\x00\x00\x00\x00\xff\xa4\x59\x6d\x73\xdb\x36\x12\xfe\xee\x5f\xb1\xe3\xe9\xf4\xec\x39\x49\x76" + + "\x72\x69\xe6\xce\xd7\x4e\xc7\xb5\x99\x9c\xda\x58\xca\xc8\xf2\x35\x99\x9b\x1b\x11\x22\x97\x12\x2a\x10\x60\xf0\x22" + + "\x46\xf7\xeb\x6f\x16\x00\x29\x52\x56\x1c\x67\xda\x0f\xa9\x48\x02\x8b\x7d\x7d\xf6\x59\x78\x38\x84\xbf\x96\x7c\xa5" + + "\x99\x45\xb8\x55\xb5\x3c\xe9\xbe\xb8\xb7\xcc\x62\x89\xd2\xfe\x82\x2b\x2e\x4f\x4e\x6e\x67\xd3\xf7\x30\xbf\xfe\xe5" + + "\x5d\x02\xa9\x46\xe3\x84\x35\xe9\x3f\x7b\x6f\x4b\x64\xc6\x69\xbf\xe7\xf0\x93\xd3\xe2\xf0\x95\x44\x5b\x2b\xbd\xa1" + + "\xd7\xc7\xcf\x4d\x64\xde\xff\xf2\x50\x3d\xa9\xe0\xcd\x2c\xb9\x9e\x27\xbd\x13\xe1\xec\x04\xfc\xcf\x05\xcf\x53\x18" + + "\x4f\xe6\xc9\xdb\x64\x06\xef\x67\xe3\xbb\xeb\xd9\x47\xf8\x2d\xf9\x08\xd7\x0f\xf3\xe9\x78\x72\x33\x4b\xee\x92\xc9" + + "\x7c\x10\x57\xa7\xf0\xef\xeb\xd9\xcd\xbf\xae\x67\x67\x2f\x7f\xf8\xe1\x1c\x26\xd3\x39\x4c\x1e\xde\xbd\x1b\xc0\x70" + + "\x08\x1f\x3e\x7c\x00\x6e\xc0\xae\xb9\x01\xa1\xe4\x0a\x50\x2a\xb7\x5a\xff\x4c\x5b\x33\x66\x71\xa5\xf4\x6e\x91\xa9" + + "\x1c\xf7\x42\x0e\x45\xcc\xd7\x08\x19\xb7\xfc\x7f\x28\x05\x5b\x42\xb3\x0b\x68\x17\x14\x4a\x83\x5d\xe3\x09\x3c\xef" + + "\xbf\xe1\x10\x0c\xb7\x38\x82\xdf\x11\x9c\x41\xda\x0a\xc6\x6a\x2e\x57\x30\x99\x4e\x12\xb0\x0a\x72\x94\xca\x7e\x8b" + + "\x40\xa9\x60\x23\x55\x2d\xfb\x9a\x8d\x4e\x1a\x5f\x66\xca\x49\xfb\xc8\xca\x97\x7b\x2b\x1b\x23\x6d\xad\x40\xa0\xb5" + + "\xa8\x21\xee\x09\x36\xd6\x6b\x9e\xad\xbd\x0b\x9f\xa7\xd5\x70\x08\x0f\xb3\x77\xb0\x44\x72\xb8\x01\xab\x4e\xce\x43" + + "\xd2\xfc\x8e\x90\x69\xa4\x64\x60\x60\xb0\x62\x3e\x2f\x2c\x5b\x8a\xe0\xc7\x26\xc5\xfc\xc3\x4b\xd0\xc8\x8c\x92\xe6" + + "\x8a\x76\xbe\x18\xc1\x1b\xa5\xc1\xa8\x12\x41\x15\xde\x6d\x5b\x8e\xb5\x81\x7a\x8d\x1a\x41\x22\xe6\xfe\xa5\x55\x96" + + "\x09\x90\xae\x5c\xa2\xa6\x85\x31\xc7\xf3\x56\xf6\x80\xa4\x71\xfb\x17\x03\x2b\x45\x5e\xb7\x0a\x96\x08\xa5\xcb\xd6" + + "\x50\x2a\x8d\x80\x45\xc1\x33\x8e\xd2\xd2\x97\x3f\x9c\xb1\x20\x94\xda\xb8\xca\x4b\xf7\x5e\x21\xb1\x5a\xd5\x06\xb8" + + "\x0c\x3e\x19\x0e\x83\x0d\x23\xfa\xf5\x72\x04\x67\xa5\x32\x16\x78\x59\x29\x6d\x99\xb4\xe7\x64\x76\xcd\x82\x44\xb6" + + "\x55\x3c\x87\xdc\x55\x82\x67\xcc\x92\x02\x0c\x96\x4e\x66\x6b\x92\xca\x65\xa1\x74\xc9\x2c\x57\x24\x99\x59\xaf\x6a" + + "\x5f\xd1\x4c\x95\x25\x7d\x55\x60\x70\x8b\x9a\x6c\x6d\x9c\x46\x0a\x3a\x83\x9a\xb6\x28\xe9\x95\x49\x3e\xb3\xb2\x12" + + "\x78\x15\x7d\x5f\xb2\x1d\xd4\xdc\xac\xbd\x22\x79\x4e\xff\xf3\x75\x11\x22\x40\xfb\x85\xca\xc2\xf1\x85\x56\x65\xe3" + + "\xe8\x4a\xab\x25\x86\x37\xf4\xf8\xf6\xfd\x3d\xc9\x53\xda\xcb\x30\xae\x22\x3b\x7d\xc8\x98\x10\xaa\xf6\xba\x36\xaa" + + "\x58\x05\xa7\x99\xd2\x1a\x33\x7b\x0a\x0c\x4a\x6e\x32\xc1\x8c\xe1\x05\xc7\x1c\x3a\xf8\x13\x05\xe6\xdc\x90\x4f\x1c" + + "\x37\x6b\x12\xb3\x44\x5b\x23\x4a\xa8\x79\xc1\x81\xc9\x1c\x4a\xb5\xe4\xe4\xe7\x3e\x74\xb4\xc8\x14\xe0\x23\x3e\x7e" + + "\x03\x84\x34\x3b\x24\x2b\xf1\x29\x2c\xb9\x0f\x85\x4a\xcb\x40\x63\xa5\xd1\xa0\xb4\x8d\xbd\x5d\x21\xb1\x62\x96\x3b" + + "\xc8\xb1\x60\x4e\x58\x8a\x49\xa5\x2a\x27\x98\xc5\x1c\x96\xcc\x60\xfe\xb5\x52\x22\x8f\x48\x2f\xf9\xfa\x7e\x32\x7a" + + "\xc6\xea\x88\x28\x9d\xca\xda\xe0\x8e\x22\xa0\xb1\x40\x8d\x32\x0b\x21\x8e\xa9\xfb\x0c\x81\xfb\xdc\x30\x03\x58\x62" + + "\xc6\x48\x7c\xdd\x4f\xa3\x53\x94\x9a\x67\xeb\xd3\xe7\x8a\xab\xb9\x8d\x85\x96\x33\xcb\x42\x09\x21\x14\xce\x3a\x8d" + + "\xa3\x6e\x2c\xec\xae\xea\xc4\xe2\xc5\xeb\x83\x50\x4c\xa5\xc7\x01\xca\x8c\x41\x4c\x0b\x8f\x77\xbc\xda\x6f\x7a\x75" + + "\xd9\xdd\x14\x02\xa8\x34\x1a\x72\x51\x88\x64\x1b\xc4\x90\xf4\xaa\x00\x26\x81\x57\xdb\x57\x94\x8c\xbc\xda\xbe\xa6" + + "\x14\xd7\x68\xcc\x73\xfc\x3f\xf7\xf5\x23\x57\x48\xc5\x5f\x51\xc4\x83\xb0\x56\x08\x08\xbe\xc1\xab\x67\x48\xba\xbc" + + "\xbc\xbc\xbc\xfa\xfa\x3f\x83\x67\x88\x0a\x89\xc8\x0d\xfc\xed\x1f\x90\xad\x99\xf6\x96\xa4\xcc\x48\x5f\x1b\x67\xaf" + + "\x3a\x1e\xea\x7a\xff\xcf\xb6\x0c\x0f\xf8\xfd\x2a\x6d\x38\x88\x2f\x52\x68\x9e\x9f\x5f\xa5\xd1\xc7\xdc\x40\xc6\x24" + + "\x41\xa1\x0a\x29\x70\x5a\xe3\x92\x5a\xaa\x39\x1d\xc0\x29\x2f\xe9\xdf\x0a\xb5\x07\x52\x99\x21\x3d\x96\x3c\xcf\x05" + + "\x2e\xd5\xe7\xd3\x10\xc6\xd4\xa2\xb1\x8b\x95\x56\xae\x3a\x28\xf9\x5e\x9a\x35\x67\xb6\x75\x95\xf3\xc2\x17\x92\x05" + + "\x63\x99\xb6\x0b\xcb\x4b\xf4\xb0\xa4\x9d\xa4\xdf\xbd\x22\x69\x01\x5f\x18\x05\x6b\xb6\xc5\x46\x9c\xcf\x7b\xab\x1a" + + "\xf4\xf3\xf9\xaf\xb6\xa8\xd7\xc8\x72\xb2\xc7\x37\xc8\xd0\x18\x34\x7a\x68\xa5\x23\x94\x5d\xa3\x86\x82\x65\x56\x69" + + "\x13\x9a\x43\x94\xb7\x52\xc0\xa5\x47\x72\x04\x32\x6c\xb4\xf7\x15\xf3\xb8\x43\xbd\x82\xed\xae\x20\xbd\x7f\xb8\x3b" + + "\x8b\xaa\x9e\xc3\x9b\xd9\xf4\x0e\x7a\x0c\x10\x6a\x2e\x04\x30\x51\xb3\x9d\x21\xff\xfe\xf8\x53\x23\x29\x8d\xbb\xc2" + + "\xa6\x7d\x20\x7d\x9f\xa3\x0f\x06\x7e\x3c\xef\x45\x75\xef\xa0\x14\x6e\xaf\xe7\xc9\x7c\x7c\x97\x1c\x78\xb6\x59\x1a" + + "\x65\xa7\x30\x4b\xae\xdf\x0d\x4e\x9a\x33\x1f\x0c\xfa\x06\xc5\x65\x4e\x9d\x12\x81\x17\xfb\xb6\xb2\x66\x06\x0c\x75" + + "\x06\x8f\x29\x41\x50\x3f\xab\xcc\x82\xe8\x01\xe6\x29\xcc\xc7\x93\x8f\x94\xea\x2f\xba\xa1\xed\xe5\x13\x55\x2a\x14" + + "\x82\xad\x48\xf8\xd1\x43\x83\x54\x5a\x98\xfb\xac\xf3\x3d\x36\x73\x9a\x92\x41\xec\x28\xfe\x92\xcb\xd5\xe8\x50\x05" + + "\x5a\xfc\x05\x05\xba\x2b\x29\x23\x16\xce\xb0\x15\x2e\x5c\x15\xfc\xf0\xf5\x95\xb9\xaa\xe5\xd1\xb5\xc3\x21\x8c\x89" + + "\xde\x50\xd7\x66\x4b\xd2\xce\xd3\xa8\xd0\xe2\x89\x36\x58\x6f\x52\xc9\x3e\xf3\xd2\x95\x20\x50\xae\xac\x87\xf2\x97" + + "\xaf\x2f\x81\x45\xa6\xec\x19\x73\x9b\xb2\x07\x6b\x55\x01\x05\x17\x08\x15\xb3\x6b\xa2\x1a\x50\x73\x99\xab\x3a\x82" + + "\x64\x77\xac\x58\xe4\x5c\x77\xe0\xe3\xf5\xe5\xa3\x18\x1c\xed\xd6\x7d\x83\x6e\xa6\x93\xfb\xf9\xec\x7a\x3c\x99\x43" + + "\x5a\x6c\x16\x9d\x0d\x11\xff\xde\x4c\x67\xc9\xf8\xed\x84\x60\xe3\xac\x2b\xef\x3c\x7e\x9f\x25\x6f\x92\x59\x32\xb9" + + "\x49\xee\x3b\x5c\xe1\x60\xe5\x63\xbc\xea\xd7\xc6\xd9\x63\xdb\xfe\x2c\x72\x5d\x35\x9f\x0a\x96\xe1\x52\xa9\xcd\xa2" + + "\x44\x63\x50\xae\x50\x37\x5f\x2c\x0a\x5c\x69\x56\x9e\xb4\x68\xce\xac\x61\x55\xd5\x3c\xaf\xad\xad\x16\x04\x1c\xa8" + + "\x17\x05\x47\x91\x2f\x4a\x26\xb9\xa7\x19\x5c\xc9\xde\x2a\x2e\xb7\x4c\xf0\x7c\xa1\xf1\x93\x23\xf8\x13\x5c\x76\x20" + + "\xc9\xac\x9b\xdf\x32\xb7\x1d\x90\xec\xc3\xe3\xeb\x57\x8f\x52\xb8\xeb\x90\xe7\x14\x7d\x77\x7d\xaf\xf2\x8f\x14\xe7" + + "\x44\xd9\x30\x17\xac\x94\x60\x72\x75\x45\xb0\xda\x54\x28\x21\x2a\xc1\xb0\xc5\x4e\x2b\x48\x43\xc1\x11\x5e\xa6\x2c" + + "\xb3\x7c\x8b\xe9\x00\x8c\x3a\xe9\x12\x10\x6e\x00\x3f\x39\xbe\x65\x22\x72\x7c\x5f\xd1\x4b\xf4\x34\x4e\x3b\x5f\xdc" + + "\x05\x13\x06\x5b\x1c\x4d\xfd\x31\x29\xcc\x93\x0f\xf3\x23\x56\x7c\xbd\xce\x63\xab\x0c\x75\xd8\x2a\xcf\x20\xc7\x80" + + "\x32\x39\x70\xb3\x70\x95\x50\x2c\xc7\xdc\x03\xd1\x00\xb8\x34\x36\x36\x04\x3f\x84\x38\xc3\xe5\xaa\x91\xd6\x2e\x5f" + + "\x14\x8c\x0b\xcc\x07\xa1\x5e\x99\x6d\xd8\x99\x54\x36\x1c\xd2\x4a\xf5\x25\xbf\xd7\x1a\x72\xd7\x46\x9f\x9a\x14\xc1" + + "\x82\xdd\x43\xd8\x81\x7d\x8d\x94\x67\x82\xe9\xe1\x59\x41\x49\xcf\x44\x9d\xf4\xd1\x69\x81\xdc\xac\x95\x13\xb9\x0f" + + "\x21\xf5\x56\xed\x97\x35\xf2\x34\x0e\x69\x03\xb7\xc7\xb5\x0a\x62\x9f\xc2\xd7\xee\x06\x5a\xed\x34\x2e\x4a\xb3\xea" + + "\x53\xfc\x06\x88\x8e\xda\xfc\x8d\x87\x74\x36\x3d\x75\x16\x41\xb4\x79\xdc\x6c\x7c\x04\x7d\x92\x56\x4c\x5b\x9e\x39" + + "\xc1\x74\xcf\x91\xd4\xf6\x96\xd4\xf6\xa2\x67\x98\xcc\xf7\xb9\x8d\x1a\x0b\x15\xf9\xc4\xc3\xd8\x43\x8d\x65\x1b\x8c" + + "\x59\x4f\x0c\x81\x65\x61\x7e\xb5\x0a\x90\x7b\x3e\xb1\xe6\x39\x02\xb7\xed\x6c\xb7\xf7\xbc\xef\x77\xd4\x42\xfd\x9c" + + "\x17\x5a\xc6\x16\xf5\x0e\x04\x32\x63\x69\x50\x6b\x67\x46\xb6\xe4\x82\xdb\x38\x69\xf4\x22\x16\xaf\x5f\x72\x45\x79" + + "\xe9\x89\x50\xc3\x8a\x62\x05\x74\x26\x13\x15\x1b\xad\x17\xd0\x31\xfa\xe7\xa3\xd1\xd1\xa8\x9d\xfc\x86\x74\x34\xa8" + + "\xb7\xa8\x87\x86\xec\x0d\xac\x6a\xc1\x73\xd0\x68\x9d\x96\x34\x90\xed\xe2\x78\x2f\x04\x12\xc3\x1a\xc1\x2f\xbb\x7e" + + "\xc9\xed\x37\x7d\x0f\x5c\x56\xce\x0e\x60\xa7\x9c\xf7\xf2\x27\x47\x7e\xf1\x9e\xa8\x38\x19\x52\xa0\x8d\xd7\x25\x5d" + + "\x43\x5a\x97\x24\x9f\xdb\x9f\x6f\x93\xb9\x47\x67\x73\x75\x71\xc1\x2a\x3e\x52\x4a\xf2\x11\x57\xf4\xfb\x62\xfb\xe2" + + "\xa2\xdb\x82\x7e\xf6\xa7\xfe\xf4\xdd\x78\xf2\xfe\x61\xfe\x7d\xab\xce\x4f\xdf\xcd\x92\xf7\xd3\xd9\x7c\x31\xbe\xdd" + + "\xcb\xb7\x9a\x65\x21\x64\x05\xd7\x34\x8d\x58\x2c\xf7\xf3\x7b\x24\x13\xff\xf9\x6f\x0a\x82\x1b\xdb\x14\xa4\x0c\x7a" + + "\xb7\x5d\xa9\x9f\xd8\x5a\xa4\x64\xda\x2a\xb2\x87\x5f\xef\xa7\x93\x70\x3d\xd0\x37\x92\xa6\xcb\x0e\x01\x45\x13\x26" + + "\x84\x2d\x13\x0e\x0d\x9c\xa5\xad\xde\xe9\x00\x52\x6f\x51\x7a\x0e\x4c\x7b\x34\x28\x9c\xd8\x7b\x8f\xb5\xdc\xa3\x23" + + "\xdc\x17\x08\x15\x01\x13\x1a\x59\xbe\x0b\xc5\x50\x69\x95\x51\xe3\x6c\xc3\x58\xf1\x0a\xa9\xbd\x0d\x3a\x58\xc2\xcb" + + "\x4a\x04\x21\x99\x40\x26\x5d\xe5\x87\xbd\x28\xa6\x45\xc9\xae\xc3\x5b\x36\xd7\x68\xdc\xaf\xe4\xc3\x9e\xee\x87\xa2" + + "\x9a\xdc\x28\x55\x43\xdc\x3d\x4b\x6a\x8a\xf6\x2b\xc3\xda\x70\x18\xaf\xc6\xf2\x51\x04\xa4\x83\x6b\xd0\xc7\x89\x4d" + + "\x28\xbf\x43\x4b\xe4\x15\x19\x0d\xd0\xcd\x85\x4d\x9b\xc7\x03\x58\x3a\xdf\x14\xc8\xc5\x95\x60\x9e\xa6\xc6\xdb\x9f" + + "\x5e\x57\x64\x36\x5c\xad\x55\x8a\x4b\xdb\x4c\xe5\x12\x99\xee\x8c\xe6\x61\x82\x46\xbc\x6a\x53\x76\xc5\xed\xda\x2d" + + "\x47\x99\x2a\x2f\x28\x73\x2f\x1a\xc7\x5f\x2c\x85\x5a\x5e\x94\xcc\x58\xd4\x17\xb9\xca\x8c\xff\x3c\x74\x8e\xe7\xa3" + + "\x32\x87\xef\xbb\xc4\xe4\x49\x39\xdc\x18\x87\xe6\xe2\xd5\xdf\x83\x47\x5a\xbb\x16\x47\x78\x18\xb1\x93\x43\x1f\x45" + + "\x64\x35\x8d\x45\x19\x33\xde\x49\x0c\x9a\xa1\xd1\x8f\x4c\x83\x90\x59\xcc\x5f\xcd\x92\x67\x69\x50\x17\xbb\x46\xd6" + + "\x52\xa8\x6c\x43\x5d\x96\xa8\x01\xc1\xa1\x84\xf1\x9d\xdf\xd8\xcc\x07\xf1\xd1\xd0\xa0\x65\x22\x14\x54\x4f\x0b\xe2" + + "\x85\xbf\x0f\x8b\x93\x29\xd4\xcc\x40\x8e\x16\x33\x9f\x00\x71\xfd\xc7\x88\x30\xe9\xaf\xd3\xf1\x24\x05\x06\xe9\xcd" + + "\xf4\x61\x32\x3f\x3b\x4f\xdb\xda\xf3\x95\xd5\x98\x17\x27\xb3\x80\xdb\xb1\x5a\x59\x7b\x69\x79\xa0\x05\x04\xfb\x95" + + "\x6e\x5f\x8c\xef\x48\xed\x70\xc7\x9b\x72\xb3\x60\x52\x95\x4c\xec\xba\x30\x7b\x64\x72\x92\xa0\x2a\xf6\xc9\x45\x4c" + + "\x30\x56\xbb\x8c\x32\x66\x10\x6f\x66\x6b\xa2\x69\xd4\x97\xba\x57\xb7\x9e\x5b\x6e\x70\x67\x5a\x62\x1b\xaf\x70\xe3" + + "\x6d\x7a\x9f\xaa\xa0\x65\x5c\x98\x78\xdf\x4b\x68\xe5\x45\x75\x7a\x94\x81\x33\xfc\x3c\xea\x36\xb0\x50\xd1\x17\x34" + + "\x24\xd1\x0f\x30\x15\x49\x57\x05\x4c\x6e\xe7\x83\xe8\x2b\xcf\xc6\x8a\xc6\x7e\x2a\x0c\x9f\x19\xe4\x96\x96\xb7\xa1" + + "\xcd\x46\xe7\x1d\x3e\x4c\x3a\xa7\xc1\xd2\x63\x0d\x07\x21\xd3\xca\x34\x17\xaa\xbd\xee\x46\x21\x0c\xb6\xd7\x2a\x5e" + + "\xaa\x81\x55\x2b\xa4\x3e\x3c\xfa\xe2\x8d\x48\xe7\x90\xc7\x23\xeb\x96\x69\xee\x0f\xf2\x4c\x82\x4b\x8b\x5a\x32\x21" + + "\x7c\x27\xa6\x16\xb0\x09\x68\xc8\xc2\x78\xe7\xef\x19\xe4\x30\xe7\x66\x73\x04\x5b\xcd\xe8\x0f\xa3\xe4\x08\xc6\xd6" + + "\x13\xc8\x92\x98\x83\x41\x69\xbc\xee\xb5\xa6\xba\x20\x9e\x1c\xe6\x3e\xd4\x80\xfe\xaa\x67\xd9\xa6\xf6\x5a\x29\xef" + + "\xc2\xbb\xdf\x7c\x84\x2a\x8d\xdb\x78\x37\xda\xd0\x0b\x12\xd2\xa0\x4f\x90\xa3\x24\xf1\x88\x4d\xbc\xa3\x2a\xd9\x5e" + + "\x18\xb1\x83\x92\xc9\x5d\x4f\x43\x7f\x6e\xe1\xef\x7f\xbb\xc8\x4c\x6f\x16\x64\xe4\xd3\x83\xe6\xc1\x14\xb9\xf7\xf5" + + "\xe3\x21\xd2\xf7\xa9\xe6\xf3\xb1\x21\xb2\xb9\x01\x39\xb6\x6e\x3a\x81\xdb\xe4\x5d\x32\x4f\xe0\xe6\xfa\xfe\xe6\xfa" + + "\x36\xf1\x9d\x62\x5c\x50\x8a\xe7\x28\xd0\x06\xde\xe3\x73\xb7\xcb\x8a\xbe\xdc\x1e\x86\x43\x60\x42\x1c\x96\x85\x89" + + "\x7f\x10\x08\x32\x73\x9a\xdc\x6b\x14\x22\xf8\xa6\x6f\x4c\x6c\x24\xe7\x3d\x1b\xfc\xdf\xdb\xf6\xdf\x68\x00\xfe\xe2" + + "\x5f\xf5\xfe\x1f\x00\x00\xff\xff\x38\xc6\x64\x22\x78\x1c\x00\x00") func bindataDataMigrations1createmsmtresultssqlBytes() ([]byte, error) { return bindataRead( diff --git a/internal/cli/app/app.go b/internal/cli/app/app.go index 0cb7a07..6dbb4d5 100644 --- a/internal/cli/app/app.go +++ b/internal/cli/app/app.go @@ -14,7 +14,7 @@ func Run() { _, err := root.Cmd.Parse(os.Args[1:]) if err != nil { log.WithError(err).Error("failure in main command") - os.Exit(1) + os.Exit(2) } return } diff --git a/internal/cli/list/list.go b/internal/cli/list/list.go index 0534a87..dd8c438 100644 --- a/internal/cli/list/list.go +++ b/internal/cli/list/list.go @@ -49,13 +49,13 @@ func init() { // We assume that since these are summary level information the first // item will contain the information necessary. if isFirst { - msmtSummary.TotalRuntime = msmt.ResultRuntime + msmtSummary.TotalRuntime = msmt.Result.Runtime msmtSummary.DataUsageUp = msmt.DataUsageUp msmtSummary.DataUsageDown = msmt.DataUsageDown msmtSummary.NetworkName = msmt.NetworkName - msmtSummary.NetworkCountryCode = msmt.NetworkCountryCode + msmtSummary.NetworkCountryCode = msmt.Network.CountryCode msmtSummary.ASN = msmt.ASN - msmtSummary.StartTime = msmt.MeasurementStartTime + msmtSummary.StartTime = msmt.Measurement.StartTime } if msmt.IsAnomaly.Bool == true { msmtSummary.AnomalyCount++ @@ -76,7 +76,7 @@ func init() { } for idx, result := range incompleteResults { output.ResultItem(output.ResultItemData{ - ID: result.ResultID, + ID: result.Result.ID, Index: idx, TotalCount: len(incompleteResults), Name: result.TestGroupName, @@ -97,16 +97,16 @@ func init() { netCount := make(map[uint]int) output.SectionTitle("Results") for idx, result := range doneResults { - totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.ResultID) + totalCount, anmlyCount, err := database.GetMeasurementCounts(ctx.DB, result.Result.ID) if err != nil { log.WithError(err).Error("failed to list measurement counts") } - testKeys, err := database.GetResultTestKeys(ctx.DB, result.ResultID) + testKeys, err := database.GetResultTestKeys(ctx.DB, result.Result.ID) if err != nil { log.WithError(err).Error("failed to get testKeys") } output.ResultItem(output.ResultItemData{ - ID: result.ResultID, + ID: result.Result.ID, Index: idx, TotalCount: len(doneResults), Name: result.TestGroupName, diff --git a/internal/database/actions.go b/internal/database/actions.go index d712e16..27583f7 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -19,29 +19,16 @@ func ListMeasurements(sess sqlbuilder.Database, resultID int64) ([]MeasurementUR measurements := []MeasurementURLNetwork{} req := sess.Select( - "measurements.id as msmt_tbl_id", - "measurements.is_done as measurement_is_done", - "measurements.start_time as measurement_start_time", - "measurements.runtime as measurement_runtime", - "networks.id as network_id", - "networks.country_code as network_country_code", - "results.id as result_id", - "results.start_time as result_start_time", - "results.is_done as result_is_done", - "results.runtime as result_runtime", - "results.test_group_name as test_group_name", - "urls.id as url_id", - "urls.country_code as url_country_code", db.Raw("networks.*"), db.Raw("urls.*"), db.Raw("measurements.*"), db.Raw("results.*"), ).From("results"). - Join("measurements").On("results.id = measurements.result_id"). - Join("networks").On("results.network_id = networks.id"). - LeftJoin("urls").On("urls.id = measurements.url_id"). - OrderBy("measurements.start_time"). - Where("results.id = ?", resultID) + Join("measurements").On("results.result_id = measurements.result_id"). + Join("networks").On("results.network_id = networks.network_id"). + LeftJoin("urls").On("urls.url_id = measurements.url_id"). + OrderBy("measurements.measurement_start_time"). + Where("results.result_id = ?", resultID) if err := req.All(&measurements); err != nil { log.Errorf("failed to run query %s: %v", req.String(), err) @@ -228,7 +215,7 @@ func CreateOrUpdateURL(sess sqlbuilder.Database, urlStr string, categoryCode str return 0, err } res := tx.Collection("urls").Find( - db.Cond{"url": urlStr, "country_code": countryCode}, + db.Cond{"url": urlStr, "url_country_code": countryCode}, ) err = res.One(&url) @@ -285,7 +272,7 @@ func AddTestKeys(sess sqlbuilder.Database, msmt *Measurement, tk interface{}) er msmt.TestKeys = string(tkBytes) msmt.IsAnomaly = sql.NullBool{Bool: isAnomaly, Valid: isAnomalyValid} - err = sess.Collection("measurements").Find("id", msmt.ID).Update(msmt) + err = sess.Collection("measurements").Find("measurement_id", msmt.ID).Update(msmt) if err != nil { log.WithError(err).Error("failed to update measurement") return errors.Wrap(err, "updating measurement") diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 152a6b8..e8d0056 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -3,6 +3,7 @@ package database import ( "database/sql" "encoding/json" + "fmt" "io/ioutil" "os" "testing" @@ -55,7 +56,7 @@ func TestMeasurementWorkflow(t *testing.T) { } var m2 Measurement - err = sess.Collection("measurements").Find("id", m1.ID).One(&m2) + err = sess.Collection("measurements").Find("measurement_id", m1.ID).One(&m2) if err != nil { t.Fatal(err) } diff --git a/internal/database/models.go b/internal/database/models.go index 87d95ef..720b1e6 100644 --- a/internal/database/models.go +++ b/internal/database/models.go @@ -13,68 +13,52 @@ import ( // ResultNetwork is used to represent the structure made from the JOIN // between the results and networks tables. type ResultNetwork struct { - Result `db:",inline"` - ResultID int64 `db:"result_id"` - Network `db:",inline"` - NetworkID int64 `db:"network_id"` + Result `db:",inline"` + Network `db:",inline"` } // MeasurementURLNetwork is used for the JOIN between Measurement and URL type MeasurementURLNetwork struct { - Measurement `db:",inline"` - MeasurementStartTime time.Time `db:"measurement_start_time"` - MeasurementIsDone bool `db:"measurement_is_done"` - MeasurementRuntime float64 `db:"measurement_runtime"` - MsmtTblID int64 `db:"msmt_tbl_id"` - - Network `db:",inline"` - NetworkID int64 `db:"network_id"` - NetworkCountryCode string `db:"network_country_code"` - - Result `db:",inline"` - ResultID int64 `db:"result_id"` - ResultRuntime float64 `db:"result_runtime"` - ResultStartTime time.Time `db:"result_start_time"` - ResultIsDone bool `db:"result_is_done"` - - URL `db:",inline"` - URLCountryCode sql.NullString `db:"url_country_code"` + Measurement `db:",inline"` + Network `db:",inline"` + Result `db:",inline"` + URL `db:",inline"` } // Network represents a network tested by the user type Network struct { - ID int64 `db:"id,omitempty"` + ID int64 `db:"network_id,omitempty"` NetworkName string `db:"network_name"` NetworkType string `db:"network_type"` IP string `db:"ip"` ASN uint `db:"asn"` - CountryCode string `db:"country_code"` + CountryCode string `db:"network_country_code"` } // URL represents URLs from the testing lists type URL struct { - ID sql.NullInt64 `db:"id,omitempty"` + ID sql.NullInt64 `db:"url_id,omitempty"` URL sql.NullString `db:"url"` CategoryCode sql.NullString `db:"category_code"` - CountryCode sql.NullString `db:"country_code"` + CountryCode sql.NullString `db:"url_country_code"` } // Measurement model type Measurement struct { - ID int64 `db:"id,omitempty"` + ID int64 `db:"measurement_id,omitempty"` TestName string `db:"test_name"` - StartTime time.Time `db:"start_time"` - Runtime float64 `db:"runtime"` // Fractional number of seconds - IsDone bool `db:"is_done"` - IsUploaded bool `db:"is_uploaded"` - IsFailed bool `db:"is_failed"` - FailureMsg sql.NullString `db:"failure_msg,omitempty"` - IsUploadFailed bool `db:"is_upload_failed"` - UploadFailureMsg sql.NullString `db:"upload_failure_msg,omitempty"` - IsRerun bool `db:"is_rerun"` + StartTime time.Time `db:"measurement_start_time"` + Runtime float64 `db:"measurement_runtime"` // Fractional number of seconds + IsDone bool `db:"measurement_is_done"` + IsUploaded bool `db:"measurement_is_uploaded"` + IsFailed bool `db:"measurement_is_failed"` + FailureMsg sql.NullString `db:"measurement_failure_msg,omitempty"` + IsUploadFailed bool `db:"measurement_is_upload_failed"` + UploadFailureMsg sql.NullString `db:"measurement_upload_failure_msg,omitempty"` + IsRerun bool `db:"measurement_is_rerun"` ReportID sql.NullString `db:"report_id,omitempty"` URLID sql.NullInt64 `db:"url_id,omitempty"` // Used to reference URL - MeasurementID sql.NullInt64 `db:"measurement_id,omitempty"` + MeasurementID sql.NullInt64 `db:"collector_measurement_id,omitempty"` IsAnomaly sql.NullBool `db:"is_anomaly,omitempty"` // FIXME we likely want to support JSON. See: https://github.com/upper/db/issues/462 TestKeys string `db:"test_keys"` @@ -84,15 +68,15 @@ type Measurement struct { // Result model type Result struct { - ID int64 `db:"id,omitempty"` + ID int64 `db:"result_id,omitempty"` TestGroupName string `db:"test_group_name"` - StartTime time.Time `db:"start_time"` - NetworkID int64 `db:"network_id"` // Used to include a Network - Runtime float64 `db:"runtime"` // Runtime is expressed in fractional seconds - IsViewed bool `db:"is_viewed"` - IsDone bool `db:"is_done"` - DataUsageUp float64 `db:"data_usage_up"` - DataUsageDown float64 `db:"data_usage_down"` + StartTime time.Time `db:"result_start_time"` + NetworkID int64 `db:"network_id"` // Used to include a Network + Runtime float64 `db:"result_runtime"` // Runtime is expressed in fractional seconds + IsViewed bool `db:"result_is_viewed"` + IsDone bool `db:"result_is_done"` + DataUsageUp float64 `db:"result_data_usage_up"` + DataUsageDown float64 `db:"result_data_usage_down"` MeasurementDir string `db:"measurement_dir"` } @@ -112,7 +96,7 @@ func (r *Result) Finished(sess sqlbuilder.Database) error { r.Runtime = time.Now().UTC().Sub(r.StartTime).Seconds() r.IsDone = true - err := sess.Collection("results").Find("id", r.ID).Update(r) + err := sess.Collection("results").Find("result_id", r.ID).Update(r) if err != nil { return errors.Wrap(err, "updating finished result") } @@ -123,7 +107,7 @@ func (r *Result) Finished(sess sqlbuilder.Database) error { func (m *Measurement) Failed(sess sqlbuilder.Database, failure string) error { m.FailureMsg = sql.NullString{String: failure, Valid: true} m.IsFailed = true - err := sess.Collection("measurements").Find("id", m.ID).Update(m) + err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -136,7 +120,7 @@ func (m *Measurement) Done(sess sqlbuilder.Database) error { m.Runtime = runtime.Seconds() m.IsDone = true - err := sess.Collection("measurements").Find("id", m.ID).Update(m) + err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -148,7 +132,7 @@ func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) err m.UploadFailureMsg = sql.NullString{String: failure, Valid: true} m.IsUploaded = false - err := sess.Collection("measurements").Find("id", m.ID).Update(m) + err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -159,7 +143,7 @@ func (m *Measurement) UploadFailed(sess sqlbuilder.Database, failure string) err func (m *Measurement) UploadSucceeded(sess sqlbuilder.Database) error { m.IsUploaded = true - err := sess.Collection("measurements").Find("id", m.ID).Update(m) + err := sess.Collection("measurements").Find("measurement_id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } @@ -184,7 +168,7 @@ func (m *Measurement) AddToResult(sess sqlbuilder.Database, result *Result) erro } m.ReportFilePath = finalPath - err = sess.Collection("measurements").Find("id", m.ID).Update(m) + err = sess.Collection("measurements").Find("measurement_id", m.ID).Update(m) if err != nil { return errors.Wrap(err, "updating measurement") } diff --git a/internal/output/output.go b/internal/output/output.go index 0081bb2..0c55225 100644 --- a/internal/output/output.go +++ b/internal/output/output.go @@ -54,25 +54,25 @@ func MeasurementItem(msmt database.MeasurementURLNetwork, isFirst bool, isLast b "is_first": isFirst, "is_last": isLast, - "id": msmt.MsmtTblID, + "id": msmt.Measurement.ID, "test_name": msmt.TestName, "test_group_name": msmt.Result.TestGroupName, - "start_time": msmt.MeasurementStartTime, + "start_time": msmt.Measurement.StartTime, "test_keys": msmt.TestKeys, - "network_country_code": msmt.NetworkCountryCode, + "network_country_code": msmt.Network.CountryCode, "network_name": msmt.Network.NetworkName, "asn": msmt.Network.ASN, - "runtime": msmt.MeasurementRuntime, + "runtime": msmt.Measurement.Runtime, "url": msmt.URL.URL.String, "url_category_code": msmt.URL.CategoryCode.String, - "url_country_code": msmt.URLCountryCode.String, + "url_country_code": msmt.URL.CountryCode.String, "is_anomaly": msmt.IsAnomaly.Bool, "is_uploaded": msmt.IsUploaded, "is_upload_failed": msmt.IsUploadFailed, "upload_failure_msg": msmt.UploadFailureMsg.String, "is_failed": msmt.IsFailed, "failure_msg": msmt.FailureMsg.String, - "is_done": msmt.MeasurementIsDone, + "is_done": msmt.Measurement.IsDone, "report_file_path": msmt.ReportFilePath, }).Info("measurement") } From 41f449d6c9317cea33ad6f79c5ee86591c38e871 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 17:35:57 +0200 Subject: [PATCH 60/65] Remove debug log from unittests --- internal/database/actions_test.go | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index e8d0056..6068ddf 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -3,7 +3,6 @@ package database import ( "database/sql" "encoding/json" - "fmt" "io/ioutil" "os" "testing" @@ -90,8 +89,7 @@ func TestDeleteResult(t *testing.T) { if err != nil { t.Fatal(err) } - fmt.Printf("%s", tmpfile.Name()) - //defer os.Remove(tmpfile.Name()) + defer os.Remove(tmpfile.Name()) tmpdir, err := ioutil.TempDir("", "oonitest") if err != nil { From aad8a395ed8f28e108ccdf6b151dc7ae386e33ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Mon, 17 Sep 2018 17:57:26 +0200 Subject: [PATCH 61/65] Fix return value of database.Connect method --- internal/database/database.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/database/database.go b/internal/database/database.go index 577e022..02aca18 100644 --- a/internal/database/database.go +++ b/internal/database/database.go @@ -40,7 +40,8 @@ func Connect(path string) (db sqlbuilder.Database, err error) { err = RunMigrations(sess.Driver().(*sql.DB)) if err != nil { - db = nil + log.WithError(err).Error("failed to run DB migration") + return nil, err } return sess, err } From e6a67ca5aaf738b0360ca3af00ed07704bb448f0 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 18 Sep 2018 09:36:26 +0200 Subject: [PATCH 62/65] Improve output of CLI --- internal/log/handlers/cli/cli.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index a167ad6..993a4af 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -106,9 +106,9 @@ func (h *Handler) TypedLog(t string, e *log.Entry) error { switch t { case "progress": perc := e.Fields.Get("percentage").(float64) * 100 - s := fmt.Sprintf(" %s\n %-25s", + s := fmt.Sprintf(" %s %-25s", bold.Sprintf("%.2f%%", perc), - bold.Sprint(e.Message)) + e.Message) fmt.Fprint(h.Writer, s) fmt.Fprintln(h.Writer) return nil From 47a2fbb88c5f1b0498cd82b1223029a00481bc3f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 18 Sep 2018 09:54:27 +0200 Subject: [PATCH 63/65] Add rm command to delete results --- cmd/ooni/main.go | 1 + internal/cli/rm/rm.go | 51 +++++++++++++++++++++++++++++++ internal/database/actions.go | 3 ++ internal/database/actions_test.go | 6 ++++ 4 files changed, 61 insertions(+) create mode 100644 internal/cli/rm/rm.go diff --git a/cmd/ooni/main.go b/cmd/ooni/main.go index 017382b..5b6a6ce 100644 --- a/cmd/ooni/main.go +++ b/cmd/ooni/main.go @@ -9,6 +9,7 @@ import ( _ "github.com/ooni/probe-cli/internal/cli/list" _ "github.com/ooni/probe-cli/internal/cli/onboard" _ "github.com/ooni/probe-cli/internal/cli/reset" + _ "github.com/ooni/probe-cli/internal/cli/rm" _ "github.com/ooni/probe-cli/internal/cli/run" _ "github.com/ooni/probe-cli/internal/cli/show" _ "github.com/ooni/probe-cli/internal/cli/upload" diff --git a/internal/cli/rm/rm.go b/internal/cli/rm/rm.go new file mode 100644 index 0000000..284a2a9 --- /dev/null +++ b/internal/cli/rm/rm.go @@ -0,0 +1,51 @@ +package rm + +import ( + "errors" + "fmt" + + "github.com/alecthomas/kingpin" + "github.com/apex/log" + "github.com/ooni/probe-cli/internal/cli/root" + "github.com/ooni/probe-cli/internal/database" + survey "gopkg.in/AlecAivazis/survey.v1" + db "upper.io/db.v3" +) + +func init() { + cmd := root.Command("rm", "Delete a result") + yes := cmd.Flag("yes", "Skip interactive prompt").Bool() + + resultID := cmd.Arg("id", "the id of the result to delete").Int64() + + cmd.Action(func(_ *kingpin.ParseContext) error { + ctx, err := root.Init() + if err != nil { + log.Errorf("%s", err) + return err + } + + if *yes == true { + err = database.DeleteResult(ctx.DB, *resultID) + if err == db.ErrNoMoreRows { + return errors.New("result not found") + } + return err + } + answer := "" + confirm := &survey.Select{ + Message: fmt.Sprintf("Are you sure you wish to delete the result #%d", *resultID), + Options: []string{"true", "false"}, + Default: "false", + } + survey.AskOne(confirm, &answer, nil) + if answer == "false" { + return errors.New("canceled by user") + } + err = database.DeleteResult(ctx.DB, *resultID) + if err == db.ErrNoMoreRows { + return errors.New("result not found") + } + return err + }) +} diff --git a/internal/database/actions.go b/internal/database/actions.go index 27583f7..205fed1 100644 --- a/internal/database/actions.go +++ b/internal/database/actions.go @@ -122,6 +122,9 @@ func DeleteResult(sess sqlbuilder.Database, resultID int64) error { var result Result res := sess.Collection("results").Find("result_id", resultID) if err := res.One(&result); err != nil { + if err == db.ErrNoMoreRows { + return err + } log.WithError(err).Error("error in obtaining the result") return err } diff --git a/internal/database/actions_test.go b/internal/database/actions_test.go index 6068ddf..5f03f69 100644 --- a/internal/database/actions_test.go +++ b/internal/database/actions_test.go @@ -8,6 +8,7 @@ import ( "testing" "github.com/ooni/probe-cli/utils" + db "upper.io/db.v3" ) func TestMeasurementWorkflow(t *testing.T) { @@ -155,6 +156,11 @@ func TestDeleteResult(t *testing.T) { if totalMeasurements != 0 { t.Fatal("measurements should be zero") } + + err = DeleteResult(sess, 20) + if err != db.ErrNoMoreRows { + t.Fatal(err) + } } func TestNetworkCreate(t *testing.T) { From 8402446e478f33f8892fdd1dbb7305b73350c1a5 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 18 Sep 2018 09:55:23 +0200 Subject: [PATCH 64/65] Use Stdout in CLI logging --- internal/log/handlers/cli/cli.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/log/handlers/cli/cli.go b/internal/log/handlers/cli/cli.go index 993a4af..a478a7f 100644 --- a/internal/log/handlers/cli/cli.go +++ b/internal/log/handlers/cli/cli.go @@ -15,7 +15,7 @@ import ( ) // Default handler outputting to stderr. -var Default = New(os.Stderr) +var Default = New(os.Stdout) // start time. var start = time.Now() From cd04ba8bf9e872a94d4099beae69bbdb291eb9ec Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Arturo=20Filast=C3=B2?= Date: Tue, 18 Sep 2018 12:10:11 +0200 Subject: [PATCH 65/65] Remove sketchy workaround to UTF-8 characters in homedir We now implement a workaround to this inside of probe-desktop. See: https://github.com/ooni/probe-desktop/pull/17/commits/d97189f312eff5462abeb56e34a2b9de36e6b51e --- nettests/nettests.go | 36 ------------------------------------ 1 file changed, 36 deletions(-) diff --git a/nettests/nettests.go b/nettests/nettests.go index 3b4455b..1348fea 100644 --- a/nettests/nettests.go +++ b/nettests/nettests.go @@ -75,47 +75,11 @@ func (c *Controller) Init(nt *mk.Nettest) error { testName := strcase.ToSnake(nt.Name) resultID := c.res.ID reportFilePath := c.msmtPath - - // This is to workaround homedirs having UTF-8 characters in them. - // See: https://github.com/measurement-kit/measurement-kit/issues/1635 geoIPCountryPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIP.dat") geoIPASNPath := filepath.Join(utils.GeoIPDir(c.Ctx.Home), "GeoIPASNum.dat") caBundlePath := getCaBundlePath() msmtPath := c.msmtPath - userHome, err := utils.GetOONIHome() - if err != nil { - log.WithError(err).Error("failed to figure out the homedir") - return err - } - // Get the parent of it - userHome = filepath.Dir(userHome) - - relPath, err := filepath.Rel(userHome, caBundlePath) - if err != nil { - log.WithError(err).Error("caBundlePath is not relative to the users home") - } else { - caBundlePath = relPath - } - relPath, err = filepath.Rel(userHome, geoIPASNPath) - if err != nil { - log.WithError(err).Error("geoIPASNPath is not relative to the users home") - } else { - geoIPASNPath = relPath - } - relPath, err = filepath.Rel(userHome, geoIPCountryPath) - if err != nil { - log.WithError(err).Error("geoIPCountryPath is not relative to the users home") - } else { - geoIPCountryPath = relPath - } - - log.Debugf("Chdir to: %s", userHome) - if err := os.Chdir(userHome); err != nil { - log.WithError(err).Errorf("failed to chdir to %s", userHome) - return err - } - log.Debugf("OutputPath: %s", msmtPath) nt.Options = mk.NettestOptions{ IncludeIP: c.Ctx.Config.Sharing.IncludeIP,