diff --git a/Cargo.lock b/Cargo.lock index 540750f..322b0bf 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -96,6 +96,12 @@ dependencies = [ "windows-sys 0.59.0", ] +[[package]] +name = "anyhow" +version = "1.0.95" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04" + [[package]] name = "autocfg" version = "1.4.0" @@ -1093,6 +1099,7 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2" name = "planet-mars" version = "0.1.0" dependencies = [ + "anyhow", "clap", "env_logger", "feed-rs", diff --git a/Cargo.toml b/Cargo.toml index 92a58eb..ab6f164 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -5,6 +5,7 @@ edition = "2021" [dependencies] # ammonia = "*" done by feed-rs +anyhow = "*" clap = { version = "*", features = ["derive"] } env_logger = "*" feed-rs = "*" diff --git a/src/feed_store.rs b/src/feed_store.rs index 6b162de..7b75e51 100644 --- a/src/feed_store.rs +++ b/src/feed_store.rs @@ -1,6 +1,7 @@ use feed_rs::model::Entry; use feed_rs::model::Feed; use serde::{Deserialize, Serialize}; +use std::convert::AsRef; use std::fs; use std::io::BufReader; use std::path::PathBuf; @@ -20,7 +21,7 @@ pub struct FeedStore { } impl FeedStore { - pub fn new(dir: String) -> Self { + pub fn new(dir: &str) -> Self { Self { dir: super::to_checked_pathbuf(dir), } @@ -66,6 +67,16 @@ impl FeedStore { false } + fn write + std::fmt::Display, C: AsRef<[u8]>>( + path: P, + contents: C, + ) -> std::io::Result<()> { + if fs::exists(&path)? { + fs::rename(&path, format!("{path}.backup"))?; + } + fs::write(path, contents) + } + pub fn store(&self, url: &Url, mut response: Response) -> bool { let headers = response.headers(); let fetchdata = FetchData { @@ -89,8 +100,9 @@ impl FeedStore { if !self.has_changed(url, &feed) { return false; } - let _ = fs::write(self.feed_path(url), body); - let _ = fs::write( + debug!("Storing feed for {url}."); + let _ = Self::write(self.feed_path(url), body); + let _ = Self::write( self.fetchdata_path(url), toml::to_string(&fetchdata).unwrap(), ); diff --git a/src/fetcher.rs b/src/fetcher.rs index 43d08bb..13c326f 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -37,10 +37,10 @@ impl Fetcher { .agent .get(url.to_string()) .header("FROM", self.from.clone()); - if fetchdata.etag != "" { + if !fetchdata.etag.is_empty() { builder = builder.header("If-None-Match", fetchdata.etag); } - if fetchdata.date != "" { + if !fetchdata.date.is_empty() { builder = builder.header("If-Modified-Since", fetchdata.date); } diff --git a/src/main.rs b/src/main.rs index 5729cd7..f479cae 100644 --- a/src/main.rs +++ b/src/main.rs @@ -3,14 +3,17 @@ extern crate log; use crate::feed_store::FeedStore; use crate::fetcher::Fetcher; +use anyhow::Result; use clap::Parser; use serde::Deserialize; +use simple_entry::SimpleEntry; use std::fs; use std::path::PathBuf; use url::Url; mod feed_store; mod fetcher; +mod simple_entry; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -21,6 +24,8 @@ struct Args { default_value_t = String::from("mars.toml") )] config: String, + #[arg(long, default_value_t = false)] + no_fetch: bool, } #[derive(Deserialize)] @@ -39,7 +44,7 @@ struct Config { templates_dir: String, } -pub fn to_checked_pathbuf(dir: String) -> PathBuf { +pub fn to_checked_pathbuf(dir: &str) -> PathBuf { let dir: PathBuf = PathBuf::from(dir); let m = dir @@ -54,7 +59,54 @@ struct FeedConfig { url: String, } -fn main() -> Result<(), Box> { +fn fetch(config: &Config, feed_store: &FeedStore) -> Result { + let fetcher = Fetcher::new(&config.bot_name, &config.from); + let mut rebuild = false; + for feed in &config.feeds { + let url = match Url::parse(&feed.url) { + Ok(x) => x, + Err(e) => { + error!("Error parsing url '{}': {e:?}", feed.url); + continue; + } + }; + rebuild |= fetcher.fetch(url, feed_store); + } + info!("Done fetching. Rebuild needed: {rebuild}"); + Ok(rebuild) +} + +fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { + let templates_dir = to_checked_pathbuf(&config.templates_dir); + let out_dir = to_checked_pathbuf(&config.out_dir); + + let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { + Ok(t) => t, + Err(e) => { + println!("Parsing error(s): {}", e); + ::std::process::exit(1); + } + }; + // disable autoescape as this would corrupt urls or the entriy contents. todo check this! + tera.autoescape_on(vec![]); + + let mut context = tera::Context::new(); + let entries: Vec = feed_store + .collect(&config.feeds) + .into_iter() + .map(SimpleEntry::from_feed_entry) + .collect(); + context.insert("entries", &entries); + + for name in tera.get_template_names() { + debug!("Processing template {name}"); + let file = fs::File::create(format!("{}/{name}", out_dir.display()))?; + tera.render_to(name, &context, file)?; + } + Ok(()) +} + +fn main() -> Result<()> { env_logger::init(); info!("starting up"); @@ -64,35 +116,20 @@ fn main() -> Result<(), Box> { panic!("Configuration file {config_path} does not exist!"); } let config: Config = toml::from_str(&fs::read_to_string(config_path)?)?; - let templates_dir = to_checked_pathbuf(config.templates_dir); - let out_dir = to_checked_pathbuf(config.out_dir); + // only check here to avoid fetching with broken config + // todo: get a config lib that provides validation! + let _ = to_checked_pathbuf(&config.templates_dir); + let _ = to_checked_pathbuf(&config.out_dir); - let feed_store = FeedStore::new(config.feed_dir); - let fetcher = Fetcher::new(&config.bot_name, &config.from); + let feed_store = FeedStore::new(&config.feed_dir); + let should_build = if args.no_fetch { + true + } else { + fetch(&config, &feed_store)? + }; - let mut rebuild = false; - for feed in &config.feeds { - let url = Url::parse(&feed.url)?; - rebuild |= fetcher.fetch(url, &feed_store); - } - info!("Done fetching. Rebuild needed: {rebuild}"); - if rebuild { - let entries = feed_store.collect(&config.feeds); - let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { - Ok(t) => t, - Err(e) => { - println!("Parsing error(s): {}", e); - ::std::process::exit(1); - } - }; - tera.autoescape_on(vec![]); - let mut context = tera::Context::new(); - context.insert("entries", &entries); - for name in tera.get_template_names() { - debug!("Processing template {name}"); - let file = fs::File::create(&format!("{}/{name}", out_dir.display()))?; - let _ = tera.render_to(name, &context, file)?; - } + if should_build { + build(&config, &feed_store)?; } Ok(()) } diff --git a/src/simple_entry.rs b/src/simple_entry.rs new file mode 100644 index 0000000..4f531bf --- /dev/null +++ b/src/simple_entry.rs @@ -0,0 +1,43 @@ +use feed_rs::model::Entry; + +/// Simplified Feed entry for easier value access in template +#[derive(serde::Serialize)] +pub struct SimpleEntry { + pub date: String, + pub content: String, + pub author: String, + pub link: String, + pub title: String, +} + +/// format for the entries timestamp +/// +const FMT: &str = "%c"; + +impl SimpleEntry { + pub fn from_feed_entry(entry: Entry) -> Self { + Self { + date: entry + .updated + .or(entry.published) + .unwrap_or_default() + .format(FMT) + .to_string(), + content: entry + .content + .map(|x| x.body.unwrap_or_default()) + .unwrap_or_default(), + author: if !entry.authors.is_empty() { + entry.authors[0].name.clone() + } else { + "".to_string() + }, + link: if !entry.links.is_empty() { + entry.links[0].href.clone() + } else { + "".to_string() + }, + title: entry.title.map(|x| x.content).unwrap_or_default(), + } + } +} diff --git a/templates/index.html b/templates/index.html index 5ecbc15..0c781cf 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,8 +1,32 @@ -hello world - -{% for entry in entries %} - {% for link in entry.links %} - {{link.href}} - {% endfor %} -{% endfor %} + + + Planet TVL + + + + + {# todo #} + + + +
+

Planet TVL

+ {% for entry in entries %} + + {% endfor %} +
+ + +