diff --git a/Cargo.lock b/Cargo.lock index 322b0bf..744b457 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -108,6 +108,12 @@ version = "1.4.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "ace50bade8e6234aa140d9a2f552bbee1db4d353f69b8217bc503490fc1a9f26" +[[package]] +name = "base64" +version = "0.21.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9d297deb1925b89f2ccc13d7635fa0714f12c87adce1c75356b39ca9b7178567" + [[package]] name = "base64" version = "0.22.1" @@ -125,6 +131,9 @@ name = "bitflags" version = "2.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "b048fb63fd8b5923fc5aa7b340d8e156aec7ec02f0c78fa8a6ddc2613f6f71de" +dependencies = [ + "serde", +] [[package]] name = "block-buffer" @@ -1104,6 +1113,8 @@ dependencies = [ "env_logger", "feed-rs", "log", + "quick-xml", + "ron", "serde", "slug", "tera", @@ -1144,6 +1155,7 @@ checksum = "165859e9e55f79d67b96c5d96f4e88b6f2695a1972849c15a6a3f5c59fc2c003" dependencies = [ "encoding_rs", "memchr", + "serde", ] [[package]] @@ -1229,6 +1241,18 @@ dependencies = [ "windows-sys 0.52.0", ] +[[package]] +name = "ron" +version = "0.8.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b91f7eff05f748767f183df4320a63d6936e9c6107d97c9e6bdd9784f4289c94" +dependencies = [ + "base64 0.21.7", + "bitflags", + "serde", + "serde_derive", +] + [[package]] name = "rustix" version = "0.38.43" @@ -1667,7 +1691,7 @@ version = "3.0.0-rc5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "77f9bd1d03fe3bf87c90b115a0e09e22535cbcff3f8d0a6487524f006325f173" dependencies = [ - "base64", + "base64 0.22.1", "brotli-decompressor", "cc", "cookie_store", @@ -1695,7 +1719,7 @@ version = "0.2.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "a27729fd2c15426f48992a911ce31bd1f29c1cd0898c2c86b410d24f51c99eb3" dependencies = [ - "base64", + "base64 0.22.1", "http", "httparse", "log", diff --git a/Cargo.toml b/Cargo.toml index 5e7894a..848af9c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -9,10 +9,11 @@ clap = { version = "*", features = ["derive"] } env_logger = "*" feed-rs = "*" log = "*" +ron = "*" # todo for development, to check atom-rs internal representation of feeds serde = { version = "*", features = ["derive"] } slug = "*" tera = "*" toml = "*" ureq = { version = "3.0.0-rc5", features = ["brotli", "charset", "gzip", "native-tls"]} url = "*" - +quick-xml = { version = "*", features = ["serialize"] } diff --git a/planet.css b/planet.css new file mode 100644 index 0000000..aca682f --- /dev/null +++ b/planet.css @@ -0,0 +1,59 @@ +p, h1, h2, h3, h4, h5, h6, small { + max-width: 48em; +} + +h1 a, h2 a, h3 a, h4 a, h5 a, h6 a { + color: inherit !important; + text-decoration: none; +} + +ul, ol { + /* account for the 1em -webkit-margin-start for the list icon */ + max-width: 45em; +} + +ul,ol,dl, p { + line-height: 1.4; +} + +body { + margin-top: 1em; + margin-bottom: 1em; +} + +#maincontainer { + display: flex; + max-width: 80em; +} + +#maincontainer main { + max-width: 50em; +} + +#maincontainer main * { + max-width: 50em; +} + +#maincontainer aside { + margin-left: 5em; + max-width: 25em; +} + +article > h2.entry_header { + margin-bottom: 3px; +} + +.entry_meta { + border: 1px thin; + padding: 3px 0; + background-color: LightBlue; +} + +hr.entry_sep { + border: none; +} +hr.entry_sep::before { + content: '* * *'; + display: block; + text-align: center; +} diff --git a/src/feed_store.rs b/src/feed_store.rs index 9e72e6c..7b86426 100644 --- a/src/feed_store.rs +++ b/src/feed_store.rs @@ -1,5 +1,7 @@ +use anyhow::Result; use feed_rs::model::Entry; use feed_rs::model::Feed; +use ron::ser::{to_string_pretty, PrettyConfig}; use serde::{Deserialize, Serialize}; use std::convert::AsRef; use std::fs; @@ -28,43 +30,47 @@ impl FeedStore { } fn slugify_url(url: &Url) -> String { - let domain = url.domain().unwrap(); + let domain = url.domain().unwrap(); // todo don't hide error let query = url.query().unwrap_or(""); slug::slugify(format!("{domain}{}{query}", url.path())) } + fn generic_path(&self, url: &Url, ext: &str) -> String { + format!("{}/{}{ext}", self.dir.display(), Self::slugify_url(url)) + } + fn feed_path(&self, url: &Url) -> String { - format!("{}/{}", self.dir.display(), Self::slugify_url(url)) + self.generic_path(url, "") } fn fetchdata_path(&self, url: &Url) -> String { - format!("{}.toml", self.feed_path(url)) + self.generic_path(url, ".toml") } - pub fn load_fetchdata(&self, url: &Url) -> FetchData { + pub fn load_fetchdata(&self, url: &Url) -> Result { let path = self.fetchdata_path(url); - if !fs::exists(path.clone()).unwrap() { - return FetchData::default(); + if !fs::exists(path.clone())? { + return Ok(FetchData::default()); } - toml::from_str(&fs::read_to_string(path).unwrap()).unwrap() + Ok(toml::from_str(&fs::read_to_string(path)?)?) } - fn has_changed(&self, url: &Url, new_feed: &Feed) -> bool { + fn has_changed(&self, url: &Url, new_feed: &Feed) -> Result { let Some(old_feed) = self.load_feed(url, false) else { - return true; + return Ok(true); }; let mut old_iter = old_feed.entries.iter(); for new in &new_feed.entries { let Some(old) = old_iter.next() else { - return true; + return Ok(true); }; if old != new { - return true; + return Ok(true); } } // ignoring any entries left in old_iter - false + Ok(false) } fn write + std::fmt::Display, C: AsRef<[u8]>>( @@ -77,7 +83,7 @@ impl FeedStore { fs::write(path, contents) } - pub fn store(&self, url: &Url, mut response: Response) -> bool { + pub fn store(&self, url: &Url, mut response: Response) -> Result { let headers = response.headers(); let fetchdata = FetchData { etag: hv(headers, "etag"), @@ -94,19 +100,24 @@ impl FeedStore { Ok(f) => f, Err(e) => { warn!("Error when parsing feed for {url}: {e:?}"); - return false; + return Ok(false); } }; - if !self.has_changed(url, &feed) { - return false; + if !self.has_changed(url, &feed)? { + return Ok(false); } debug!("Storing feed for {url}."); - let _ = Self::write(self.feed_path(url), body); - let _ = Self::write( + // todo don't serialize to string but to writer + Self::write( + self.generic_path(url, ".ron"), + to_string_pretty(&feed, PrettyConfig::default())?, + )?; + Self::write(self.feed_path(url), body)?; + Self::write( self.fetchdata_path(url), toml::to_string(&fetchdata).unwrap(), - ); - true + )?; + Ok(true) } fn load_feed(&self, url: &Url, sanitize: bool) -> Option { @@ -132,6 +143,7 @@ impl FeedStore { warn!("Problem parsing feed file for feed {}", feed_config.url); continue; }; + entries.append(&mut feed.entries); // todo also trim mid-way when length > something, trading cpu for memory } diff --git a/src/fetcher.rs b/src/fetcher.rs index 8cc469a..289bbe1 100644 --- a/src/fetcher.rs +++ b/src/fetcher.rs @@ -1,3 +1,4 @@ +use anyhow::Result; use std::time::Instant; use ureq::tls::{TlsConfig, TlsProvider}; use ureq::Agent; @@ -31,8 +32,8 @@ impl Fetcher { } } - pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> bool { - let fetchdata = feed_store.load_fetchdata(&url); + pub fn fetch(&self, url: Url, feed_store: &FeedStore) -> Result { + let fetchdata = feed_store.load_fetchdata(&url)?; let mut builder = self .agent .get(url.to_string()) @@ -48,7 +49,7 @@ impl Fetcher { let result = builder.call(); let duration = start_instant.elapsed(); - let response = result.unwrap(); // todo log and return false + let response = result?; // todo log and return false debug!( "fetched with status {} in {} ms: {url}", response.status(), @@ -56,14 +57,14 @@ impl Fetcher { ); let status = response.status(); match status.as_u16() { - 304 => false, // Not Modified -> nothing to do + 304 => Ok(false), // Not Modified -> nothing to do 200 => feed_store.store(&url, response), _ => { warn!( "HTTP Status {} not implemented for {url}", response.status() ); - false + Ok(false) } } } diff --git a/src/main.rs b/src/main.rs index f479cae..296cf9e 100644 --- a/src/main.rs +++ b/src/main.rs @@ -6,14 +6,14 @@ use crate::fetcher::Fetcher; use anyhow::Result; use clap::Parser; use serde::Deserialize; -use simple_entry::SimpleEntry; use std::fs; use std::path::PathBuf; use url::Url; +//mod atom_serializer; mod feed_store; mod fetcher; -mod simple_entry; +mod template_engine; #[derive(Parser)] #[command(author, version, about, long_about = None)] @@ -70,42 +70,12 @@ fn fetch(config: &Config, feed_store: &FeedStore) -> Result { continue; } }; - rebuild |= fetcher.fetch(url, feed_store); + rebuild |= fetcher.fetch(url, feed_store)?; } info!("Done fetching. Rebuild needed: {rebuild}"); Ok(rebuild) } -fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { - let templates_dir = to_checked_pathbuf(&config.templates_dir); - let out_dir = to_checked_pathbuf(&config.out_dir); - - let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) { - Ok(t) => t, - Err(e) => { - println!("Parsing error(s): {}", e); - ::std::process::exit(1); - } - }; - // disable autoescape as this would corrupt urls or the entriy contents. todo check this! - tera.autoescape_on(vec![]); - - let mut context = tera::Context::new(); - let entries: Vec = feed_store - .collect(&config.feeds) - .into_iter() - .map(SimpleEntry::from_feed_entry) - .collect(); - context.insert("entries", &entries); - - for name in tera.get_template_names() { - debug!("Processing template {name}"); - let file = fs::File::create(format!("{}/{name}", out_dir.display()))?; - tera.render_to(name, &context, file)?; - } - Ok(()) -} - fn main() -> Result<()> { env_logger::init(); info!("starting up"); @@ -129,7 +99,7 @@ fn main() -> Result<()> { }; if should_build { - build(&config, &feed_store)?; + template_engine::build(&config, &feed_store)?; } Ok(()) } diff --git a/src/simple_entry.rs b/src/simple_entry.rs deleted file mode 100644 index 4f531bf..0000000 --- a/src/simple_entry.rs +++ /dev/null @@ -1,43 +0,0 @@ -use feed_rs::model::Entry; - -/// Simplified Feed entry for easier value access in template -#[derive(serde::Serialize)] -pub struct SimpleEntry { - pub date: String, - pub content: String, - pub author: String, - pub link: String, - pub title: String, -} - -/// format for the entries timestamp -/// -const FMT: &str = "%c"; - -impl SimpleEntry { - pub fn from_feed_entry(entry: Entry) -> Self { - Self { - date: entry - .updated - .or(entry.published) - .unwrap_or_default() - .format(FMT) - .to_string(), - content: entry - .content - .map(|x| x.body.unwrap_or_default()) - .unwrap_or_default(), - author: if !entry.authors.is_empty() { - entry.authors[0].name.clone() - } else { - "".to_string() - }, - link: if !entry.links.is_empty() { - entry.links[0].href.clone() - } else { - "".to_string() - }, - title: entry.title.map(|x| x.content).unwrap_or_default(), - } - } -} diff --git a/src/template_engine.rs b/src/template_engine.rs new file mode 100644 index 0000000..b1d9bd8 --- /dev/null +++ b/src/template_engine.rs @@ -0,0 +1,35 @@ +use crate::feed_store::FeedStore; +use crate::to_checked_pathbuf; +use crate::Config; +use anyhow::Result; +use feed_rs::model::Entry; +use std::fs::File; +use tera::Tera; + +pub fn build(config: &Config, feed_store: &FeedStore) -> Result<()> { + let tera = create_tera(&config.templates_dir)?; + let out_dir = to_checked_pathbuf(&config.out_dir); + + let mut context = tera::Context::new(); + let feed_entries: Vec = feed_store.collect(&config.feeds); + context.insert("entries", &feed_entries); + context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS")); + context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE")); + context.insert("PKG_NAME", env!("CARGO_PKG_NAME")); + context.insert("PKG_VERSION", env!("CARGO_PKG_VERSION")); + + for name in tera.get_template_names() { + debug!("Processing template {name}"); + let file = File::create(format!("{}/{name}", out_dir.display()))?; + tera.render_to(name, &context, file)?; + } + Ok(()) +} + +fn create_tera(templates_dir: &str) -> Result { + let dir = to_checked_pathbuf(templates_dir); + let mut tera = tera::Tera::new(&format!("{}/*", &dir.display()))?; + // disable autoescape as this would corrupt urls or the entriy contents. todo check this! + tera.autoescape_on(vec![]); + Ok(tera) +} diff --git a/templates/atom.xml b/templates/atom.xml new file mode 100644 index 0000000..bc11aa9 --- /dev/null +++ b/templates/atom.xml @@ -0,0 +1,52 @@ + + + Planet TVL + + {{now()|date(format="%Y-%m-%dT%H:%M:%SZ")}} + https::/planet.tvl.fyi + + {{ PKG_NAME }} by {{ PKG_AUTHORS }} + + https://planet.tvl.fyi/logo.svg + + {% for entry in entries %} + + {{ entry.id }}/planet.tvl.fyi + {% if entry.title -%} + {{ entry.title.content }} + {% endif -%} + {% for link in entry.links %} + + {% endfor %} + {% if entry.updated %} + {{ entry.updated }} + {% endif %} + {% if entry.published %} + {{ entry.published }} + {% endif %} + {% if entry.summary -%} + + {{ entry.summary.content|escape }} + + {% endif -%} + {% for author in entry.authors %} + + {% if author.name -%} + {{ author.name }} + {% endif -%} + {% if author.email -%} + {{ author.email }} + {% endif -%} + + {% if author.email -%} + {{ author.uri }} + {% endif -%} + {% endfor %} + {% if entry.content -%} + + {{ entry.content.body|escape }} + + {% endif -%} + + {% endfor %} + diff --git a/templates/index.html b/templates/index.html index c82073e..0b5c1f1 100644 --- a/templates/index.html +++ b/templates/index.html @@ -1,3 +1,5 @@ +{% set dateformat = "%d.%m.%Y %H:%M" -%} + Planet TVL @@ -5,7 +7,6 @@ - {# todo #} @@ -14,17 +15,50 @@
- {% for entry in entries %} + {% for entry in entries -%} + {% if loop.index > 1 -%} +
+ {% endif -%} + {% if entry.links.0 -%} + {% set link = entry.links.0.href -%} + {% else -%} + {% set link = "" -%} + {% endif -%}
-

{{ entry.title|striptags }}

- {% if entry.published %}{{ entry.published | date(format="%Y-%m-%d %H:%M", timezone="Europe/Moscow") }}{% endif %}{# todo: maybe group posts by day? #} -

{{ entry.author|striptags }}

-
- {{ entry.content }} +

+ + {% if entry.title -%} + {{ entry.title.content|striptags }} + {% else -%} + NO TITLE + {% endif -%} + +

+
+ + {% if entry.updated -%} + {{ entry.updated | date(format=dateformat) }} + {% else -%} + {{ entry.published | date(format=dateformat) }} + {% endif -%} + + {% if entry.authors -%} + — {{ entry.authors.0.name | striptags }} + {% endif -%}
-

full post

+ + {% if entry.summary -%} +
+ {{ entry.summary.content }} +
+ {% endif -%} + {% if entry.content -%} +
+ {{ entry.content.body }} +
+ {% endif -%}
- {% endfor %} + {% endfor -%}