selfhoster1312/planet-mars
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: selfhoster1312:5271c4c9aaf567afa3bd207a79cf6cae4b3bc363
Branches Tags
selfhoster1312:squat
selfhoster1312:fork
...
compare: selfhoster1312:90f29bd2a420dba333d3bd81bb5b21a0629e582a
Branches Tags
selfhoster1312:squat
selfhoster1312:fork

4 Commits
5271c4c9aa ... 90f29bd2a4

Author SHA1 Message Date
selfhoster1312
90f29bd2a4 feat: Copy THEME/assets to out_dir 2025-04-15 15:26:12 +02:00
selfhoster1312
6ee952133d feat: Add config.lang for translation links 2025-04-15 13:30:05 +02:00
selfhoster1312
6e288b3525 feat: add config.theme for templates subdirectory 2025-04-15 13:29:43 +02:00
selfhoster1312
001ca4879a feat: add --force flag to force rebuild 2025-04-15 13:27:51 +02:00
3 changed files with 86 additions and 30 deletions
Show Stats Expand all files Collapse all files

37
src/feed_store.rs
View File

@ -1,4 +1,4 @@
use anyhow::Result; use anyhow::{bail, Result};
use camino::{Utf8Path, Utf8PathBuf}; use camino::{Utf8Path, Utf8PathBuf};
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use feed_rs::model::Entry; use feed_rs::model::Entry;
@ -99,33 +99,33 @@ impl FeedStoreFeed {
self.info.fetch_data.as_ref() self.info.fetch_data.as_ref()
} }
pub fn load_feed(&self, sanitize: bool) -> Option<Feed> { pub fn load_feed(&self, sanitize: bool) -> Result<Feed> {
if let Some(raw_feed) = &self.raw_feed { if let Some(raw_feed) = &self.raw_feed {
let parser = feed_rs::parser::Builder::new() let parser = feed_rs::parser::Builder::new()
.sanitize_content(sanitize) .sanitize_content(sanitize)
.build(); .build();
Some(parser.parse(raw_feed.as_bytes()).unwrap()) Ok(parser.parse(raw_feed.as_bytes())?)
} else { } else {
None bail!("Feed not loaded yet: {}", self.url);
} }
} }
pub fn has_changed(&self, new_feed: &Feed) -> bool { pub fn has_changed(&self, new_feed: &Feed) -> Result<bool> {
let Some(old_feed) = self.load_feed(false) else { let Ok(old_feed) = self.load_feed(false) else {
return true; return Ok(true);
}; };
let mut old_iter = old_feed.entries.iter(); let mut old_iter = old_feed.entries.iter();
for new in &new_feed.entries { for new in &new_feed.entries {
let Some(old) = old_iter.next() else { let Some(old) = old_iter.next() else {
return true; return Ok(true);
}; };
if old != new { if old != new {
return true; return Ok(true);
} }
} }
// ignoring any entries left in old_iter // ignoring any entries left in old_iter
false Ok(false)
} }
pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> { pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> {
@ -149,7 +149,7 @@ impl FeedStoreFeed {
self.info.fetch_data = Some(fetchdata); self.info.fetch_data = Some(fetchdata);
Self::write(&self.path_settings, toml::to_string(&self.info)?)?; Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
if !self.has_changed(&feed) { if !self.has_changed(&feed)? {
return Ok(false); return Ok(false);
} }
debug!("Storing feed for {}.", self.url); debug!("Storing feed for {}.", self.url);
@ -235,7 +235,7 @@ pub struct FeedStore {
} }
impl FeedStore { impl FeedStore {
pub fn new(dir: &str, feedlist: &Vec<super::FeedConfig>) -> Self { pub fn new(dir: &Utf8Path, feedlist: &Vec<super::FeedConfig>) -> Self {
let dir = super::to_checked_pathbuf(dir); let dir = super::to_checked_pathbuf(dir);
let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new(); let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new();
@ -248,20 +248,25 @@ impl FeedStore {
} }
pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) { pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) {
debug!("Collecting feeds");
let mut feeds = HashMap::new(); let mut feeds = HashMap::new();
let mut entries = Vec::new(); let mut entries = Vec::new();
for (feed_url, feed_store_feed) in self.feeds.iter_mut() { for (feed_url, feed_store_feed) in self.feeds.iter_mut() {
let Some(mut feed) = feed_store_feed.load_feed(true) else { debug!("Collecting {feed_url}");
warn!("Problem parsing feed file for feed {}", feed_url); let mut feed = match feed_store_feed.load_feed(true) {
continue; Ok(feed) => feed,
Err(e) => {
warn!("Problem parsing feed file for feed {}: {}", feed_url, e);
continue;
}
}; };
for entry in &mut feed.entries { for entry in &mut feed.entries {
entry.source = Some(feed_url.to_string()); entry.source = Some(feed_url.to_string());
} }
entries.append(&mut std::mem::take(&mut feed.entries)); entries.extend(feed.entries.clone());
if entries.len() > 4 * max_entries { if entries.len() > 4 * max_entries {
entries = trim_entries(entries, max_entries); entries = trim_entries(entries, max_entries);
} }

38
src/main.rs
View File

@ -26,10 +26,11 @@ extern crate log;
use crate::feed_store::FeedStore; use crate::feed_store::FeedStore;
use crate::fetcher::Fetcher; use crate::fetcher::Fetcher;
use anyhow::Result; use anyhow::Result;
use camino::Utf8PathBuf; use camino::{Utf8Path, Utf8PathBuf};
use clap::Parser; use clap::Parser;
use serde::Deserialize; use serde::{Deserialize, Serialize};
use std::fs; use std::fs;
use url::Url;
//mod atom_serializer; //mod atom_serializer;
mod feed_store; mod feed_store;
@ -48,6 +49,8 @@ struct Args {
config: String, config: String,
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
no_fetch: bool, no_fetch: bool,
#[arg(long, default_value_t = false)]
force: bool,
} }
/// Config to be parsed from toml file given as cmdline option /// Config to be parsed from toml file given as cmdline option
@ -56,29 +59,42 @@ struct Config {
/// to be used as part of the fetchers username header /// to be used as part of the fetchers username header
bot_name: String, bot_name: String,
/// where to store downloaded feeds and their metadata /// where to store downloaded feeds and their metadata
feed_dir: String, feed_dir: Utf8PathBuf,
/// feeds to be agregated /// feeds to be agregated
feeds: Vec<FeedConfig>, feeds: Vec<FeedConfig>,
/// Email adress to use for the from header when fetching feeds /// Email adress to use for the from header when fetching feeds
from: String, from: String,
/// where to build the output files /// where to build the output files
out_dir: String, out_dir: Utf8PathBuf,
/// templates folder /// templates folder
templates_dir: String, templates_dir: Utf8PathBuf,
/// How many feed entries should be included in the planet /// How many feed entries should be included in the planet
max_entries: usize, max_entries: usize,
/// How soon to refresh, in hours /// How soon to refresh, in hours
refresh: usize, refresh: usize,
/// A theme to apply, if any.
///
/// This is a folder in the templates_dir. If an assets directory
/// exists within, the contents will be copied over to the out_dir.
theme: Option<String>,
/// List of languages for translations
#[serde(default)]
lang: Vec<Lang>,
} }
pub fn to_checked_pathbuf(dir: &str) -> Utf8PathBuf { #[derive(Clone, Debug, Deserialize, Serialize)]
let dir = Utf8PathBuf::from(dir); pub struct Lang {
code: String,
name: String,
link: Url,
}
pub fn to_checked_pathbuf(dir: &Utf8Path) -> Utf8PathBuf {
let m = dir let m = dir
.metadata() .metadata()
.unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}")); .unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}"));
assert!(m.is_dir(), "Not a dir: {dir}"); assert!(m.is_dir(), "Not a dir: {dir}");
dir dir.to_path_buf()
} }
/// Config for one individual input feed /// Config for one individual input feed
@ -91,10 +107,10 @@ struct FeedConfig {
url: String, url: String,
} }
fn fetch(config: &Config, feed_store: &mut FeedStore) -> Result<bool> { fn fetch(config: &Config, feed_store: &mut FeedStore, force_rebuild: bool) -> Result<bool> {
let fetcher = Fetcher::new(&config.bot_name, &config.from); let fetcher = Fetcher::new(&config.bot_name, &config.from);
let rebuild = feed_store.fetch(&fetcher, config.refresh)?; let rebuild = feed_store.fetch(&fetcher, config.refresh)?;
Ok(rebuild) Ok(rebuild || force_rebuild)
} }
fn main() -> Result<()> { fn main() -> Result<()> {
@ -116,7 +132,7 @@ fn main() -> Result<()> {
let should_build = if args.no_fetch { let should_build = if args.no_fetch {
true true
} else { } else {
fetch(&config, &mut feed_store)? fetch(&config, &mut feed_store, args.force)?
}; };
if should_build { if should_build {

41
src/template_engine.rs
View File

@ -2,19 +2,25 @@ use crate::feed_store::FeedStore;
use crate::to_checked_pathbuf; use crate::to_checked_pathbuf;
use crate::Config; use crate::Config;
use anyhow::Result; use anyhow::Result;
use camino::Utf8Path;
use feed_rs::model::Feed; use feed_rs::model::Feed;
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::File; use std::fs::{copy, create_dir_all, File};
use tera::{from_value, Tera}; use tera::{from_value, Tera};
pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> { pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
let mut tera = create_tera(&config.templates_dir)?; let mut tera = if let Some(theme) = &config.theme {
create_tera(&config.templates_dir.join(theme))?
} else {
create_tera(&config.templates_dir)?
};
let out_dir = to_checked_pathbuf(&config.out_dir); let out_dir = to_checked_pathbuf(&config.out_dir);
let mut context = tera::Context::new(); let mut context = tera::Context::new();
let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries); let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries);
context.insert("feeds", &feeds); context.insert("feeds", &feeds);
context.insert("entries", &entries); context.insert("entries", &entries);
context.insert("lang", &config.lang);
context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS")); context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS"));
context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE")); context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE"));
context.insert("PKG_NAME", env!("CARGO_PKG_NAME")); context.insert("PKG_NAME", env!("CARGO_PKG_NAME"));
@ -26,10 +32,39 @@ pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
let file = File::create(format!("{out_dir}/{name}"))?; let file = File::create(format!("{out_dir}/{name}"))?;
tera.render_to(name, &context, file)?; tera.render_to(name, &context, file)?;
} }
// Copy static assets from theme, if any
if let Some(theme) = &config.theme {
let assets_dir = config.templates_dir.join(theme).join("assets");
if assets_dir.is_dir() {
copy_assets(&assets_dir, &out_dir)?;
}
}
Ok(()) Ok(())
} }
fn create_tera(templates_dir: &str) -> Result<Tera> { /// Recursively copy assets from one dir to another
///
/// Symlinks are ignored.
fn copy_assets(orig: &Utf8Path, dest: &Utf8Path) -> Result<()> {
if orig.is_dir() {
if !dest.is_dir() {
create_dir_all(dest)?;
}
for entry in orig.read_dir_utf8()? {
let entry = entry?;
copy_assets(entry.path(), &dest.join(entry.file_name()))?;
}
} else if orig.is_file() {
copy(orig, dest)?;
}
Ok(())
}
fn create_tera(templates_dir: &Utf8Path) -> Result<Tera> {
let dir = to_checked_pathbuf(templates_dir); let dir = to_checked_pathbuf(templates_dir);
let mut tera = tera::Tera::new(&format!("{dir}/*"))?; let mut tera = tera::Tera::new(&format!("{dir}/*"))?;
// disable autoescape as this would corrupt urls or the entriy contents. todo check this! // disable autoescape as this would corrupt urls or the entriy contents. todo check this!