selfhoster1312/planet-mars
1
0
Fork 0
Code Issues Pull Requests Packages Projects Releases Wiki Activity

Compare commits

base: selfhoster1312:90f29bd2a420dba333d3bd81bb5b21a0629e582a
Branches Tags
selfhoster1312:squat
selfhoster1312:fork
..
compare: selfhoster1312:5271c4c9aaf567afa3bd207a79cf6cae4b3bc363
Branches Tags
selfhoster1312:squat
selfhoster1312:fork

No commits in common. "90f29bd2a420dba333d3bd81bb5b21a0629e582a" and "5271c4c9aaf567afa3bd207a79cf6cae4b3bc363" have entirely different histories.
90f29bd2a4 ... 5271c4c9aa

3 changed files with 30 additions and 86 deletions
Show Stats Expand all files Collapse all files

37
src/feed_store.rs
View File

@ -1,4 +1,4 @@
use anyhow::{bail, Result}; use anyhow::Result;
use camino::{Utf8Path, Utf8PathBuf}; use camino::{Utf8Path, Utf8PathBuf};
use chrono::{DateTime, Duration, Utc}; use chrono::{DateTime, Duration, Utc};
use feed_rs::model::Entry; use feed_rs::model::Entry;
@ -99,33 +99,33 @@ impl FeedStoreFeed {
self.info.fetch_data.as_ref() self.info.fetch_data.as_ref()
} }
pub fn load_feed(&self, sanitize: bool) -> Result<Feed> { pub fn load_feed(&self, sanitize: bool) -> Option<Feed> {
if let Some(raw_feed) = &self.raw_feed { if let Some(raw_feed) = &self.raw_feed {
let parser = feed_rs::parser::Builder::new() let parser = feed_rs::parser::Builder::new()
.sanitize_content(sanitize) .sanitize_content(sanitize)
.build(); .build();
Ok(parser.parse(raw_feed.as_bytes())?) Some(parser.parse(raw_feed.as_bytes()).unwrap())
} else { } else {
bail!("Feed not loaded yet: {}", self.url); None
} }
} }
pub fn has_changed(&self, new_feed: &Feed) -> Result<bool> { pub fn has_changed(&self, new_feed: &Feed) -> bool {
let Ok(old_feed) = self.load_feed(false) else { let Some(old_feed) = self.load_feed(false) else {
return Ok(true); return true;
}; };
let mut old_iter = old_feed.entries.iter(); let mut old_iter = old_feed.entries.iter();
for new in &new_feed.entries { for new in &new_feed.entries {
let Some(old) = old_iter.next() else { let Some(old) = old_iter.next() else {
return Ok(true); return true;
}; };
if old != new { if old != new {
return Ok(true); return true;
} }
} }
// ignoring any entries left in old_iter // ignoring any entries left in old_iter
Ok(false) false
} }
pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> { pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> {
@ -149,7 +149,7 @@ impl FeedStoreFeed {
self.info.fetch_data = Some(fetchdata); self.info.fetch_data = Some(fetchdata);
Self::write(&self.path_settings, toml::to_string(&self.info)?)?; Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
if !self.has_changed(&feed)? { if !self.has_changed(&feed) {
return Ok(false); return Ok(false);
} }
debug!("Storing feed for {}.", self.url); debug!("Storing feed for {}.", self.url);
@ -235,7 +235,7 @@ pub struct FeedStore {
} }
impl FeedStore { impl FeedStore {
pub fn new(dir: &Utf8Path, feedlist: &Vec<super::FeedConfig>) -> Self { pub fn new(dir: &str, feedlist: &Vec<super::FeedConfig>) -> Self {
let dir = super::to_checked_pathbuf(dir); let dir = super::to_checked_pathbuf(dir);
let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new(); let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new();
@ -248,25 +248,20 @@ impl FeedStore {
} }
pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) { pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) {
debug!("Collecting feeds");
let mut feeds = HashMap::new(); let mut feeds = HashMap::new();
let mut entries = Vec::new(); let mut entries = Vec::new();
for (feed_url, feed_store_feed) in self.feeds.iter_mut() { for (feed_url, feed_store_feed) in self.feeds.iter_mut() {
debug!("Collecting {feed_url}"); let Some(mut feed) = feed_store_feed.load_feed(true) else {
let mut feed = match feed_store_feed.load_feed(true) { warn!("Problem parsing feed file for feed {}", feed_url);
Ok(feed) => feed, continue;
Err(e) => {
warn!("Problem parsing feed file for feed {}: {}", feed_url, e);
continue;
}
}; };
for entry in &mut feed.entries { for entry in &mut feed.entries {
entry.source = Some(feed_url.to_string()); entry.source = Some(feed_url.to_string());
} }
entries.extend(feed.entries.clone()); entries.append(&mut std::mem::take(&mut feed.entries));
if entries.len() > 4 * max_entries { if entries.len() > 4 * max_entries {
entries = trim_entries(entries, max_entries); entries = trim_entries(entries, max_entries);
} }

38
src/main.rs
View File

@ -26,11 +26,10 @@ extern crate log;
use crate::feed_store::FeedStore; use crate::feed_store::FeedStore;
use crate::fetcher::Fetcher; use crate::fetcher::Fetcher;
use anyhow::Result; use anyhow::Result;
use camino::{Utf8Path, Utf8PathBuf}; use camino::Utf8PathBuf;
use clap::Parser; use clap::Parser;
use serde::{Deserialize, Serialize}; use serde::Deserialize;
use std::fs; use std::fs;
use url::Url;
//mod atom_serializer; //mod atom_serializer;
mod feed_store; mod feed_store;
@ -49,8 +48,6 @@ struct Args {
config: String, config: String,
#[arg(long, default_value_t = false)] #[arg(long, default_value_t = false)]
no_fetch: bool, no_fetch: bool,
#[arg(long, default_value_t = false)]
force: bool,
} }
/// Config to be parsed from toml file given as cmdline option /// Config to be parsed from toml file given as cmdline option
@ -59,42 +56,29 @@ struct Config {
/// to be used as part of the fetchers username header /// to be used as part of the fetchers username header
bot_name: String, bot_name: String,
/// where to store downloaded feeds and their metadata /// where to store downloaded feeds and their metadata
feed_dir: Utf8PathBuf, feed_dir: String,
/// feeds to be agregated /// feeds to be agregated
feeds: Vec<FeedConfig>, feeds: Vec<FeedConfig>,
/// Email adress to use for the from header when fetching feeds /// Email adress to use for the from header when fetching feeds
from: String, from: String,
/// where to build the output files /// where to build the output files
out_dir: Utf8PathBuf, out_dir: String,
/// templates folder /// templates folder
templates_dir: Utf8PathBuf, templates_dir: String,
/// How many feed entries should be included in the planet /// How many feed entries should be included in the planet
max_entries: usize, max_entries: usize,
/// How soon to refresh, in hours /// How soon to refresh, in hours
refresh: usize, refresh: usize,
/// A theme to apply, if any.
///
/// This is a folder in the templates_dir. If an assets directory
/// exists within, the contents will be copied over to the out_dir.
theme: Option<String>,
/// List of languages for translations
#[serde(default)]
lang: Vec<Lang>,
} }
#[derive(Clone, Debug, Deserialize, Serialize)] pub fn to_checked_pathbuf(dir: &str) -> Utf8PathBuf {
pub struct Lang { let dir = Utf8PathBuf::from(dir);
code: String,
name: String,
link: Url,
}
pub fn to_checked_pathbuf(dir: &Utf8Path) -> Utf8PathBuf {
let m = dir let m = dir
.metadata() .metadata()
.unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}")); .unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}"));
assert!(m.is_dir(), "Not a dir: {dir}"); assert!(m.is_dir(), "Not a dir: {dir}");
dir.to_path_buf() dir
} }
/// Config for one individual input feed /// Config for one individual input feed
@ -107,10 +91,10 @@ struct FeedConfig {
url: String, url: String,
} }
fn fetch(config: &Config, feed_store: &mut FeedStore, force_rebuild: bool) -> Result<bool> { fn fetch(config: &Config, feed_store: &mut FeedStore) -> Result<bool> {
let fetcher = Fetcher::new(&config.bot_name, &config.from); let fetcher = Fetcher::new(&config.bot_name, &config.from);
let rebuild = feed_store.fetch(&fetcher, config.refresh)?; let rebuild = feed_store.fetch(&fetcher, config.refresh)?;
Ok(rebuild || force_rebuild) Ok(rebuild)
} }
fn main() -> Result<()> { fn main() -> Result<()> {
@ -132,7 +116,7 @@ fn main() -> Result<()> {
let should_build = if args.no_fetch { let should_build = if args.no_fetch {
true true
} else { } else {
fetch(&config, &mut feed_store, args.force)? fetch(&config, &mut feed_store)?
}; };
if should_build { if should_build {

41
src/template_engine.rs
View File

@ -2,25 +2,19 @@ use crate::feed_store::FeedStore;
use crate::to_checked_pathbuf; use crate::to_checked_pathbuf;
use crate::Config; use crate::Config;
use anyhow::Result; use anyhow::Result;
use camino::Utf8Path;
use feed_rs::model::Feed; use feed_rs::model::Feed;
use std::collections::HashMap; use std::collections::HashMap;
use std::fs::{copy, create_dir_all, File}; use std::fs::File;
use tera::{from_value, Tera}; use tera::{from_value, Tera};
pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> { pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
let mut tera = if let Some(theme) = &config.theme { let mut tera = create_tera(&config.templates_dir)?;
create_tera(&config.templates_dir.join(theme))?
} else {
create_tera(&config.templates_dir)?
};
let out_dir = to_checked_pathbuf(&config.out_dir); let out_dir = to_checked_pathbuf(&config.out_dir);
let mut context = tera::Context::new(); let mut context = tera::Context::new();
let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries); let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries);
context.insert("feeds", &feeds); context.insert("feeds", &feeds);
context.insert("entries", &entries); context.insert("entries", &entries);
context.insert("lang", &config.lang);
context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS")); context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS"));
context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE")); context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE"));
context.insert("PKG_NAME", env!("CARGO_PKG_NAME")); context.insert("PKG_NAME", env!("CARGO_PKG_NAME"));
@ -32,39 +26,10 @@ pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
let file = File::create(format!("{out_dir}/{name}"))?; let file = File::create(format!("{out_dir}/{name}"))?;
tera.render_to(name, &context, file)?; tera.render_to(name, &context, file)?;
} }
// Copy static assets from theme, if any
if let Some(theme) = &config.theme {
let assets_dir = config.templates_dir.join(theme).join("assets");
if assets_dir.is_dir() {
copy_assets(&assets_dir, &out_dir)?;
}
}
Ok(()) Ok(())
} }
/// Recursively copy assets from one dir to another fn create_tera(templates_dir: &str) -> Result<Tera> {
///
/// Symlinks are ignored.
fn copy_assets(orig: &Utf8Path, dest: &Utf8Path) -> Result<()> {
if orig.is_dir() {
if !dest.is_dir() {
create_dir_all(dest)?;
}
for entry in orig.read_dir_utf8()? {
let entry = entry?;
copy_assets(entry.path(), &dest.join(entry.file_name()))?;
}
} else if orig.is_file() {
copy(orig, dest)?;
}
Ok(())
}
fn create_tera(templates_dir: &Utf8Path) -> Result<Tera> {
let dir = to_checked_pathbuf(templates_dir); let dir = to_checked_pathbuf(templates_dir);
let mut tera = tera::Tera::new(&format!("{dir}/*"))?; let mut tera = tera::Tera::new(&format!("{dir}/*"))?;
// disable autoescape as this would corrupt urls or the entriy contents. todo check this! // disable autoescape as this would corrupt urls or the entriy contents. todo check this!