Compare commits
4 Commits
5271c4c9aa
...
90f29bd2a4
Author | SHA1 | Date | |
---|---|---|---|
90f29bd2a4 | |||
6ee952133d | |||
6e288b3525 | |||
001ca4879a |
@ -1,4 +1,4 @@
|
|||||||
use anyhow::Result;
|
use anyhow::{bail, Result};
|
||||||
use camino::{Utf8Path, Utf8PathBuf};
|
use camino::{Utf8Path, Utf8PathBuf};
|
||||||
use chrono::{DateTime, Duration, Utc};
|
use chrono::{DateTime, Duration, Utc};
|
||||||
use feed_rs::model::Entry;
|
use feed_rs::model::Entry;
|
||||||
@ -99,33 +99,33 @@ impl FeedStoreFeed {
|
|||||||
self.info.fetch_data.as_ref()
|
self.info.fetch_data.as_ref()
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn load_feed(&self, sanitize: bool) -> Option<Feed> {
|
pub fn load_feed(&self, sanitize: bool) -> Result<Feed> {
|
||||||
if let Some(raw_feed) = &self.raw_feed {
|
if let Some(raw_feed) = &self.raw_feed {
|
||||||
let parser = feed_rs::parser::Builder::new()
|
let parser = feed_rs::parser::Builder::new()
|
||||||
.sanitize_content(sanitize)
|
.sanitize_content(sanitize)
|
||||||
.build();
|
.build();
|
||||||
Some(parser.parse(raw_feed.as_bytes()).unwrap())
|
Ok(parser.parse(raw_feed.as_bytes())?)
|
||||||
} else {
|
} else {
|
||||||
None
|
bail!("Feed not loaded yet: {}", self.url);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn has_changed(&self, new_feed: &Feed) -> bool {
|
pub fn has_changed(&self, new_feed: &Feed) -> Result<bool> {
|
||||||
let Some(old_feed) = self.load_feed(false) else {
|
let Ok(old_feed) = self.load_feed(false) else {
|
||||||
return true;
|
return Ok(true);
|
||||||
};
|
};
|
||||||
|
|
||||||
let mut old_iter = old_feed.entries.iter();
|
let mut old_iter = old_feed.entries.iter();
|
||||||
for new in &new_feed.entries {
|
for new in &new_feed.entries {
|
||||||
let Some(old) = old_iter.next() else {
|
let Some(old) = old_iter.next() else {
|
||||||
return true;
|
return Ok(true);
|
||||||
};
|
};
|
||||||
if old != new {
|
if old != new {
|
||||||
return true;
|
return Ok(true);
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// ignoring any entries left in old_iter
|
// ignoring any entries left in old_iter
|
||||||
false
|
Ok(false)
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> {
|
pub fn store(&mut self, mut response: Response<Body>) -> Result<bool> {
|
||||||
@ -149,7 +149,7 @@ impl FeedStoreFeed {
|
|||||||
self.info.fetch_data = Some(fetchdata);
|
self.info.fetch_data = Some(fetchdata);
|
||||||
Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
|
Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
|
||||||
|
|
||||||
if !self.has_changed(&feed) {
|
if !self.has_changed(&feed)? {
|
||||||
return Ok(false);
|
return Ok(false);
|
||||||
}
|
}
|
||||||
debug!("Storing feed for {}.", self.url);
|
debug!("Storing feed for {}.", self.url);
|
||||||
@ -235,7 +235,7 @@ pub struct FeedStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
impl FeedStore {
|
impl FeedStore {
|
||||||
pub fn new(dir: &str, feedlist: &Vec<super::FeedConfig>) -> Self {
|
pub fn new(dir: &Utf8Path, feedlist: &Vec<super::FeedConfig>) -> Self {
|
||||||
let dir = super::to_checked_pathbuf(dir);
|
let dir = super::to_checked_pathbuf(dir);
|
||||||
let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new();
|
let mut feeds: BTreeMap<Url, FeedStoreFeed> = BTreeMap::new();
|
||||||
|
|
||||||
@ -248,20 +248,25 @@ impl FeedStore {
|
|||||||
}
|
}
|
||||||
|
|
||||||
pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) {
|
pub fn collect(&mut self, max_entries: usize) -> (HashMap<String, Feed>, Vec<Entry>) {
|
||||||
|
debug!("Collecting feeds");
|
||||||
let mut feeds = HashMap::new();
|
let mut feeds = HashMap::new();
|
||||||
let mut entries = Vec::new();
|
let mut entries = Vec::new();
|
||||||
|
|
||||||
for (feed_url, feed_store_feed) in self.feeds.iter_mut() {
|
for (feed_url, feed_store_feed) in self.feeds.iter_mut() {
|
||||||
let Some(mut feed) = feed_store_feed.load_feed(true) else {
|
debug!("Collecting {feed_url}");
|
||||||
warn!("Problem parsing feed file for feed {}", feed_url);
|
let mut feed = match feed_store_feed.load_feed(true) {
|
||||||
|
Ok(feed) => feed,
|
||||||
|
Err(e) => {
|
||||||
|
warn!("Problem parsing feed file for feed {}: {}", feed_url, e);
|
||||||
continue;
|
continue;
|
||||||
|
}
|
||||||
};
|
};
|
||||||
|
|
||||||
for entry in &mut feed.entries {
|
for entry in &mut feed.entries {
|
||||||
entry.source = Some(feed_url.to_string());
|
entry.source = Some(feed_url.to_string());
|
||||||
}
|
}
|
||||||
|
|
||||||
entries.append(&mut std::mem::take(&mut feed.entries));
|
entries.extend(feed.entries.clone());
|
||||||
if entries.len() > 4 * max_entries {
|
if entries.len() > 4 * max_entries {
|
||||||
entries = trim_entries(entries, max_entries);
|
entries = trim_entries(entries, max_entries);
|
||||||
}
|
}
|
||||||
|
38
src/main.rs
38
src/main.rs
@ -26,10 +26,11 @@ extern crate log;
|
|||||||
use crate::feed_store::FeedStore;
|
use crate::feed_store::FeedStore;
|
||||||
use crate::fetcher::Fetcher;
|
use crate::fetcher::Fetcher;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
use camino::Utf8PathBuf;
|
use camino::{Utf8Path, Utf8PathBuf};
|
||||||
use clap::Parser;
|
use clap::Parser;
|
||||||
use serde::Deserialize;
|
use serde::{Deserialize, Serialize};
|
||||||
use std::fs;
|
use std::fs;
|
||||||
|
use url::Url;
|
||||||
|
|
||||||
//mod atom_serializer;
|
//mod atom_serializer;
|
||||||
mod feed_store;
|
mod feed_store;
|
||||||
@ -48,6 +49,8 @@ struct Args {
|
|||||||
config: String,
|
config: String,
|
||||||
#[arg(long, default_value_t = false)]
|
#[arg(long, default_value_t = false)]
|
||||||
no_fetch: bool,
|
no_fetch: bool,
|
||||||
|
#[arg(long, default_value_t = false)]
|
||||||
|
force: bool,
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Config to be parsed from toml file given as cmdline option
|
/// Config to be parsed from toml file given as cmdline option
|
||||||
@ -56,29 +59,42 @@ struct Config {
|
|||||||
/// to be used as part of the fetchers username header
|
/// to be used as part of the fetchers username header
|
||||||
bot_name: String,
|
bot_name: String,
|
||||||
/// where to store downloaded feeds and their metadata
|
/// where to store downloaded feeds and their metadata
|
||||||
feed_dir: String,
|
feed_dir: Utf8PathBuf,
|
||||||
/// feeds to be agregated
|
/// feeds to be agregated
|
||||||
feeds: Vec<FeedConfig>,
|
feeds: Vec<FeedConfig>,
|
||||||
/// Email adress to use for the from header when fetching feeds
|
/// Email adress to use for the from header when fetching feeds
|
||||||
from: String,
|
from: String,
|
||||||
/// where to build the output files
|
/// where to build the output files
|
||||||
out_dir: String,
|
out_dir: Utf8PathBuf,
|
||||||
/// templates folder
|
/// templates folder
|
||||||
templates_dir: String,
|
templates_dir: Utf8PathBuf,
|
||||||
/// How many feed entries should be included in the planet
|
/// How many feed entries should be included in the planet
|
||||||
max_entries: usize,
|
max_entries: usize,
|
||||||
/// How soon to refresh, in hours
|
/// How soon to refresh, in hours
|
||||||
refresh: usize,
|
refresh: usize,
|
||||||
|
/// A theme to apply, if any.
|
||||||
|
///
|
||||||
|
/// This is a folder in the templates_dir. If an assets directory
|
||||||
|
/// exists within, the contents will be copied over to the out_dir.
|
||||||
|
theme: Option<String>,
|
||||||
|
/// List of languages for translations
|
||||||
|
#[serde(default)]
|
||||||
|
lang: Vec<Lang>,
|
||||||
}
|
}
|
||||||
|
|
||||||
pub fn to_checked_pathbuf(dir: &str) -> Utf8PathBuf {
|
#[derive(Clone, Debug, Deserialize, Serialize)]
|
||||||
let dir = Utf8PathBuf::from(dir);
|
pub struct Lang {
|
||||||
|
code: String,
|
||||||
|
name: String,
|
||||||
|
link: Url,
|
||||||
|
}
|
||||||
|
|
||||||
|
pub fn to_checked_pathbuf(dir: &Utf8Path) -> Utf8PathBuf {
|
||||||
let m = dir
|
let m = dir
|
||||||
.metadata()
|
.metadata()
|
||||||
.unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}"));
|
.unwrap_or_else(|_| panic!("Could not get metadata of dir: {dir}"));
|
||||||
assert!(m.is_dir(), "Not a dir: {dir}");
|
assert!(m.is_dir(), "Not a dir: {dir}");
|
||||||
dir
|
dir.to_path_buf()
|
||||||
}
|
}
|
||||||
|
|
||||||
/// Config for one individual input feed
|
/// Config for one individual input feed
|
||||||
@ -91,10 +107,10 @@ struct FeedConfig {
|
|||||||
url: String,
|
url: String,
|
||||||
}
|
}
|
||||||
|
|
||||||
fn fetch(config: &Config, feed_store: &mut FeedStore) -> Result<bool> {
|
fn fetch(config: &Config, feed_store: &mut FeedStore, force_rebuild: bool) -> Result<bool> {
|
||||||
let fetcher = Fetcher::new(&config.bot_name, &config.from);
|
let fetcher = Fetcher::new(&config.bot_name, &config.from);
|
||||||
let rebuild = feed_store.fetch(&fetcher, config.refresh)?;
|
let rebuild = feed_store.fetch(&fetcher, config.refresh)?;
|
||||||
Ok(rebuild)
|
Ok(rebuild || force_rebuild)
|
||||||
}
|
}
|
||||||
|
|
||||||
fn main() -> Result<()> {
|
fn main() -> Result<()> {
|
||||||
@ -116,7 +132,7 @@ fn main() -> Result<()> {
|
|||||||
let should_build = if args.no_fetch {
|
let should_build = if args.no_fetch {
|
||||||
true
|
true
|
||||||
} else {
|
} else {
|
||||||
fetch(&config, &mut feed_store)?
|
fetch(&config, &mut feed_store, args.force)?
|
||||||
};
|
};
|
||||||
|
|
||||||
if should_build {
|
if should_build {
|
||||||
|
@ -2,19 +2,25 @@ use crate::feed_store::FeedStore;
|
|||||||
use crate::to_checked_pathbuf;
|
use crate::to_checked_pathbuf;
|
||||||
use crate::Config;
|
use crate::Config;
|
||||||
use anyhow::Result;
|
use anyhow::Result;
|
||||||
|
use camino::Utf8Path;
|
||||||
use feed_rs::model::Feed;
|
use feed_rs::model::Feed;
|
||||||
use std::collections::HashMap;
|
use std::collections::HashMap;
|
||||||
use std::fs::File;
|
use std::fs::{copy, create_dir_all, File};
|
||||||
use tera::{from_value, Tera};
|
use tera::{from_value, Tera};
|
||||||
|
|
||||||
pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
|
pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
|
||||||
let mut tera = create_tera(&config.templates_dir)?;
|
let mut tera = if let Some(theme) = &config.theme {
|
||||||
|
create_tera(&config.templates_dir.join(theme))?
|
||||||
|
} else {
|
||||||
|
create_tera(&config.templates_dir)?
|
||||||
|
};
|
||||||
let out_dir = to_checked_pathbuf(&config.out_dir);
|
let out_dir = to_checked_pathbuf(&config.out_dir);
|
||||||
|
|
||||||
let mut context = tera::Context::new();
|
let mut context = tera::Context::new();
|
||||||
let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries);
|
let (feeds, entries): (HashMap<String, Feed>, _) = feed_store.collect(config.max_entries);
|
||||||
context.insert("feeds", &feeds);
|
context.insert("feeds", &feeds);
|
||||||
context.insert("entries", &entries);
|
context.insert("entries", &entries);
|
||||||
|
context.insert("lang", &config.lang);
|
||||||
context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS"));
|
context.insert("PKG_AUTHORS", env!("CARGO_PKG_AUTHORS"));
|
||||||
context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE"));
|
context.insert("PKG_HOMEPAGE", env!("CARGO_PKG_HOMEPAGE"));
|
||||||
context.insert("PKG_NAME", env!("CARGO_PKG_NAME"));
|
context.insert("PKG_NAME", env!("CARGO_PKG_NAME"));
|
||||||
@ -26,10 +32,39 @@ pub fn build(config: &Config, feed_store: &mut FeedStore) -> Result<()> {
|
|||||||
let file = File::create(format!("{out_dir}/{name}"))?;
|
let file = File::create(format!("{out_dir}/{name}"))?;
|
||||||
tera.render_to(name, &context, file)?;
|
tera.render_to(name, &context, file)?;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Copy static assets from theme, if any
|
||||||
|
if let Some(theme) = &config.theme {
|
||||||
|
let assets_dir = config.templates_dir.join(theme).join("assets");
|
||||||
|
if assets_dir.is_dir() {
|
||||||
|
copy_assets(&assets_dir, &out_dir)?;
|
||||||
|
}
|
||||||
|
}
|
||||||
|
|
||||||
Ok(())
|
Ok(())
|
||||||
}
|
}
|
||||||
|
|
||||||
fn create_tera(templates_dir: &str) -> Result<Tera> {
|
/// Recursively copy assets from one dir to another
|
||||||
|
///
|
||||||
|
/// Symlinks are ignored.
|
||||||
|
fn copy_assets(orig: &Utf8Path, dest: &Utf8Path) -> Result<()> {
|
||||||
|
if orig.is_dir() {
|
||||||
|
if !dest.is_dir() {
|
||||||
|
create_dir_all(dest)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
for entry in orig.read_dir_utf8()? {
|
||||||
|
let entry = entry?;
|
||||||
|
copy_assets(entry.path(), &dest.join(entry.file_name()))?;
|
||||||
|
}
|
||||||
|
} else if orig.is_file() {
|
||||||
|
copy(orig, dest)?;
|
||||||
|
}
|
||||||
|
|
||||||
|
Ok(())
|
||||||
|
}
|
||||||
|
|
||||||
|
fn create_tera(templates_dir: &Utf8Path) -> Result<Tera> {
|
||||||
let dir = to_checked_pathbuf(templates_dir);
|
let dir = to_checked_pathbuf(templates_dir);
|
||||||
let mut tera = tera::Tera::new(&format!("{dir}/*"))?;
|
let mut tera = tera::Tera::new(&format!("{dir}/*"))?;
|
||||||
// disable autoescape as this would corrupt urls or the entriy contents. todo check this!
|
// disable autoescape as this would corrupt urls or the entriy contents. todo check this!
|
||||||
|
Loading…
x
Reference in New Issue
Block a user