generating html from 12 feeds works
This commit is contained in:
Thomas Koch
parent
0f134ad61b
commit
cd1237cfc6
7
Cargo.lock
generated
7
Cargo.lock
generated
@ -96,6 +96,12 @@ dependencies = [
|
||||
"windows-sys 0.59.0",
|
||||
]
|
||||
|
||||
[[package]]
|
||||
name = "anyhow"
|
||||
version = "1.0.95"
|
||||
source = "registry+https://github.com/rust-lang/crates.io-index"
|
||||
checksum = "34ac096ce696dc2fcabef30516bb13c0a68a11d30131d3df6f04711467681b04"
|
||||
|
||||
[[package]]
|
||||
name = "autocfg"
|
||||
version = "1.4.0"
|
||||
@ -1093,6 +1099,7 @@ checksum = "953ec861398dccce10c670dfeaf3ec4911ca479e9c02154b3a215178c5f566f2"
|
||||
name = "planet-mars"
|
||||
version = "0.1.0"
|
||||
dependencies = [
|
||||
"anyhow",
|
||||
"clap",
|
||||
"env_logger",
|
||||
"feed-rs",
|
||||
|
||||
@ -5,6 +5,7 @@ edition = "2021"
|
||||
|
||||
[dependencies]
|
||||
# ammonia = "*" done by feed-rs
|
||||
anyhow = "*"
|
||||
clap = { version = "*", features = ["derive"] }
|
||||
env_logger = "*"
|
||||
feed-rs = "*"
|
||||
|
||||
@ -1,6 +1,7 @@
|
||||
use feed_rs::model::Entry;
|
||||
use feed_rs::model::Feed;
|
||||
use serde::{Deserialize, Serialize};
|
||||
use std::convert::AsRef;
|
||||
use std::fs;
|
||||
use std::io::BufReader;
|
||||
use std::path::PathBuf;
|
||||
@ -20,7 +21,7 @@ pub struct FeedStore {
|
||||
}
|
||||
|
||||
impl FeedStore {
|
||||
pub fn new(dir: String) -> Self {
|
||||
pub fn new(dir: &str) -> Self {
|
||||
Self {
|
||||
dir: super::to_checked_pathbuf(dir),
|
||||
}
|
||||
@ -66,6 +67,16 @@ impl FeedStore {
|
||||
false
|
||||
}
|
||||
|
||||
fn write<P: AsRef<std::path::Path> + std::fmt::Display, C: AsRef<[u8]>>(
|
||||
path: P,
|
||||
contents: C,
|
||||
) -> std::io::Result<()> {
|
||||
if fs::exists(&path)? {
|
||||
fs::rename(&path, format!("{path}.backup"))?;
|
||||
}
|
||||
fs::write(path, contents)
|
||||
}
|
||||
|
||||
pub fn store(&self, url: &Url, mut response: Response<Body>) -> bool {
|
||||
let headers = response.headers();
|
||||
let fetchdata = FetchData {
|
||||
@ -89,8 +100,9 @@ impl FeedStore {
|
||||
if !self.has_changed(url, &feed) {
|
||||
return false;
|
||||
}
|
||||
let _ = fs::write(self.feed_path(url), body);
|
||||
let _ = fs::write(
|
||||
debug!("Storing feed for {url}.");
|
||||
let _ = Self::write(self.feed_path(url), body);
|
||||
let _ = Self::write(
|
||||
self.fetchdata_path(url),
|
||||
toml::to_string(&fetchdata).unwrap(),
|
||||
);
|
||||
|
||||
@ -37,10 +37,10 @@ impl Fetcher {
|
||||
.agent
|
||||
.get(url.to_string())
|
||||
.header("FROM", self.from.clone());
|
||||
if fetchdata.etag != "" {
|
||||
if !fetchdata.etag.is_empty() {
|
||||
builder = builder.header("If-None-Match", fetchdata.etag);
|
||||
}
|
||||
if fetchdata.date != "" {
|
||||
if !fetchdata.date.is_empty() {
|
||||
builder = builder.header("If-Modified-Since", fetchdata.date);
|
||||
}
|
||||
|
||||
|
||||
95
src/main.rs
95
src/main.rs
@ -3,14 +3,17 @@ extern crate log;
|
||||
|
||||
use crate::feed_store::FeedStore;
|
||||
use crate::fetcher::Fetcher;
|
||||
use anyhow::Result;
|
||||
use clap::Parser;
|
||||
use serde::Deserialize;
|
||||
use simple_entry::SimpleEntry;
|
||||
use std::fs;
|
||||
use std::path::PathBuf;
|
||||
use url::Url;
|
||||
|
||||
mod feed_store;
|
||||
mod fetcher;
|
||||
mod simple_entry;
|
||||
|
||||
#[derive(Parser)]
|
||||
#[command(author, version, about, long_about = None)]
|
||||
@ -21,6 +24,8 @@ struct Args {
|
||||
default_value_t = String::from("mars.toml")
|
||||
)]
|
||||
config: String,
|
||||
#[arg(long, default_value_t = false)]
|
||||
no_fetch: bool,
|
||||
}
|
||||
|
||||
#[derive(Deserialize)]
|
||||
@ -39,7 +44,7 @@ struct Config {
|
||||
templates_dir: String,
|
||||
}
|
||||
|
||||
pub fn to_checked_pathbuf(dir: String) -> PathBuf {
|
||||
pub fn to_checked_pathbuf(dir: &str) -> PathBuf {
|
||||
let dir: PathBuf = PathBuf::from(dir);
|
||||
|
||||
let m = dir
|
||||
@ -54,7 +59,54 @@ struct FeedConfig {
|
||||
url: String,
|
||||
}
|
||||
|
||||
fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
fn fetch(config: &Config, feed_store: &FeedStore) -> Result<bool> {
|
||||
let fetcher = Fetcher::new(&config.bot_name, &config.from);
|
||||
let mut rebuild = false;
|
||||
for feed in &config.feeds {
|
||||
let url = match Url::parse(&feed.url) {
|
||||
Ok(x) => x,
|
||||
Err(e) => {
|
||||
error!("Error parsing url '{}': {e:?}", feed.url);
|
||||
continue;
|
||||
}
|
||||
};
|
||||
rebuild |= fetcher.fetch(url, feed_store);
|
||||
}
|
||||
info!("Done fetching. Rebuild needed: {rebuild}");
|
||||
Ok(rebuild)
|
||||
}
|
||||
|
||||
fn build(config: &Config, feed_store: &FeedStore) -> Result<()> {
|
||||
let templates_dir = to_checked_pathbuf(&config.templates_dir);
|
||||
let out_dir = to_checked_pathbuf(&config.out_dir);
|
||||
|
||||
let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
println!("Parsing error(s): {}", e);
|
||||
::std::process::exit(1);
|
||||
}
|
||||
};
|
||||
// disable autoescape as this would corrupt urls or the entriy contents. todo check this!
|
||||
tera.autoescape_on(vec![]);
|
||||
|
||||
let mut context = tera::Context::new();
|
||||
let entries: Vec<SimpleEntry> = feed_store
|
||||
.collect(&config.feeds)
|
||||
.into_iter()
|
||||
.map(SimpleEntry::from_feed_entry)
|
||||
.collect();
|
||||
context.insert("entries", &entries);
|
||||
|
||||
for name in tera.get_template_names() {
|
||||
debug!("Processing template {name}");
|
||||
let file = fs::File::create(format!("{}/{name}", out_dir.display()))?;
|
||||
tera.render_to(name, &context, file)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
fn main() -> Result<()> {
|
||||
env_logger::init();
|
||||
info!("starting up");
|
||||
|
||||
@ -64,35 +116,20 @@ fn main() -> Result<(), Box<dyn std::error::Error>> {
|
||||
panic!("Configuration file {config_path} does not exist!");
|
||||
}
|
||||
let config: Config = toml::from_str(&fs::read_to_string(config_path)?)?;
|
||||
let templates_dir = to_checked_pathbuf(config.templates_dir);
|
||||
let out_dir = to_checked_pathbuf(config.out_dir);
|
||||
// only check here to avoid fetching with broken config
|
||||
// todo: get a config lib that provides validation!
|
||||
let _ = to_checked_pathbuf(&config.templates_dir);
|
||||
let _ = to_checked_pathbuf(&config.out_dir);
|
||||
|
||||
let feed_store = FeedStore::new(config.feed_dir);
|
||||
let fetcher = Fetcher::new(&config.bot_name, &config.from);
|
||||
let feed_store = FeedStore::new(&config.feed_dir);
|
||||
let should_build = if args.no_fetch {
|
||||
true
|
||||
} else {
|
||||
fetch(&config, &feed_store)?
|
||||
};
|
||||
|
||||
let mut rebuild = false;
|
||||
for feed in &config.feeds {
|
||||
let url = Url::parse(&feed.url)?;
|
||||
rebuild |= fetcher.fetch(url, &feed_store);
|
||||
}
|
||||
info!("Done fetching. Rebuild needed: {rebuild}");
|
||||
if rebuild {
|
||||
let entries = feed_store.collect(&config.feeds);
|
||||
let mut tera = match tera::Tera::new(&format!("{}/*", &templates_dir.display())) {
|
||||
Ok(t) => t,
|
||||
Err(e) => {
|
||||
println!("Parsing error(s): {}", e);
|
||||
::std::process::exit(1);
|
||||
}
|
||||
};
|
||||
tera.autoescape_on(vec![]);
|
||||
let mut context = tera::Context::new();
|
||||
context.insert("entries", &entries);
|
||||
for name in tera.get_template_names() {
|
||||
debug!("Processing template {name}");
|
||||
let file = fs::File::create(&format!("{}/{name}", out_dir.display()))?;
|
||||
let _ = tera.render_to(name, &context, file)?;
|
||||
}
|
||||
if should_build {
|
||||
build(&config, &feed_store)?;
|
||||
}
|
||||
Ok(())
|
||||
}
|
||||
|
||||
43
src/simple_entry.rs
Normal file
43
src/simple_entry.rs
Normal file
@ -0,0 +1,43 @@
|
||||
use feed_rs::model::Entry;
|
||||
|
||||
/// Simplified Feed entry for easier value access in template
|
||||
#[derive(serde::Serialize)]
|
||||
pub struct SimpleEntry {
|
||||
pub date: String,
|
||||
pub content: String,
|
||||
pub author: String,
|
||||
pub link: String,
|
||||
pub title: String,
|
||||
}
|
||||
|
||||
/// format for the entries timestamp
|
||||
/// <https://docs.rs/chrono/latest/chrono/format/strftime>
|
||||
const FMT: &str = "%c";
|
||||
|
||||
impl SimpleEntry {
|
||||
pub fn from_feed_entry(entry: Entry) -> Self {
|
||||
Self {
|
||||
date: entry
|
||||
.updated
|
||||
.or(entry.published)
|
||||
.unwrap_or_default()
|
||||
.format(FMT)
|
||||
.to_string(),
|
||||
content: entry
|
||||
.content
|
||||
.map(|x| x.body.unwrap_or_default())
|
||||
.unwrap_or_default(),
|
||||
author: if !entry.authors.is_empty() {
|
||||
entry.authors[0].name.clone()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
link: if !entry.links.is_empty() {
|
||||
entry.links[0].href.clone()
|
||||
} else {
|
||||
"".to_string()
|
||||
},
|
||||
title: entry.title.map(|x| x.content).unwrap_or_default(),
|
||||
}
|
||||
}
|
||||
}
|
||||
@ -1,8 +1,32 @@
|
||||
hello world
|
||||
|
||||
{% for entry in entries %}
|
||||
{% for link in entry.links %}
|
||||
{{link.href}}
|
||||
{% endfor %}
|
||||
{% endfor %}
|
||||
<html>
|
||||
<head>
|
||||
<title>Planet TVL</title>
|
||||
<meta http-equiv="Content-Type" content="text/html; charset=utf-8">
|
||||
<meta name="generator" content="planet-mars">
|
||||
<link rel="shortcut icon" href="/favicon.ico">
|
||||
<link rel="stylesheet" href="planet.css" type="text/css">
|
||||
{# todo <link rel="alternate" type="application/rss+xml" title="Planet Haskell RSS Feed" href="rss20.xml"> #}
|
||||
<link rel="alternate" type="application/xml+atom" title="Planet Haskell Atom Feed" href="atom.xml">
|
||||
</head>
|
||||
<body>
|
||||
<main>
|
||||
<h1 class="tophead">Planet TVL</h1>
|
||||
{% for entry in entries %}
|
||||
<article>
|
||||
<h2><a href="{{entry.link}}">{{ entry.title|striptags }}<a></h2>
|
||||
<date>{% if entry.published %}{{ entry.published | date(format="%Y-%m-%d %H:%M", timezone="Europe/Moscow") }}{% endif %}</date>{# todo: maybe group posts by day? #}
|
||||
<p class="entry_author">{{ entry.author|striptags }}</p>
|
||||
<div class="entry_content">
|
||||
{{ entry.content }}
|
||||
</div>
|
||||
<p><a href="{{entry.link}}">full post<a></p>
|
||||
</article>
|
||||
{% endfor %}
|
||||
</main>
|
||||
|
||||
<aside>
|
||||
<img src="logo.svg">
|
||||
<p>Last updated: {{now()|date(format="%Y-%m-%d %H:%M")}}
|
||||
</aside>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
Loading…
x
Reference in New Issue
Block a user