feat: Add refresh config to prevent hammering servers

src/feed_store.rs

@@ -1,6 +1,6 @@
 use anyhow::Result;
 use camino::{Utf8Path, Utf8PathBuf};
-use chrono::{DateTime, Utc};
+use chrono::{DateTime, Duration, Utc};
 use feed_rs::model::Entry;
 use feed_rs::model::Feed;
 use ron::ser::{to_string_pretty, PrettyConfig};
@@ -144,6 +144,11 @@ impl FeedStoreFeed {
                 return Ok(false);
             }
         };
+
+        debug!("Storing fetchdata for {}", self.url);
+        self.info.fetch_data = Some(fetchdata);
+        Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
+
         if !self.has_changed(&feed) {
             return Ok(false);
         }
@@ -154,13 +159,11 @@ impl FeedStoreFeed {
             to_string_pretty(&feed, PrettyConfig::default())?,
         )?;
         Self::write(&self.path_feed, body)?;
-        // Save info
-        self.info.fetch_data = Some(fetchdata);
-        Self::write(&self.path_settings, toml::to_string(&self.info)?)?;
         Ok(true)
     }
 
-    pub fn fetch(&mut self, fetcher: &super::Fetcher) -> Result<bool> {
+    /// refresh in hours
+    pub fn fetch(&mut self, fetcher: &super::Fetcher, refresh: usize) -> Result<bool> {
         let mut builder = fetcher
             .agent
             .get(self.url.to_string())
@@ -173,6 +176,12 @@ impl FeedStoreFeed {
             if !fetchdata.last_modified.is_empty() {
                 builder = builder.header("If-Modified-Since", fetchdata.last_modified.clone());
             }
+
+            // Check if we have hit time for refresh
+            if fetchdata.when + Duration::try_hours(refresh as i64).unwrap() >= Utc::now() {
+                // No need to rebuild, check again later
+                return Ok(false);
+            }
         }
 
         let start_instant = Instant::now();
@@ -262,10 +271,10 @@ impl FeedStore {
         (feeds, trim_entries(entries, max_entries))
     }
 
-    pub fn fetch(&mut self, fetcher: &super::Fetcher) -> Result<bool> {
+    pub fn fetch(&mut self, fetcher: &super::Fetcher, refresh: usize) -> Result<bool> {
         let mut rebuild = false;
         for (_url, feed) in self.feeds.iter_mut() {
-            rebuild |= feed.fetch(fetcher)?;
+            rebuild |= feed.fetch(fetcher, refresh)?;
         }
 
         Ok(rebuild)

src/main.rs

@@ -67,6 +67,8 @@ struct Config {
     templates_dir: String,
     /// How many feed entries should be included in the planet
     max_entries: usize,
+    /// How soon to refresh, in hours
+    refresh: usize,
 }
 
 pub fn to_checked_pathbuf(dir: &str) -> Utf8PathBuf {
@@ -91,7 +93,7 @@ struct FeedConfig {
 
 fn fetch(config: &Config, feed_store: &mut FeedStore) -> Result<bool> {
     let fetcher = Fetcher::new(&config.bot_name, &config.from);
-    let rebuild = feed_store.fetch(&fetcher)?;
+    let rebuild = feed_store.fetch(&fetcher, config.refresh)?;
     Ok(rebuild)
 }