~nickbp/force-rss

ed3e67317f40792b2ffc70463d47a3fa613a9d04 — Nick Parker 8 months ago 0209e0a main
Add 'test' mode for evaluating a local file
4 files changed, 81 insertions(+), 10 deletions(-)

M src/config.rs
M src/main.rs
M src/rss.rs
M src/server.rs
M src/config.rs => src/config.rs +5 -1
@@ 187,10 187,14 @@ impl SitesConfig {
        self.site_configs_by_domain.keys().cloned().collect()
    }

    pub fn domain_config(self: &SitesConfig, query_url: &Url) -> Result<&SiteConfig> {
    pub fn config_for_url(self: &SitesConfig, query_url: &Url) -> Result<&SiteConfig> {
        let domain = query_url
            .domain()
            .with_context(|| "Query url is missing a domain name")?;
        self.config_for_domain(domain)
    }

    pub fn config_for_domain(self: &SitesConfig, domain: &str) -> Result<&SiteConfig> {
        self.site_configs_by_domain.get(domain).with_context(|| {
            format!(
                "No config found for domain={}, available domains are: {}",

M src/main.rs => src/main.rs +57 -6
@@ 7,6 7,8 @@ mod site;
mod twitch;
mod twitch_auth;

use std::fs::File;
use std::io::prelude::*;
use std::path::PathBuf;

use anyhow::{Context, Result};


@@ 33,6 35,9 @@ enum Commands {

    /// Fetches the RSS feed for a single twitch user and exits
    Twitch(TwitchArgs),

    /// Runs the provided filter config against a local file for testing
    Test(TestArgs),
}

#[derive(Args)]


@@ 64,6 69,21 @@ struct TwitchArgs {
    config: Option<PathBuf>,
}

#[derive(Args)]
struct TestArgs {
    /// Path to the html file to test against
    #[arg(index = 1)]
    file: PathBuf,

    /// URL where the test file would be located online
    #[arg(index = 2)]
    url: Url,

    /// Path to the toml config file containing matching site config
    #[arg(index = 3)]
    config: PathBuf,
}

#[tokio::main]
async fn main() -> Result<()> {
    logging::init_logging();


@@ 99,10 119,40 @@ async fn main() -> Result<()> {
                );
            }
            Ok(())
        },
        Commands::Test(args) => {
            // Test local file
            let config = config::Config::from_toml(Some(args.config), true)?;
            if let Err(e) = main_test(&args.file, &args.url, &config).await {
                error!(
                    "Failed to generate feed for test file {:?}: {}",
                    args.file, e
                );
            }
            Ok(())
        }
    }
}

async fn main_site(query_url: &Url, config: &config::Config) -> Result<()> {
    // Check for valid/matching domain config before making an HTTP query
    let domain_config = config.sites_config.config_for_url(query_url)?;
    let html = site::fetch_content(query_url).await?;
    let content = &site::scrape_content(&html, domain_config).await?;
    println!(
        "{}",
        rss::site_rss(
            content,
            query_url,
            &config.sites_config,
            domain_config,
            &Utc::now(),
            true
        )?
    );
    Ok(())
}

async fn main_twitch(
    username: &str,
    config: &config::TwitchConfig,


@@ 112,15 162,15 @@ async fn main_twitch(
    let videos = &twitch::get_videos(auth, &user.id).await?;
    println!(
        "{}",
        rss::twitch_rss(&user, videos, config, &None, &Utc::now())?
        rss::twitch_rss(&user, videos, config, &None, &Utc::now(), true)?
    );
    Ok(())
}

async fn main_site(query_url: &Url, config: &config::Config) -> Result<()> {
    // Check for valid/matching domain config before making an HTTP query
    let domain_config = config.sites_config.domain_config(query_url)?;
    let html = site::fetch_content(query_url).await?;
async fn main_test(file: &PathBuf, query_url: &Url, config: &config::Config) -> Result<()> {
    let domain_config = config.sites_config.config_for_url(query_url)?;
    let mut html = String::new();
    File::open(&file)?.read_to_string(&mut html)?;
    let content = &site::scrape_content(&html, domain_config).await?;
    println!(
        "{}",


@@ 129,7 179,8 @@ async fn main_site(query_url: &Url, config: &config::Config) -> Result<()> {
            query_url,
            &config.sites_config,
            domain_config,
            &Utc::now()
            &Utc::now(),
            true,
        )?
    );
    Ok(())

M src/rss.rs => src/rss.rs +14 -2
@@ 13,6 13,7 @@ pub fn twitch_rss(
    config: &config::TwitchConfig,
    embed_host: &Option<String>,
    last_updated: &DateTime<Utc>,
    pretty_print: bool,
) -> Result<String> {
    let mut items = vec![];
    for video in videos {


@@ 52,7 53,7 @@ pub fn twitch_rss(
    if !user.description.is_empty() {
        channel_builder.description(user.description.clone());
    }
    Ok(channel_builder.build().to_string())
    to_string(channel_builder, pretty_print)
}

fn twitch_content_html(


@@ 93,6 94,7 @@ pub fn site_rss(
    config: &config::SitesConfig,
    site_config: &config::SiteConfig,
    last_updated: &DateTime<Utc>,
    pretty_print: bool,
) -> Result<String> {
    let mut items = vec![];
    for article in &content.articles {


@@ 176,5 178,15 @@ pub fn site_rss(
    if let Some(title) = &content.title {
        channel_builder.title(title);
    }
    Ok(channel_builder.build().to_string())
    to_string(channel_builder, pretty_print)
}

fn to_string(channel_builder: ChannelBuilder, pretty_print: bool) -> Result<String> {
    let channel = channel_builder.build();
    if pretty_print {
        let buf = channel.pretty_write_to(Vec::new(), b' ', 4)?;
        Ok(String::from_utf8(buf)?)
    } else {
        Ok(channel.to_string())
    }
}

M src/server.rs => src/server.rs +5 -1
@@ 209,7 209,7 @@ fn handle_index(

async fn handle_site(query: SiteQuery, state: &mut SiteState) -> Result<String> {
    // Ensure we have a config for the url domain before making any HTTP queries
    let site_config = state.config.domain_config(&query.url)?;
    let site_config = state.config.config_for_url(&query.url)?;

    if let Some(read_guard) = state.url_to_rss_cache.get(&query.url).await {
        // Return cached content


@@ 219,6 219,7 @@ async fn handle_site(query: SiteQuery, state: &mut SiteState) -> Result<String> 
            &state.config,
            site_config,
            &read_guard.last_updated,
            false,
        );
    }



@@ 231,6 232,7 @@ async fn handle_site(query: SiteQuery, state: &mut SiteState) -> Result<String> 
        &state.config,
        site_config,
        &last_updated,
        false,
    )?;
    // Only add content to cache AFTER we've successfully converted it to RSS
    if !state.config.site_cache_ttl.is_zero() {


@@ 277,6 279,7 @@ async fn handle_twitch(query: TwitchQuery, state: &mut TwitchState) -> Result<St
            &state.config,
            &query.embed_host,
            &read_guard.last_updated,
            false,
        );
    }



@@ 288,6 291,7 @@ async fn handle_twitch(query: TwitchQuery, state: &mut TwitchState) -> Result<St
        &state.config,
        &query.embed_host,
        &last_updated,
        false,
    )?;
    // Only add content to cache AFTER we've successfully converted it to RSS
    if !state.config.video_cache_ttl.is_zero() {