~nickbp/kapiti

3d6e5d90626b2e78895b3fee22148d93c9d5149d — Nick Parker 7 months ago 7f276e5
Restructure config to support multiple filter blocks and HTTP listeners (#30/#32)
4 files changed, 97 insertions(+), 46 deletions(-)

M examples/update_specs.rs
M src/config.rs
M src/runner.rs
M src/specs/version_generated.rs
M examples/update_specs.rs => examples/update_specs.rs +1 -3
@@ 39,8 39,6 @@ fn main() -> Result<()> {
    // Output paths for generated code
    let specs_dir = Path::new("src").join("specs");

    // TODO get crc32 or something of ordered dns-specs/*.csv content, store as 'schema version' for redis keys

    // Generate src/specs/enums_generated.rs from CSVs in dns-specs/*
    generate_enums_rs(&specs_dir.join("enums_generated.rs"), &csv_dir, &sections)?;



@@ 573,5 571,5 @@ fn generate_version_rs(version_rs: &Path, specs_dir: &Path) -> Result<()> {
    info!("Generating {:?}", version_rs);
    enumsfile.write(b"// This file is autogenerated by update_specs.rs. Don't touch.\n")?;
    enumsfile.write(format!("pub const VERSION_HASH: &str = \"{}\";\n", hash_trunc).as_bytes())?;
    Ok(()) // TODO
    Ok(())
}

M src/config.rs => src/config.rs +75 -25
@@ 1,3 1,4 @@
use std::collections::HashMap;
use std::env;
use std::fmt;
use std::fs::File;


@@ 10,11 11,48 @@ use serde::de;
use serde::{Deserialize, Deserializer};
use tracing::{trace, warn};

/// The struct representation of a Kapiti filter section
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct ConfigFilter {
    /// Zero or more file paths or URLs for block files, where URLs will be automatically updated periodically.
    /// Each listed file should look similar to a `/etc/hosts` file: a newline-separated list of hostnames paired with IP destinations.
    #[serde(default)]
    #[serde(alias = "override")]
    #[serde(deserialize_with = "string_or_seq_string")]
    pub overrides: Vec<String>,

    /// Zero or more file paths or URLs for block files, where URLs will be automatically updated periodically.
    /// Each listed file should contain a newline-separated list of hostnames to be blocked (along with their subdomains).
    /// This cannot be combined with `allows` in the same filter object.
    #[serde(default)]
    #[serde(alias = "block")]
    #[serde(deserialize_with = "string_or_seq_string")]
    pub blocks: Vec<String>,

    /// Zero or more file paths or URLs for allow files, where URLs will be automatically updated periodically.
    /// Each listed file should contain a newline-separated list of hostnames to be allowed (along with their subdomains).
    /// This cannot be combined with `blocks` in the same filter object.
    #[serde(default)]
    #[serde(alias = "allow")]
    #[serde(deserialize_with = "string_or_seq_string")]
    pub allows: Vec<String>,

    /// The hosts that this filter block should apply to, or empty/unset for all requests (unless another filter has a closer `applies_to`).
    /// May either be a single IP (`192.16.1.2`) or a range of IPs via CIDR notation (`192.16.1.0/24`).
    /// No two filter blocks may have the same `applies_to` setting.
    /// The filter block with the "closest" match for a given host is the one that will be evaluated.
    #[serde(default)]
    #[serde(alias = "apply_to")]
    pub applies_to: String,
}

/// The struct representation of a Kapiti TOML config file.
#[derive(Debug, Deserialize)]
#[serde(deny_unknown_fields)]
pub struct Config {
    /// Where temporary files, such as downloaded override/block files, should be stored. Defaults to `/tmp/kapiti`.
    /// Where temporary files, such as downloaded override/block files, should be stored.
    /// Defaults to `/tmp/kapiti`.
    #[serde(default = "default_storage")]
    pub storage: String,



@@ 23,9 61,26 @@ pub struct Config {
    #[serde(default = "default_user")]
    pub user: String,

    /// Listen endpoint for the server. Currently defaults to `127.0.0.1:53` but may switch to `0.0.0.0:53` in the future.
    #[serde(default = "default_listen")]
    pub listen: String,
    /// IP+Port endpoint for the DNS server.
    /// Defaults to `0.0.0.0:53`.
    #[serde(default = "default_listen_dns")]
    pub listen_dns: String,

    /// IP+Port endpoint for the Web server for dashboards and HTTP connection snooping.
    /// Defaults to `0.0.0.0:80`.
    #[serde(default = "default_listen_http")]
    pub listen_http: String,

    /// IP+Port endpoint for the Web server for dashboards and HTTPS connection snooping.
    /// Defaults to `0.0.0.0:443`.
    #[serde(default = "default_listen_https")]
    pub listen_https: String,

    /// Filters to apply against request.
    /// Each filter block may optionally specify applicable client IPs for that block.
    #[serde(default)]
    #[serde(alias = "filter")]
    pub filters: HashMap<String, ConfigFilter>,

    /// One or more DNS server endpoints.
    /// These may be provided as hostnames (required for DNS-over-HTTPS), but at least one upstream entry must be provided as an IP to "bootstrap" any hostname entries.


@@ 41,23 96,9 @@ pub struct Config {
    #[serde(deserialize_with = "string_or_seq_string")]
    pub upstreams: Vec<String>,

    /// Zero or more file paths or URLs for block files, where URLs will be automatically updated periodically.
    /// Each listed file should look similar to a `/etc/hosts` file: a newline-separated list of hostnames paired with IP destinations.
    #[serde(default)]
    #[serde(alias = "override")]
    #[serde(deserialize_with = "string_or_seq_string")]
    pub overrides: Vec<String>,

    /// Zero or more file paths or URLs for block files, where URLs will be automatically updated periodically.
    /// Each listed file should contain a newline-separated list of hostnames to be blocked (along with their subdomains).
    #[serde(default)]
    #[serde(alias = "block")]
    #[serde(deserialize_with = "string_or_seq_string")]
    pub blocks: Vec<String>,

    /// URL endpoint for a Redis cache.
    /// For example `redis://127.0.0.1:6379/0`, or `redis://user:password@redis.local:6379/0`.
    /// Or empty/unset to disable caching.
    /// Or empty/unset to disable Redis in favor of an internal in-memory cache.
    #[serde(default)]
    pub redis: String,
}


@@ 66,14 107,16 @@ impl Config {
    /// Returns a new `Config` instance suitable for use in benchmark tests.
    /// Most values are empty or left as their defaults, while the "listen" value is set to `127.0.0.1:0` for an ephemeral port.
    pub fn new_for_test(storage: &str, upstream: String) -> Config {
        let listen_random = "127.0.0.1:0".to_string();
        Config {
            storage: storage.to_string(),
            // Disable user downgrade to avoid system-specific issues (what if 'nobody' doesn't exist in the test environment?)
            user: "".to_string(),
            listen: "127.0.0.1:0".to_string(),
            listen_dns: listen_random.clone(),
            listen_http: listen_random.clone(),
            listen_https: listen_random,
            upstreams: vec![upstream],
            overrides: vec![],
            blocks: vec![],
            filters: HashMap::new(),
            redis: "".to_string(),
        }
    }


@@ 87,9 130,16 @@ fn default_user() -> String {
    "nobody".to_string()
}

fn default_listen() -> String {
    // TODO(#8) switch default to 0.0.0.0:53 if/when confident about service (rate limiting?)
    "127.0.0.1:53".to_string()
fn default_listen_dns() -> String {
    "0.0.0.0:53".to_string()
}

fn default_listen_http() -> String {
    "0.0.0.0:80".to_string()
}

fn default_listen_https() -> String {
    "0.0.0.0:443".to_string()
}

/// A config value that can be provided either as a string or a list of strings in the TOML file.

M src/runner.rs => src/runner.rs +20 -17
@@ 45,11 45,11 @@ impl Runner {
    /// Creates a new `Runner` instance after setting up any listen sockets.
    pub async fn new(config_path: String, config: config::Config) -> Result<Runner> {
        // Initialize listen socket up-front so that upstream can quickly downgrade the user to non-root if needed.
        let listen_host = config.listen.trim();
        let listen_addr = listen_host
        let dns_listen_host = config.listen_dns.trim();
        let dns_listen_addr = dns_listen_host
            .to_socket_addrs()?
            .next()
            .with_context(|| format!("Invalid listen address: {}", listen_host))?;
            .with_context(|| format!("Invalid listen_dns address: {}", dns_listen_host))?;

        let storage_dir = PathBuf::from(config.storage.trim());
        if !storage_dir.exists() {


@@ 66,12 66,12 @@ impl Runner {

        // Set up sockets up-front. This is mainly to support listening on an ephemeral port (:0),
        // where tcp_addr/udp_addr are unknown until the listeners have been initialized.
        let tcp_listener = TcpListener::bind(listen_addr)
        let tcp_listener = TcpListener::bind(dns_listen_addr)
            .await
            .with_context(|| format!("Failed to listen on TCP {}", listen_addr))?;
        let udp_sock = UdpSocket::bind(listen_addr)
            .with_context(|| format!("Failed to listen on TCP {}", dns_listen_addr))?;
        let udp_sock = UdpSocket::bind(dns_listen_addr)
            .await
            .with_context(|| format!("Failed to listen on UDP {}", listen_addr))?;
            .with_context(|| format!("Failed to listen on UDP {}", dns_listen_addr))?;

        Ok(Runner {
            config,


@@ 146,17 146,20 @@ impl Runner {
        // - refrain from killing the process if a download fails or something
        let resolver = client::upstream::parse_upstreams(cache_tx.clone(), &self.config.upstreams)?;
        let mut filter = filter::Filter::new(self.storage_dir.join("filters"), resolver)?;
        // Allow these path lists to be unset/empty
        for entry in &self.config.overrides {
            filter.update_override(entry)?;
        }
        {
            let span = tracing::info_span!("update-filters");
            let _enter = span.enter();
            for entry in &self.config.blocks {
                filter.update_block(entry, 10000).await?;
        // TODO(#30): Implement advanced filter blocks with applies_to support, and with allow support
        for (_name, conf) in &self.config.filters {
            // Allow these path lists to be unset/empty
            for entry in &conf.overrides {
                filter.update_override(entry)?;
            }
        };
            {
                let span = tracing::info_span!("update-filters");
                let _enter = span.enter();
                for entry in &conf.blocks {
                    filter.update_block(entry, 10000).await?;
                }
            };
        }
        let filter = Arc::new(Mutex::new(filter));

        // Start independent threads to handle received requests, query upstreams, and send back responses

M src/specs/version_generated.rs => src/specs/version_generated.rs +1 -1
@@ 1,2 1,2 @@
// This file is autogenerated by update_specs.rs. Don't touch.
pub const VERSION_HASH: &str = "5b67e69c6f604df0";
pub const VERSION_HASH: &str = "88ed579d6ebf37f5";