~vpzom/lotide

daea3783bd37636691a51641b14fc2cf7bd381c9 — Colin Reeder 13 days ago 754c40d + eb570cc
Merge branch 'ratelimit' into master
4 files changed, 183 insertions(+), 6 deletions(-)

M Cargo.lock
M Cargo.toml
M src/main.rs
A src/ratelimit.rs
M Cargo.lock => Cargo.lock +55 -3
@@ 26,6 26,15 @@ dependencies = [
]

[[package]]
name = "ahash"
version = "0.3.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e8fd72866655d1904d6b0997d0b07ba561047d070fbe29de039031c641b61217"
dependencies = [
 "const-random",
]

[[package]]
name = "aho-corasick"
version = "0.7.13"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 84,7 93,7 @@ dependencies = [
 "base64",
 "blowfish",
 "byteorder",
 "getrandom",
 "getrandom 0.1.14",
]

[[package]]


@@ 220,6 229,26 @@ dependencies = [
]

[[package]]
name = "const-random"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "02dc82c12dc2ee6e1ded861cf7d582b46f66f796d1b6c93fa28b911ead95da02"
dependencies = [
 "const-random-macro",
 "proc-macro-hack",
]

[[package]]
name = "const-random-macro"
version = "0.1.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "fc757bbb9544aa296c2ae00c679e81f886b37e28e59097defe0cf524306f6685"
dependencies = [
 "getrandom 0.2.0",
 "proc-macro-hack",
]

[[package]]
name = "core-foundation"
version = "0.7.0"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 274,6 303,17 @@ dependencies = [
]

[[package]]
name = "dashmap"
version = "3.11.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0f260e2fc850179ef410018660006951c1b55b79e8087e87111a2c388994b9b5"
dependencies = [
 "ahash",
 "cfg-if",
 "num_cpus",
]

[[package]]
name = "deadpool"
version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 597,6 637,17 @@ dependencies = [
]

[[package]]
name = "getrandom"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee8025cf36f917e6a52cce185b7c7177689b838b7ec138364e50cc2277a56cf4"
dependencies = [
 "cfg-if",
 "libc",
 "wasi",
]

[[package]]
name = "h2"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 949,6 1000,7 @@ dependencies = [
 "bumpalo",
 "bytes",
 "chrono",
 "dashmap",
 "deadpool-postgres",
 "either",
 "fast_chemail",


@@ 1406,7 1458,7 @@ version = "0.7.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6a6b1679d49b24bbfe0c803429aa1874472f50d9b363131f0e89fc356b544d03"
dependencies = [
 "getrandom",
 "getrandom 0.1.14",
 "libc",
 "rand_chacha",
 "rand_core",


@@ 1429,7 1481,7 @@ version = "0.5.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "90bde5296fc891b0cef12a6d03ddccc162ce7b2aff54160af9338f8d40df6d19"
dependencies = [
 "getrandom",
 "getrandom 0.1.14",
]

[[package]]

M Cargo.toml => Cargo.toml +1 -0
@@ 47,6 47,7 @@ rand = "0.7.3"
bs58 = "0.3.1"
bumpalo = "3.4.0"
tokio-util = "0.3.1"
dashmap = "3.11.10"

[dev-dependencies]
rand = "0.7.3"

M src/main.rs => src/main.rs +45 -3
@@ 8,6 8,7 @@ use std::sync::Arc;
use trout::hyper::RoutingFailureExtHyper;

mod apub_util;
mod ratelimit;
mod routes;
mod tasks;
mod worker;


@@ 139,6 140,7 @@ pub struct BaseContext {
    pub http_client: HttpClient,
    pub apub_proxy_rewrites: bool,
    pub media_location: Option<std::path::PathBuf>,
    pub api_ratelimit: ratelimit::RatelimitBucket<std::net::IpAddr>,

    pub local_hostname: String,
}


@@ 851,6 853,12 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
        Err(other) => Err(other).expect("Failed to parse APUB_PROXY_REWRITES"),
    };

    let allow_forwarded = match std::env::var("ALLOW_FORWARDED") {
        Ok(value) => value.parse().expect("Failed to parse ALLOW_FORWARDED"),
        Err(std::env::VarError::NotPresent) => false,
        Err(other) => Err(other).expect("Failed to parse ALLOW_FORWARDED"),
    };

    let db_pool = deadpool_postgres::Pool::new(
        deadpool_postgres::Manager::new(
            std::env::var("DATABASE_URL")


@@ 929,6 937,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
        host_url_apub,
        http_client: hyper::Client::builder().build(hyper_tls::HttpsConnector::new()),
        apub_proxy_rewrites,
        api_ratelimit: ratelimit::RatelimitBucket::new(300),
    });

    let worker_trigger = worker::start_worker(base_context.clone());


@@ 939,15 948,48 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
    });

    let server = hyper::Server::bind(&(std::net::Ipv6Addr::UNSPECIFIED, port).into()).serve(
        hyper::service::make_service_fn(|_| {
        hyper::service::make_service_fn(|sock: &hyper::server::conn::AddrStream| {
            let addr_direct = sock.remote_addr().ip();
            let routes = routes.clone();
            let context = context.clone();
            async {
            async move {
                Ok::<_, hyper::Error>(hyper::service::service_fn(move |req| {
                    let routes = routes.clone();
                    let context = context.clone();
                    async move {
                        let result = if req.method() == hyper::Method::OPTIONS
                        let addr = if allow_forwarded {
                            if let Some(value) = req
                                .headers()
                                .get(hyper::header::HeaderName::from_static("x-forwarded-for"))
                            {
                                match value
                                    .to_str()
                                    .map_err(|_| ())
                                    .and_then(|value| value.split(", ").next().ok_or(()))
                                    .and_then(|value| value.parse().map_err(|_| ()))
                                {
                                    Err(_) => {
                                        return Ok(simple_response(
                                            hyper::StatusCode::BAD_REQUEST,
                                            "Invalid X-Forwarded-For value",
                                        ));
                                    }
                                    Ok(value) => value,
                                }
                            } else {
                                addr_direct
                            }
                        } else {
                            addr_direct
                        };

                        let ratelimit_ok = context.api_ratelimit.try_call(addr).await;
                        let result = if !ratelimit_ok {
                            Ok(simple_response(
                                hyper::StatusCode::TOO_MANY_REQUESTS,
                                "Ratelimit exceeded.",
                            ))
                        } else if req.method() == hyper::Method::OPTIONS
                            && req.uri().path().starts_with("/api")
                        {
                            hyper::Response::builder()

A src/ratelimit.rs => src/ratelimit.rs +82 -0
@@ 0,0 1,82 @@
use std::sync::atomic::AtomicU16;

pub struct RatelimitBucket<K> {
    cap: u16,
    inner: tokio::sync::RwLock<Inner<K>>,
}

impl<K: Eq + std::hash::Hash + std::fmt::Debug> RatelimitBucket<K> {
    pub fn new(cap: u16) -> Self {
        Self {
            cap,
            inner: tokio::sync::RwLock::new(Inner {
                divider_time: std::time::Instant::now(),
                last_minute: None,
                current_minute: dashmap::DashMap::new(),
            }),
        }
    }

    pub async fn try_call(&self, key: K) -> bool {
        let now = std::time::Instant::now();
        let inner = self.inner.read().await;
        let seconds_into = now.duration_since(inner.divider_time).as_secs();
        if seconds_into >= 60 {
            std::mem::drop(inner);
            let mut inner = self.inner.write().await;

            let seconds_into_new = now.duration_since(inner.divider_time).as_secs();

            // check again
            if seconds_into_new >= 120 {
                // more than two minutes elapsed, reset
                inner.last_minute = None;
                inner.current_minute = dashmap::DashMap::new();
                inner.divider_time = now;

                self.try_for_current(0, &inner, key).await
            } else if seconds_into_new >= 60 {
                let mut tmp = dashmap::DashMap::new();
                std::mem::swap(&mut tmp, &mut inner.current_minute);
                inner.last_minute = Some(tmp.into_read_only());
                inner.divider_time += std::time::Duration::new(60, 0);

                self.try_for_current(seconds_into_new - 60, &inner, key)
                    .await
            } else {
                self.try_for_current(seconds_into_new, &inner, key).await
            }
        } else {
            self.try_for_current(seconds_into, &inner, key).await
        }
    }

    async fn try_for_current(&self, seconds_into: u64, inner: &Inner<K>, key: K) -> bool {
        let prev_count = if let Some(last_minute) = &inner.last_minute {
            if let Some(prev_count) = last_minute.get(&key) {
                (u64::from(prev_count.load(std::sync::atomic::Ordering::Relaxed))
                    * (60 - seconds_into)
                    / 60) as u16
            } else {
                0
            }
        } else {
            0
        };

        let count = prev_count
            + inner
                .current_minute
                .entry(key)
                .or_default()
                .fetch_add(1, std::sync::atomic::Ordering::SeqCst);

        count < self.cap
    }
}

struct Inner<K> {
    divider_time: std::time::Instant,
    last_minute: Option<dashmap::ReadOnlyView<K, AtomicU16>>,
    current_minute: dashmap::DashMap<K, AtomicU16>,
}