~koehr/k0r

ea6b84ed50d66c6c3ad8100b5c1e36597bd9ce73 — koehr 7 months ago 45970d2
get rid of all this SQL nonsense and use sled
8 files changed, 141 insertions(+), 257 deletions(-)

M .gitignore
M Cargo.lock
M Cargo.toml
M README.md
R src/{db.rs => db.old.rs}
M src/main.rs
M src/server.rs
M src/templates/index.rs.html
M .gitignore => .gitignore +1 -0
@@ 2,3 2,4 @@
*.db
*.db-shm
*.db-wal
/test_db

M Cargo.lock => Cargo.lock +60 -108
@@ 277,12 277,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"

[[package]]
name = "ahash"
version = "0.4.7"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"

[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 525,6 519,30 @@ dependencies = [
]

[[package]]
name = "crossbeam-epoch"
version = "0.9.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
dependencies = [
 "cfg-if 1.0.0",
 "crossbeam-utils",
 "lazy_static",
 "memoffset",
 "scopeguard",
]

[[package]]
name = "crossbeam-utils"
version = "0.8.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
dependencies = [
 "autocfg",
 "cfg-if 1.0.0",
 "lazy_static",
]

[[package]]
name = "derive_more"
version = "0.99.11"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 619,18 637,6 @@ dependencies = [
]

[[package]]
name = "fallible-iterator"
version = "0.2.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"

[[package]]
name = "fallible-streaming-iterator"
version = "0.1.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"

[[package]]
name = "flate2"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 659,6 665,16 @@ dependencies = [
]

[[package]]
name = "fs2"
version = "0.4.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
dependencies = [
 "libc",
 "winapi 0.3.9",
]

[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 841,18 857,6 @@ name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
dependencies = [
 "ahash",
]

[[package]]
name = "hashlink"
version = "0.6.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8"
dependencies = [
 "hashbrown",
]

[[package]]
name = "heck"


@@ 1004,14 1008,11 @@ dependencies = [
 "log",
 "mime",
 "pretty_env_logger",
 "r2d2",
 "r2d2_sqlite",
 "radix_fmt",
 "ructe",
 "rusqlite",
 "serde",
 "serde_json",
 "text_io",
 "sled",
 "url",
 "uuid",
]


@@ 1058,17 1059,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"

[[package]]
name = "libsqlite3-sys"
version = "0.20.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd"
dependencies = [
 "cc",
 "pkg-config",
 "vcpkg",
]

[[package]]
name = "linked-hash-map"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1126,6 1116,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"

[[package]]
name = "memoffset"
version = "0.6.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d"
dependencies = [
 "autocfg",
]

[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1351,12 1350,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"

[[package]]
name = "pkg-config"
version = "0.3.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"

[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1409,27 1402,6 @@ dependencies = [
]

[[package]]
name = "r2d2"
version = "0.8.9"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
dependencies = [
 "log",
 "parking_lot",
 "scheduled-thread-pool",
]

[[package]]
name = "r2d2_sqlite"
version = "0.17.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "227ab35ff4cbb01fa76da8f062590fe677b93c8d9e8415eb5fa981f2c1dba9d8"
dependencies = [
 "r2d2",
 "rusqlite",
]

[[package]]
name = "radix_fmt"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1525,21 1497,6 @@ dependencies = [
]

[[package]]
name = "rusqlite"
version = "0.24.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d5f38ee71cbab2c827ec0ac24e76f82eca723cee92c509a65f67dee393c25112"
dependencies = [
 "bitflags",
 "fallible-iterator",
 "fallible-streaming-iterator",
 "hashlink",
 "libsqlite3-sys",
 "memchr",
 "smallvec",
]

[[package]]
name = "rustc-demangle"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1561,15 1518,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"

[[package]]
name = "scheduled-thread-pool"
version = "0.2.5"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
dependencies = [
 "parking_lot",
]

[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1668,6 1616,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"

[[package]]
name = "sled"
version = "0.34.6"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
dependencies = [
 "crc32fast",
 "crossbeam-epoch",
 "crossbeam-utils",
 "fs2",
 "fxhash",
 "libc",
 "log",
 "parking_lot",
]

[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 1781,12 1745,6 @@ dependencies = [
]

[[package]]
name = "text_io"
version = "0.1.8"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "6cb170b4f47dc48835fbc56259c12d8963e542b05a24be2e3a1f5a6c320fd2d4"

[[package]]
name = "thiserror"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"


@@ 2060,12 2018,6 @@ dependencies = [
]

[[package]]
name = "vcpkg"
version = "0.2.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"

[[package]]
name = "version_check"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"

M Cargo.toml => Cargo.toml +1 -4
@@ 20,9 20,6 @@ serde_json = "1"
radix_fmt = "1"
mime = "0.3"
url = "2.2"
r2d2 = "0.8"
r2d2_sqlite = "0.17"
rusqlite = { version = "0.24", features = ["bundled"] }
futures = "0.3"
log = { version = "0.4", features = ["max_level_debug", "release_max_level_info"] }
pretty_env_logger = "0.4"


@@ 31,8 28,8 @@ failure = "0.1.8"
failure_derive = "0.1.1"
exitcode = "1.1.2"
human-panic = "1.0.3"
text_io = "0.1.8"
chrono = "0.4.19"
sled = "0.34.6"

[build-dependencies]
ructe = { version = "0.13", features = ["mime03"] }

M README.md => README.md +4 -5
@@ 5,8 5,8 @@

A very performant URL shortener service for individuals and small groups.

The service builds upon the [Actix] web framework and [Rusqlite] for data
handling. Thanks to Actix and the speed optimized SQLite database you can
The service builds upon the [Actix] web framework and [Sled] for data
handling. Thanks to Actix and the in-memory database you can
expect 100k requests handled per second on consumer hardware (my laptop).

# Quick Start


@@ 15,8 15,7 @@ The database will be automatically initialized with a super user if it is not ye

```
$ k0r
Database file k0r.db not found. Create it? [y/N]
y
Database file k0r.db not found. Will be created!
Added first user with api key 859b397c-a933-461d-a9b1-86dd20084c02
Server is listening on 127.0.0.1:8080
```


@@ 72,6 71,6 @@ not yet implemented. See the [todo list] for more information about the
planned features and current state of implementation.

[Actix]: https://actix.rs/
[Rusqlite]: https://docs.rs/rusqlite/
[Sled]: https://docs.rs/sled/
[250kb.club]: https://git.sr.ht/~koehr/the-250kb-club/tree/main/item/pages.txt
[todo list]: https://todo.sr.ht/~koehr/k0r-planned-features

R src/db.rs => src/db.old.rs +0 -0
M src/main.rs => src/main.rs +9 -85
@@ 1,6 1,6 @@
//! A very performant URL shortener service for individuals and small groups.
//!
//! The service builds upon the [Actix] web framework and [Rusqlite] for data
//! The service builds upon the [Actix] web framework and [Sled] for data
//! handling. Thanks to Actix and the speed optimized SQLite database you can
//! expect 100k requests handled per second on consumer hardware (my laptop).
//!


@@ 74,7 74,7 @@
//!
//!
//! [Actix]: https://actix.rs/
//! [Rusqlite]: https://docs.rs/rusqlite/
//! [Sled]: https://docs.rs/sled/
//! [250kb.club]: https://git.sr.ht/~koehr/the-250kb-club/tree/main/item/pages.txt
//! [todo list]: https://todo.sr.ht/~koehr/k0r-planned-features
//!


@@ 85,96 85,16 @@ extern crate log;
extern crate pretty_env_logger;
use futures::executor::block_on;
use human_panic::setup_panic;
use r2d2_sqlite::SqliteConnectionManager;
use std::path::PathBuf;
use text_io::read;

mod actix_ructe;
mod db;
mod server;
mod response_types;
mod short_code;

use db::DBValue;

// This includes the template code generated by ructe
include!(concat!(env!("OUT_DIR"), "/templates.rs"));

/// Prompts and stops process on negative response.
fn prompt_or_exit(msg: &str, err_msg: &str) {
    println!("{}", msg);
    let input: String = read!("{}\n");
    if input != "y" && input != "Y" {
        error!("{}", err_msg);
        std::process::exit(exitcode::CANTCREAT);
    }
}

/// Builds the database path with some heuristics,
/// for example /foo/ becomes /foo/k0r.db,
/// and checks for its existence. If non-existent, a prompt asks for a decision
fn build_db_path(path_str: &str) -> PathBuf {
    let mut db_path = PathBuf::from(path_str);

    // append k0r.db as filename if db_path is a directory
    if db_path.is_dir() {
        db_path.push("k0r.db");
        debug!("Expanded given argument \"{}\" to {:?}", &path_str, db_path);
    }

    if !db_path.is_file() {
        let msg = format!("Database file {} not found. Create it? [y/N]", path_str);
        prompt_or_exit(&msg, "DB not created. Exiting.");
    }

    db_path
}

/// Initializes the database connection and pool.
/// The database is configured for performance.
async fn init_db_pool(path_str: String) -> db::Pool {
    let db_path = build_db_path(&path_str);

    debug!("Initializing database...");
    let db_manager = SqliteConnectionManager::file(db_path).with_init(|c| {
        c.execute_batch(
            "
            PRAGMA journal_mode = WAL;
            PRAGMA synchronous = normal;
            PRAGMA temp_store = memory;
            PRAGMA mmap_size = 314572800;
            ",
        )
    });
    let db_pool = db::Pool::new(db_manager).unwrap();

    if (db::query(&db_pool, db::Queries::NeedsInit).await).is_err() {
        debug!("New database. Initializing schema...");
        let _ = db::query(&db_pool, db::Queries::InitDB).await;
    }

    match db::query(&db_pool, db::Queries::CountUsers).await {
        Ok(DBValue::Number(0)) => {
            match db::query(&db_pool, db::Queries::CreateUser(0, true)).await {
                Ok(DBValue::String(api_key)) => println!("Added first user with api key {}", api_key),
                Ok(v) => debug!("Got unexpected value after user creation: {:#?}", v),
                Err(err) => panic!("Failed to create super user! {}", err),
            }
        }
        Ok(DBValue::Number(_)) => { /* nothing to do */ }
        Ok(v) => debug!("Got unexpected value when counting users: {:#?}", v),
        Err(err) => panic!("Failed to create super user! {}", err),
    }

    match db::query(&db_pool, db::Queries::ForceSync).await {
        Ok(_) => { /* all is good */ }
        Err(err) => panic!("Failed to sync database! {}", err)
    }

    db_pool
}

fn main() -> Result<(), std::io::Error> {
fn main() -> std::io::Result<()>  {
    pretty_env_logger::init();
    setup_panic!();



@@ 188,10 108,14 @@ fn main() -> Result<(), std::io::Error> {
        std::process::exit(exitcode::USAGE);
    }

    debug!("Initializing database...");
    let db = sled::open(arg)?;

    // TODO: create default user if data.users.is_empty()

    let serv = async {
        let db_pool = init_db_pool(arg).await;
        debug!("Starting server...");
        server::start(db_pool)
        server::start(db)
    };

    block_on(serv)

M src/server.rs => src/server.rs +65 -54
@@ 1,4 1,4 @@
use super::db::{self, DBValue};
use sled::Tree;
use super::render;
use super::templates::{self, statics::StaticFile};
use actix_web::{


@@ 10,6 10,7 @@ use actix_web::{
use std::time::{Duration, SystemTime};
use url::Url;
use super::response_types::Error;
use super::short_code::ShortCode;

const CONTENT_TYPE_HTML: &str = "text/html; charset=utf-8";
const CONTENT_TYPE_JSON: &str = "application/json; charset=utf-8";


@@ 21,10 22,26 @@ const FAR: Duration = Duration::from_secs(180 * 24 * 60 * 60);
/// Common, unsoliticed queries by browsers that should be ignored
const IGNORED_SHORT_CODES: &[&str] = &["favicon.ico"];

type DB = web::Data<db::Pool>;
type JSON = web::Json<db::UrlPostData>;
/// Describes the expected structure for posting new URLs
#[derive(serde::Serialize, serde::Deserialize)]
pub struct UrlPostData {
    pub url: String,
    pub title: Option<String>,
    pub description: Option<String>,
    pub expiry: Option<i64>,
    pub key: String,
}

#[derive(Clone)]
pub struct DBTrees {
    users: Tree,
    urls: Tree,
}

fn get_request_origin(req: &HttpRequest) -> String {
type State = web::Data<DBTrees>;
type JSON = web::Json<UrlPostData>;

fn req_info(req: &HttpRequest) -> String {
    req.connection_info()
        .remote_addr()
        .unwrap_or("unkown origin")


@@ 67,35 84,28 @@ fn static_file(path: web::Path<String>) -> HttpResponse {
/// Asks the database for the URL matching short_code and responds
/// with a redirect or, if not found, a JSON error
#[actix_web::get("/{short_code}")]
async fn redirect(req: HttpRequest, db: DB) -> Result<HttpResponse, Error> {
async fn redirect(req: HttpRequest, db: State) -> Result<HttpResponse, Error> {
    let short_code = req.match_info().get("short_code").unwrap_or("0");

    if IGNORED_SHORT_CODES.contains(&short_code) {
        debug!(
            "{} queried {}: IGNORED",
            get_request_origin(&req),
            short_code
        );
        debug!("{} queried {}: IGNORED", req_info(&req), &short_code);
        Err(Error::not_found())
    } else if let Ok(DBValue::String(url)) =
        db::query(&db, db::Queries::GetURL(short_code.to_owned())).await
    {
        debug!(
            "{} queried {}, got {}",
            get_request_origin(&req),
            &short_code,
            &url
        );
        Ok(HttpResponse::MovedPermanently()
            .header(LOCATION, url.clone())
            .content_type(CONTENT_TYPE_HTML)
            .body(render!(templates::redirect, "redirect", &url)))

    } else if let Ok(Some(url)) = db.urls.get(&short_code) {

        if let Ok(url) = serde_json::from_slice::<UrlPostData>(&url) {
            debug!("{} queried {}, got {}", req_info(&req), &short_code, &url.url);

            Ok(HttpResponse::MovedPermanently()
                .header(LOCATION, url.url.clone())
                .content_type(CONTENT_TYPE_HTML)
                .body(render!(templates::redirect, "redirect", &url.url)))
        } else {
            debug!("{} queried {}, got Serialization Error", req_info(&req), &short_code);
            Err(Error::internal())
        }
    } else {
        debug!(
            "{} queried {}, got Not Found",
            get_request_origin(&req),
            short_code
        );
        debug!("{} queried {}, got Not Found", req_info(&req), &short_code);
        Err(Error::not_found())
    }
}


@@ 108,40 118,36 @@ async fn redirect(req: HttpRequest, db: DB) -> Result<HttpResponse, Error> {
///   description?: an optional description for the URL, defaults to empty string,
///   key: the API key
#[actix_web::post("/")]
async fn add_url(_req: HttpRequest, data: JSON, db: DB) -> Result<impl Responder, Error> {
async fn add_url(_req: HttpRequest, data: JSON, db: State) -> Result<impl Responder, Error> {
    match Url::parse(&data.url) {
        Ok(parsed_url) => {
            if !parsed_url.has_authority() {
                debug!(
                    "{} posted \"{}\", got Invalid, no authority.",
                    get_request_origin(&_req),
                    &data.url
                );
                debug!("{} posted \"{}\", got Invalid, no authority.", req_info(&_req), &data.url);
                return Err(Error::new("Invalid URL, cannot be path only or data URL"));
            }

            let query_result = db::query(&db, db::Queries::StoreNewURL(data.into_inner())).await;

            match query_result {
                Ok(DBValue::String(code)) => Ok(HttpResponse::Created()
                    .content_type(CONTENT_TYPE_JSON)
                    .body(format!("{{\"status\": \"ok\", \"message\": \"{}\"}}", code))),
                Err(_) => Err(Error::new("Invalid API key")),
                _ => {
                    debug!(
                        "Got unexpected type back from StoreNewURL query: {:#?}",
                        query_result
                    );
                    Err(Error::internal())
            if let Ok(serialized) = serde_json::to_string(&*data) {
                // TODO: see https://docs.rs/sled/0.34.6/sled/struct.Tree.html#method.len
                // tree.len performs a full O(n) scan under the hood so instead of asking
                // for the length all the time a separate "global" atomic counter should
                // be used that is initialized with the tree len on server start
                let key = ShortCode::new(db.urls.len()).code;
                match db.urls.insert(&key, serialized.as_bytes()) {
                    Ok(_) => Ok(HttpResponse::Created()
                        .content_type(CONTENT_TYPE_JSON)
                        .body(format!("{{\"status\": \"ok\", \"message\": \"{}\"}}", key ))),
                    Err(err) => {
                        debug!("{} posted \"{}\", got Insertion Error: {}.", req_info(&_req), &data.url, err);
                        Err(Error::new("Insertion Error"))
                    }
                }
            } else {
                debug!("{} posted \"{}\", got Invalid, Serialization Error.", req_info(&_req), &data.url);
                Err(Error::new("Serialization Error"))
            }
        }
        Err(_) => {
            debug!(
                "{} posted \"{}\", got Invalid, Parser Error.",
                get_request_origin(&_req),
                &data.url
            );
            debug!("{} posted \"{}\", got Invalid, Parser Error.", req_info(&_req), &data.url);
            Err(Error::new("Invalid URL"))
        }
    }


@@ 149,13 155,18 @@ async fn add_url(_req: HttpRequest, data: JSON, db: DB) -> Result<impl Responder

/// the web service initiator
#[actix_web::main]
pub async fn start(db_pool: db::Pool) -> std::io::Result<()> {
pub async fn start(db: sled::Db) -> std::io::Result<()> {
    println!("Server is listening on 127.0.0.1:8080");

    let data = DBTrees {
        urls: db.open_tree(b"urls")?,
        users: db.open_tree(b"users")?,
    };

    actix_web::HttpServer::new(move || {
        actix_web::App::new()
            .wrap(Logger::default())
            .data(db_pool.clone())
            .data(data.clone())
            .service(static_file) // GET /static/file.xyz
            .service(index) // GET /
            .service(redirect) // GET /123

M src/templates/index.rs.html => src/templates/index.rs.html +1 -1
@@ 54,7 54,7 @@
      <a target="_blank" href="https://actix.rs/" title="Actix actor framework" rel="noopener">Actix</a>
      <a target="_blank" href="https://serde.rs/" title="Serde De-/Serialization Framework" rel="noopener">Serde</a>
      <a target="_blank" href="https://github.com/sfackler/r2d2" title="r2d2 Generic Connection Pool" rel="noopener">r2d2</a>
      <a target="_blank" href="https://github.com/rusqlite/rusqlite" title="Rusqlite SQlite wrapper" rel="noopener">Rusqlite</a>
      <a target="_blank" href="https://github.com/spacejam/sled" title="Sled embedded database" rel="noopener">Sled</a>
      <a target="_blank" href="https://rust-lang.github.io/futures-rs" title="Futures, async programming for Rust" rel="noopener">Futures-rs</a>
      and <a target="_blank" href="https://git.sr.ht/~koehr/k0r/tree/main/item/Cargo.toml" title="used crates" rel="noopener">more</a>.
    </div>