M .gitignore => .gitignore +1 -0
@@ 2,3 2,4 @@
*.db
*.db-shm
*.db-wal
+/test_db
M Cargo.lock => Cargo.lock +60 -108
@@ 277,12 277,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "ee2a4ec343196209d6594e19543ae87a39f96d5534d7174822a3ad825dd6ed7e"
[[package]]
-name = "ahash"
-version = "0.4.7"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "739f4a8db6605981345c5654f3a85b056ce52f37a39d34da03f25bf2151ea16e"
-
-[[package]]
name = "aho-corasick"
version = "0.7.15"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 525,6 519,30 @@ dependencies = [
]
[[package]]
+name = "crossbeam-epoch"
+version = "0.9.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2584f639eb95fea8c798496315b297cf81b9b58b6d30ab066a75455333cf4b12"
+dependencies = [
+ "cfg-if 1.0.0",
+ "crossbeam-utils",
+ "lazy_static",
+ "memoffset",
+ "scopeguard",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e7e9d99fa91428effe99c5c6d4634cdeba32b8cf784fc428a2a687f61a952c49"
+dependencies = [
+ "autocfg",
+ "cfg-if 1.0.0",
+ "lazy_static",
+]
+
+[[package]]
name = "derive_more"
version = "0.99.11"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 619,18 637,6 @@ dependencies = [
]
[[package]]
-name = "fallible-iterator"
-version = "0.2.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "4443176a9f2c162692bd3d352d745ef9413eec5782a80d8fd6f8a1ac692a07f7"
-
-[[package]]
-name = "fallible-streaming-iterator"
-version = "0.1.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "7360491ce676a36bf9bb3c56c1aa791658183a54d2744120f27285738d90465a"
-
-[[package]]
name = "flate2"
version = "1.0.19"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 659,6 665,16 @@ dependencies = [
]
[[package]]
+name = "fs2"
+version = "0.4.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9564fc758e15025b46aa6643b1b77d047d1a56a1aea6e01002ac0c7026876213"
+dependencies = [
+ "libc",
+ "winapi 0.3.9",
+]
+
+[[package]]
name = "fuchsia-zircon"
version = "0.3.3"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 841,18 857,6 @@ name = "hashbrown"
version = "0.9.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "d7afe4a420e3fe79967a00898cc1f4db7c8a49a9333a29f8a4bd76a253d5cd04"
-dependencies = [
- "ahash",
-]
-
-[[package]]
-name = "hashlink"
-version = "0.6.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d99cf782f0dc4372d26846bec3de7804ceb5df083c2d4462c0b8d2330e894fa8"
-dependencies = [
- "hashbrown",
-]
[[package]]
name = "heck"
@@ 1004,14 1008,11 @@ dependencies = [
"log",
"mime",
"pretty_env_logger",
- "r2d2",
- "r2d2_sqlite",
"radix_fmt",
"ructe",
- "rusqlite",
"serde",
"serde_json",
- "text_io",
+ "sled",
"url",
"uuid",
]
@@ 1058,17 1059,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "89203f3fba0a3795506acaad8ebce3c80c0af93f994d5a1d7a0b1eeb23271929"
[[package]]
-name = "libsqlite3-sys"
-version = "0.20.1"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "64d31059f22935e6c31830db5249ba2b7ecd54fd73a9909286f0a67aa55c2fbd"
-dependencies = [
- "cc",
- "pkg-config",
- "vcpkg",
-]
-
-[[package]]
name = "linked-hash-map"
version = "0.5.4"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1126,6 1116,15 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "0ee1c47aaa256ecabcaea351eae4a9b01ef39ed810004e298d2511ed284b1525"
[[package]]
+name = "memoffset"
+version = "0.6.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f83fb6581e8ed1f85fd45c116db8405483899489e38406156c25eb743554361d"
+dependencies = [
+ "autocfg",
+]
+
+[[package]]
name = "mime"
version = "0.3.16"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1351,12 1350,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
[[package]]
-name = "pkg-config"
-version = "0.3.19"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "3831453b3449ceb48b6d9c7ad7c96d5ea673e9b470a1dc578c2ce6521230884c"
-
-[[package]]
name = "ppv-lite86"
version = "0.2.10"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1409,27 1402,6 @@ dependencies = [
]
[[package]]
-name = "r2d2"
-version = "0.8.9"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "545c5bc2b880973c9c10e4067418407a0ccaa3091781d1671d46eb35107cb26f"
-dependencies = [
- "log",
- "parking_lot",
- "scheduled-thread-pool",
-]
-
-[[package]]
-name = "r2d2_sqlite"
-version = "0.17.0"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "227ab35ff4cbb01fa76da8f062590fe677b93c8d9e8415eb5fa981f2c1dba9d8"
-dependencies = [
- "r2d2",
- "rusqlite",
-]
-
-[[package]]
name = "radix_fmt"
version = "1.0.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1525,21 1497,6 @@ dependencies = [
]
[[package]]
-name = "rusqlite"
-version = "0.24.2"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "d5f38ee71cbab2c827ec0ac24e76f82eca723cee92c509a65f67dee393c25112"
-dependencies = [
- "bitflags",
- "fallible-iterator",
- "fallible-streaming-iterator",
- "hashlink",
- "libsqlite3-sys",
- "memchr",
- "smallvec",
-]
-
-[[package]]
name = "rustc-demangle"
version = "0.1.18"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1561,15 1518,6 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "71d301d4193d031abdd79ff7e3dd721168a9572ef3fe51a1517aba235bd8f86e"
[[package]]
-name = "scheduled-thread-pool"
-version = "0.2.5"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "dc6f74fd1204073fa02d5d5d68bec8021be4c38690b61264b2fdb48083d0e7d7"
-dependencies = [
- "parking_lot",
-]
-
-[[package]]
name = "scopeguard"
version = "1.1.0"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1668,6 1616,22 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
checksum = "c111b5bd5695e56cffe5129854aa230b39c93a305372fdbb2668ca2394eea9f8"
[[package]]
+name = "sled"
+version = "0.34.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1d0132f3e393bcb7390c60bb45769498cf4550bcb7a21d7f95c02b69f6362cdc"
+dependencies = [
+ "crc32fast",
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "fs2",
+ "fxhash",
+ "libc",
+ "log",
+ "parking_lot",
+]
+
+[[package]]
name = "smallvec"
version = "1.6.1"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 1781,12 1745,6 @@ dependencies = [
]
[[package]]
-name = "text_io"
-version = "0.1.8"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "6cb170b4f47dc48835fbc56259c12d8963e542b05a24be2e3a1f5a6c320fd2d4"
-
-[[package]]
name = "thiserror"
version = "1.0.23"
source = "registry+https://github.com/rust-lang/crates.io-index"
@@ 2060,12 2018,6 @@ dependencies = [
]
[[package]]
-name = "vcpkg"
-version = "0.2.11"
-source = "registry+https://github.com/rust-lang/crates.io-index"
-checksum = "b00bca6106a5e23f3eee943593759b7fcddb00554332e856d990c893966879fb"
-
-[[package]]
name = "version_check"
version = "0.9.2"
source = "registry+https://github.com/rust-lang/crates.io-index"
M Cargo.toml => Cargo.toml +1 -4
@@ 20,9 20,6 @@ serde_json = "1"
radix_fmt = "1"
mime = "0.3"
url = "2.2"
-r2d2 = "0.8"
-r2d2_sqlite = "0.17"
-rusqlite = { version = "0.24", features = ["bundled"] }
futures = "0.3"
log = { version = "0.4", features = ["max_level_debug", "release_max_level_info"] }
pretty_env_logger = "0.4"
@@ 31,8 28,8 @@ failure = "0.1.8"
failure_derive = "0.1.1"
exitcode = "1.1.2"
human-panic = "1.0.3"
-text_io = "0.1.8"
chrono = "0.4.19"
+sled = "0.34.6"
[build-dependencies]
ructe = { version = "0.13", features = ["mime03"] }
M README.md => README.md +4 -5
@@ 5,8 5,8 @@
A very performant URL shortener service for individuals and small groups.
-The service builds upon the [Actix] web framework and [Rusqlite] for data
-handling. Thanks to Actix and the speed optimized SQLite database you can
+The service builds upon the [Actix] web framework and [Sled] for data
+handling. Thanks to Actix and the in-memory database you can
expect 100k requests handled per second on consumer hardware (my laptop).
# Quick Start
@@ 15,8 15,7 @@ The database will be automatically initialized with a super user if it is not ye
```
$ k0r
-Database file k0r.db not found. Create it? [y/N]
-y
+Database file k0r.db not found. Will be created!
Added first user with api key 859b397c-a933-461d-a9b1-86dd20084c02
Server is listening on 127.0.0.1:8080
```
@@ 72,6 71,6 @@ not yet implemented. See the [todo list] for more information about the
planned features and current state of implementation.
[Actix]: https://actix.rs/
-[Rusqlite]: https://docs.rs/rusqlite/
+[Sled]: https://docs.rs/sled/
[250kb.club]: https://git.sr.ht/~koehr/the-250kb-club/tree/main/item/pages.txt
[todo list]: https://todo.sr.ht/~koehr/k0r-planned-features
R src/db.rs => src/db.old.rs +0 -0
M src/main.rs => src/main.rs +9 -85
@@ 1,6 1,6 @@
//! A very performant URL shortener service for individuals and small groups.
//!
-//! The service builds upon the [Actix] web framework and [Rusqlite] for data
+//! The service builds upon the [Actix] web framework and [Sled] for data
//! handling. Thanks to Actix and the speed optimized SQLite database you can
//! expect 100k requests handled per second on consumer hardware (my laptop).
//!
@@ 74,7 74,7 @@
//!
//!
//! [Actix]: https://actix.rs/
-//! [Rusqlite]: https://docs.rs/rusqlite/
+//! [Sled]: https://docs.rs/sled/
//! [250kb.club]: https://git.sr.ht/~koehr/the-250kb-club/tree/main/item/pages.txt
//! [todo list]: https://todo.sr.ht/~koehr/k0r-planned-features
//!
@@ 85,96 85,16 @@ extern crate log;
extern crate pretty_env_logger;
use futures::executor::block_on;
use human_panic::setup_panic;
-use r2d2_sqlite::SqliteConnectionManager;
-use std::path::PathBuf;
-use text_io::read;
mod actix_ructe;
-mod db;
mod server;
mod response_types;
mod short_code;
-use db::DBValue;
-
// This includes the template code generated by ructe
include!(concat!(env!("OUT_DIR"), "/templates.rs"));
-/// Prompts and stops process on negative response.
-fn prompt_or_exit(msg: &str, err_msg: &str) {
- println!("{}", msg);
- let input: String = read!("{}\n");
- if input != "y" && input != "Y" {
- error!("{}", err_msg);
- std::process::exit(exitcode::CANTCREAT);
- }
-}
-
-/// Builds the database path with some heuristics,
-/// for example /foo/ becomes /foo/k0r.db,
-/// and checks for its existence. If non-existent, a prompt asks for a decision
-fn build_db_path(path_str: &str) -> PathBuf {
- let mut db_path = PathBuf::from(path_str);
-
- // append k0r.db as filename if db_path is a directory
- if db_path.is_dir() {
- db_path.push("k0r.db");
- debug!("Expanded given argument \"{}\" to {:?}", &path_str, db_path);
- }
-
- if !db_path.is_file() {
- let msg = format!("Database file {} not found. Create it? [y/N]", path_str);
- prompt_or_exit(&msg, "DB not created. Exiting.");
- }
-
- db_path
-}
-
-/// Initializes the database connection and pool.
-/// The database is configured for performance.
-async fn init_db_pool(path_str: String) -> db::Pool {
- let db_path = build_db_path(&path_str);
-
- debug!("Initializing database...");
- let db_manager = SqliteConnectionManager::file(db_path).with_init(|c| {
- c.execute_batch(
- "
- PRAGMA journal_mode = WAL;
- PRAGMA synchronous = normal;
- PRAGMA temp_store = memory;
- PRAGMA mmap_size = 314572800;
- ",
- )
- });
- let db_pool = db::Pool::new(db_manager).unwrap();
-
- if (db::query(&db_pool, db::Queries::NeedsInit).await).is_err() {
- debug!("New database. Initializing schema...");
- let _ = db::query(&db_pool, db::Queries::InitDB).await;
- }
-
- match db::query(&db_pool, db::Queries::CountUsers).await {
- Ok(DBValue::Number(0)) => {
- match db::query(&db_pool, db::Queries::CreateUser(0, true)).await {
- Ok(DBValue::String(api_key)) => println!("Added first user with api key {}", api_key),
- Ok(v) => debug!("Got unexpected value after user creation: {:#?}", v),
- Err(err) => panic!("Failed to create super user! {}", err),
- }
- }
- Ok(DBValue::Number(_)) => { /* nothing to do */ }
- Ok(v) => debug!("Got unexpected value when counting users: {:#?}", v),
- Err(err) => panic!("Failed to create super user! {}", err),
- }
-
- match db::query(&db_pool, db::Queries::ForceSync).await {
- Ok(_) => { /* all is good */ }
- Err(err) => panic!("Failed to sync database! {}", err)
- }
-
- db_pool
-}
-
-fn main() -> Result<(), std::io::Error> {
+fn main() -> std::io::Result<()> {
pretty_env_logger::init();
setup_panic!();
@@ 188,10 108,14 @@ fn main() -> Result<(), std::io::Error> {
std::process::exit(exitcode::USAGE);
}
+ debug!("Initializing database...");
+ let db = sled::open(arg)?;
+
+ // TODO: create default user if data.users.is_empty()
+
let serv = async {
- let db_pool = init_db_pool(arg).await;
debug!("Starting server...");
- server::start(db_pool)
+ server::start(db)
};
block_on(serv)
M src/server.rs => src/server.rs +65 -54
@@ 1,4 1,4 @@
-use super::db::{self, DBValue};
+use sled::Tree;
use super::render;
use super::templates::{self, statics::StaticFile};
use actix_web::{
@@ 10,6 10,7 @@ use actix_web::{
use std::time::{Duration, SystemTime};
use url::Url;
use super::response_types::Error;
+use super::short_code::ShortCode;
const CONTENT_TYPE_HTML: &str = "text/html; charset=utf-8";
const CONTENT_TYPE_JSON: &str = "application/json; charset=utf-8";
@@ 21,10 22,26 @@ const FAR: Duration = Duration::from_secs(180 * 24 * 60 * 60);
/// Common, unsoliticed queries by browsers that should be ignored
const IGNORED_SHORT_CODES: &[&str] = &["favicon.ico"];
-type DB = web::Data<db::Pool>;
-type JSON = web::Json<db::UrlPostData>;
+/// Describes the expected structure for posting new URLs
+#[derive(serde::Serialize, serde::Deserialize)]
+pub struct UrlPostData {
+ pub url: String,
+ pub title: Option<String>,
+ pub description: Option<String>,
+ pub expiry: Option<i64>,
+ pub key: String,
+}
+
+#[derive(Clone)]
+pub struct DBTrees {
+ users: Tree,
+ urls: Tree,
+}
-fn get_request_origin(req: &HttpRequest) -> String {
+type State = web::Data<DBTrees>;
+type JSON = web::Json<UrlPostData>;
+
+fn req_info(req: &HttpRequest) -> String {
req.connection_info()
.remote_addr()
.unwrap_or("unkown origin")
@@ 67,35 84,28 @@ fn static_file(path: web::Path<String>) -> HttpResponse {
/// Asks the database for the URL matching short_code and responds
/// with a redirect or, if not found, a JSON error
#[actix_web::get("/{short_code}")]
-async fn redirect(req: HttpRequest, db: DB) -> Result<HttpResponse, Error> {
+async fn redirect(req: HttpRequest, db: State) -> Result<HttpResponse, Error> {
let short_code = req.match_info().get("short_code").unwrap_or("0");
if IGNORED_SHORT_CODES.contains(&short_code) {
- debug!(
- "{} queried {}: IGNORED",
- get_request_origin(&req),
- short_code
- );
+ debug!("{} queried {}: IGNORED", req_info(&req), &short_code);
Err(Error::not_found())
- } else if let Ok(DBValue::String(url)) =
- db::query(&db, db::Queries::GetURL(short_code.to_owned())).await
- {
- debug!(
- "{} queried {}, got {}",
- get_request_origin(&req),
- &short_code,
- &url
- );
- Ok(HttpResponse::MovedPermanently()
- .header(LOCATION, url.clone())
- .content_type(CONTENT_TYPE_HTML)
- .body(render!(templates::redirect, "redirect", &url)))
+
+ } else if let Ok(Some(url)) = db.urls.get(&short_code) {
+
+ if let Ok(url) = serde_json::from_slice::<UrlPostData>(&url) {
+ debug!("{} queried {}, got {}", req_info(&req), &short_code, &url.url);
+
+ Ok(HttpResponse::MovedPermanently()
+ .header(LOCATION, url.url.clone())
+ .content_type(CONTENT_TYPE_HTML)
+ .body(render!(templates::redirect, "redirect", &url.url)))
+ } else {
+ debug!("{} queried {}, got Serialization Error", req_info(&req), &short_code);
+ Err(Error::internal())
+ }
} else {
- debug!(
- "{} queried {}, got Not Found",
- get_request_origin(&req),
- short_code
- );
+ debug!("{} queried {}, got Not Found", req_info(&req), &short_code);
Err(Error::not_found())
}
}
@@ 108,40 118,36 @@ async fn redirect(req: HttpRequest, db: DB) -> Result<HttpResponse, Error> {
/// description?: an optional description for the URL, defaults to empty string,
/// key: the API key
#[actix_web::post("/")]
-async fn add_url(_req: HttpRequest, data: JSON, db: DB) -> Result<impl Responder, Error> {
+async fn add_url(_req: HttpRequest, data: JSON, db: State) -> Result<impl Responder, Error> {
match Url::parse(&data.url) {
Ok(parsed_url) => {
if !parsed_url.has_authority() {
- debug!(
- "{} posted \"{}\", got Invalid, no authority.",
- get_request_origin(&_req),
- &data.url
- );
+ debug!("{} posted \"{}\", got Invalid, no authority.", req_info(&_req), &data.url);
return Err(Error::new("Invalid URL, cannot be path only or data URL"));
}
- let query_result = db::query(&db, db::Queries::StoreNewURL(data.into_inner())).await;
-
- match query_result {
- Ok(DBValue::String(code)) => Ok(HttpResponse::Created()
- .content_type(CONTENT_TYPE_JSON)
- .body(format!("{{\"status\": \"ok\", \"message\": \"{}\"}}", code))),
- Err(_) => Err(Error::new("Invalid API key")),
- _ => {
- debug!(
- "Got unexpected type back from StoreNewURL query: {:#?}",
- query_result
- );
- Err(Error::internal())
+ if let Ok(serialized) = serde_json::to_string(&*data) {
+ // TODO: see https://docs.rs/sled/0.34.6/sled/struct.Tree.html#method.len
+ // tree.len performs a full O(n) scan under the hood so instead of asking
+ // for the length all the time a separate "global" atomic counter should
+ // be used that is initialized with the tree len on server start
+ let key = ShortCode::new(db.urls.len()).code;
+ match db.urls.insert(&key, serialized.as_bytes()) {
+ Ok(_) => Ok(HttpResponse::Created()
+ .content_type(CONTENT_TYPE_JSON)
+ .body(format!("{{\"status\": \"ok\", \"message\": \"{}\"}}", key ))),
+ Err(err) => {
+ debug!("{} posted \"{}\", got Insertion Error: {}.", req_info(&_req), &data.url, err);
+ Err(Error::new("Insertion Error"))
+ }
}
+ } else {
+ debug!("{} posted \"{}\", got Invalid, Serialization Error.", req_info(&_req), &data.url);
+ Err(Error::new("Serialization Error"))
}
}
Err(_) => {
- debug!(
- "{} posted \"{}\", got Invalid, Parser Error.",
- get_request_origin(&_req),
- &data.url
- );
+ debug!("{} posted \"{}\", got Invalid, Parser Error.", req_info(&_req), &data.url);
Err(Error::new("Invalid URL"))
}
}
@@ 149,13 155,18 @@ async fn add_url(_req: HttpRequest, data: JSON, db: DB) -> Result<impl Responder
/// the web service initiator
#[actix_web::main]
-pub async fn start(db_pool: db::Pool) -> std::io::Result<()> {
+pub async fn start(db: sled::Db) -> std::io::Result<()> {
println!("Server is listening on 127.0.0.1:8080");
+ let data = DBTrees {
+ urls: db.open_tree(b"urls")?,
+ users: db.open_tree(b"users")?,
+ };
+
actix_web::HttpServer::new(move || {
actix_web::App::new()
.wrap(Logger::default())
- .data(db_pool.clone())
+ .data(data.clone())
.service(static_file) // GET /static/file.xyz
.service(index) // GET /
.service(redirect) // GET /123
M src/templates/index.rs.html => src/templates/index.rs.html +1 -1
@@ 54,7 54,7 @@
<a target="_blank" href="https://actix.rs/" title="Actix actor framework" rel="noopener">Actix</a>
<a target="_blank" href="https://serde.rs/" title="Serde De-/Serialization Framework" rel="noopener">Serde</a>
<a target="_blank" href="https://github.com/sfackler/r2d2" title="r2d2 Generic Connection Pool" rel="noopener">r2d2</a>
- <a target="_blank" href="https://github.com/rusqlite/rusqlite" title="Rusqlite SQlite wrapper" rel="noopener">Rusqlite</a>
+ <a target="_blank" href="https://github.com/spacejam/sled" title="Sled embedded database" rel="noopener">Sled</a>
<a target="_blank" href="https://rust-lang.github.io/futures-rs" title="Futures, async programming for Rust" rel="noopener">Futures-rs</a>
and <a target="_blank" href="https://git.sr.ht/~koehr/k0r/tree/main/item/Cargo.toml" title="used crates" rel="noopener">more</a>.
</div>