~vpzom/lotide

da817f5d37e261f876b8c927df0e1b6a210bd6d8 — Colin Reeder 6 months ago c44b438
Add initial support for media uploads
M .gitignore => .gitignore +1 -0
@@ 1,1 1,2 @@
/target
/files

M Cargo.lock => Cargo.lock +1 -0
@@ 976,6 976,7 @@ dependencies = [
 "thiserror",
 "tokio",
 "tokio-postgres",
 "tokio-util",
 "trout",
 "unic-char-range",
 "unic-langid",

M Cargo.toml => Cargo.toml +1 -0
@@ 46,6 46,7 @@ lettre = { version = "0.10.0-alpha.2", features = ["tokio02", "tokio02-native-tl
rand = "0.7.3"
bs58 = "0.3.1"
bumpalo = "3.4.0"
tokio-util = "0.3.1"

[dev-dependencies]
rand = "0.7.3"

A migrations/20201007034916_media/down.sql => migrations/20201007034916_media/down.sql +3 -0
@@ 0,0 1,3 @@
BEGIN;
	DROP TABLE media;
COMMIT;

A migrations/20201007034916_media/up.sql => migrations/20201007034916_media/up.sql +8 -0
@@ 0,0 1,8 @@
BEGIN;
	CREATE TABLE media (
		id INTEGER PRIMARY KEY,
		path TEXT NOT NULL,
		person BIGINT NOT NULL REFERENCES person,
		mime TEXT NOT NULL
	);
COMMIT;

M res/lang/en.ftl => res/lang/en.ftl +3 -0
@@ 4,6 4,9 @@ community_edit_denied = You are not authorized to modify this community
community_name_disallowed_chars = Community name contains disallowed characters
email_content_forgot_password = Hi { $username }, if you requested a password reset from lotide, use this code: { $key }
email_not_configured = Email is not configured on this server
media_upload_not_configured = Media Upload is not configured on this server
media_upload_not_image = Media upload is only available for images
missing_content_type = Missing Content-Type
moderators_only_local = Only local users can be community moderators
must_be_moderator = You must be a community moderator to perform this action
name_in_use = That name is already in use

M src/main.rs => src/main.rs +59 -0
@@ 1,3 1,4 @@
use rand::Rng;
use serde_derive::{Deserialize, Serialize};
use std::borrow::Cow;
use std::collections::{HashMap, HashSet};


@@ 89,6 90,43 @@ impl Into<activitystreams::primitives::OneOrMany<activitystreams::base::AnyBase>
#[derive(Serialize, Default)]
pub struct Empty {}

pub struct Pineapple {
    value: i32,
}

impl Pineapple {
    pub fn generate() -> Self {
        Self {
            value: rand::thread_rng().gen(),
        }
    }

    pub fn as_int(&self) -> i32 {
        self.value
    }
}

// implementing this trait is discouraged in favor of Display, but bs58 doesn't do streaming output
impl std::string::ToString for Pineapple {
    fn to_string(&self) -> String {
        bs58::encode(&self.value.to_be_bytes()).into_string()
    }
}

impl std::str::FromStr for Pineapple {
    type Err = bs58::decode::Error;

    fn from_str(src: &str) -> Result<Self, Self::Err> {
        let src = src.trim_matches(|c: char| !c.is_alphanumeric());

        let mut buf = [0; 4];
        bs58::decode(src).into(&mut buf)?;
        Ok(Self {
            value: i32::from_be_bytes(buf),
        })
    }
}

pub type DbPool = deadpool_postgres::Pool;
pub type HttpClient = hyper::Client<hyper_tls::HttpsConnector<hyper::client::HttpConnector>>;



@@ 100,10 138,28 @@ pub struct BaseContext {
    pub host_url_apub: BaseURL,
    pub http_client: HttpClient,
    pub apub_proxy_rewrites: bool,
    pub media_location: Option<std::path::PathBuf>,

    pub local_hostname: String,
}

impl BaseContext {
    pub fn process_href_opt<'a>(
        &self,
        href: Option<&'a str>,
        post_id: PostLocalID,
    ) -> Option<Cow<'a, str>> {
        match href {
            Some(href) => Some(if href.starts_with("local-media://") {
                format!("{}/unstable/posts/{}/href", self.host_url_api, post_id).into()
            } else {
                href.into()
            }),
            None => None,
        }
    }
}

pub struct RouteContext {
    base: Arc<BaseContext>,
    worker_trigger: tokio::sync::mpsc::Sender<()>,


@@ 814,6 870,8 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
        .try_into()
        .expect("HOST_URL_ACTIVITYPUB is not a valid base URL");

    let media_location = std::env::var_os("MEDIA_LOCATION").map(std::path::PathBuf::from);

    let smtp_url: Option<url::Url> = match std::env::var("SMTP_URL") {
        Ok(value) => Some(value.parse().expect("Failed to parse SMTP_URL")),
        Err(std::env::VarError::NotPresent) => None,


@@ 862,6 920,7 @@ async fn main() -> Result<(), Box<dyn std::error::Error>> {
        db_pool,
        mailer,
        mail_from,
        media_location,
        host_url_api,
        host_url_apub,
        http_client: hyper::Client::builder().build(hyper_tls::HttpsConnector::new()),

M src/routes/api/communities.rs => src/routes/api/communities.rs +1 -1
@@ 625,7 625,7 @@ async fn route_unstable_communities_posts_list(
            let post = RespPostListPost {
                id,
                title,
                href,
                href: ctx.process_href_opt(href, id),
                content_text,
                content_html,
                author: author.as_ref(),

M src/routes/api/forgot_password.rs => src/routes/api/forgot_password.rs +1 -37
@@ 1,46 1,10 @@
use crate::UserLocalID;
use lettre::Tokio02Transport;
use rand::Rng;
use serde_derive::Deserialize;
use std::borrow::Cow;
use std::sync::Arc;

struct ForgotPasswordKey {
    value: i32,
}

impl ForgotPasswordKey {
    pub fn generate() -> Self {
        Self {
            value: rand::thread_rng().gen(),
        }
    }

    pub fn as_int(&self) -> i32 {
        self.value
    }
}

// implementing this trait is discouraged in favor of Display, but bs58 doesn't do streaming output
impl std::string::ToString for ForgotPasswordKey {
    fn to_string(&self) -> String {
        bs58::encode(&self.value.to_be_bytes()).into_string()
    }
}

impl std::str::FromStr for ForgotPasswordKey {
    type Err = bs58::decode::Error;

    fn from_str(src: &str) -> Result<Self, Self::Err> {
        let src = src.trim_matches(|c: char| !c.is_alphanumeric());

        let mut buf = [0; 4];
        bs58::decode(src).into(&mut buf)?;
        Ok(Self {
            value: i32::from_be_bytes(buf),
        })
    }
}
type ForgotPasswordKey = crate::Pineapple;

async fn route_unstable_forgot_password_keys_create(
    _: (),

A src/routes/api/media.rs => src/routes/api/media.rs +68 -0
@@ 0,0 1,68 @@
use std::sync::Arc;

async fn route_unstable_media_create(
    _: (),
    ctx: Arc<crate::RouteContext>,
    req: hyper::Request<hyper::Body>,
) -> Result<hyper::Response<hyper::Body>, crate::Error> {
    let lang = crate::get_lang_for_req(&req);

    let content_type = req
        .headers()
        .get(hyper::header::CONTENT_TYPE)
        .ok_or_else(|| {
            crate::Error::UserError(crate::simple_response(
                hyper::StatusCode::BAD_REQUEST,
                lang.tr("missing_content_type", None).into_owned(),
            ))
        })?;
    let content_type = std::str::from_utf8(content_type.as_ref())?;
    let content_type: mime::Mime = content_type.parse()?;

    if content_type.type_() != mime::IMAGE {
        return Err(crate::Error::UserError(crate::simple_response(
            hyper::StatusCode::BAD_REQUEST,
            lang.tr("media_upload_not_image", None).into_owned(),
        )));
    }

    let db = ctx.db_pool.get().await?;

    let user = crate::require_login(&req, &db).await?;

    if let Some(media_location) = &ctx.media_location {
        let filename = uuid::Uuid::new_v4().to_string();
        let path = media_location.join(&filename);

        {
            use futures::TryStreamExt;
            use tokio::io::AsyncWriteExt;
            let file = tokio::fs::File::create(path).await?;
            req.into_body()
                .map_err(|err| std::io::Error::new(std::io::ErrorKind::Other, err))
                .try_fold(file, |mut file, chunk| async move {
                    file.write_all(chunk.as_ref()).await.map(|_| file)
                })
                .await?;
        }

        let id = crate::Pineapple::generate();

        db.execute(
            "INSERT INTO media (id, path, person, mime) VALUES ($1, $2, $3, $4)",
            &[&id.as_int(), &filename, &user, &content_type.as_ref()],
        )
        .await?;

        crate::json_response(&serde_json::json!({"id": id.to_string()}))
    } else {
        Err(crate::Error::UserError(crate::simple_response(
            hyper::StatusCode::INTERNAL_SERVER_ERROR,
            lang.tr("media_upload_not_configured", None).into_owned(),
        )))
    }
}

pub fn route_media() -> crate::RouteNode<()> {
    crate::RouteNode::new().with_handler_async("POST", route_unstable_media_create)
}

M src/routes/api/mod.rs => src/routes/api/mod.rs +7 -5
@@ 9,6 9,7 @@ use std::sync::Arc;
mod comments;
mod communities;
mod forgot_password;
mod media;
mod posts;
mod users;



@@ 87,7 88,7 @@ struct RespMinimalPostInfo<'a> {
struct RespPostListPost<'a> {
    id: PostLocalID,
    title: &'a str,
    href: Option<&'a str>,
    href: Option<Cow<'a, str>>,
    content_text: Option<&'a str>,
    content_html: Option<&'a str>,
    author: Option<&'a RespMinimalAuthorInfo<'a>>,


@@ 157,6 158,7 @@ pub fn route_api() -> crate::RouteNode<()> {
                            .with_handler_async("DELETE", route_unstable_logins_current_delete),
                    ),
            )
            .with_child("media", media::route_media())
            .with_child(
                "nodeinfo/2.0",
                crate::RouteNode::new().with_handler_async("GET", route_unstable_nodeinfo_20_get),


@@ 706,7 708,7 @@ async fn route_unstable_misc_render_markdown(
async fn handle_common_posts_list(
    stream: impl futures::stream::TryStream<Ok = tokio_postgres::Row, Error = tokio_postgres::Error>
        + Send,
    local_hostname: &str,
    ctx: &crate::RouteContext,
) -> Result<Vec<serde_json::Value>, crate::Error> {
    use futures::stream::TryStreamExt;



@@ 737,7 739,7 @@ async fn handle_common_posts_list(
                    host: crate::get_actor_host_or_unknown(
                        author_local,
                        author_ap_id,
                        &local_hostname,
                        &ctx.local_hostname,
                    ),
                    remote_url: author_ap_id.map(|x| x.to_owned().into()),
                    avatar: author_avatar.map(|url| RespAvatarInfo {


@@ 753,7 755,7 @@ async fn handle_common_posts_list(
                host: crate::get_actor_host_or_unknown(
                    community_local,
                    community_ap_id,
                    &local_hostname,
                    &ctx.local_hostname,
                ),
                remote_url: community_ap_id,
            };


@@ 761,7 763,7 @@ async fn handle_common_posts_list(
            let post = RespPostListPost {
                id,
                title,
                href,
                href: ctx.process_href_opt(href, id),
                content_text,
                content_html,
                author: author.as_ref(),

M src/routes/api/posts.rs => src/routes/api/posts.rs +87 -2
@@ 110,7 110,7 @@ async fn route_unstable_posts_list(
        ([limit]).iter().map(|x| x as _),
    ).await?;

    let posts = super::handle_common_posts_list(stream, &ctx.local_hostname).await?;
    let posts = super::handle_common_posts_list(stream, &ctx).await?;

    crate::json_response(&posts)
}


@@ 334,7 334,7 @@ async fn route_unstable_posts_get(
            let post = RespPostListPost {
                id: post_id,
                title,
                href,
                href: ctx.process_href_opt(href, post_id),
                content_text,
                content_html,
                author: author.as_ref(),


@@ 429,6 429,86 @@ async fn route_unstable_posts_delete(
    }
}

async fn route_unstable_posts_href_get(
    params: (PostLocalID,),
    ctx: Arc<crate::RouteContext>,
    req: hyper::Request<hyper::Body>,
) -> Result<hyper::Response<hyper::Body>, crate::Error> {
    let (post_id,) = params;

    let lang = crate::get_lang_for_req(&req);
    let db = ctx.db_pool.get().await?;

    let row = db
        .query_opt("SELECT href FROM post WHERE id=$1", &[&post_id])
        .await?;
    match row {
        None => Ok(crate::simple_response(
            hyper::StatusCode::NOT_FOUND,
            lang.tr("no_such_post", None).into_owned(),
        )),
        Some(row) => {
            let href: Option<String> = row.get(0);
            match href {
                None => Ok(crate::simple_response(
                    hyper::StatusCode::NOT_FOUND,
                    lang.tr("post_not_link", None).into_owned(),
                )),
                Some(href) => {
                    if href.starts_with("local-media://") {
                        // local media, serve file content

                        let media_id: crate::Pineapple = (&href[14..]).parse()?;

                        let media_row = db
                            .query_opt(
                                "SELECT path, mime FROM media WHERE id=$1",
                                &[&media_id.as_int()],
                            )
                            .await?;
                        match media_row {
                            None => Ok(crate::simple_response(
                                hyper::StatusCode::NOT_FOUND,
                                lang.tr("media_upload_missing", None).into_owned(),
                            )),
                            Some(media_row) => {
                                let path: &str = media_row.get(0);
                                let mime: &str = media_row.get(1);

                                if let Some(media_location) = &ctx.media_location {
                                    let path = media_location.join(path);

                                    let file = tokio::fs::File::open(path).await?;
                                    let body = hyper::Body::wrap_stream(
                                        tokio_util::codec::FramedRead::new(
                                            file,
                                            tokio_util::codec::BytesCodec::new(),
                                        ),
                                    );

                                    Ok(crate::common_response_builder()
                                        .header(hyper::header::CONTENT_TYPE, mime)
                                        .body(body)?)
                                } else {
                                    Ok(crate::simple_response(
                                        hyper::StatusCode::NOT_FOUND,
                                        lang.tr("media_upload_missing", None).into_owned(),
                                    ))
                                }
                            }
                        }
                    } else {
                        Ok(crate::common_response_builder()
                            .status(hyper::StatusCode::FOUND)
                            .header(hyper::header::LOCATION, &href)
                            .body(href.into())?)
                    }
                }
            }
        }
    }
}

async fn route_unstable_posts_like(
    params: (PostLocalID,),
    ctx: Arc<crate::RouteContext>,


@@ 793,6 873,11 @@ pub fn route_posts() -> crate::RouteNode<()> {
                .with_handler_async("GET", route_unstable_posts_get)
                .with_handler_async("DELETE", route_unstable_posts_delete)
                .with_child(
                    "href",
                    crate::RouteNode::new()
                        .with_handler_async("GET", route_unstable_posts_href_get),
                )
                .with_child(
                    "like",
                    crate::RouteNode::new().with_handler_async("POST", route_unstable_posts_like),
                )

M src/routes/api/users.rs => src/routes/api/users.rs +1 -1
@@ 260,7 260,7 @@ async fn route_unstable_users_following_posts_list(
        values.iter().map(|s| *s as _)
    ).await?;

    let posts = handle_common_posts_list(stream, &ctx.local_hostname).await?;
    let posts = handle_common_posts_list(stream, &ctx).await?;

    crate::json_response(&posts)
}