~vpzom/lotide

41930ed2f2ca2b4f597d7532d35cc16366decb91 — Colin Reeder 10 days ago 8d9100a
Use pagination for posts list
M src/main.rs => src/main.rs +15 -5
@@ 154,17 154,22 @@ pub struct BaseContext {
}

impl BaseContext {
    pub fn process_href<'a>(&self, href: &'a str, post_id: PostLocalID) -> Cow<'a, str> {
    pub fn process_href<'a>(
        &self,
        href: impl Into<Cow<'a, str>>,
        post_id: PostLocalID,
    ) -> Cow<'a, str> {
        let href = href.into();
        if href.starts_with("local-media://") {
            format!("{}/stable/posts/{}/href", self.host_url_api, post_id).into()
        } else {
            href.into()
            href
        }
    }

    pub fn process_href_opt<'a>(
        &self,
        href: Option<&'a str>,
        href: Option<Cow<'a, str>>,
        post_id: PostLocalID,
    ) -> Option<Cow<'a, str>> {
        match href {


@@ 192,11 197,16 @@ impl BaseContext {
        }
    }

    pub fn process_avatar_href<'a>(&self, href: &'a str, user_id: UserLocalID) -> Cow<'a, str> {
    pub fn process_avatar_href<'a>(
        &self,
        href: impl Into<Cow<'a, str>>,
        user_id: UserLocalID,
    ) -> Cow<'a, str> {
        let href = href.into();
        if href.starts_with("local-media://") {
            format!("{}/stable/users/{}/avatar/href", self.host_url_api, user_id,).into()
        } else {
            href.into()
            href
        }
    }


M src/routes/api/communities.rs => src/routes/api/communities.rs +12 -11
@@ 155,7 155,7 @@ async fn route_unstable_communities_list(
            .iter()
            .map(|row| {
                let id = CommunityLocalID(row.get(0));
                let name = row.get(1);
                let name: &str = row.get(1);
                let local = row.get(2);
                let ap_id = row.get(3);



@@ 167,10 167,10 @@ async fn route_unstable_communities_list(
                RespCommunityInfo {
                    base: RespMinimalCommunityInfo {
                        id,
                        name,
                        name: Cow::Borrowed(name),
                        local,
                        host,
                        remote_url: ap_id,
                        remote_url: ap_id.map(Cow::Borrowed),
                    },

                    description,


@@ 325,7 325,7 @@ async fn route_unstable_communities_get(
    let info = RespCommunityInfo {
        base: RespMinimalCommunityInfo {
            id: community_id,
            name: row.get(0),
            name: Cow::Borrowed(row.get(0)),
            local: community_local,
            host: if community_local {
                (&ctx.local_hostname).into()


@@ 335,7 335,7 @@ async fn route_unstable_communities_get(
                    None => "[unknown]".into(),
                }
            },
            remote_url: community_ap_id,
            remote_url: community_ap_id.map(Cow::Borrowed),
        },
        description,
        description_html,


@@ 783,7 783,7 @@ async fn route_unstable_communities_posts_list(

        RespMinimalCommunityInfo {
            id: community_id,
            name: row.get(0),
            name: Cow::Borrowed(row.get(0)),
            local: community_local,
            host: if community_local {
                (&ctx.local_hostname).into()


@@ 793,7 793,7 @@ async fn route_unstable_communities_posts_list(
                    None => "[unknown]".into(),
                }
            },
            remote_url: community_ap_id,
            remote_url: community_ap_id.map(Cow::Borrowed),
        }
    };



@@ 855,13 855,14 @@ async fn route_unstable_communities_posts_list(

            let post = RespPostListPost {
                id,
                title,
                href: ctx.process_href_opt(href, id),
                content_text,
                title: Cow::Borrowed(title),
                href: ctx.process_href_opt(href.map(Cow::Borrowed), id),
                content_text: content_text.map(Cow::Borrowed),
                content_html_safe: content_html.map(|html| crate::clean_html(&html)),
                author: author.as_ref(),
                author: author.map(Cow::Owned),
                created: Cow::Owned(created.to_rfc3339()),
                community: Cow::Borrowed(&community),
                relevance: None,
                replies_count_total: Some(row.get(12)),
                score: row.get(11),
                sticky: row.get(13),

M src/routes/api/mod.rs => src/routes/api/mod.rs +68 -27
@@ 96,6 96,7 @@ impl SortType {
    pub fn handle_page(
        &self,
        page: Option<&str>,
        table: &str,
        mut value_out: ValueConsumer,
    ) -> Result<(Option<String>, Option<String>), InvalidPage> {
        match page {


@@ 131,8 132,8 @@ impl SortType {

                    Ok((
                        Some(format!(
                            " AND created < ${0} OR (created = ${0} AND id <= ${1})",
                            idx1, idx2
                            " AND ({2}.created < ${0} OR ({2}.created = ${0} AND {2}.id <= ${1}))",
                            idx1, idx2, table,
                        )),
                        None,
                    ))


@@ 158,7 159,37 @@ impl SortType {
            SortType::New => {
                let ts: chrono::DateTime<chrono::offset::FixedOffset> =
                    comment.created.parse().unwrap();
                format!("{},{}", ts, comment.base.id)
                format!("{},{}", ts.timestamp_nanos(), comment.base.id)
            }
        }
    }

    fn get_next_posts_page(
        &self,
        post: &RespPostListPost<'_>,
        sort_sticky: bool,
        limit: u8,
        current_page: Option<&str>,
    ) -> String {
        match self {
            SortType::Hot => format_number_58(
                i64::from(limit)
                    + match current_page {
                        None => 0,
                        Some(current_page) => parse_number_58(current_page).unwrap(),
                    },
            ),
            SortType::New => {
                let ts: chrono::DateTime<chrono::offset::FixedOffset> =
                    post.created.parse().unwrap();

                let ts = ts.timestamp_nanos();

                if sort_sticky {
                    format!("{},{},{}", post.sticky, ts, post.id)
                } else {
                    format!("{},{}", ts, post.id)
                }
            }
        }
    }


@@ 896,39 927,42 @@ async fn handle_common_posts_list(
        + Send,
    ctx: &crate::RouteContext,
    include_your: bool,
) -> Result<Vec<serde_json::Value>, crate::Error> {
    relevance: bool,
) -> Result<Vec<RespPostListPost<'static>>, crate::Error> {
    use futures::stream::TryStreamExt;

    let posts: Vec<serde_json::Value> = stream
    let posts: Vec<_> = stream
        .map_err(crate::Error::from)
        .and_then(|row| {
        .map_ok(|row| {
            let id = PostLocalID(row.get(0));
            let author_id = row.get::<_, Option<_>>(1).map(UserLocalID);
            let href: Option<&str> = row.get(2);
            let content_text: Option<&str> = row.get(3);
            let content_html: Option<&str> = row.get(6);
            let title: &str = row.get(4);
            let href: Option<String> = row.get(2);
            let content_text: Option<String> = row.get(3);
            let content_html: Option<String> = row.get(6);
            let title: String = row.get(4);
            let created: chrono::DateTime<chrono::FixedOffset> = row.get(5);
            let community_id = CommunityLocalID(row.get(7));
            let community_name: &str = row.get(8);
            let community_name: String = row.get(8);
            let community_local: bool = row.get(9);
            let community_ap_id: Option<&str> = row.get(10);
            let community_ap_id: Option<String> = row.get(10);

            let author = author_id.map(|id| {
                let author_name: &str = row.get(11);
                let author_name: String = row.get(11);
                let author_local: bool = row.get(12);
                let author_ap_id: Option<&str> = row.get(13);
                let author_avatar: Option<&str> = row.get(14);
                let author_ap_id: Option<String> = row.get(13);
                let author_avatar: Option<String> = row.get(14);
                RespMinimalAuthorInfo {
                    id,
                    username: author_name.into(),
                    local: author_local,
                    host: crate::get_actor_host_or_unknown(
                        author_local,
                        author_ap_id,
                        author_ap_id.as_deref(),
                        &ctx.local_hostname,
                    ),
                    remote_url: author_ap_id.map(|x| x.to_owned().into()),
                    )
                    .into_owned()
                    .into(),
                    remote_url: author_ap_id.map(Cow::Owned),
                    avatar: author_avatar.map(|url| RespAvatarInfo {
                        url: ctx.process_avatar_href(url, id).into_owned().into(),
                    }),


@@ 937,27 971,34 @@ async fn handle_common_posts_list(

            let community = RespMinimalCommunityInfo {
                id: community_id,
                name: community_name,
                name: Cow::Owned(community_name),
                local: community_local,
                host: crate::get_actor_host_or_unknown(
                    community_local,
                    community_ap_id,
                    community_ap_id.as_deref(),
                    &ctx.local_hostname,
                ),
                remote_url: community_ap_id,
                )
                .into_owned()
                .into(),
                remote_url: community_ap_id.map(Cow::Owned),
            };

            let post = RespPostListPost {
                id,
                title,
                href: ctx.process_href_opt(href, id),
                content_text,
                title: Cow::Owned(title),
                href: ctx.process_href_opt(href.map(Cow::Owned), id),
                content_text: content_text.map(Cow::Owned),
                content_html_safe: content_html.map(|html| crate::clean_html(&html)),
                author: author.as_ref(),
                author: author.map(Cow::Owned),
                created: Cow::Owned(created.to_rfc3339()),
                community: Cow::Owned(community),
                score: row.get(15),
                sticky: row.get(17),
                relevance: if relevance {
                    row.get(if include_your { 19 } else { 18 })
                } else {
                    None
                },
                replies_count_total: Some(row.get(16)),
                your_vote: if include_your {
                    Some(if row.get(18) {


@@ 970,7 1011,7 @@ async fn handle_common_posts_list(
                },
            };

            futures::future::ready(serde_json::to_value(&post).map_err(Into::into))
            post
        })
        .try_collect()
        .await?;

M src/routes/api/posts.rs => src/routes/api/posts.rs +149 -24
@@ 1,5 1,5 @@
use super::{
    JustURL, RespAvatarInfo, RespList, RespMinimalAuthorInfo, RespMinimalCommentInfo,
    InvalidPage, JustURL, RespAvatarInfo, RespList, RespMinimalAuthorInfo, RespMinimalCommentInfo,
    RespMinimalCommunityInfo, RespPostCommentInfo, RespPostListPost, ValueConsumer,
};
use crate::types::{


@@ 43,6 43,7 @@ async fn get_post_comments<'a>(
    let (page_part1, page_part2) = sort
        .handle_page(
            page,
            "reply",
            ValueConsumer {
                targets: vec![&mut con1, &mut con2],
                start_idx: values.len(),


@@ 167,12 168,63 @@ async fn route_unstable_posts_list(
        Extra(PostsListExtraSortType),
    }

    impl PostsListSortType {
        fn get_next_posts_page(
            &self,
            post: &RespPostListPost<'_>,
            sort_sticky: bool,
            limit: u8,
            current_page: Option<&str>,
        ) -> String {
            match self {
                Self::Normal(inner) => {
                    inner.get_next_posts_page(post, sort_sticky, limit, current_page)
                }
                Self::Extra(PostsListExtraSortType::Relevant) => super::format_number_58(
                    i64::from(limit)
                        + match current_page {
                            None => 0,
                            Some(current_page) => super::parse_number_58(current_page).unwrap(),
                        },
                ),
            }
        }

        pub fn handle_page(
            &self,
            page: Option<&str>,
            mut value_out: ValueConsumer,
        ) -> Result<(Option<String>, Option<String>), InvalidPage> {
            match self {
                Self::Extra(sort) => {
                    match page {
                        None => Ok((None, None)),
                        Some(page) => match sort {
                            PostsListExtraSortType::Relevant => {
                                // TODO maybe
                                let page: i64 =
                                    super::parse_number_58(page).map_err(|_| InvalidPage)?;
                                let idx = value_out.push(page);
                                Ok((None, Some(format!(" OFFSET ${}", idx))))
                            }
                        },
                    }
                }
                Self::Normal(sort) => sort.handle_page(page, "post", value_out),
            }
        }
    }

    impl Default for PostsListSortType {
        fn default() -> Self {
            Self::Normal(super::SortType::Hot)
        }
    }

    fn default_limit() -> u8 {
        30
    }

    #[derive(Deserialize)]
    struct PostsListQuery<'a> {
        in_any_local_community: Option<bool>,


@@ 181,6 233,11 @@ async fn route_unstable_posts_list(
        use_aggregate_filters: bool,
        community: Option<CommunityLocalID>,

        #[serde(default = "default_limit")]
        limit: u8,

        page: Option<Cow<'a, str>>,

        #[serde(default)]
        include_your: bool,



@@ 203,11 260,16 @@ async fn route_unstable_posts_list(
        None
    };

    let mut search_value_idx = None;
    let limit_plus_1: i64 = (query.limit + 1).into();

    let limit: i64 = 30;
    let mut values: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = vec![&limit_plus_1];

    let mut values: Vec<&(dyn tokio_postgres::types::ToSql + Sync)> = vec![&limit];
    let search_value_idx = if let Some(search) = &query.search {
        values.push(search);
        Some(values.len())
    } else {
        None
    };

    let include_your_idx = if let Some(user) = &include_your_for {
        values.push(user);


@@ 221,14 283,23 @@ async fn route_unstable_posts_list(
        super::common_posts_list_query(include_your_idx)
    );

    let relevance_sql = if let Some(search_value_idx) = search_value_idx {
        Some(format!("ts_rank_cd(to_tsvector('english', title || ' ' || COALESCE(content_text, content_markdown, content_html, '')), plainto_tsquery('english', ${}))", search_value_idx))
    } else {
        None
    };

    if let Some(relevance_sql) = &relevance_sql {
        sql.push_str(", ");
        sql.push_str(relevance_sql);
    }

    sql.push_str( " FROM community, post LEFT OUTER JOIN person ON (person.id = post.author) WHERE post.community = community.id AND deleted=FALSE");
    if query.use_aggregate_filters {
        sql.push_str(" AND community.hide_posts_from_aggregates=FALSE");
    }
    if let Some(search) = &query.search {
        values.push(search);
        search_value_idx = Some(values.len());
        write!(sql, " AND to_tsvector('english', title || ' ' || COALESCE(content_text, content_markdown, content_html, '')) @@ plainto_tsquery('english', ${})", values.len()).unwrap();
    if let Some(search_value_idx) = &search_value_idx {
        write!(sql, " AND to_tsvector('english', title || ' ' || COALESCE(content_text, content_markdown, content_html, '')) @@ plainto_tsquery('english', ${})", search_value_idx).unwrap();
    }
    if let Some(value) = query.in_any_local_community {
        write!(


@@ 242,15 313,40 @@ async fn route_unstable_posts_list(
        values.push(value);
        write!(sql, " AND community.id=${} AND post.approved", values.len(),).unwrap();
    }

    let mut con1 = None;
    let mut con2 = None;
    let (page_part1, page_part2) = query
        .sort
        .handle_page(
            query.page.as_deref(),
            ValueConsumer {
                targets: vec![&mut con1, &mut con2],
                start_idx: values.len(),
                used: 0,
            },
        )
        .map_err(super::InvalidPage::into_user_error)?;
    if let Some(value) = &con1 {
        values.push(value.as_ref());
        if let Some(value) = &con2 {
            values.push(value.as_ref());
        }
    }

    if let Some(part) = page_part1 {
        sql.push_str(&part);
    }

    sql.push_str(" ORDER BY ");
    if query.sort_sticky {
        sql.push_str("sticky DESC, ");
    }
    match query.sort {
    match &query.sort {
        PostsListSortType::Normal(ty) => sql.push_str(ty.post_sort_sql()),
        PostsListSortType::Extra(PostsListExtraSortType::Relevant) => {
            if let Some(search_value_idx) = search_value_idx {
                write!(sql, "ts_rank_cd(to_tsvector('english', title || ' ' || COALESCE(content_text, content_markdown, content_html, '')), plainto_tsquery('english', ${})) DESC", search_value_idx).unwrap();
            if let Some(relevance_sql) = relevance_sql {
                write!(sql, "{} DESC, post.id DESC", relevance_sql).unwrap();
            } else {
                return Err(crate::Error::UserError(crate::simple_response(
                    hyper::StatusCode::BAD_REQUEST,


@@ 261,13 357,41 @@ async fn route_unstable_posts_list(
    }
    sql.push_str(" LIMIT $1");

    if let Some(part) = page_part2 {
        sql.push_str(&part);
    }

    let sql: &str = &sql;

    let stream = crate::query_stream(&db, sql, &values).await?;

    let posts = super::handle_common_posts_list(stream, &ctx, include_your_for.is_some()).await?;
    let posts = super::handle_common_posts_list(
        stream,
        &ctx,
        include_your_for.is_some(),
        search_value_idx.is_some(),
    )
    .await?;
    let output = if posts.len() > query.limit as usize {
        let last_post = &posts[posts.len() - 1];

        RespList {
            next_page: Some(Cow::Owned(query.sort.get_next_posts_page(
                last_post,
                query.sort_sticky,
                query.limit,
                query.page.as_deref(),
            ))),
            items: Cow::Borrowed(&posts[0..(posts.len() - 1)]),
        }
    } else {
        RespList {
            items: posts.into(),
            next_page: None,
        }
    };

    crate::json_response(&posts)
    crate::json_response(&output)
}

async fn route_unstable_posts_replies_list(


@@ 483,15 607,15 @@ async fn route_unstable_posts_get(
            lang.tr("no_such_post", None).into_owned(),
        )),
        Some(row) => {
            let href = row.get(1);
            let content_text = row.get(2);
            let href: Option<&str> = row.get(1);
            let content_text: Option<&str> = row.get(2);
            let content_html: Option<&str> = row.get(5);
            let title = row.get(3);
            let title: &str = row.get(3);
            let created: chrono::DateTime<chrono::FixedOffset> = row.get(4);
            let community_id = CommunityLocalID(row.get(6));
            let community_name = row.get(7);
            let community_name: &str = row.get(7);
            let community_local = row.get(8);
            let community_ap_id = row.get(9);
            let community_ap_id: Option<&str> = row.get(9);

            let author = match row.get(10) {
                Some(author_username) => {


@@ 519,25 643,26 @@ async fn route_unstable_posts_get(

            let community = RespMinimalCommunityInfo {
                id: community_id,
                name: community_name,
                name: Cow::Borrowed(community_name),
                local: community_local,
                host: crate::get_actor_host_or_unknown(
                    community_local,
                    community_ap_id,
                    &ctx.local_hostname,
                ),
                remote_url: community_ap_id,
                remote_url: community_ap_id.map(Cow::Borrowed),
            };

            let post = RespPostListPost {
                id: post_id,
                title,
                href: ctx.process_href_opt(href, post_id),
                content_text,
                title: Cow::Borrowed(title),
                href: ctx.process_href_opt(href.map(Cow::Borrowed), post_id),
                content_text: content_text.map(Cow::Borrowed),
                content_html_safe: content_html.map(|html| crate::clean_html(&html)),
                author: author.as_ref(),
                author: author.map(Cow::Owned),
                created: Cow::Owned(created.to_rfc3339()),
                community: Cow::Owned(community),
                relevance: None,
                replies_count_total: None,
                score: row.get(13),
                sticky: row.get(17),

M src/routes/api/stable.rs => src/routes/api/stable.rs +1 -1
@@ 155,7 155,7 @@ async fn route_stable_communities_feed_get(
            let title: String = row.get(4);

            let href_raw: Option<&str> = row.get(2);
            let href = ctx.process_href_opt(href_raw, post_id);
            let href = ctx.process_href_opt(href_raw.map(Cow::Borrowed), post_id);

            let post_ap_id = if row.get(8) {
                crate::apub_util::get_local_post_apub_id(post_id, &ctx.host_url_apub).to_string()

M src/routes/api/users.rs => src/routes/api/users.rs +9 -5
@@ 347,7 347,7 @@ async fn route_unstable_users_following_posts_list(
        values,
    ).await?;

    let posts = super::handle_common_posts_list(stream, &ctx, true).await?;
    let posts = super::handle_common_posts_list(stream, &ctx, true, false).await?;

    crate::json_response(&posts)
}


@@ 604,20 604,24 @@ async fn route_unstable_users_things_list(

                RespThingInfo::Post(RespPostListPost {
                    id: post_id,
                    href: ctx.process_href_opt(row.get(2), post_id),
                    title: row.get(3),
                    href: ctx.process_href_opt(
                        row.get::<_, Option<&str>>(2).map(Cow::Borrowed),
                        post_id,
                    ),
                    title: Cow::Borrowed(row.get(3)),
                    created: Cow::Owned(created),
                    community: Cow::Owned(RespMinimalCommunityInfo {
                        id: CommunityLocalID(row.get(5)),
                        name: row.get(6),
                        name: Cow::Borrowed(row.get(6)),
                        local: community_local,
                        host: crate::get_actor_host_or_unknown(
                            community_local,
                            community_ap_id,
                            &ctx.local_hostname,
                        ),
                        remote_url: community_ap_id,
                        remote_url: community_ap_id.map(Cow::Borrowed),
                    }),
                    relevance: None,
                    replies_count_total: row.get(10),
                    sticky: row.get(11),
                    score: row.get(9),

M types/src/lib.rs => types/src/lib.rs +8 -6
@@ 175,10 175,10 @@ pub struct JustUser<'a> {
#[derive(Serialize, Clone)]
pub struct RespMinimalCommunityInfo<'a> {
    pub id: CommunityLocalID,
    pub name: &'a str,
    pub name: Cow<'a, str>,
    pub local: bool,
    pub host: Cow<'a, str>,
    pub remote_url: Option<&'a str>,
    pub remote_url: Option<Cow<'a, str>>,
}

#[derive(Serialize)]


@@ 187,19 187,21 @@ pub struct RespMinimalPostInfo<'a> {
    pub title: &'a str,
}

#[derive(Serialize)]
#[derive(Serialize, Clone)]
pub struct RespPostListPost<'a> {
    pub id: PostLocalID,
    pub title: &'a str,
    pub title: Cow<'a, str>,
    pub href: Option<Cow<'a, str>>,
    pub content_text: Option<&'a str>,
    pub content_text: Option<Cow<'a, str>>,
    #[serde(rename = "content_html")]
    pub content_html_safe: Option<String>,
    pub author: Option<&'a RespMinimalAuthorInfo<'a>>,
    pub author: Option<Cow<'a, RespMinimalAuthorInfo<'a>>>,
    pub created: Cow<'a, str>,
    pub community: Cow<'a, RespMinimalCommunityInfo<'a>>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub replies_count_total: Option<i64>,
    #[serde(skip_serializing_if = "Option::is_none")]
    pub relevance: Option<f32>,
    pub score: i64,
    pub sticky: bool,
    #[serde(skip_serializing_if = "Option::is_none")]