diff --git a/blog-server-api/Cargo.toml b/blog-server-api/Cargo.toml index d20ea68..8fe722c 100644 --- a/blog-server-api/Cargo.toml +++ b/blog-server-api/Cargo.toml @@ -43,9 +43,9 @@ tokio = { version = "1.27.0", features = ["full"] } hyper = { version = "0.14.26", features = ["full"] } serde = { version = "1.0.160", features = ["derive"] } serde_json = { version = "1.0.104" } -rbs = { version = "4.3" } -rbatis = { version = "4.3" } -rbdc-pg = { version = "4.3" } +rbs = { version = "4.5.2" } +rbatis = { version = "4.5.7" } +rbdc-pg = { version = "4.5.2" } async-trait = { version = "0.1.68" } password-hash = "0.5.0" argon2 = "0.5.0" @@ -57,5 +57,4 @@ validator = { version = "0.16.1" } reqwest = { version = "0.11.20", features = ["json"], optional = true } sha2 = { version = "0.10.0", optional = true } hmac = { version = "0.12.1", optional = true } -hex = { version = "0.4.3", optional = true } -ammonia = "3.3.0" \ No newline at end of file +hex = { version = "0.4.3", optional = true } \ No newline at end of file diff --git a/blog-server-api/src/endpoints/create_post/handler.rs b/blog-server-api/src/endpoints/create_post/handler.rs index 8b1f4ed..1aa091d 100644 --- a/blog-server-api/src/endpoints/create_post/handler.rs +++ b/blog-server-api/src/endpoints/create_post/handler.rs @@ -6,8 +6,6 @@ use super::response_content_failure::CreatePostContentFailure; use super::response_content_failure::CreatePostContentFailure::*; use super::response_content_success::CreatePostContentSuccess; -use crate::utils::html; - pub async fn http_handler( (CreatePostRequestContent { new_post_data, @@ -25,10 +23,9 @@ pub async fn http_handler( return Err(CreatingForbidden); } - let mut base_post = new_post_data.map_err(|e| ValidationError { + let base_post = new_post_data.map_err(|e| ValidationError { reason: e.to_string(), })?; - base_post.content = base_post.content.map(|c| html::clean(&c)); if let Some(err) = base_post.validate().err() { return Err(ValidationError { diff --git a/blog-server-api/src/endpoints/telegram_login/handler.rs b/blog-server-api/src/endpoints/telegram_login/handler.rs index 1a6eb60..902fc24 100644 --- a/blog-server-api/src/endpoints/telegram_login/handler.rs +++ b/blog-server-api/src/endpoints/telegram_login/handler.rs @@ -78,10 +78,7 @@ pub async fn http_handler( } let telegram_base_minimal_author = BaseMinimalAuthor { - slug: author_slug_utils::extend( - &username.unwrap_or(id.to_string()), - &"t".to_string(), - ), + slug: author_slug_utils::extend(&username.unwrap_or(id.to_string()), &"t".to_string()), first_name, last_name, image_url: photo_url.map(|u| { diff --git a/blog-server-api/src/endpoints/update_post/handler.rs b/blog-server-api/src/endpoints/update_post/handler.rs index bf7f64b..37a7a98 100644 --- a/blog-server-api/src/endpoints/update_post/handler.rs +++ b/blog-server-api/src/endpoints/update_post/handler.rs @@ -6,8 +6,6 @@ use super::response_content_failure::UpdatePostContentFailure; use super::response_content_failure::UpdatePostContentFailure::*; use super::response_content_success::UpdatePostContentSuccess; -use crate::utils::html; - pub async fn http_handler( (UpdatePostRequestContent { id, @@ -50,10 +48,9 @@ pub async fn http_handler( return Err(EditingForbidden); } - let mut base_post = updated_post_data.map_err(|e| ValidationError { + let base_post = updated_post_data.map_err(|e| ValidationError { reason: e.to_string(), })?; - base_post.content = base_post.content.map(|c| html::clean(&c)); if let Some(err) = base_post.validate().err() { return Err(ValidationError { diff --git a/blog-server-api/src/main.rs b/blog-server-api/src/main.rs index 11f3907..0791d9c 100644 --- a/blog-server-api/src/main.rs +++ b/blog-server-api/src/main.rs @@ -2,6 +2,7 @@ mod endpoints; mod extensions; +mod migrations; mod router; mod utils; @@ -33,19 +34,19 @@ async fn main() -> screw_components::dyn_result::DResult<()> { } pub async fn init_db() -> rbatis::RBatis { - let rb = rbatis::RBatis::new_with_opt(rbatis::RBatisOption::default()); - rb.init(rbdc_pg::driver::PgDriver {}, PG_URL).unwrap(); - - let sql = std::fs::read_to_string("./table_pg.sql").unwrap(); - rb.exec(&sql, vec![]).await.expect("DB migration failed"); + let rb = rbatis::RBatis::new(); + rb.init(rbdc_pg::driver::PgDriver {}, PG_URL) + .expect("DB init failed"); + migrations::exec(&rb).await.expect("DB migration failed"); return rb; } pub async fn init_rabbit( ) -> Box { - if RABBIT_URL.is_empty() { - blog_server_services::impls::create_rabbit_event_bus_service(None).await + blog_server_services::impls::create_rabbit_event_bus_service(if RABBIT_URL.is_empty() { + None } else { - blog_server_services::impls::create_rabbit_event_bus_service(Some(RABBIT_URL)).await - } + Some(RABBIT_URL) + }) + .await } diff --git a/blog-server-api/src/migrations/base.rs b/blog-server-api/src/migrations/base.rs new file mode 100644 index 0000000..0c6e390 --- /dev/null +++ b/blog-server-api/src/migrations/base.rs @@ -0,0 +1,5 @@ +pub async fn exec(rb: &rbatis::RBatis) -> Result<(), Box> { + let sql = std::fs::read_to_string("./table_pg.sql")?; + rb.exec(&sql, vec![]).await?; + Ok(()) +} diff --git a/blog-server-api/src/migrations/content_formatting.rs b/blog-server-api/src/migrations/content_formatting.rs new file mode 100644 index 0000000..4ddf16e --- /dev/null +++ b/blog-server-api/src/migrations/content_formatting.rs @@ -0,0 +1,45 @@ +const KEY: &'static str = "content_formatting"; + +pub async fn exec(rb: &rbatis::RBatis) -> Result<(), Box> { + let is_content_migrated: bool = rb + .query_decode::( + "select count(1) as count from migration where key=?", + vec![rbs::to_value!(KEY)], + ) + .await? + > 0; + + if !is_content_migrated { + let posts: Vec = + rb.query_decode("select * from post", vec![]).await?; + for post in posts { + let content = post + .base + .content + .as_ref() + .map(|c| blog_server_services::utils::html::clean(c)); + let plain_text_content = content + .as_ref() + .map(|c| blog_server_services::utils::html::to_plain(c)); + rb.query( + "update post set content=?, plain_text_content=? where id=?", + vec![ + rbs::to_value!(content), + rbs::to_value!(plain_text_content), + rbs::to_value!(post.id), + ], + ) + .await?; + } + rb.query( + "insert into migration (key, created_at) values (?, to_timestamp(?))", + vec![ + rbs::to_value!(KEY), + rbs::to_value!(blog_server_services::utils::time_utils::now_as_secs()), + ], + ) + .await?; + } + + Ok(()) +} diff --git a/blog-server-api/src/migrations/mod.rs b/blog-server-api/src/migrations/mod.rs new file mode 100644 index 0000000..88af5d1 --- /dev/null +++ b/blog-server-api/src/migrations/mod.rs @@ -0,0 +1,8 @@ +mod base; +mod content_formatting; + +pub async fn exec(rb: &rbatis::RBatis) -> Result<(), Box> { + base::exec(rb).await?; + content_formatting::exec(rb).await?; + Ok(()) +} diff --git a/blog-server-api/src/utils/mod.rs b/blog-server-api/src/utils/mod.rs index 1920dfc..6ec6460 100644 --- a/blog-server-api/src/utils/mod.rs +++ b/blog-server-api/src/utils/mod.rs @@ -1,4 +1,3 @@ pub mod auth; -pub mod html; pub mod jwt; pub mod password; diff --git a/blog-server-services/Cargo.toml b/blog-server-services/Cargo.toml index 63798f3..987df00 100644 --- a/blog-server-services/Cargo.toml +++ b/blog-server-services/Cargo.toml @@ -10,11 +10,13 @@ package = "screw-components" [dependencies] blog-generic = { path = "../blog-generic" } -rbs = { version = "4.3" } -rbatis = { version = "4.3" } +rbs = { version = "4.5.2" } +rbatis = { version = "4.5.7" } amqprs = { version = "1.5.1", features = ["urispec"] } async-trait = { version = "0.1.68" } serde = { version = "1.0.160", features = ["derive"] } serde_repr = { version = "0.1.12" } serde_json = { version = "1.0.96" } -translit = { version = "0.5.0" } \ No newline at end of file +translit = { version = "0.5.0" } +ammonia = "3.3.0" +html2text = "0.9.0" \ No newline at end of file diff --git a/blog-server-services/src/impls/rbatis_post_service.rs b/blog-server-services/src/impls/rbatis_post_service.rs index ffee3e6..71ab6aa 100644 --- a/blog-server-services/src/impls/rbatis_post_service.rs +++ b/blog-server-services/src/impls/rbatis_post_service.rs @@ -97,10 +97,16 @@ impl Post { } #[py_sql( " - SELECT COUNT(1) \ - FROM post \ - WHERE post.title ILIKE '%' || #{query} || '%' OR post.summary ILIKE '%' || #{query} || '%' OR post.content ILIKE '%' || #{query} || '%' \ - AND post.published = 1 \ + SELECT \ + COUNT(1) \ + FROM \ + post, \ + plainto_tsquery('russian', LOWER(#{query})) query, \ + to_tsvector('russian', LOWER(post.title || ' ' || post.summary || ' ' || post.plain_text_content)) textsearch \ + WHERE \ + textsearch @@ query \ + AND \ + post.published = 1 \ " )] async fn count_by_query(rb: &RBatis, query: &String) -> rbatis::Result { @@ -201,13 +207,23 @@ impl Post { #[py_sql( " SELECT \ - post.* \ - FROM post \ - WHERE post.title ILIKE '%' || #{query} || '%' OR post.summary ILIKE '%' || #{query} || '%' OR post.content ILIKE '%' || #{query} || '%' \ - AND post.published = 1 \ - ORDER BY post.id DESC \ - LIMIT #{limit} \ - OFFSET #{offset} \ + post.*, \ + ts_rank_cd(textsearch, query) AS rank \ + FROM \ + post, \ + plainto_tsquery('russian', LOWER(#{query})) query, \ + to_tsvector('russian', LOWER(post.title || ' ' || post.summary || ' ' || post.plain_text_content)) textsearch \ + WHERE \ + textsearch @@ query \ + AND \ + post.published = 1 \ + ORDER BY \ + rank \ + DESC \ + LIMIT \ + #{limit} \ + OFFSET \ + #{offset} \ " )] async fn select_by_query_with_limit_and_offset( @@ -312,9 +328,9 @@ impl RbatisPostService { #[py_sql( " INSERT INTO post - (author_id,title,slug,summary,published,created_at,content,image_url) + (author_id,title,slug,summary,published,created_at,content,plain_text_content,image_url) VALUES - (#{post.author_id},#{post.title},#{post.slug},#{post.summary},#{post.published},to_timestamp(#{post.created_at}),#{post.content},#{post.image_url}) + (#{post.author_id},#{post.title},#{post.slug},#{post.summary},#{post.published},to_timestamp(#{post.created_at}),#{post.content},#{post.plain_text_content},#{post.image_url}) RETURNING id " )] @@ -331,6 +347,7 @@ impl RbatisPostService { summary = #{post_data.summary}, \ published = #{post_data.published}, \ content = #{post_data.content}, \ + plain_text_content = #{post_data.plain_text_content}, \ image_url = #{post_data.image_url} \ WHERE id = #{post_id} \ RETURNING id diff --git a/blog-server-services/src/traits/post_service.rs b/blog-server-services/src/traits/post_service.rs index ed484d1..04ceac5 100644 --- a/blog-server-services/src/traits/post_service.rs +++ b/blog-server-services/src/traits/post_service.rs @@ -33,27 +33,32 @@ pub struct BasePost { pub published: u8, pub created_at: u64, pub content: Option, + pub plain_text_content: Option, pub image_url: Option, } impl From<(u64, ECommonPost)> for BasePost { - fn from(value: (u64, ECommonPost)) -> Self { + fn from((author_id, post): (u64, ECommonPost)) -> Self { + let slug = { + let transliterated = transliteration::ru_to_latin_single( + post.title.clone(), + transliteration::TranslitOption::ToLowerCase, + ) + .transliterated; + string_filter::remove_non_latin_or_number_chars(&transliterated) + }; + let content = post.content.as_ref().map(|c| html::clean(c)); + let plain_text_content = content.as_ref().map(|c| html::to_plain(c)); BasePost { - author_id: value.0, + author_id, created_at: time_utils::now_as_secs(), - slug: { - let transliterated = transliteration::ru_to_latin_single( - value.1.title.clone(), - transliteration::TranslitOption::ToLowerCase, - ) - .transliterated; - string_filter::remove_non_latin_or_number_chars(&transliterated) - }, - title: value.1.title, - summary: value.1.summary, - published: value.1.published, - content: value.1.content, - image_url: value.1.image_url, + slug, + title: post.title, + summary: post.summary, + published: post.published, + content, + plain_text_content, + image_url: post.image_url, } } } diff --git a/blog-server-api/src/utils/html.rs b/blog-server-services/src/utils/html.rs similarity index 90% rename from blog-server-api/src/utils/html.rs rename to blog-server-services/src/utils/html.rs index 4d60663..663eba3 100644 --- a/blog-server-api/src/utils/html.rs +++ b/blog-server-services/src/utils/html.rs @@ -25,3 +25,7 @@ pub fn clean(src: &str) -> String { .clean(src) .to_string() } + +pub fn to_plain(src: &str) -> String { + html2text::from_read(src.as_bytes(), usize::MAX) +} diff --git a/blog-server-services/src/utils/mod.rs b/blog-server-services/src/utils/mod.rs index d0232fd..9a2d754 100644 --- a/blog-server-services/src/utils/mod.rs +++ b/blog-server-services/src/utils/mod.rs @@ -1,3 +1,4 @@ +pub mod html; pub mod string_filter; pub mod time_utils; pub mod transliteration; diff --git a/table_pg.sql b/table_pg.sql index b71bb53..dbf615f 100644 --- a/table_pg.sql +++ b/table_pg.sql @@ -215,3 +215,23 @@ DO $$ BEGIN END $$ ; + +DO $$ BEGIN + IF NOT EXISTS(select * from information_schema.columns where table_name = 'post' and column_name = 'plain_text_content') THEN + ALTER TABLE post ADD COLUMN plain_text_content TEXT; + END IF; +END $$ + +; + +DO $$ BEGIN + IF NOT EXISTS(select * from pg_tables where schemaname = 'public' and tablename = 'migration') THEN + CREATE TABLE migration ( + key VARCHAR(100) NOT NULL, + created_at TIMESTAMP(0) NOT NULL, + PRIMARY KEY (key) + ); + END IF; +END $$ + +; \ No newline at end of file