From f8f68932c6ca0f5350e738f95c4a64ad96016f84 Mon Sep 17 00:00:00 2001 From: Beast Date: Mon, 22 Dec 2025 19:17:32 +0800 Subject: [PATCH 1/4] feat: make whitelist configgurable - add flag ignore to tweet author - remove whitelist from config - add endpoints for managing whitelist --- config/default.toml | 1 - config/example.toml | 1 - config/test.toml | 1 - ...d_is_ignore_col_to_tweet_authors_table.sql | 5 + src/config.rs | 2 - src/handlers/tweet_author.rs | 184 +++++++++++++++++- src/models/tweet_author.rs | 7 + src/repositories/tweet_author.rs | 64 ++++++ src/routes/tweet_author.rs | 19 +- src/services/tweet_synchronizer_service.rs | 7 +- 10 files changed, 273 insertions(+), 18 deletions(-) create mode 100644 migrations/008_add_is_ignore_col_to_tweet_authors_table.sql diff --git a/config/default.toml b/config/default.toml index 9438f70..6e33593 100644 --- a/config/default.toml +++ b/config/default.toml @@ -58,7 +58,6 @@ client_secret = "lfXc45dZLqYTzP62Ms32EhXinGQzxcIP9TvjJml2B-h0T1nIJK" api_key = "some-key" interval_in_hours = 24 keywords = "quantum" -whitelist = ["username"] [tg_bot] base_url = "https://api.telegram.org" diff --git a/config/example.toml b/config/example.toml index 2c8f04e..916a4c8 100644 --- a/config/example.toml +++ b/config/example.toml @@ -68,7 +68,6 @@ client_secret = "example-secret" api_key = "some-key" interval_in_hours = 24 keywords = "example" -whitelist = ["example-username"] [tg_bot] base_url = "https://api.telegram.org" diff --git a/config/test.toml b/config/test.toml index ac1fb8d..3a040cd 100644 --- a/config/test.toml +++ b/config/test.toml @@ -58,7 +58,6 @@ client_secret = "test-secret" api_key = "some-key" interval_in_hours = 24 keywords = "test" -whitelist = ["test-username"] [tg_bot] base_url = "https://api.telegram.org" diff --git a/migrations/008_add_is_ignore_col_to_tweet_authors_table.sql b/migrations/008_add_is_ignore_col_to_tweet_authors_table.sql new file mode 100644 index 0000000..63032cf --- /dev/null +++ b/migrations/008_add_is_ignore_col_to_tweet_authors_table.sql @@ -0,0 +1,5 @@ +ALTER TABLE tweet_authors +ADD COLUMN is_ignored BOOLEAN NOT NULL DEFAULT false; + +CREATE INDEX IF NOT EXISTS idx_tweet_authors_is_ignored ON tweet_authors(is_ignored) +WHERE is_ignored = false; \ No newline at end of file diff --git a/src/config.rs b/src/config.rs index 57e0b03..4539884 100644 --- a/src/config.rs +++ b/src/config.rs @@ -71,7 +71,6 @@ pub struct JwtConfig { #[derive(Debug, Clone, Serialize, Deserialize)] pub struct TweetSyncConfig { - pub whitelist: Vec, pub interval_in_hours: u64, pub keywords: String, pub api_key: String, @@ -207,7 +206,6 @@ impl Default for Config { client_secret: "example".to_string(), }, tweet_sync: TweetSyncConfig { - whitelist: vec![], interval_in_hours: 24, keywords: "hello".to_string(), api_key: "key".to_string(), diff --git a/src/handlers/tweet_author.rs b/src/handlers/tweet_author.rs index d8b9e48..c788d0f 100644 --- a/src/handlers/tweet_author.rs +++ b/src/handlers/tweet_author.rs @@ -1,15 +1,19 @@ use axum::{ - extract::{self, Query, State}, + extract::{self, Path, Query, State}, + http::StatusCode, + response::NoContent, Extension, Json, }; use crate::{ db_persistence::DbError, - handlers::{calculate_total_pages, ListQueryParams, PaginatedResponse, PaginationMetadata, SuccessResponse}, + handlers::{ + calculate_total_pages, HandlerError, ListQueryParams, PaginatedResponse, PaginationMetadata, SuccessResponse, + }, http_server::AppState, models::{ admin::Admin, - tweet_author::{AuthorFilter, AuthorSortColumn, TweetAuthor}, + tweet_author::{AuthorFilter, AuthorSortColumn, CreateTweetAuthorInput, NewAuthorPayload, TweetAuthor}, }, AppError, }; @@ -40,6 +44,48 @@ pub async fn handle_get_tweet_authors( Ok(Json(response)) } +/// POST /tweet-authors +pub async fn handle_create_tweet_author( + State(state): State, + Extension(_): Extension, + Json(payload): Json, +) -> Result<(StatusCode, Json>), AppError> { + let author_response = state.twitter_gateway.users().get_by_username(&payload.username).await?; + let Some(author) = author_response.data else { + return Err(AppError::Handler(HandlerError::InvalidBody(format!( + "Tweet Author {} not found", + payload.username + )))); + }; + + let new_author = NewAuthorPayload::new(author); + let create_response = state.db.tweet_authors.upsert(&new_author).await?; + + Ok((StatusCode::CREATED, SuccessResponse::new(create_response))) +} + +/// PUT /tweet-authors/:id/ignore +pub async fn handle_ignore_tweet_author( + State(state): State, + Extension(_): Extension, + Path(id): Path, +) -> Result { + state.db.tweet_authors.make_ignored_from_whitelist(&id).await?; + + Ok(NoContent) +} + +/// PUT /tweet-authors/:id/watch +pub async fn handle_watch_tweet_author( + State(state): State, + Extension(_): Extension, + Path(id): Path, +) -> Result { + state.db.tweet_authors.make_watched_in_whitelist(&id).await?; + + Ok(NoContent) +} + /// GET /tweet-authors/:id /// Gets a single author by their X ID pub async fn handle_get_tweet_author_by_id( @@ -60,12 +106,30 @@ pub async fn handle_get_tweet_author_by_id( #[cfg(test)] mod tests { - use axum::{body::Body, extract::Request, http::StatusCode, routing::get, Extension, Router}; + use std::sync::Arc; + + use axum::{ + body::Body, + extract::Request, + http::StatusCode, + routing::{get, post, put}, + Extension, Router, + }; + use rusx::{ + resources::{ + user::{User, UserApi, UserPublicMetrics}, + TwitterApiResponse, + }, + MockTwitterGateway, MockUserApi, + }; use serde_json::Value; use tower::ServiceExt; use crate::{ - handlers::tweet_author::{handle_get_tweet_author_by_id, handle_get_tweet_authors}, + handlers::tweet_author::{ + handle_create_tweet_author, handle_get_tweet_author_by_id, handle_get_tweet_authors, + handle_ignore_tweet_author, handle_watch_tweet_author, + }, models::tweet_author::NewAuthorPayload, utils::{ test_app_state::create_test_app_state, @@ -263,7 +327,6 @@ mod tests { async fn test_get_tweet_author_by_id_not_found() { let state = create_test_app_state().await; reset_database(&state.db.pool).await; - // No authors seeded let router = Router::new() .route("/tweet-authors/:id", get(handle_get_tweet_author_by_id)) @@ -282,4 +345,113 @@ mod tests { assert_eq!(response.status(), 404); } + + #[tokio::test] + async fn test_create_tweet_author_success() { + let mut state = create_test_app_state().await; + reset_database(&state.db.pool).await; + + // --- Setup Twitter Mock --- + let mut mock_gateway = MockTwitterGateway::new(); + let mut mock_user = MockUserApi::new(); + + mock_user.expect_get_by_username().returning(|_| { + Ok(TwitterApiResponse { + data: Some(User { + id: "hello".to_string(), + name: "hello".to_string(), + username: "test_user".to_string(), + public_metrics: Some(UserPublicMetrics { + followers_count: 100, + following_count: 50, + tweet_count: 10, + listed_count: 5, + like_count: Some(0), + media_count: Some(0), + }), + }), + includes: None, + meta: None, + }) + }); + + let user_api_arc: Arc = Arc::new(mock_user); + + mock_gateway.expect_users().times(1).return_const(user_api_arc); + + state.twitter_gateway = Arc::new(mock_gateway); + + let router = Router::new() + .route("/tweet-authors", post(handle_create_tweet_author)) + .layer(Extension(create_mock_admin())) + .with_state(state); + + let payload = serde_json::json!({ + "username": "test_user" + }); + + let response = router + .oneshot( + Request::builder() + .method("POST") + .uri("/tweet-authors") + .header("Content-Type", "application/json") + .body(Body::from(serde_json::to_vec(&payload).unwrap())) + .unwrap(), + ) + .await + .unwrap(); + + assert!(response.status() == StatusCode::CREATED); + } + + #[tokio::test] + async fn test_ignore_and_watch_tweet_author() { + let state = create_test_app_state().await; + reset_database(&state.db.pool).await; + seed_authors(&state).await; + + let router = Router::new() + .route("/tweet-authors/:id/ignore", put(handle_ignore_tweet_author)) + .route("/tweet-authors/:id/watch", put(handle_watch_tweet_author)) + .layer(Extension(create_mock_admin())) + .with_state(state.clone()); + + // 1. Test Ignore + let ignore_res = router + .clone() + .oneshot( + Request::builder() + .method("PUT") + .uri("/tweet-authors/auth_1/ignore") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(ignore_res.status(), StatusCode::NO_CONTENT); + + // Verify in DB + let author = state.db.tweet_authors.find_by_id("auth_1").await.unwrap().unwrap(); + assert!(author.is_ignored); + + // 2. Test Watch (Un-ignore) + let watch_res = router + .oneshot( + Request::builder() + .method("PUT") + .uri("/tweet-authors/auth_1/watch") + .body(Body::empty()) + .unwrap(), + ) + .await + .unwrap(); + + assert_eq!(watch_res.status(), StatusCode::NO_CONTENT); + + // Verify in DB + let author_updated = state.db.tweet_authors.find_by_id("auth_1").await.unwrap().unwrap(); + assert!(!author_updated.is_ignored); + } } diff --git a/src/models/tweet_author.rs b/src/models/tweet_author.rs index d352e8c..323d11b 100644 --- a/src/models/tweet_author.rs +++ b/src/models/tweet_author.rs @@ -8,6 +8,7 @@ pub struct TweetAuthor { pub id: String, pub name: String, pub username: String, + pub is_ignored: bool, pub followers_count: i32, pub following_count: i32, pub tweet_count: i32, @@ -22,6 +23,7 @@ impl<'r> FromRow<'r, PgRow> for TweetAuthor { id: row.try_get("id")?, name: row.try_get("name")?, username: row.try_get("username")?, + is_ignored: row.try_get("is_ignored")?, followers_count: row.try_get("followers_count")?, following_count: row.try_get("following_count")?, tweet_count: row.try_get("tweet_count")?, @@ -105,3 +107,8 @@ impl NewAuthorPayload { new_author } } + +#[derive(Debug, Deserialize, Serialize, Clone)] +pub struct CreateTweetAuthorInput { + pub username: String, +} diff --git a/src/repositories/tweet_author.rs b/src/repositories/tweet_author.rs index 7ad0271..b4b3848 100644 --- a/src/repositories/tweet_author.rs +++ b/src/repositories/tweet_author.rs @@ -107,6 +107,70 @@ impl TweetAuthorRepository { Ok(authors) } + pub async fn get_whitelist(&self) -> Result, DbError> { + let authors = sqlx::query_as::<_, TweetAuthor>("SELECT * FROM tweet_authors WHERE is_ignored = false") + .fetch_all(&self.pool) + .await?; + + let whitelist: Vec = authors.iter().map(|f| f.id.clone()).collect(); + + Ok(whitelist) + } + + pub async fn make_ignored_from_whitelist(&self, id: &str) -> Result<(), DbError> { + sqlx::query("UPDATE tweet_authors SET is_ignored = true WHERE id = $1") + .bind(id) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn make_watched_in_whitelist(&self, id: &str) -> Result<(), DbError> { + sqlx::query("UPDATE tweet_authors SET is_ignored = false WHERE id = $1") + .bind(id) + .execute(&self.pool) + .await?; + + Ok(()) + } + + pub async fn upsert(&self, payload: &NewAuthorPayload) -> DbResult { + let author = sqlx::query_as::<_, TweetAuthor>( + r#" + INSERT INTO tweet_authors ( + id, name, username, followers_count, following_count, + tweet_count, listed_count, like_count, media_count, fetched_at + ) + VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, NOW()) + ON CONFLICT (id) DO UPDATE SET + name = EXCLUDED.name, + username = EXCLUDED.username, + followers_count = EXCLUDED.followers_count, + following_count = EXCLUDED.following_count, + tweet_count = EXCLUDED.tweet_count, + listed_count = EXCLUDED.listed_count, + like_count = EXCLUDED.like_count, + media_count = EXCLUDED.media_count, + fetched_at = NOW() + RETURNING * + "#, + ) + .bind(&payload.id) + .bind(&payload.name) + .bind(&payload.username) + .bind(payload.followers_count) + .bind(payload.following_count) + .bind(payload.tweet_count) + .bind(payload.listed_count) + .bind(payload.like_count) + .bind(payload.media_count) + .fetch_one(&self.pool) + .await?; + + Ok(author.id) + } + /// Batch Upsert for Authors pub async fn upsert_many(&self, authors: &Vec) -> DbResult { if authors.is_empty() { diff --git a/src/routes/tweet_author.rs b/src/routes/tweet_author.rs index 4343c85..1241b5f 100644 --- a/src/routes/tweet_author.rs +++ b/src/routes/tweet_author.rs @@ -1,7 +1,10 @@ use axum::{handler::Handler, middleware, routing::get, Router}; use crate::{ - handlers::tweet_author::{handle_get_tweet_author_by_id, handle_get_tweet_authors}, + handlers::tweet_author::{ + handle_create_tweet_author, handle_get_tweet_author_by_id, handle_get_tweet_authors, + handle_ignore_tweet_author, handle_watch_tweet_author, + }, http_server::AppState, middlewares::jwt_auth, }; @@ -10,12 +13,22 @@ pub fn tweet_author_routes(state: AppState) -> Router { Router::new() .route( "/tweet-authors", - get(handle_get_tweet_authors - .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), + get(handle_get_tweet_authors.layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))) + .post(handle_create_tweet_author.layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), ) .route( "/tweet-authors/:id", get(handle_get_tweet_author_by_id .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), ) + .route( + "/tweet-authors/:id/ignore", + get(handle_ignore_tweet_author + .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), + ) + .route( + "/tweet-authors/:id/watch", + get(handle_watch_tweet_author + .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), + ) } diff --git a/src/services/tweet_synchronizer_service.rs b/src/services/tweet_synchronizer_service.rs index 0edbf95..3f4007a 100644 --- a/src/services/tweet_synchronizer_service.rs +++ b/src/services/tweet_synchronizer_service.rs @@ -170,11 +170,10 @@ impl TweetSynchronizerService { pub async fn sync_relevant_tweets(&self) -> Result<(), AppError> { let last_id = self.db.relevant_tweets.get_newest_tweet_id().await?; + let whitelist = self.db.tweet_authors.get_whitelist().await?; - let whitelist_queries = SearchParams::build_batched_whitelist_queries( - &self.config.tweet_sync.whitelist, - Some(&self.config.tweet_sync.keywords), - ); + let whitelist_queries = + SearchParams::build_batched_whitelist_queries(&whitelist, Some(&self.config.tweet_sync.keywords)); for query in whitelist_queries { let mut params = SearchParams::new(query); From 620a91f87152b2f3132f9eced7b2665f127d9d95 Mon Sep 17 00:00:00 2001 From: Beast Date: Wed, 24 Dec 2025 20:16:29 +0800 Subject: [PATCH 2/4] fix: wrong method for updating --- src/routes/tweet_author.rs | 11 ++++++++--- 1 file changed, 8 insertions(+), 3 deletions(-) diff --git a/src/routes/tweet_author.rs b/src/routes/tweet_author.rs index 1241b5f..ea11689 100644 --- a/src/routes/tweet_author.rs +++ b/src/routes/tweet_author.rs @@ -1,4 +1,9 @@ -use axum::{handler::Handler, middleware, routing::get, Router}; +use axum::{ + handler::Handler, + middleware, + routing::{get, put}, + Router, +}; use crate::{ handlers::tweet_author::{ @@ -23,12 +28,12 @@ pub fn tweet_author_routes(state: AppState) -> Router { ) .route( "/tweet-authors/:id/ignore", - get(handle_ignore_tweet_author + put(handle_ignore_tweet_author .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), ) .route( "/tweet-authors/:id/watch", - get(handle_watch_tweet_author + put(handle_watch_tweet_author .layer(middleware::from_fn_with_state(state.clone(), jwt_auth::jwt_admin_auth))), ) } From 52dad59f79c71edd79d0511898bc0cf9a5332686 Mon Sep 17 00:00:00 2001 From: Beast Date: Wed, 24 Dec 2025 20:30:50 +0800 Subject: [PATCH 3/4] fix: handle on none value --- src/models/tweet_author.rs | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/models/tweet_author.rs b/src/models/tweet_author.rs index 323d11b..e09614f 100644 --- a/src/models/tweet_author.rs +++ b/src/models/tweet_author.rs @@ -100,8 +100,8 @@ impl NewAuthorPayload { following_count: public_metrics.following_count as i32, tweet_count: public_metrics.tweet_count as i32, listed_count: public_metrics.listed_count as i32, - media_count: public_metrics.media_count.unwrap() as i32, - like_count: public_metrics.like_count.unwrap() as i32, + media_count: public_metrics.media_count.unwrap_or(0) as i32, + like_count: public_metrics.like_count.unwrap_or(0) as i32, }; new_author From b4e5900a0756d6508af3d1c0428736d28b324675 Mon Sep 17 00:00:00 2001 From: Beast Date: Wed, 24 Dec 2025 21:11:59 +0800 Subject: [PATCH 4/4] fix: fetching author public metrics --- Cargo.lock | 4 ++-- Cargo.toml | 4 ++-- src/handlers/tweet_author.rs | 17 +++++++++++++++-- src/models/tweet_author.rs | 11 ++--------- 4 files changed, 21 insertions(+), 15 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 81ae7ac..0f416a0 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5000,8 +5000,8 @@ checksum = "b39cdef0fa800fc44525c84ccb54a029961a8215f9619753635a9c0d2538d46d" [[package]] name = "rusx" -version = "0.5.0" -source = "git+https://github.com/Quantus-Network/rusx?tag=v0.5.0#c94af4059c58b942a5740fc1e3ab234f44d837eb" +version = "0.6.0" +source = "git+https://github.com/Quantus-Network/rusx?tag=v0.6.0#1f5383af185509649fcbc117fc97e166a6cb47d3" dependencies = [ "async-trait", "mockall 0.12.1", diff --git a/Cargo.toml b/Cargo.toml index 42475fd..7c4325d 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -24,7 +24,7 @@ path = "src/bin/create_raid.rs" qp-human-checkphrase = "0.1.2" qp-rusty-crystals-dilithium = "2.0.0" quantus-cli = "0.3.0" -rusx = {git = "https://github.com/Quantus-Network/rusx", tag = "v0.5.0"} +rusx = {git = "https://github.com/Quantus-Network/rusx", tag = "v0.6.0"} # Async runtime tokio = {version = "1.46", features = ["full", "test-util"]} @@ -90,4 +90,4 @@ tiny-keccak = {version = "2.0.2", features = ["keccak"]} mockall = "0.13" wiremock = "0.5" # Enable the testing feature ONLY for tests -rusx = {git = "https://github.com/Quantus-Network/rusx", tag = "v0.5.0", features = ["testing"]} +rusx = {git = "https://github.com/Quantus-Network/rusx", tag = "v0.6.0", features = ["testing"]} diff --git a/src/handlers/tweet_author.rs b/src/handlers/tweet_author.rs index c788d0f..1fff6e3 100644 --- a/src/handlers/tweet_author.rs +++ b/src/handlers/tweet_author.rs @@ -4,6 +4,7 @@ use axum::{ response::NoContent, Extension, Json, }; +use rusx::resources::{user::UserParams, UserField}; use crate::{ db_persistence::DbError, @@ -50,7 +51,19 @@ pub async fn handle_create_tweet_author( Extension(_): Extension, Json(payload): Json, ) -> Result<(StatusCode, Json>), AppError> { - let author_response = state.twitter_gateway.users().get_by_username(&payload.username).await?; + let mut params = UserParams::new(); + params.user_fields = Some(vec![ + UserField::PublicMetrics, + UserField::Id, + UserField::Name, + UserField::Username, + ]); + + let author_response = state + .twitter_gateway + .users() + .get_by_username(&payload.username, Some(params.clone())) + .await?; let Some(author) = author_response.data else { return Err(AppError::Handler(HandlerError::InvalidBody(format!( "Tweet Author {} not found", @@ -355,7 +368,7 @@ mod tests { let mut mock_gateway = MockTwitterGateway::new(); let mut mock_user = MockUserApi::new(); - mock_user.expect_get_by_username().returning(|_| { + mock_user.expect_get_by_username().returning(|_, _| { Ok(TwitterApiResponse { data: Some(User { id: "hello".to_string(), diff --git a/src/models/tweet_author.rs b/src/models/tweet_author.rs index e09614f..42cae43 100644 --- a/src/models/tweet_author.rs +++ b/src/models/tweet_author.rs @@ -1,5 +1,5 @@ use chrono::{DateTime, Utc}; -use rusx::resources::user::{User as TwitterUser, UserPublicMetrics}; +use rusx::resources::user::User as TwitterUser; use serde::{Deserialize, Serialize}; use sqlx::{postgres::PgRow, FromRow, Row}; @@ -83,14 +83,7 @@ pub struct NewAuthorPayload { impl NewAuthorPayload { pub fn new(author: TwitterUser) -> Self { - let public_metrics = author - .public_metrics - .ok_or_else(|| UserPublicMetrics { - media_count: Some(0), - like_count: Some(0), - ..Default::default() - }) - .unwrap(); + let public_metrics = author.public_metrics.unwrap_or_default(); let new_author = NewAuthorPayload { id: author.id,