use models::admin::admin_user::AdminRole; use models::chat::webhook_deliveries::{self, WebhookDeliveryStatus}; use models::import::job; use models::repo::pipeline_run::{self, PipelineRunState}; use models::search::index_job; use models::system::system_tasks::{self, TaskStatus}; use models::users::user_deletion_requests::{self, UserDeletionRequestStatus}; use sea_orm::{ ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect, Set, }; use serde::{Deserialize, Serialize}; use serde_json::{Value, json}; use time::OffsetDateTime; use utoipa::{IntoParams, ToSchema}; use uuid::Uuid; use crate::errors::AdminError; use crate::services::audit::{AuditCtx, AuditRecord}; use crate::services::service::AdminService; use crate::share::roles::AdminActor; pub const SOURCE_IMPORT: &str = "import"; pub const SOURCE_SEARCH_INDEX: &str = "search_index"; pub const SOURCE_CHAT_WEBHOOK: &str = "chat_webhook"; pub const SOURCE_PIPELINE: &str = "pipeline"; pub const SOURCE_DELETION: &str = "deletion"; pub const SOURCE_SYSTEM_TASKS: &str = "system_tasks"; #[derive(Debug, Clone, Deserialize, IntoParams)] pub struct ListTasksQuery { pub source: Option, pub state: Option, pub q: Option, pub from: Option, pub to: Option, pub limit: Option, pub offset: Option, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskDto { pub id: Uuid, pub source: String, pub title: String, pub state: String, pub created_at: OffsetDateTime, pub updated_at: Option, pub finished_at: Option, pub last_error: Option, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskPageDto { pub items: Vec, pub total: i64, pub limit: u64, pub offset: u64, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskDetailDto { pub task: TaskDto, pub detail: Value, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskActionResp { pub source: String, pub id: Uuid, pub state: String, } fn pipeline_state_str(state: PipelineRunState) -> String { match state { PipelineRunState::Queued => "queued", PipelineRunState::Running => "running", PipelineRunState::Success => "success", PipelineRunState::Failure => "failure", PipelineRunState::CancelRequested => "cancel_requested", PipelineRunState::Cancelled => "cancelled", } .to_string() } fn deletion_state_str(state: UserDeletionRequestStatus) -> String { match state { UserDeletionRequestStatus::Pending => "pending", UserDeletionRequestStatus::Scheduled => "scheduled", UserDeletionRequestStatus::Completed => "completed", UserDeletionRequestStatus::Cancelled => "cancelled", } .to_string() } fn task_state_str(state: TaskStatus) -> String { match state { TaskStatus::Pending => "pending", TaskStatus::Running => "running", TaskStatus::Success => "success", TaskStatus::Failed => "failed", } .to_string() } fn webhook_state_str(state: WebhookDeliveryStatus) -> String { match state { WebhookDeliveryStatus::Pending => "pending", WebhookDeliveryStatus::Delivering => "delivering", WebhookDeliveryStatus::Done => "done", WebhookDeliveryStatus::Failed => "failed", } .to_string() } fn short_sha(sha: &str) -> String { sha.chars().take(8).collect() } fn parse_pipeline_state(value: &str) -> Option { match value { "queued" => Some(PipelineRunState::Queued), "running" => Some(PipelineRunState::Running), "success" => Some(PipelineRunState::Success), "failure" => Some(PipelineRunState::Failure), "cancel_requested" => Some(PipelineRunState::CancelRequested), "cancelled" => Some(PipelineRunState::Cancelled), _ => None, } } fn parse_deletion_state(value: &str) -> Option { match value { "pending" => Some(UserDeletionRequestStatus::Pending), "scheduled" => Some(UserDeletionRequestStatus::Scheduled), "completed" => Some(UserDeletionRequestStatus::Completed), "cancelled" => Some(UserDeletionRequestStatus::Cancelled), _ => None, } } fn parse_task_status(value: &str) -> Option { match value { "pending" => Some(TaskStatus::Pending), "running" => Some(TaskStatus::Running), "success" => Some(TaskStatus::Success), "failed" => Some(TaskStatus::Failed), _ => None, } } fn parse_webhook_state(value: &str) -> Option { match value { "pending" => Some(WebhookDeliveryStatus::Pending), "delivering" => Some(WebhookDeliveryStatus::Delivering), "done" => Some(WebhookDeliveryStatus::Done), "failed" => Some(WebhookDeliveryStatus::Failed), _ => None, } } impl AdminService { fn requested_sources(&self, source: &Option) -> Vec<&'static str> { match source.as_deref() { Some(SOURCE_IMPORT) => vec![SOURCE_IMPORT], Some(SOURCE_SEARCH_INDEX) => vec![SOURCE_SEARCH_INDEX], Some(SOURCE_CHAT_WEBHOOK) => vec![SOURCE_CHAT_WEBHOOK], Some(SOURCE_PIPELINE) => vec![SOURCE_PIPELINE], Some(SOURCE_DELETION) => vec![SOURCE_DELETION], Some(SOURCE_SYSTEM_TASKS) => vec![SOURCE_SYSTEM_TASKS], _ => vec![ SOURCE_IMPORT, SOURCE_SEARCH_INDEX, SOURCE_CHAT_WEBHOOK, SOURCE_PIPELINE, SOURCE_DELETION, SOURCE_SYSTEM_TASKS, ], } } pub async fn list_tasks(&self, query: ListTasksQuery) -> Result { let limit = query.limit.unwrap_or(50).clamp(1, 100); let offset = query.offset.unwrap_or(0); let fetch_limit = (offset + limit).clamp(1, 500); let mut tasks: Vec = Vec::new(); for source in self.requested_sources(&query.source) { tasks.extend( self.fetch_source(source, fetch_limit, query.state.as_deref(), query.from, query.to) .await?, ); } if let Some(needle) = query .q .as_deref() .map(str::trim) .filter(|value| !value.is_empty()) { let needle = needle.to_ascii_lowercase(); tasks.retain(|task| { task.title.to_ascii_lowercase().contains(&needle) || task.id.to_string().contains(&needle) || task .last_error .as_deref() .is_some_and(|error| error.to_ascii_lowercase().contains(&needle)) }); } tasks.sort_by(|left, right| right.created_at.cmp(&left.created_at)); let total = tasks.len() as i64; let items = tasks .into_iter() .skip(offset as usize) .take(limit as usize) .collect(); Ok(TaskPageDto { items, total, limit, offset, }) } async fn fetch_source( &self, source: &str, fetch_limit: u64, state: Option<&str>, from: Option, to: Option, ) -> Result, AdminError> { let mut tasks = Vec::new(); match source { SOURCE_IMPORT => { let mut select = job::Entity::find().order_by_desc(job::Column::CreatedAt); if let Some(state) = state { select = select.filter(job::Column::State.eq(state)); } if let Some(from) = from { select = select.filter(job::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(job::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_IMPORT.to_string(), title: format!("import {}", row.repo_name), state: row.state, created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.finished_at, last_error: row.last_error, }); } } SOURCE_SEARCH_INDEX => { let mut select = index_job::Entity::find().order_by_desc(index_job::Column::CreatedAt); if let Some(state) = state { select = select.filter(index_job::Column::State.eq(state)); } if let Some(from) = from { select = select.filter(index_job::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(index_job::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_SEARCH_INDEX.to_string(), title: format!( "index {} {}", row.kind, row.target_id .map(|value| value.to_string()) .unwrap_or_else(|| "-".to_string()) ), state: row.state, created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: None, last_error: row.last_error, }); } } SOURCE_CHAT_WEBHOOK => { let mut select = webhook_deliveries::Entity::find() .order_by_desc(webhook_deliveries::Column::CreatedAt); if let Some(state) = state { select = select.filter( webhook_deliveries::Column::Status.eq(parse_webhook_state(state).ok_or_else( || AdminError::BadRequest("invalid webhook state".to_string()), )?), ); } if let Some(from) = from { select = select.filter(webhook_deliveries::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(webhook_deliveries::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_CHAT_WEBHOOK.to_string(), title: format!("{} {}", row.event_type, row.webhook_id), state: webhook_state_str(row.status), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: None, last_error: row.last_error, }); } } SOURCE_PIPELINE => { let mut select = pipeline_run::Entity::find().order_by_desc(pipeline_run::Column::CreatedAt); if let Some(state) = state { select = select.filter( pipeline_run::Column::State.eq(parse_pipeline_state(state).ok_or_else( || AdminError::BadRequest("invalid pipeline state".to_string()), )?), ); } if let Some(from) = from { select = select.filter(pipeline_run::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(pipeline_run::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_PIPELINE.to_string(), title: format!("run {}@{}", short_sha(&row.commit_sha), row.branch), state: pipeline_state_str(row.state), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.finished_at, last_error: None, }); } } SOURCE_DELETION => { let mut select = user_deletion_requests::Entity::find() .order_by_desc(user_deletion_requests::Column::CreatedAt); if let Some(state) = state { select = select.filter( user_deletion_requests::Column::Status.eq( parse_deletion_state(state).ok_or_else(|| { AdminError::BadRequest("invalid deletion state".to_string()) })?, ), ); } if let Some(from) = from { select = select.filter(user_deletion_requests::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(user_deletion_requests::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_DELETION.to_string(), title: format!("deletion request {}", row.user_id), state: deletion_state_str(row.status), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.completed_at, last_error: None, }); } } SOURCE_SYSTEM_TASKS => { let mut select = system_tasks::Entity::find().order_by_desc(system_tasks::Column::CreatedAt); if let Some(state) = state { select = select.filter( system_tasks::Column::Status.eq(parse_task_status(state).ok_or_else(|| { AdminError::BadRequest("invalid task state".to_string()) })?), ); } if let Some(from) = from { select = select.filter(system_tasks::Column::CreatedAt.gte(from)); } if let Some(to) = to { select = select.filter(system_tasks::Column::CreatedAt.lte(to)); } for row in select.limit(fetch_limit).all(self.db().read()).await? { tasks.push(TaskDto { id: row.id, source: SOURCE_SYSTEM_TASKS.to_string(), title: row.task_type, state: task_state_str(row.status), created_at: row.created_at, updated_at: row.updated_at, finished_at: row.finished_at, last_error: row.error, }); } } _ => {} } Ok(tasks) } pub async fn get_task(&self, source: &str, id: Uuid) -> Result { let (task, detail) = match source { SOURCE_IMPORT => { let row = job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "provider": row.provider, "source_url": row.source_url, "clone_url": row.clone_url, "space_id": row.space_id, "repo_id": row.repo_id, "repo_name": row.repo_name, "state": row.state, "stage": row.stage, "attempts": row.attempts, "stats": row.stats, "options": row.options, "run_after": row.run_after, "locked_at": row.locked_at, "started_at": row.started_at, "finished_at": row.finished_at, "token": "[redacted]", }); let task = TaskDto { id: row.id, source: SOURCE_IMPORT.to_string(), title: format!("import {}", row.repo_name), state: row.state, created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.finished_at, last_error: row.last_error, }; (task, detail) } SOURCE_SEARCH_INDEX => { let row = index_job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "kind": row.kind, "target_id": row.target_id, "repo_id": row.repo_id, "attempts": row.attempts, "run_after": row.run_after, "locked_at": row.locked_at, }); let task = TaskDto { id: row.id, source: SOURCE_SEARCH_INDEX.to_string(), title: format!("index {}", row.kind), state: row.state, created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: None, last_error: row.last_error, }; (task, detail) } SOURCE_CHAT_WEBHOOK => { let row = webhook_deliveries::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "webhook_id": row.webhook_id, "room_id": row.room_id, "event_type": row.event_type, "attempts": row.attempts, "next_retry_at": row.next_retry_at, "payload": "[redacted]", }); let task = TaskDto { id: row.id, source: SOURCE_CHAT_WEBHOOK.to_string(), title: format!("{} {}", row.event_type, row.webhook_id), state: webhook_state_str(row.status), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: None, last_error: row.last_error, }; (task, detail) } SOURCE_PIPELINE => { let row = pipeline_run::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "repo_id": row.repo_id, "trigger_user_id": row.trigger_user_id, "commit_sha": row.commit_sha, "branch": row.branch, "started_at": row.started_at, }); let task = TaskDto { id: row.id, source: SOURCE_PIPELINE.to_string(), title: format!("run {}@{}", short_sha(&row.commit_sha), row.branch), state: pipeline_state_str(row.state), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.finished_at, last_error: None, }; (task, detail) } SOURCE_DELETION => { let row = user_deletion_requests::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "user_id": row.user_id, "reason": row.reason, "scheduled_for": row.scheduled_for, "completed_at": row.completed_at, "cancelled_at": row.cancelled_at, }); let task = TaskDto { id: row.id, source: SOURCE_DELETION.to_string(), title: format!("deletion request {}", row.user_id), state: deletion_state_str(row.status), created_at: row.created_at, updated_at: Some(row.updated_at), finished_at: row.completed_at, last_error: None, }; (task, detail) } SOURCE_SYSTEM_TASKS => { let row = system_tasks::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let detail = json!({ "task_type": row.task_type, "payload": row.payload, "retry_count": row.retry_count, "next_run_at": row.next_run_at, "started_at": row.started_at, }); let task = TaskDto { id: row.id, source: SOURCE_SYSTEM_TASKS.to_string(), title: row.task_type, state: task_state_str(row.status), created_at: row.created_at, updated_at: row.updated_at, finished_at: row.finished_at, last_error: row.error, }; (task, detail) } _ => return Err(AdminError::BadRequest("unknown task source".to_string())), }; Ok(TaskDetailDto { task, detail }) } pub async fn retry_task( &self, actor: &AdminActor, source: &str, id: Uuid, ctx: &AuditCtx, ) -> Result { actor.require(AdminRole::Operator)?; let now = OffsetDateTime::now_utc(); let state = match source { SOURCE_IMPORT => { let row = job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!(row.state.as_str(), "failed" | "canceled") { return Err(AdminError::Conflict( "only failed or canceled imports can be retried".to_string(), )); } let mut am: job::ActiveModel = row.into(); am.state = Set("queued".to_string()); am.run_after = Set(now); am.locked_at = Set(None); am.finished_at = Set(None); am.updated_at = Set(now); am.update(self.db().write()).await?; "queued".to_string() } SOURCE_SEARCH_INDEX => { let row = index_job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; let mut am: index_job::ActiveModel = row.into(); am.state = Set("queued".to_string()); am.run_after = Set(now); am.locked_at = Set(None); am.updated_at = Set(now); am.update(self.db().write()).await?; "queued".to_string() } SOURCE_CHAT_WEBHOOK => { let row = webhook_deliveries::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if row.status != WebhookDeliveryStatus::Failed { return Err(AdminError::Conflict( "only failed webhook deliveries can be retried".to_string(), )); } let mut am: webhook_deliveries::ActiveModel = row.into(); am.status = Set(WebhookDeliveryStatus::Pending); am.next_retry_at = Set(now); am.updated_at = Set(now); am.update(self.db().write()).await?; "pending".to_string() } SOURCE_PIPELINE => { let row = pipeline_run::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!( row.state, PipelineRunState::Failure | PipelineRunState::Cancelled ) { return Err(AdminError::Conflict( "only failed or cancelled pipeline runs can be retried".to_string(), )); } let mut am: pipeline_run::ActiveModel = row.into(); am.state = Set(PipelineRunState::Queued); am.started_at = Set(None); am.finished_at = Set(None); am.updated_at = Set(now); am.update(self.db().write()).await?; "queued".to_string() } SOURCE_DELETION => { let row = user_deletion_requests::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if row.status != UserDeletionRequestStatus::Cancelled { return Err(AdminError::Conflict( "only cancelled deletion requests can be retried".to_string(), )); } let mut am: user_deletion_requests::ActiveModel = row.into(); am.status = Set(UserDeletionRequestStatus::Scheduled); am.scheduled_for = Set(Some(now)); am.cancelled_at = Set(None); am.updated_at = Set(now); am.update(self.db().write()).await?; "scheduled".to_string() } SOURCE_SYSTEM_TASKS => { let row = system_tasks::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if row.status != TaskStatus::Failed { return Err(AdminError::Conflict( "only failed system tasks can be retried".to_string(), )); } let mut am: system_tasks::ActiveModel = row.into(); am.status = Set(TaskStatus::Pending); am.error = Set(None); am.next_run_at = Set(Some(now)); am.started_at = Set(None); am.finished_at = Set(None); am.updated_at = Set(Some(now)); am.update(self.db().write()).await?; "pending".to_string() } _ => return Err(AdminError::BadRequest("unknown task source".to_string())), }; self.audit( Some(actor), AuditRecord { action: "task.retried", target: Some((source, id.to_string())), detail: json!({ "state": state }), }, ctx, ) .await?; Ok(TaskActionResp { source: source.to_string(), id, state, }) } pub async fn cancel_task( &self, actor: &AdminActor, source: &str, id: Uuid, ctx: &AuditCtx, ) -> Result { actor.require(AdminRole::Operator)?; let now = OffsetDateTime::now_utc(); let state = match source { SOURCE_IMPORT => { let row = job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if row.state == "processing" { return Err(AdminError::Conflict( "import is already processing and cannot be cancelled".to_string(), )); } if row.state != "queued" { return Err(AdminError::Conflict( "only queued imports can be cancelled".to_string(), )); } let mut am: job::ActiveModel = row.into(); am.state = Set("canceled".to_string()); am.finished_at = Set(Some(now)); am.updated_at = Set(now); am.update(self.db().write()).await?; "canceled".to_string() } SOURCE_SEARCH_INDEX => { let row = index_job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!(row.state.as_str(), "queued" | "processing") { return Err(AdminError::Conflict( "only queued or processing index jobs can be cancelled".to_string(), )); } let mut am: index_job::ActiveModel = row.into(); am.state = Set("cancelled".to_string()); am.locked_at = Set(None); am.updated_at = Set(now); am.update(self.db().write()).await?; "cancelled".to_string() } SOURCE_CHAT_WEBHOOK => { let row = webhook_deliveries::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!( row.status, WebhookDeliveryStatus::Pending | WebhookDeliveryStatus::Delivering ) { return Err(AdminError::Conflict( "only pending webhook deliveries can be cancelled".to_string(), )); } let mut am: webhook_deliveries::ActiveModel = row.into(); am.status = Set(WebhookDeliveryStatus::Failed); am.last_error = Set(Some("cancelled by admin".to_string())); am.updated_at = Set(now); am.update(self.db().write()).await?; "failed".to_string() } SOURCE_PIPELINE => { let row = pipeline_run::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!(row.state, PipelineRunState::Queued | PipelineRunState::Running) { return Err(AdminError::Conflict( "only queued or running pipeline runs can be cancelled".to_string(), )); } let mut am: pipeline_run::ActiveModel = row.into(); am.state = Set(PipelineRunState::CancelRequested); am.updated_at = Set(now); am.update(self.db().write()).await?; "cancel_requested".to_string() } SOURCE_DELETION => { let row = user_deletion_requests::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!( row.status, UserDeletionRequestStatus::Pending | UserDeletionRequestStatus::Scheduled ) { return Err(AdminError::Conflict( "only pending deletion requests can be cancelled".to_string(), )); } let mut am: user_deletion_requests::ActiveModel = row.into(); am.status = Set(UserDeletionRequestStatus::Cancelled); am.cancelled_at = Set(Some(now)); am.updated_at = Set(now); am.update(self.db().write()).await?; "cancelled".to_string() } SOURCE_SYSTEM_TASKS => { let row = system_tasks::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; if !matches!(row.status, TaskStatus::Pending | TaskStatus::Running) { return Err(AdminError::Conflict( "only pending or running tasks can be cancelled".to_string(), )); } let mut am: system_tasks::ActiveModel = row.into(); am.status = Set(TaskStatus::Failed); am.error = Set(Some("cancelled by admin".to_string())); am.finished_at = Set(Some(now)); am.updated_at = Set(Some(now)); am.update(self.db().write()).await?; "failed".to_string() } _ => return Err(AdminError::BadRequest("unknown task source".to_string())), }; self.audit( Some(actor), AuditRecord { action: "task.cancelled", target: Some((source, id.to_string())), detail: json!({ "state": state }), }, ctx, ) .await?; Ok(TaskActionResp { source: source.to_string(), id, state, }) } pub async fn delete_task( &self, actor: &AdminActor, source: &str, id: Uuid, ctx: &AuditCtx, ) -> Result<(), AdminError> { actor.require(AdminRole::SuperAdmin)?; let terminal = match source { SOURCE_IMPORT => { let row = job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!(row.state.as_str(), "succeeded" | "failed" | "canceled") } SOURCE_SEARCH_INDEX => { let row = index_job::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!(row.state.as_str(), "succeeded" | "cancelled") } SOURCE_CHAT_WEBHOOK => { let row = webhook_deliveries::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!( row.status, WebhookDeliveryStatus::Done | WebhookDeliveryStatus::Failed ) } SOURCE_PIPELINE => { let row = pipeline_run::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!( row.state, PipelineRunState::Success | PipelineRunState::Failure | PipelineRunState::Cancelled ) } SOURCE_DELETION => { let row = user_deletion_requests::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!( row.status, UserDeletionRequestStatus::Completed | UserDeletionRequestStatus::Cancelled ) } SOURCE_SYSTEM_TASKS => { let row = system_tasks::Entity::find_by_id(id) .one(self.db().read()) .await? .ok_or(AdminError::NotFound)?; matches!(row.status, TaskStatus::Success | TaskStatus::Failed) } _ => return Err(AdminError::BadRequest("unknown task source".to_string())), }; if !terminal { return Err(AdminError::Conflict( "only finished tasks can be deleted".to_string(), )); } match source { SOURCE_IMPORT => { job::Entity::delete_by_id(id).exec(self.db().write()).await?; } SOURCE_SEARCH_INDEX => { index_job::Entity::delete_by_id(id) .exec(self.db().write()) .await?; } SOURCE_CHAT_WEBHOOK => { webhook_deliveries::Entity::delete_by_id(id) .exec(self.db().write()) .await?; } SOURCE_PIPELINE => { pipeline_run::Entity::delete_by_id(id) .exec(self.db().write()) .await?; } SOURCE_DELETION => { user_deletion_requests::Entity::delete_by_id(id) .exec(self.db().write()) .await?; } SOURCE_SYSTEM_TASKS => { system_tasks::Entity::delete_by_id(id) .exec(self.db().write()) .await?; } _ => {} } self.audit( Some(actor), AuditRecord { action: "task.deleted", target: Some((source, id.to_string())), detail: json!({}), }, ctx, ) .await?; Ok(()) } } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskSourceStatDto { pub source: String, pub queued: i64, pub running: i64, pub succeeded: i64, pub failed: i64, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct TaskStatsDto { pub sources: Vec, pub total: i64, pub generated_at: OffsetDateTime, } #[derive(Debug, Clone, Deserialize, ToSchema)] pub struct BulkTaskCmd { pub action: String, pub ids: Vec, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct BulkTaskFailure { pub id: Uuid, pub error: String, } #[derive(Debug, Clone, Serialize, Deserialize, ToSchema)] pub struct BulkTaskResp { pub source: String, pub action: String, pub succeeded: Vec, pub failed: Vec, } impl AdminService { pub async fn task_stats(&self) -> Result { let mut sources: Vec = Vec::new(); for source in self.requested_sources(&None) { let stat = match source { SOURCE_IMPORT => TaskSourceStatDto { source: source.to_string(), queued: job::Entity::find() .filter(job::Column::State.eq("queued")) .count(self.db().read()) .await? as i64, running: job::Entity::find() .filter(job::Column::State.eq("processing")) .count(self.db().read()) .await? as i64, succeeded: job::Entity::find() .filter(job::Column::State.eq("succeeded")) .count(self.db().read()) .await? as i64, failed: job::Entity::find() .filter(job::Column::State.eq("failed")) .count(self.db().read()) .await? as i64, }, SOURCE_SEARCH_INDEX => TaskSourceStatDto { source: source.to_string(), queued: index_job::Entity::find() .filter(index_job::Column::State.eq("queued")) .count(self.db().read()) .await? as i64, running: index_job::Entity::find() .filter(index_job::Column::State.eq("processing")) .count(self.db().read()) .await? as i64, succeeded: index_job::Entity::find() .filter(index_job::Column::State.eq("succeeded")) .count(self.db().read()) .await? as i64, failed: index_job::Entity::find() .filter(index_job::Column::LastError.is_not_null()) .count(self.db().read()) .await? as i64, }, SOURCE_CHAT_WEBHOOK => TaskSourceStatDto { source: source.to_string(), queued: webhook_deliveries::Entity::find() .filter(webhook_deliveries::Column::Status.eq(WebhookDeliveryStatus::Pending)) .count(self.db().read()) .await? as i64, running: webhook_deliveries::Entity::find() .filter( webhook_deliveries::Column::Status.eq(WebhookDeliveryStatus::Delivering), ) .count(self.db().read()) .await? as i64, succeeded: webhook_deliveries::Entity::find() .filter(webhook_deliveries::Column::Status.eq(WebhookDeliveryStatus::Done)) .count(self.db().read()) .await? as i64, failed: webhook_deliveries::Entity::find() .filter(webhook_deliveries::Column::Status.eq(WebhookDeliveryStatus::Failed)) .count(self.db().read()) .await? as i64, }, SOURCE_PIPELINE => TaskSourceStatDto { source: source.to_string(), queued: pipeline_run::Entity::find() .filter(pipeline_run::Column::State.eq(PipelineRunState::Queued)) .count(self.db().read()) .await? as i64, running: pipeline_run::Entity::find() .filter(pipeline_run::Column::State.eq(PipelineRunState::Running)) .count(self.db().read()) .await? as i64, succeeded: pipeline_run::Entity::find() .filter(pipeline_run::Column::State.eq(PipelineRunState::Success)) .count(self.db().read()) .await? as i64, failed: pipeline_run::Entity::find() .filter(pipeline_run::Column::State.eq(PipelineRunState::Failure)) .count(self.db().read()) .await? as i64, }, SOURCE_DELETION => TaskSourceStatDto { source: source.to_string(), queued: user_deletion_requests::Entity::find() .filter( user_deletion_requests::Column::Status .is_in([UserDeletionRequestStatus::Pending, UserDeletionRequestStatus::Scheduled]), ) .count(self.db().read()) .await? as i64, running: 0, succeeded: user_deletion_requests::Entity::find() .filter( user_deletion_requests::Column::Status .eq(UserDeletionRequestStatus::Completed), ) .count(self.db().read()) .await? as i64, failed: 0, }, _ => TaskSourceStatDto { source: source.to_string(), queued: system_tasks::Entity::find() .filter(system_tasks::Column::Status.eq(TaskStatus::Pending)) .count(self.db().read()) .await? as i64, running: system_tasks::Entity::find() .filter(system_tasks::Column::Status.eq(TaskStatus::Running)) .count(self.db().read()) .await? as i64, succeeded: system_tasks::Entity::find() .filter(system_tasks::Column::Status.eq(TaskStatus::Success)) .count(self.db().read()) .await? as i64, failed: system_tasks::Entity::find() .filter(system_tasks::Column::Status.eq(TaskStatus::Failed)) .count(self.db().read()) .await? as i64, }, }; sources.push(stat); } let total = sources .iter() .map(|stat| stat.queued + stat.running + stat.succeeded + stat.failed) .sum(); Ok(TaskStatsDto { sources, total, generated_at: OffsetDateTime::now_utc(), }) } pub async fn bulk_tasks( &self, actor: &AdminActor, source: &str, cmd: BulkTaskCmd, ctx: &AuditCtx, ) -> Result { actor.require(AdminRole::Operator)?; if ![ SOURCE_IMPORT, SOURCE_SEARCH_INDEX, SOURCE_CHAT_WEBHOOK, SOURCE_PIPELINE, SOURCE_DELETION, SOURCE_SYSTEM_TASKS, ] .contains(&source) { return Err(AdminError::BadRequest(format!( "unknown task source: {source}" ))); } let action = cmd.action.trim().to_ascii_lowercase(); match action.as_str() { "retry" | "cancel" => {} other => { return Err(AdminError::BadRequest(format!( "unsupported bulk action: {other}" ))); } } if cmd.ids.is_empty() { return Err(AdminError::BadRequest("ids must not be empty".to_string())); } if cmd.ids.len() > 100 { return Err(AdminError::BadRequest( "at most 100 ids per request".to_string(), )); } let mut succeeded: Vec = Vec::new(); let mut failed: Vec = Vec::new(); for id in cmd.ids { let outcome = if action == "retry" { self.retry_task(actor, source, id, ctx).await.map(|_| ()) } else { self.cancel_task(actor, source, id, ctx).await.map(|_| ()) }; match outcome { Ok(()) => succeeded.push(id), Err(AdminError::NotFound) => failed.push(BulkTaskFailure { id, error: "not found".to_string(), }), Err(error) => failed.push(BulkTaskFailure { id, error: error.to_string(), }), } } self.audit( Some(actor), AuditRecord { action: "task.bulk_action", target: Some(("task", source.to_string())), detail: json!({ "action": action, "source": source, "succeeded": succeeded.len(), "failed": failed.len(), }), }, ctx, ) .await?; Ok(BulkTaskResp { source: source.to_string(), action, succeeded, failed, }) } }