feat(ci): disable stale runners and revoke their credentials

Modified app/webhook/src/main.rs +5 −0 View app/webhook/src/main.rs

@@ -209,6 +209,11 @@

209209 {

210210 warn!(%error, "retention prune failed");

211211 }

212 if ticks.is_multiple_of(30)

213 && let Err(error) = service.reap_stale_runners().await

214 {

215 warn!(%error, "stale runner sweep failed");

216 }

212217 }

213218 info!("scheduler stopped");

214219}

Modified lib/ci/src/runners.rs +44 −1 View lib/ci/src/runners.rs

@@ -8,7 +8,7 @@

88use models::ci::workflow_job;

99use models::ci::workflow_run;

1010use models::ci::workflow_step;

11use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, QueryOrder, Set};

11use sea_orm::{ActiveModelTrait, ColumnTrait, Condition, EntityTrait, QueryFilter, QueryOrder, Set};

1212use sha2::{Digest, Sha256};

1313use time::OffsetDateTime;

1414use uuid::Uuid;

@@ -282,6 +282,49 @@

282282 self.cancel_list_for_runner(runner_id).await

283283 }

284284

285 /// Disables runners that stopped heartbeating and revokes their credentials.

286 pub async fn reap_stale_runners(&self) -> CiResult<u64> {

287 let now = OffsetDateTime::now_utc();

288 let stale_secs = self.config().ci.runner_stale_secs.max(1) as i64;

289 let cutoff = now - time::Duration::seconds(stale_secs);

290 let stale = runner::Entity::find()

291 .filter(runner::Column::State.eq(RunnerState::Active))

292 .filter(

293 Condition::any()

294 .add(runner::Column::LastSeenAt.lt(cutoff))

295 .add(runner::Column::LastSeenAt.is_null()),

296 )

297 .all(self.db_write())

298 .await?;

299 let mut ids = Vec::with_capacity(stale.len());

300 for row in stale {

301 let id = row.id;

302 let mut am: runner::ActiveModel = row.into();

303 am.state = Set(RunnerState::Disabled);

304 am.updated_at = Set(now);

305 am.update(self.db_write()).await?;

306 ids.push(id);

307 }

308 if ids.is_empty() {

309 return Ok(0);

310 }

311 runner_token::Entity::update_many()

312 .col_expr(

313 runner_token::Column::RevokedAt,

314 sea_orm::sea_query::Expr::value(Some(now)),

315 )

316 .col_expr(

317 runner_token::Column::UpdatedAt,

318 sea_orm::sea_query::Expr::value(now),

319 )

320 .filter(runner_token::Column::RunnerId.is_in(ids.clone()))

321 .filter(runner_token::Column::RevokedAt.is_null())

322 .exec(self.db_write())

323 .await?;

324 tracing::info!(count = ids.len(), "stale runners disabled");

325 Ok(ids.len() as u64)

326 }

327

285328 pub async fn runner_labels(&self, runner_id: Uuid) -> CiResult<Vec<String>> {

286329 let runner = runner::Entity::find_by_id(runner_id)

287330 .one(self.db_read())

Modified lib/config/src/ci.rs +3 −0 View lib/config/src/ci.rs

@@ -23,6 +23,7 @@

2323 pub outbound_timeout_secs: u64,

2424 pub scheduler_interval_secs: u64,

2525 pub cancel_grace_secs: u64,

26 pub runner_stale_secs: u64,

2627 pub actions_engine_binary: String,

2728 pub actions_engine_timeout_secs: u64,

2829 pub runtime_base_url: String,

@@ -54,6 +55,7 @@

5455 outbound_timeout_secs: 10,

5556 scheduler_interval_secs: 2,

5657 cancel_grace_secs: 120,

58 runner_stale_secs: 180,

5759 actions_engine_binary: "gitdata-actions-engine".into(),

5860 actions_engine_timeout_secs: 30,

5961 runtime_base_url: String::new(),

@@ -93,6 +95,7 @@

9395 || self.job_timeout_max_secs < self.job_timeout_default_secs

9496 || self.log_chunk_max_bytes == 0

9597 || self.scheduler_interval_secs == 0

98 || self.runner_stale_secs == 0

9699 {

97100 return Err("ci timing and size limits must be greater than zero".to_string());

98101 }