feat(ci): add the actions runtime, scheduler and triggers

Added lib/ci/examples/actions_fixture.rs +376 −0 View lib/ci/examples/actions_fixture.rs

@@ -0,0 +1,376 @@

1use std::{collections::BTreeMap, path::PathBuf};

2

3use anyhow::{Context, ensure};

4use ci::{CiService, LoadedWorkflow, actions_environments::EnvironmentRules, runs::RunSeed};

5use models::{

6 ci::{runner as runners, workflow_run},

7 repo::repos,

8 space::spaces,

9};

10use sea_orm::{ActiveModelTrait, ColumnTrait, Database, EntityTrait, QueryFilter, Set};

11use serde_json::json;

12use time::OffsetDateTime;

13use uuid::Uuid;

14

15const WORKFLOW: &str = r#"name: Fixture

16on: [push, workflow_dispatch]

17jobs:

18 plan:

19 runs-on: self-hosted

20 outputs:

21 matrix: ${{ steps.plan.outputs.matrix }}

22 steps:

23 - id: plan

24 run: echo 'matrix={"value":["a","b"]}' >> "$GITHUB_OUTPUT"

25 verify:

26 needs: plan

27 runs-on: self-hosted

28 name: verify-${{ matrix.value }}

29 strategy:

30 max-parallel: 1

31 matrix: ${{ fromJSON(needs.plan.outputs.matrix) }}

32 outputs:

33 value: ${{ steps.result.outputs.value }}

34 steps:

35 - id: result

36 run: echo 'value=${{ matrix.value }}' >> "$GITHUB_OUTPUT"

37 cleanup:

38 needs: verify

39 if: always()

40 runs-on: self-hosted

41 steps:

42 - run: echo 'fixture completed'

43"#;

44

45#[tokio::main]

46async fn main() -> anyhow::Result<()> {

47 let directory = std::env::current_dir()?.join("target/actions-fixture");

48 tokio::fs::create_dir_all(&directory).await?;

49 let mut config = config::AppConfig::default();

50 config.database.url =

51 Some("postgres://postgres:fixture-password@127.0.0.1:45432/gitdata_ci_fixture".into());

52 config.database.max_connections = Some(20);

53 config.database.sqlx_logging = Some(false);

54 config.redis.url = Some("redis://127.0.0.1:46379/9".into());

55 config.s3.bucket = Some("ci-fixture".into());

56 config.s3.region = Some("us-east-1".into());

57 config.s3.endpoint = Some("http://127.0.0.1:49000".into());

58 config.s3.access_key_id = Some("fixture-access".into());

59 config.s3.secret_access_key = Some("fixture-secret-password".into());

60 config.s3.path_style = Some(true);

61 config.email.enabled = Some(false);

62 config.email.from_address = Some("fixture@localhost".into());

63 config.ci.enabled = true;

64 config.ci.secret_encryption_key = "QUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUFBQUE=".into();

65 config.ci.actions_engine_binary = std::env::current_dir()?

66 .join("target/gitdata-actions-engine")

67 .to_string_lossy()

68 .into_owned();

69 config.ci.runtime_base_url = "http://127.0.0.1:43120".into();

70 config.ci.event_queue_topic = "gitdata:ci-fixture:events".into();

71 config.ci.lease_secs = 120;

72 config.git.repo_root = directory.join("repos").to_string_lossy().into_owned();

73 config.git.base_url = "http://127.0.0.1:43121".into();

74 let database = Database::connect(config.database.url.clone().unwrap()).await?;

75 models::migrate::sync_schema(&database)

76 .await

77 .context("fixture schema sync")?;

78 let state = export::AppState::with_config(config).await?;

79 let service = CiService::new(state);

80 let (repo, space, owner, reviewer) = seed(&service).await?;

81 let http = axum::Router::new()

82 .nest(

83 "/api/v1",

84 ci::routes::runner_router().with_state(service.clone()),

85 )

86 .merge(ci::actions_runtime::router().with_state(service.clone()));

87 let listener = tokio::net::TcpListener::bind("127.0.0.1:43120").await?;

88 let server = tokio::spawn(async move { axum::serve(listener, http).await });

89 let git_service = git::service::GitService::new(service.state.clone());

90 let smart = git::smart::service::SmartService::new(git_service);

91 let git_listener = tokio::net::TcpListener::bind("127.0.0.1:43121").await?;

92 let git_server =

93 tokio::spawn(

94 async move { axum::serve(git_listener, git::smart::handler::router(smart)).await },

95 );

96 let runner_id = Uuid::now_v7();

97 let now = OffsetDateTime::now_utc();

98 runners::ActiveModel {

99 id: Set(runner_id),

100 name: Set("actions-fixture".into()),

101 labels: Set(json!(["self-hosted"])),

102 capacity: Set(4),

103 version: Set(Some("fixture".into())),

104 state: Set(runners::RunnerState::Active),

105 space_id: Set(Some(space.id)),

106 last_seen_at: Set(Some(now)),

107 created_at: Set(now),

108 updated_at: Set(now),

109 protocol_version: Set(Some(2)),

110 capabilities: Set(Some(json!(["actions-v2", "docker"]))),

111 }

112 .insert(service.db_write())

113 .await?;

114 let token_id = Uuid::now_v7();

115 let (token, hash, salt, last_eight) = ci::runners::generate_credential(token_id);

116 models::ci::runner_token::ActiveModel {

117 id: Set(token_id),

118 runner_id: Set(None),

119 name: Set("fixture-client".into()),

120 token_hash: Set(hash),

121 token_salt: Set(salt),

122 token_last_eight: Set(last_eight),

123 space_id: Set(Some(space.id)),

124 single_use: Set(true),

125 used_at: Set(None),

126 revoked_at: Set(None),

127 expires_at: Set(Some(now + time::Duration::days(1))),

128 created_at: Set(now),

129 updated_at: Set(now),

130 }

131 .insert(service.db_write())

132 .await?;

133 let mut runner_config = config::runner::RunnerConfig::default();

134 runner_config.server_url = "http://127.0.0.1:43120".into();

135 runner_config.registration_token = token;

136 runner_config.name = "fixture-client".into();

137 runner_config.actions_engine_binary = service.config().ci.actions_engine_binary.clone();

138 runner_config.workspace_root = directory.join("workspaces").to_string_lossy().into_owned();

139 runner_config.actions_node_runtimes = BTreeMap::from([(

140 "node24".into(),

141 std::env::current_dir()?

142 .join("target/node24/bin/node")

143 .to_string_lossy()

144 .into_owned(),

145 )]);

146 runner_config.actions_cache_dir = directory

147 .join("action-cache")

148 .to_string_lossy()

149 .into_owned();

150 let client = runner::client::RunnerClient::new(&runner_config)?;

151 let registration = client

152 .register(&ci_protocol::runner::RegisterRequest {

153 token: runner_config.registration_token.clone(),

154 name: runner_config.name.clone(),

155 labels: vec!["self-hosted".into()],

156 capacity: 4,

157 version: Some("fixture".into()),

158 protocol_versions: vec![2, 1],

159 capabilities: vec!["actions-v2".into(), "docker".into()],

160 })

161 .await?;

162 let client = client.with_credential(registration.credential);

163 let run_id = create(&service, &repo, &space, owner, WORKFLOW).await?;

164 let count = drive(&service, &client, &runner_config, run_id).await?;

165 let result = workflow_run::Entity::find_by_id(run_id)

166 .one(service.db_write())

167 .await?

168 .unwrap();

169 ensure!(

170 result.state == models::ci::workflow_run::WorkflowRunState::Success,

171 "matrix run failed: {:?}",

172 service.run_detail(run_id).await?

173 );

174 ensure!(count == 4, "expected 4 physical jobs, got {count}");

175 let jobs = models::ci::workflow_job::Entity::find()

176 .filter(models::ci::workflow_job::Column::RunId.eq(run_id))

177 .all(service.db_write())

178 .await?;

179 ensure!(

180 jobs.iter()

181 .find(|job| job.job_key == "verify")

182 .and_then(|job| job.outputs.as_ref())

183 .is_some_and(|value| value["value"] == "b")

184 );

185 println!(

186 "PASS: protocol v2, persisted DAG, dynamic matrix, cross-job outputs, terminal step events"

187 );

188 let rerun = service.rerun(run_id, owner).await?;

189 drive(&service, &client, &runner_config, rerun).await?;

190 println!("PASS: snapshot rerun");

191 let rules = EnvironmentRules {

192 reviewers: vec![reviewer],

193 wait_minutes: 0,

194 ..Default::default()

195 };

196 service

197 .upsert_actions_environment(space.id, repo.id, "production", &rules)

198 .await?;

199 let approval_workflow = "on: push\njobs:\n deploy:\n runs-on: self-hosted\n environment: production\n steps:\n - run: echo approved\n";

200 let approved = create(&service, &repo, &space, owner, approval_workflow).await?;

201 service.advance_jobs().await?;

202 let claim = service

203 .claim_job(registration.runner_id, &["self-hosted".into()])

204 .await?;

205 ensure!(claim.is_none(), "unapproved environment job was claimed");

206 let approvals = service.actions_approvals(repo.id, approved).await?;

207 ensure!(approvals.len() == 1);

208 service

209 .review_actions_environment(repo.id, approved, approvals[0].id, reviewer, true, None)

210 .await?;

211 drive(&service, &client, &runner_config, approved).await?;

212 println!("PASS: environment approval gate");

213 server.abort();

214 git_server.abort();

215 Ok(())

216}

217

218async fn create(

219 service: &CiService,

220 repo: &repos::Model,

221 space: &spaces::Model,

222 owner: Uuid,

223 source: &str,

224) -> anyhow::Result<Uuid> {

225 let actions = service

226 .parse_actions_workflow(".github/workflows/fixture.yml", source)

227 .await?;

228 let workflow = LoadedWorkflow {

229 file: actions.file.clone(),

230 name: actions.name().unwrap_or("fixture".into()),

231 definition: actions.projection(),

232 actions: Some(actions),

233 };

234 let seed = RunSeed {

235 event: models::ci::workflow_run::WorkflowEventKind::Push,

236 event_payload: json!({"event_id":Uuid::now_v7()}),

237 commit_sha: "a".repeat(40),

238 git_ref: "refs/heads/main".into(),

239 branch: Some("main".into()),

240 tag: None,

241 base_sha: None,

242 base_ref: None,

243 pr_number: None,

244 trigger_user_id: Some(owner),

245 };

246 Ok(service.create_run(repo, space, &seed, &workflow).await?)

247}

248

249async fn drive(

250 service: &CiService,

251 client: &runner::client::RunnerClient,

252 config: &config::runner::RunnerConfig,

253 run_id: Uuid,

254) -> anyhow::Result<usize> {

255 let mut count = 0;

256 for _ in 0..100 {

257 service.advance_jobs().await?;

258 service.enforce_concurrency().await?;

259 let result = client

260 .claim(&ci_protocol::runner::ClaimRequest { wait_secs: Some(0) })

261 .await?;

262 if let Some(payload) = result.job {

263 let finish = runner::actions::execute(client, config, &payload).await;

264 client.finish(&payload.job_id, &finish).await?;

265 count += 1;

266 }

267 service.advance_jobs().await?;

268 service.finalize_runs().await?;

269 let run = workflow_run::Entity::find_by_id(run_id)

270 .one(service.db_write())

271 .await?

272 .unwrap();

273 if matches!(

274 run.state,

275 models::ci::workflow_run::WorkflowRunState::Success

276 | models::ci::workflow_run::WorkflowRunState::Failure

277 | models::ci::workflow_run::WorkflowRunState::Cancelled

278 ) {

279 return Ok(count);

280 }

281 }

282 anyhow::bail!("fixture run did not reach a terminal state")

283}

284

285async fn seed(service: &CiService) -> anyhow::Result<(repos::Model, spaces::Model, Uuid, Uuid)> {

286 let owner = Uuid::now_v7();

287 let reviewer = Uuid::now_v7();

288 let now = OffsetDateTime::now_utc();

289 for (id, name) in [

290 (owner, format!("fixture-{}", owner.simple())),

291 (reviewer, format!("fixture-{}", reviewer.simple())),

292 ] {

293 models::users::user::ActiveModel {

294 uid: Set(id),

295 username: Set(name.clone()),

296 lower_name: Set(name.clone()),

297 display_name: Set(name),

298 avatar_url: Set(String::new()),

299 is_active: Set(true),

300 is_restricted: Set(false),

301 prohibit_login: Set(false),

302 visibility: Set(Default::default()),

303 version: Set(1),

304 created_at: Set(now),

305 updated_at: Set(now),

306 ..Default::default()

307 }

308 .insert(service.db_write())

309 .await?;

310 }

311 let space_id = Uuid::now_v7();

312 let name = format!("fixture-{}", space_id.simple());

313 let space = spaces::ActiveModel {

314 id: Set(space_id),

315 lower_name: Set(name.clone()),

316 name: Set(name),

317 owner_id: Set(owner),

318 is_active: Set(true),

319 allow_join: Set(false),

320 allow_invite: Set(false),

321 nums_members: Set(1),

322 allow_join_level: Set(Default::default()),

323 member_must_enable_2fa: Set(false),

324 created_at: Set(now),

325 updated_at: Set(now),

326 ..Default::default()

327 }

328 .insert(service.db_write())

329 .await?;

330 let dir = Uuid::now_v7();

331 models::dir::dirs::ActiveModel {

332 id: Set(dir),

333 space_id: Set(Some(space.id)),

334 name: Set("repos".into()),

335 kind: Set(Default::default()),

336 position: Set(0),

337 created_by_uid: Set(owner),

338 created_at: Set(now),

339 updated_at: Set(now),

340 ..Default::default()

341 }

342 .insert(service.db_write())

343 .await?;

344 let id = Uuid::now_v7();

345 let name = format!("fixture-{}", id.simple());

346 let repo = repos::ActiveModel {

347 id: Set(id),

348 dir_id: Set(dir),

349 lower_name: Set(name.clone()),

350 name: Set(name),

351 default_branch: Set("main".into()),

352 nums_watch: Set(0),

353 nums_star: Set(0),

354 nums_pr: Set(0),

355 nums_commits: Set(0),

356 nums_branches: Set(0),

357 nums_tags: Set(0),

358 enable_lfs: Set(false),

359 size: Set(0),

360 git_size: Set(0),

361 lfs_size: Set(0),

362 created_at: Set(now),

363 updated_at: Set(now),

364 ..Default::default()

365 }

366 .insert(service.db_write())

367 .await?;

368 let directory = PathBuf::from(&service.config().git.repo_root).join(repo.id.to_string());

369 tokio::fs::create_dir_all(&directory).await?;

370 let status = std::process::Command::new("git")

371 .args(["init", "--bare"])

372 .arg(directory)

373 .status()?;

374 ensure!(status.success());

375 Ok((repo, space, owner, reviewer))

376}

Added lib/ci/src/handlers/actions.rs +242 −0 View lib/ci/src/handlers/actions.rs

@@ -0,0 +1,242 @@

1use axum::{

2 Json,

3 extract::{Path, State},

4 response::Redirect,

5};

6use export::rest::AppResponseData;

7use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set};

8use session_layer::{Session, SessionUser};

9use uuid::Uuid;

10

11use crate::{

12 CiService,

13 actions_environments::{ApprovalDto, EnvironmentRules, RepoEnvironmentDto},

14 errors::{ApiError, CiError},

15};

16

17#[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)]

18pub struct ReviewCmd {

19 pub approved: bool,

20 pub comment: Option<String>,

21}

22

23#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]

24pub struct VariableDto {

25 pub name: String,

26 pub value: String,

27}

28

29#[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)]

30pub struct VariableCmd {

31 pub value: String,

32}

33

34#[utoipa::path(post,path="/repos/{space_name}/{repo_name}/pipelines/dispatches",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path)),request_body=crate::actions_events::DispatchCmd,responses((status=200,body=AppResponseData<Vec<Uuid>>)))]

35pub async fn dispatch(

36 State(service): State<CiService>,

37 session: Session,

38 Path((space, repo)): Path<(String, String)>,

39 Json(request): Json<crate::actions_events::DispatchCmd>,

40) -> Result<AppResponseData<Vec<Uuid>>, ApiError> {

41 let (repo, space) = service.resolve_repo(&space, &repo).await?;

42 let caller = service.require_write(&session, &repo).await?;

43 Ok(AppResponseData::ok(

44 service

45 .repository_dispatch(&repo, &space, caller, &request)

46 .await?,

47 ))

48}

49

50#[utoipa::path(get,path="/repos/{space_name}/{repo_name}/environments",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path)),responses((status=200,body=AppResponseData<Vec<RepoEnvironmentDto>>)))]

51pub async fn environments(

52 State(service): State<CiService>,

53 session: Session,

54 Path((space, repo)): Path<(String, String)>,

55) -> Result<AppResponseData<Vec<RepoEnvironmentDto>>, ApiError> {

56 let (repo, _) = service.resolve_repo(&space, &repo).await?;

57 service.require_read(&session, &repo).await?;

58 Ok(AppResponseData::ok(

59 service.repo_actions_environments(repo.id).await?,

60 ))

61}

62

63#[utoipa::path(put,path="/repos/{space_name}/{repo_name}/environments/{name}",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("name"=String,Path)),request_body=EnvironmentRules,responses((status=200,body=AppResponseData<RepoEnvironmentDto>)))]

64pub async fn upsert_environment(

65 State(service): State<CiService>,

66 session: Session,

67 Path((space, repo, name)): Path<(String, String, String)>,

68 Json(rules): Json<EnvironmentRules>,

69) -> Result<AppResponseData<RepoEnvironmentDto>, ApiError> {

70 let (repo, space) = service.resolve_repo(&space, &repo).await?;

71 let caller = session.uid().ok_or(CiError::Unauthorized)?;

72 let context = service.git_context(&repo).await?;

73 git::share::access::authorize_repo_access(

74 &git::service::GitService::new(service.state.clone()),

75 caller,

76 &context,

77 true,

78 true,

79 )

80 .await

81 .map_err(CiError::from)?;

82 Ok(AppResponseData::ok(

83 service

84 .upsert_actions_environment(space.id, repo.id, &name, &rules)

85 .await?,

86 ))

87}

88

89#[utoipa::path(get,path="/repos/{space_name}/{repo_name}/pipelines/{run_id}/approvals",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("run_id"=Uuid,Path)),responses((status=200,body=AppResponseData<Vec<ApprovalDto>>)))]

90pub async fn approvals(

91 State(service): State<CiService>,

92 session: Session,

93 Path((space, repo, run)): Path<(String, String, Uuid)>,

94) -> Result<AppResponseData<Vec<ApprovalDto>>, ApiError> {

95 let (repo, _) = service.resolve_repo(&space, &repo).await?;

96 service.require_read(&session, &repo).await?;

97 Ok(AppResponseData::ok(

98 service.actions_approvals(repo.id, run).await?,

99 ))

100}

101

102#[utoipa::path(post,path="/repos/{space_name}/{repo_name}/pipelines/{run_id}/approvals/{review_id}",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("run_id"=Uuid,Path),("review_id"=Uuid,Path)),request_body=ReviewCmd,responses((status=200,body=AppResponseData<Uuid>)))]

103pub async fn review(

104 State(service): State<CiService>,

105 session: Session,

106 Path((space, repo, run, review)): Path<(String, String, Uuid, Uuid)>,

107 Json(request): Json<ReviewCmd>,

108) -> Result<AppResponseData<Uuid>, ApiError> {

109 let (repo, _) = service.resolve_repo(&space, &repo).await?;

110 let caller = service

111 .require_read(&session, &repo)

112 .await?

113 .ok_or(CiError::Unauthorized)?;

114 service

115 .review_actions_environment(

116 repo.id,

117 run,

118 review,

119 caller,

120 request.approved,

121 request.comment,

122 )

123 .await?;

124 Ok(AppResponseData::ok(review))

125}

126

127#[utoipa::path(get,path="/repos/{space_name}/{repo_name}/variables",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path)),responses((status=200,body=AppResponseData<Vec<VariableDto>>)))]

128pub async fn variables(

129 State(service): State<CiService>,

130 session: Session,

131 Path((space, repo)): Path<(String, String)>,

132) -> Result<AppResponseData<Vec<VariableDto>>, ApiError> {

133 let (repo, _) = service.resolve_repo(&space, &repo).await?;

134 service.require_read(&session, &repo).await?;

135 Ok(AppResponseData::ok(

136 service

137 .actions_variables(repo.id)

138 .await?

139 .into_iter()

140 .map(|(name, value)| VariableDto { name, value })

141 .collect(),

142 ))

143}

144

145#[utoipa::path(put,path="/repos/{space_name}/{repo_name}/variables/{name}",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("name"=String,Path)),request_body=VariableCmd,responses((status=200,body=AppResponseData<VariableDto>)))]

146pub async fn upsert_variable(

147 State(service): State<CiService>,

148 session: Session,

149 Path((space, repo, name)): Path<(String, String, String)>,

150 Json(request): Json<VariableCmd>,

151) -> Result<AppResponseData<VariableDto>, ApiError> {

152 let (repo, _) = service.resolve_repo(&space, &repo).await?;

153 service.require_write(&session, &repo).await?;

154 let name = name.trim().to_uppercase();

155 if name.is_empty()

156 || name.len() > 128

157 || !name

158 .chars()

159 .all(|ch| ch.is_ascii_alphanumeric() || ch == '_')

160 || request.value.len() > 48 * 1024

161 {

162 return Err(CiError::InvalidArgument("invalid variable name or value".into()).into());

163 }

164 let row = models::ci::variable::Entity::find()

165 .filter(models::ci::variable::Column::RepoId.eq(repo.id))

166 .filter(models::ci::variable::Column::Name.eq(&name))

167 .one(service.db_write())

168 .await

169 .map_err(CiError::from)?;

170 if let Some(row) = row {

171 let mut active: models::ci::variable::ActiveModel = row.into();

172 active.value = Set(request.value.clone());

173 active.updated_at = Set(time::OffsetDateTime::now_utc());

174 active

175 .update(service.db_write())

176 .await

177 .map_err(CiError::from)?;

178 } else {

179 models::ci::variable::ActiveModel {

180 id: Set(Uuid::now_v7()),

181 repo_id: Set(repo.id),

182 name: Set(name.clone()),

183 value: Set(request.value.clone()),

184 updated_at: Set(time::OffsetDateTime::now_utc()),

185 }

186 .insert(service.db_write())

187 .await

188 .map_err(CiError::from)?;

189 }

190 Ok(AppResponseData::ok(VariableDto {

191 name,

192 value: request.value,

193 }))

194}

195

196#[utoipa::path(delete,path="/repos/{space_name}/{repo_name}/variables/{name}",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("name"=String,Path)),responses((status=200,body=AppResponseData<String>)))]

197pub async fn delete_variable(

198 State(service): State<CiService>,

199 session: Session,

200 Path((space, repo, name)): Path<(String, String, String)>,

201) -> Result<AppResponseData<String>, ApiError> {

202 let (repo, _) = service.resolve_repo(&space, &repo).await?;

203 service.require_write(&session, &repo).await?;

204 models::ci::variable::Entity::delete_many()

205 .filter(models::ci::variable::Column::RepoId.eq(repo.id))

206 .filter(models::ci::variable::Column::Name.eq(name.to_uppercase()))

207 .exec(service.db_write())

208 .await

209 .map_err(CiError::from)?;

210 Ok(AppResponseData::ok(name))

211}

212

213#[utoipa::path(get,path="/repos/{space_name}/{repo_name}/pipelines/actions-artifacts/{artifact_id}/download",tag="pipelines",params(("space_name"=String,Path),("repo_name"=String,Path),("artifact_id"=i64,Path)),responses((status=307,description="artifact download")))]

214pub async fn download_actions_artifact(

215 State(service): State<CiService>,

216 session: Session,

217 Path((space, repo, id)): Path<(String, String, i64)>,

218) -> Result<Redirect, ApiError> {

219 let (repo, _) = service.resolve_repo(&space, &repo).await?;

220 service.require_read(&session, &repo).await?;

221 let identity = models::ci::actions_identity::Entity::find_by_id(id)

222 .one(service.db_read())

223 .await

224 .map_err(CiError::from)?

225 .ok_or(CiError::NotFound)?;

226 let artifact = models::ci::workflow_artifact::Entity::find_by_id(identity.entity_id)

227 .one(service.db_read())

228 .await

229 .map_err(CiError::from)?

230 .ok_or(CiError::NotFound)?;

231 let run = models::ci::workflow_run::Entity::find_by_id(artifact.run_id)

232 .one(service.db_read())

233 .await

234 .map_err(CiError::from)?

235 .ok_or(CiError::NotFound)?;

236 if run.repo_id != repo.id {

237 return Err(ApiError::not_found());

238 }

239 Ok(Redirect::temporary(

240 &service.artifact_download_url(artifact.id).await?,

241 ))

242}

Modified lib/ci/src/handlers/mod.rs +1 −0 View lib/ci/src/handlers/mod.rs

@@ -1,3 +1,4 @@

1pub mod actions;

12pub mod artifacts;

23pub mod logs;

34pub mod runner_protocol;

Modified lib/ci/src/handlers/runner_protocol.rs +31 −2 View lib/ci/src/handlers/runner_protocol.rs

@@ -25,7 +25,7 @@

2525 State(service): State<CiService>,

2626 Json(request): Json<RegisterRequest>,

2727) -> Result<AppResponseData<RegisterResponse>, ApiError> {

28 let response = service

28 let mut response = service

2929 .register_runner(

3030 &request.token,

3131 &request.name,

@@ -34,6 +34,13 @@

3434 request.version,

3535 )

3636 .await?;

37 response.protocol_version = service

38 .configure_runner_protocol(

39 response.runner_id,

40 &request.protocol_versions,

41 &request.capabilities,

42 )

43 .await?;

3744 Ok(AppResponseData::ok(response))

3845}

3946

@@ -48,9 +55,18 @@

4855pub async fn heartbeat(

4956 State(service): State<CiService>,

5057 auth: RunnerAuth,

51 Json(_request): Json<HeartbeatRequest>,

58 Json(request): Json<HeartbeatRequest>,

5259) -> Result<AppResponseData<HeartbeatResponse>, ApiError> {

5360 let cancel = service.runner_heartbeat(auth.runner_id).await?;

61 if !request.protocol_versions.is_empty() {

62 service

63 .configure_runner_protocol(

64 auth.runner_id,

65 &request.protocol_versions,

66 &request.capabilities,

67 )

68 .await?;

69 }

5470 Ok(AppResponseData::ok(HeartbeatResponse {

5571 cancel,

5672 next_heartbeat_secs: service.config().ci.heartbeat_interval_secs,

@@ -175,6 +191,19 @@

175191 Ok(AppResponseData::ok(run_id))

176192}

177193

194#[utoipa::path(post, path="/runner/jobs/{job_id}/events", tag="runner", params(("job_id"=Uuid,Path)), request_body=ci_protocol::actions::EngineEvent, responses((status=200,body=AppResponseData<Uuid>)))]

195pub async fn event(

196 State(service): State<CiService>,

197 auth: RunnerAuth,

198 Path(job_id): Path<Uuid>,

199 Json(request): Json<ci_protocol::actions::EngineEvent>,

200) -> Result<AppResponseData<Uuid>, ApiError> {

201 service

202 .report_actions_event(job_id, auth.runner_id, &request)

203 .await?;

204 Ok(AppResponseData::ok(job_id))

205}

206

178207#[utoipa::path(

179208 post,

180209 path = "/runner/jobs/{job_id}/renew",

Modified lib/ci/src/handlers/runs.rs +1 −1 View lib/ci/src/handlers/runs.rs

@@ -136,7 +136,7 @@

136136 get,

137137 path = "/repos/{space_name}/{repo_name}/pipelines/workflows",

138138 tag = "pipelines",

139 description = "读取指定修订版本下 `.workflows/` 中的工作流清单与解析状态。",

139 description = "读取指定修订版本下 `.github/workflows/` 中的工作流清单与解析状态。",

140140 params(RepoPath, WorkflowListQuery),

141141 responses((status = 200, description = "workflow list", body = AppResponseData<WorkflowSummaryList>))

142142)]

Added lib/ci/src/actions.rs +211 −0 View lib/ci/src/actions.rs

@@ -0,0 +1,211 @@

1use ci_protocol::actions::{ActionsWorkflow, EngineEvent};

2use serde_json::{Value, json};

3use tokio::io::AsyncWriteExt;

4use tokio::process::Command;

5

6use crate::errors::{CiError, CiResult};

7use crate::service::CiService;

8

9pub async fn identity<C: sea_orm::ConnectionTrait>(

10 db: &C,

11 entity_id: uuid::Uuid,

12 kind: &str,

13) -> CiResult<i64> {

14 use sea_orm::{ActiveModelTrait, Set};

15 let row = models::ci::actions_identity::ActiveModel {

16 entity_id: Set(entity_id),

17 kind: Set(kind.to_owned()),

18 ..Default::default()

19 }

20 .insert(db)

21 .await?;

22 Ok(row.id)

23}

24

25impl CiService {

26 pub async fn report_actions_event(

27 &self,

28 job_id: uuid::Uuid,

29 runner_id: uuid::Uuid,

30 event: &EngineEvent,

31 ) -> CiResult<()> {

32 use models::ci::workflow_step::WorkflowStepState as State;

33 use models::ci::{workflow_job, workflow_step};

34 use sea_orm::{

35 ActiveModelTrait, ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, Set,

36 };

37 let job = workflow_job::Entity::find_by_id(job_id)

38 .one(self.db_write())

39 .await?

40 .ok_or(CiError::NotFound)?;

41 if job.runner_id != Some(runner_id) || job.state.is_terminal() {

42 return Err(CiError::Forbidden);

43 }

44 if event.lease_token != job.lease_token {

45 return Err(CiError::Conflict("job lease was superseded".into()));

46 }

47 let key = event.data["step_id"].as_str().unwrap_or("--job");

48 let stage = event.data["stage"].as_str().unwrap_or("main");

49 let existing = workflow_step::Entity::find()

50 .filter(workflow_step::Column::JobId.eq(job_id))

51 .filter(workflow_step::Column::StepKey.eq(key))

52 .filter(workflow_step::Column::Stage.eq(stage))

53 .one(self.db_write())

54 .await?;

55 let is_new = existing.is_none();

56 let now = time::OffsetDateTime::now_utc();

57 let mut active = match existing {

58 Some(row) => row.into(),

59 None => workflow_step::ActiveModel {

60 id: Set(uuid::Uuid::now_v7()),

61 job_id: Set(job_id),

62 step_index: Set(workflow_step::Entity::find()

63 .filter(workflow_step::Column::JobId.eq(job_id))

64 .count(self.db_write())

65 .await? as i32),

66 step_key: Set(Some(key.into())),

67 stage: Set(Some(stage.into())),

68 name: Set(event.data["name"].as_str().unwrap_or(key).to_owned()),

69 script: Set(String::new()),

70 state: Set(State::Pending),

71 created_at: Set(now),

72 updated_at: Set(now),

73 ..Default::default()

74 },

75 };

76 match event.event.as_str() {

77 "step_started" => {

78 active.state = Set(State::Running);

79 active.started_at = Set(Some(now));

80 }

81 "step_finished" => {

82 let conclusion = event.data["conclusion"].as_str().unwrap_or("failure");

83 active.state = Set(match conclusion {

84 "success" => State::Success,

85 "skipped" => State::Skipped,

86 "cancelled" => State::Cancelled,

87 _ => State::Failure,

88 });

89 active.outcome = Set(event.data["outcome"].as_str().map(str::to_owned));

90 active.conclusion = Set(Some(conclusion.into()));

91 active.outputs = Set(Some(

92 event.data.get("outputs").cloned().unwrap_or(json!({})),

93 ));

94 active.finished_at = Set(Some(now));

95 active.exit_code = Set((conclusion != "skipped")

96 .then_some(if conclusion == "success" { 0 } else { 1 }));

97 }

98 "summary" => {

99 let mut summary = active

100 .summary

101 .clone()

102 .take()

103 .unwrap_or(None)

104 .unwrap_or_default();

105 summary.push_str(event.data["text"].as_str().unwrap_or_default());

106 if summary.len() > 1024 * 1024 {

107 return Err(CiError::InvalidArgument(

108 "step summary exceeds 1 MiB".into(),

109 ));

110 }

111 active.summary = Set(Some(summary));

112 }

113 "annotation" => {

114 let mut annotations: Vec<Value> = active

115 .annotations

116 .clone()

117 .take()

118 .unwrap_or(None)

119 .and_then(|v| serde_json::from_value(v).ok())

120 .unwrap_or_default();

121 if annotations.len() < 1000 {

122 annotations.push(event.data.clone());

123 }

124 active.annotations = Set(Some(json!(annotations)));

125 }

126 _ => {

127 return Err(CiError::InvalidArgument(format!(

128 "unknown Actions event `{}`",

129 event.event

130 )));

131 }

132 }

133 if let Some(name) = event.data["name"].as_str() {

134 active.name = Set(name.into());

135 }

136 active.updated_at = Set(now);

137 if is_new {

138 active.insert(self.db_write()).await?;

139 } else {

140 active.update(self.db_write()).await?;

141 }

142 Ok(())

143 }

144

145 pub async fn actions_engine(&self, request: Value) -> CiResult<Value> {

146 let config = &self.config().ci;

147 let mut child = Command::new(&config.actions_engine_binary)

148 .stdin(std::process::Stdio::piped())

149 .stdout(std::process::Stdio::piped())

150 .stderr(std::process::Stdio::piped())

151 .kill_on_drop(true)

152 .spawn()

153 .map_err(|error| {

154 CiError::InvalidArgument(format!(

155 "Actions engine `{}`: {error}",

156 config.actions_engine_binary

157 ))

158 })?;

159 let mut input = child

160 .stdin

161 .take()

162 .ok_or_else(|| CiError::Internal("engine stdin missing".into()))?;

163 input

164 .write_all(&serde_json::to_vec(&request)?)

165 .await

166 .map_err(|error| CiError::Internal(error.to_string()))?;

167 input

168 .write_all(b"\n")

169 .await

170 .map_err(|error| CiError::Internal(error.to_string()))?;

171 drop(input);

172 let output = tokio::time::timeout(

173 std::time::Duration::from_secs(config.actions_engine_timeout_secs.max(1)),

174 child.wait_with_output(),

175 )

176 .await

177 .map_err(|_| CiError::InvalidArgument("Actions engine request timed out".into()))?

178 .map_err(|error| CiError::Internal(error.to_string()))?;

179 let result = output

180 .stdout

181 .split(|byte| *byte == b'\n')

182 .filter_map(|line| serde_json::from_slice::<EngineEvent>(line).ok())

183 .find(|message| message.event == "result")

184 .ok_or_else(|| {

185 CiError::InvalidArgument(format!(

186 "Actions engine exited {}: {}",

187 output.status,

188 String::from_utf8_lossy(&output.stderr)

189 ))

190 })?;

191 if let Some(error) = result.error {

192 return Err(CiError::InvalidArgument(error));

193 }

194 Ok(result.data)

195 }

196

197 pub async fn parse_actions_workflow(

198 &self,

199 file: &str,

200 source: &str,

201 ) -> CiResult<ActionsWorkflow> {

202 let document = self

203 .actions_engine(json!({"op":"parse", "file":file, "source":source}))

204 .await?;

205 Ok(ActionsWorkflow {

206 file: file.to_owned(),

207 source: source.to_owned(),

208 document,

209 })

210 }

211}

Added lib/ci/src/actions_environments.rs +350 −0 View lib/ci/src/actions_environments.rs

@@ -0,0 +1,350 @@

1use models::ci::{ci_environment, environment_review, workflow_job, workflow_run};

2use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set};

3use serde::{Deserialize, Serialize};

4use serde_json::json;

5use time::OffsetDateTime;

6use uuid::Uuid;

7

8use crate::{

9 CiService,

10 errors::{CiError, CiResult},

11};

12

13fn default_prevent_self_review() -> bool {

14 true

15}

16

17#[derive(Debug, Clone, Serialize, Deserialize, utoipa::ToSchema)]

18#[serde(default)]

19pub struct EnvironmentRules {

20 pub reviewers: Vec<Uuid>,

21 pub wait_minutes: u32,

22 #[serde(default = "default_prevent_self_review")]

23 pub prevent_self_review: bool,

24 pub branches: Vec<String>,

25 pub tags: Vec<String>,

26}

27

28impl Default for EnvironmentRules {

29 fn default() -> Self {

30 Self {

31 reviewers: Vec::new(),

32 wait_minutes: 0,

33 prevent_self_review: true,

34 branches: Vec::new(),

35 tags: Vec::new(),

36 }

37 }

38}

39

40#[derive(Debug, Clone, Serialize, utoipa::ToSchema)]

41pub struct RepoEnvironmentDto {

42 pub id: Uuid,

43 pub name: String,

44 pub rules: EnvironmentRules,

45}

46

47#[derive(Debug, Clone, Serialize, utoipa::ToSchema)]

48pub struct ApprovalDto {

49 pub id: Uuid,

50 pub job_id: Uuid,

51 pub environment: String,

52 pub state: String,

53 pub eligible_at: OffsetDateTime,

54 pub reviewer_id: Option<Uuid>,

55 pub reviewers: Vec<Uuid>,

56 pub prevent_self_review: bool,

57}

58

59impl CiService {

60 pub async fn find_actions_environment(

61 &self,

62 space_id: Uuid,

63 repo_id: Uuid,

64 name: &str,

65 ) -> CiResult<Option<ci_environment::Model>> {

66 let rows = ci_environment::Entity::find()

67 .filter(ci_environment::Column::SpaceId.eq(space_id))

68 .all(self.db_write())

69 .await?;

70 let lowered = name.to_lowercase();

71 Ok(rows

72 .iter()

73 .find(|row| row.repo_id == Some(repo_id) && row.name.to_lowercase() == lowered)

74 .cloned()

75 .or_else(|| {

76 rows.into_iter()

77 .find(|row| row.repo_id.is_none() && row.name.to_lowercase() == lowered)

78 }))

79 }

80

81 pub async fn upsert_actions_environment(

82 &self,

83 space_id: Uuid,

84 repo_id: Uuid,

85 name: &str,

86 rules: &EnvironmentRules,

87 ) -> CiResult<RepoEnvironmentDto> {

88 let name = name.trim().to_lowercase();

89 if name.is_empty()

90 || name.len() > 255

91 || rules.wait_minutes > 43200

92 || rules.reviewers.len() > 6

93 {

94 return Err(CiError::InvalidArgument(

95 "invalid environment name, wait time or reviewers".into(),

96 ));

97 }

98 for reviewer in &rules.reviewers {

99 if models::users::user::Entity::find_by_id(*reviewer)

100 .one(self.db_read())

101 .await?

102 .is_none()

103 {

104 return Err(CiError::InvalidArgument(format!(

105 "reviewer `{reviewer}` was not found"

106 )));

107 }

108 }

109 let existing = ci_environment::Entity::find()

110 .filter(ci_environment::Column::RepoId.eq(repo_id))

111 .filter(ci_environment::Column::Name.eq(&name))

112 .one(self.db_write())

113 .await?;

114 let now = OffsetDateTime::now_utc();

115 let row = if let Some(row) = existing {

116 let mut active: ci_environment::ActiveModel = row.into();

117 active.rules = Set(Some(json!(rules)));

118 active.updated_at = Set(now);

119 active.update(self.db_write()).await?

120 } else {

121 ci_environment::ActiveModel {

122 id: Set(Uuid::now_v7()),

123 space_id: Set(space_id),

124 repo_id: Set(Some(repo_id)),

125 name: Set(name),

126 description: Set(None),

127 required_labels: Set(json!([])),

128 rules: Set(Some(json!(rules))),

129 created_at: Set(now),

130 updated_at: Set(now),

131 }

132 .insert(self.db_write())

133 .await?

134 };

135 Ok(RepoEnvironmentDto {

136 id: row.id,

137 name: row.name,

138 rules: rules.clone(),

139 })

140 }

141

142 pub async fn repo_actions_environments(

143 &self,

144 repo_id: Uuid,

145 ) -> CiResult<Vec<RepoEnvironmentDto>> {

146 let rows = ci_environment::Entity::find()

147 .filter(ci_environment::Column::RepoId.eq(repo_id))

148 .all(self.db_read())

149 .await?;

150 Ok(rows

151 .into_iter()

152 .map(|row| RepoEnvironmentDto {

153 id: row.id,

154 name: row.name,

155 rules: row

156 .rules

157 .and_then(|value| serde_json::from_value(value).ok())

158 .unwrap_or_default(),

159 })

160 .collect())

161 }

162

163 pub async fn actions_environment_available(&self, job: &workflow_job::Model) -> CiResult<bool> {

164 let Some(name) = job.environment.as_deref() else {

165 return Ok(true);

166 };

167 let Some(environment) = self

168 .find_actions_environment(job.space_id, job.repo_id, name)

169 .await?

170 else {

171 return Ok(true);

172 };

173 let rules: EnvironmentRules = environment

174 .rules

175 .clone()

176 .and_then(|value| serde_json::from_value(value).ok())

177 .unwrap_or_default();

178 let run = workflow_run::Entity::find_by_id(job.run_id)

179 .one(self.db_write())

180 .await?

181 .ok_or(CiError::NotFound)?;

182 let patterns = if run.tag.is_some() {

183 &rules.tags

184 } else {

185 &rules.branches

186 };

187 if !patterns.is_empty() {

188 let ref_name = run

189 .tag

190 .as_deref()

191 .or(run.branch.as_deref())

192 .unwrap_or_default();

193 let matched = self

194 .actions_engine(json!({"op":"match","patterns":patterns,"paths":[ref_name]}))

195 .await?

196 .as_bool()

197 .unwrap_or(false);

198 if !matched {

199 let mut active: workflow_job::ActiveModel = job.clone().into();

200 active.state = Set(workflow_job::WorkflowJobState::Skipped);

201 active.outcome = Set(Some("skipped".into()));

202 active.error = Set(Some(format!(

203 "ref `{ref_name}` does not match environment deployment rules"

204 )));

205 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

206 active.update(self.db_write()).await?;

207 return Ok(false);

208 }

209 }

210 if rules.reviewers.is_empty() && rules.wait_minutes == 0 {

211 return Ok(true);

212 }

213 let review = environment_review::Entity::find()

214 .filter(environment_review::Column::JobId.eq(job.id))

215 .one(self.db_write())

216 .await?;

217 let review = if let Some(review) = review {

218 review

219 } else {

220 let now = OffsetDateTime::now_utc();

221 environment_review::ActiveModel {

222 id: Set(Uuid::now_v7()),

223 job_id: Set(job.id),

224 repo_id: Set(job.repo_id),

225 environment_id: Set(environment.id),

226 state: Set(if rules.reviewers.is_empty() {

227 "approved"

228 } else {

229 "pending"

230 }

231 .into()),

232 reviewer_id: Set(None),

233 comment: Set(None),

234 requested_at: Set(now),

235 eligible_at: Set(now + time::Duration::minutes(i64::from(rules.wait_minutes))),

236 updated_at: Set(now),

237 }

238 .insert(self.db_write())

239 .await?

240 };

241 if review.state == "rejected" {

242 let mut active: workflow_job::ActiveModel = job.clone().into();

243 active.state = Set(workflow_job::WorkflowJobState::Failure);

244 active.outcome = Set(Some("failure".into()));

245 active.error = Set(Some("environment deployment was rejected".into()));

246 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

247 active.update(self.db_write()).await?;

248 return Ok(false);

249 }

250 Ok(review.state == "approved" && review.eligible_at <= OffsetDateTime::now_utc())

251 }

252

253 pub async fn actions_approvals(

254 &self,

255 repo_id: Uuid,

256 run_id: Uuid,

257 ) -> CiResult<Vec<ApprovalDto>> {

258 let jobs = workflow_job::Entity::find()

259 .filter(workflow_job::Column::RepoId.eq(repo_id))

260 .filter(workflow_job::Column::RunId.eq(run_id))

261 .all(self.db_read())

262 .await?;

263 let mut result = Vec::new();

264 for job in jobs {

265 if let Some(review) = environment_review::Entity::find()

266 .filter(environment_review::Column::JobId.eq(job.id))

267 .one(self.db_read())

268 .await?

269 {

270 let environment = ci_environment::Entity::find_by_id(review.environment_id)

271 .one(self.db_read())

272 .await?

273 .ok_or(CiError::NotFound)?;

274 let rules: EnvironmentRules = environment

275 .rules

276 .and_then(|value| serde_json::from_value(value).ok())

277 .unwrap_or_default();

278 result.push(ApprovalDto {

279 id: review.id,

280 job_id: job.id,

281 environment: environment.name,

282 state: review.state,

283 eligible_at: review.eligible_at,

284 reviewer_id: review.reviewer_id,

285 reviewers: rules.reviewers,

286 prevent_self_review: rules.prevent_self_review,

287 });

288 }

289 }

290 Ok(result)

291 }

292

293 pub async fn review_actions_environment(

294 &self,

295 repo_id: Uuid,

296 run_id: Uuid,

297 review_id: Uuid,

298 caller: Uuid,

299 approved: bool,

300 comment: Option<String>,

301 ) -> CiResult<()> {

302 use sea_orm::{QuerySelect, TransactionTrait};

303 let txn = self.db_write().begin().await?;

304 let review = environment_review::Entity::find_by_id(review_id)

305 .lock_exclusive()

306 .one(&txn)

307 .await?

308 .ok_or(CiError::NotFound)?;

309 let job = workflow_job::Entity::find_by_id(review.job_id)

310 .one(&txn)

311 .await?

312 .ok_or(CiError::NotFound)?;

313 if review.repo_id != repo_id || job.run_id != run_id {

314 return Err(CiError::NotFound);

315 }

316 if review.state != "pending" || job.state.is_terminal() {

317 return Err(CiError::Conflict(

318 "environment review is already finished".into(),

319 ));

320 }

321 let env = ci_environment::Entity::find_by_id(review.environment_id)

322 .one(&txn)

323 .await?

324 .ok_or(CiError::NotFound)?;

325 let rules: EnvironmentRules = env

326 .rules

327 .and_then(|value| serde_json::from_value(value).ok())

328 .unwrap_or_default();

329 if !rules.reviewers.contains(&caller) {

330 return Err(CiError::Forbidden);

331 }

332 let run = workflow_run::Entity::find_by_id(run_id)

333 .one(&txn)

334 .await?

335 .ok_or(CiError::NotFound)?;

336 if rules.prevent_self_review && run.trigger_user_id == Some(caller) {

337 return Err(CiError::InvalidArgument(

338 "self review is disabled for this environment".into(),

339 ));

340 }

341 let mut active: environment_review::ActiveModel = review.into();

342 active.state = Set(if approved { "approved" } else { "rejected" }.into());

343 active.reviewer_id = Set(Some(caller));

344 active.comment = Set(comment);

345 active.updated_at = Set(OffsetDateTime::now_utc());

346 active.update(&txn).await?;

347 txn.commit().await?;

348 Ok(())

349 }

350}

Added lib/ci/src/actions_events.rs +359 −0 View lib/ci/src/actions_events.rs

@@ -0,0 +1,359 @@

1use ci_protocol::events::{CiEvent, TriggerEvent};

2use models::ci::workflow_run::WorkflowEventKind;

3use models::ci::{actions_schedule, workflow_run};

4use sea_orm::{

5 ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, QuerySelect, Set, TransactionTrait,

6};

7use serde_json::{Value, json};

8use time::OffsetDateTime;

9use uuid::Uuid;

10

11use crate::{

12 CiService, LoadedWorkflow,

13 errors::{CiError, CiResult},

14 runs::RunSeed,

15};

16

17#[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)]

18pub struct DispatchCmd {

19 pub event_type: String,

20 #[serde(default)]

21 pub client_payload: Value,

22}

23

24impl CiService {

25 pub async fn repository_dispatch(

26 &self,

27 repo: &models::repo::repos::Model,

28 space: &models::space::spaces::Model,

29 caller: Uuid,

30 command: &DispatchCmd,

31 ) -> CiResult<Vec<Uuid>> {

32 self.ensure_enabled()?;

33 if command.event_type.trim().is_empty()

34 || command.event_type.len() > 100

35 || serde_json::to_vec(&command.client_payload)?.len() > 65536

36 {

37 return Err(CiError::InvalidArgument(

38 "invalid repository_dispatch event type or payload".into(),

39 ));

40 }

41 let event = TriggerEvent {

42 id: Uuid::now_v7(),

43 repo_id: repo.id,

44 space_id: space.id,

45 actor_id: Some(caller),

46 event: "repository_dispatch".into(),

47 payload: json!({"action":command.event_type,"client_payload":command.client_payload}),

48 };

49 self.handle_trigger_event(&event).await

50 }

51

52 pub async fn handle_trigger_event(&self, event: &TriggerEvent) -> CiResult<Vec<Uuid>> {

53 let repo = self.repo_by_id(event.repo_id).await?;

54 let space = self.space_of_repo(&repo).await?;

55 if space.id != event.space_id {

56 return Err(CiError::InvalidArgument(

57 "event space does not match repository".into(),

58 ));

59 }

60 let git = git::service::GitService::new(self.state.clone());

61 let sha = git

62 .rev_parse(

63 space.lower_name.clone(),

64 repo.name.clone(),

65 repo.default_branch.clone(),

66 )

67 .await?;

68 let workflows = self.load_valid_workflows(&space, &repo.name, &sha).await?;

69 let kind = match event.event.as_str() {

70 "repository_dispatch" => WorkflowEventKind::RepositoryDispatch,

71 "workflow_run" => WorkflowEventKind::WorkflowRun,

72 _ => return Err(CiError::InvalidArgument("unsupported event source".into())),

73 };

74 let mut payload = event.payload.clone();

75 payload["event_id"] = json!(event.id);

76 let seed = RunSeed {

77 event: kind,

78 event_payload: payload,

79 commit_sha: sha,

80 git_ref: format!("refs/heads/{}", repo.default_branch),

81 branch: event.payload["workflow_run"]["head_branch"]

82 .as_str()

83 .map(str::to_owned)

84 .or_else(|| Some(repo.default_branch.clone())),

85 tag: None,

86 base_sha: None,

87 base_ref: None,

88 pr_number: None,

89 trigger_user_id: event.actor_id,

90 };

91 let mut created = Vec::new();

92 for workflow in workflows {

93 if self

94 .actions_event_matches(&workflow, &seed, Vec::new())

95 .await?

96 {

97 created.push(self.create_run(&repo, &space, &seed, &workflow).await?)

98 }

99 }

100 Ok(created)

101 }

102

103 pub async fn actions_event_matches(

104 &self,

105 workflow: &LoadedWorkflow,

106 seed: &RunSeed,

107 paths: Vec<String>,

108 ) -> CiResult<bool> {

109 let Some(actions) = &workflow.actions else {

110 return Ok(false);

111 };

112 let action = seed.event_payload["action"].as_str().unwrap_or_default();

113 let result=self.actions_engine(json!({"op":"trigger","source":actions.source,"file":actions.file,"trigger_event":seed.event.as_str(),"branch":if seed.event==WorkflowEventKind::PullRequest{seed.base_ref.as_deref()}else{seed.branch.as_deref()},"tag":seed.tag,"action":action,"paths":paths,"expression":seed.event_payload["workflow_run"]["name"].as_str().unwrap_or_default()})).await?;

114 Ok(result.as_bool().unwrap_or(false))

115 }

116

117 pub async fn actions_changed_paths(

118 &self,

119 repo_id: Uuid,

120 seed: &RunSeed,

121 ) -> CiResult<Vec<String>> {

122 let before = seed

123 .base_sha

124 .as_deref()

125 .or_else(|| seed.event_payload["old_oid"].as_str());

126 let mut command = tokio::process::Command::new("git");

127 command.arg("-C").arg(self.repo_dir(repo_id));

128 if let Some(before) =

129 before.filter(|oid| !oid.is_empty() && !oid.chars().all(|ch| ch == '0'))

130 {

131 command.arg("diff").arg("--name-only").arg("-z");

132 command

133 .arg(if seed.event == WorkflowEventKind::PullRequest {

134 format!("{before}...{}", seed.commit_sha)

135 } else {

136 format!("{before}..{}", seed.commit_sha)

137 })

138 .arg("--");

139 } else {

140 command

141 .arg("ls-tree")

142 .arg("-r")

143 .arg("--name-only")

144 .arg("-z")

145 .arg(&seed.commit_sha);

146 }

147 let output = command

148 .output()

149 .await

150 .map_err(|error| CiError::Internal(error.to_string()))?;

151 if !output.status.success() {

152 return Err(CiError::InvalidArgument(format!(

153 "workflow path diff failed: {}",

154 String::from_utf8_lossy(&output.stderr)

155 )));

156 }

157 Ok(output

158 .stdout

159 .split(|byte| *byte == 0)

160 .filter(|path| !path.is_empty())

161 .map(|path| String::from_utf8_lossy(path).into_owned())

162 .collect())

163 }

164

165 pub async fn sync_actions_schedules(

166 &self,

167 repo_id: Uuid,

168 workflows: &[LoadedWorkflow],

169 ) -> CiResult<()> {

170 let existing = actions_schedule::Entity::find()

171 .filter(actions_schedule::Column::RepoId.eq(repo_id))

172 .all(self.db_write())

173 .await?;

174 let now = OffsetDateTime::now_utc();

175 let mut wanted = std::collections::BTreeSet::new();

176 for workflow in workflows {

177 let Some(actions) = &workflow.actions else {

178 continue;

179 };

180 for schedule in actions.document["on"]["schedule"]

181 .as_array()

182 .into_iter()

183 .flatten()

184 {

185 let cron = schedule["cron"]

186 .as_str()

187 .ok_or_else(|| CiError::InvalidArgument("schedule cron is required".into()))?;

188 wanted.insert((workflow.file.clone(), cron.to_owned()));

189 if existing

190 .iter()

191 .any(|row| row.workflow_file == workflow.file && row.cron == cron)

192 {

193 continue;

194 }

195 let next = self

196 .actions_engine(

197 json!({"op":"cron","expression":cron,"value":now.unix_timestamp()}),

198 )

199 .await?

200 .as_i64()

201 .ok_or_else(|| CiError::InvalidArgument("invalid cron next time".into()))?;

202 actions_schedule::ActiveModel {

203 id: Set(Uuid::now_v7()),

204 repo_id: Set(repo_id),

205 workflow_file: Set(workflow.file.clone()),

206 cron: Set(cron.into()),

207 next_at: Set(OffsetDateTime::from_unix_timestamp(next)

208 .map_err(|error| CiError::InvalidArgument(error.to_string()))?),

209 updated_at: Set(now),

210 }

211 .insert(self.db_write())

212 .await?;

213 }

214 }

215 for row in existing {

216 if !wanted.contains(&(row.workflow_file, row.cron)) {

217 actions_schedule::Entity::delete_by_id(row.id)

218 .exec(self.db_write())

219 .await?;

220 }

221 }

222 Ok(())

223 }

224

225 pub async fn discover_actions_schedules(&self) -> CiResult<()> {

226 let repos = models::repo::repos::Entity::find()

227 .all(self.db_read())

228 .await?;

229 for repo in repos {

230 let result: CiResult<()> = async {

231 let space = self.space_of_repo(&repo).await?;

232 let workflows = self

233 .load_valid_workflows(&space, &repo.name, &repo.default_branch)

234 .await?;

235 self.sync_actions_schedules(repo.id, &workflows).await

236 }

237 .await;

238 if let Err(error) = result {

239 tracing::warn!(repo=%repo.id,%error,"schedule discovery failed");

240 }

241 }

242 Ok(())

243 }

244

245 pub async fn run_due_actions_schedules(&self) -> CiResult<u64> {

246 let now = OffsetDateTime::now_utc();

247 let due = actions_schedule::Entity::find()

248 .filter(actions_schedule::Column::NextAt.lte(now))

249 .all(self.db_write())

250 .await?;

251 let mut count = 0;

252 for due in due {

253 let txn = self.db_write().begin().await?;

254 let Some(current) = actions_schedule::Entity::find_by_id(due.id)

255 .lock_exclusive()

256 .one(&txn)

257 .await?

258 else {

259 continue;

260 };

261 if current.next_at > now {

262 continue;

263 }

264 let repo = self.repo_by_id(current.repo_id).await?;

265 let space = self.space_of_repo(&repo).await?;

266 let git = git::service::GitService::new(self.state.clone());

267 let sha = git

268 .rev_parse(

269 space.lower_name.clone(),

270 repo.name.clone(),

271 repo.default_branch.clone(),

272 )

273 .await?;

274 let workflow = self

275 .load_workflow_by_file(&space, &repo.name, &sha, &current.workflow_file)

276 .await?;

277 let still_declared = workflow.actions.as_ref().is_some_and(|actions| {

278 actions.document["on"]["schedule"]

279 .as_array()

280 .is_some_and(|values| {

281 values

282 .iter()

283 .any(|value| value["cron"].as_str() == Some(current.cron.as_str()))

284 })

285 });

286 if !still_declared {

287 actions_schedule::Entity::delete_by_id(current.id)

288 .exec(&txn)

289 .await?;

290 txn.commit().await?;

291 continue;

292 }

293 let seed = RunSeed {

294 event: WorkflowEventKind::Schedule,

295 event_payload: json!({"schedule":current.cron,"event_id":format!("schedule:{}:{}",current.id,current.next_at.unix_timestamp())}),

296 commit_sha: sha,

297 git_ref: format!("refs/heads/{}", repo.default_branch),

298 branch: Some(repo.default_branch.clone()),

299 tag: None,

300 base_sha: None,

301 base_ref: None,

302 pr_number: None,

303 trigger_user_id: None,

304 };

305 self.create_run(&repo, &space, &seed, &workflow).await?;

306 let next = self

307 .actions_engine(

308 json!({"op":"cron","expression":current.cron,"value":now.unix_timestamp()}),

309 )

310 .await?

311 .as_i64()

312 .ok_or_else(|| CiError::InvalidArgument("invalid cron next time".into()))?;

313 let mut active: actions_schedule::ActiveModel = current.into();

314 active.next_at = Set(OffsetDateTime::from_unix_timestamp(next)

315 .map_err(|error| CiError::InvalidArgument(error.to_string()))?);

316 active.updated_at = Set(now);

317 active.update(&txn).await?;

318 txn.commit().await?;

319 count += 1;

320 }

321 Ok(count)

322 }

323

324 pub async fn emit_actions_workflow_run(

325 &self,

326 run: &workflow_run::Model,

327 action: &str,

328 ) -> CiResult<()> {

329 if run.event_payload["_workflow_run_depth"]

330 .as_u64()

331 .unwrap_or(0)

332 >= 3

333 {

334 return Ok(());

335 }

336 let repo = self.repo_by_id(run.repo_id).await?;

337 let space = self.space_of_repo(&repo).await?;

338 use sha2::Digest;

339 let hash = sha2::Sha256::digest(

340 format!("workflow_run:{}:{action}:{}", run.id, run.attempt).as_bytes(),

341 );

342 let mut identity = [0u8; 16];

343 identity.copy_from_slice(&hash[..16]);

344 let event = CiEvent::Trigger(TriggerEvent {

345 id: Uuid::from_bytes(identity),

346 repo_id: repo.id,

347 space_id: space.id,

348 actor_id: run.trigger_user_id,

349 event: "workflow_run".into(),

350 payload: json!({"action":action,"_workflow_run_depth":run.event_payload["_workflow_run_depth"].as_u64().unwrap_or(0)+1,"workflow_run":{"id":run.actions_id,"name":run.workflow_name,"head_branch":run.branch,"head_sha":run.commit_sha,"status":if action=="completed"{"completed"}else if action=="requested"{"queued"}else{"in_progress"},"conclusion":if action=="completed"{Some(crate::share::states::run_state_str(run.state))}else{None},"event":run.event.as_str()}}),

351 });

352 crate::events::publish_event(

353 &self.state.redis,

354 &self.config().ci.event_queue_topic,

355 &event,

356 )

357 .await

358 }

359}

Added lib/ci/src/actions_payload.rs +193 −0 View lib/ci/src/actions_payload.rs

@@ -0,0 +1,193 @@

1use std::collections::BTreeMap;

2

3use ci_protocol::actions::{ActionsExecution, ActionsWorkflow, NeedResult};

4use ci_protocol::runner::{JobPayload, LeaseDto};

5use models::ci::{job_token, workflow_job, workflow_run};

6use sea_orm::{ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, Set};

7use serde_json::{Value, json};

8use time::OffsetDateTime;

9

10use crate::{

11 CiService,

12 errors::{CiError, CiResult},

13};

14

15impl CiService {

16 pub async fn build_actions_job_payload(

17 &self,

18 job: &workflow_job::Model,

19 run: &workflow_run::Model,

20 lease_token: String,

21 ) -> CiResult<JobPayload> {

22 let context = job

23 .actions_context

24 .as_ref()

25 .ok_or_else(|| CiError::Internal("Actions context missing".into()))?;

26 let workflow: ActionsWorkflow = serde_json::from_value(context["workflow"].clone())?;

27 let repo = self.repo_by_id(job.repo_id).await?;

28 let space = self.space_of_repo(&repo).await?;

29 let mut secrets = self.actions_job_secrets(job, run, context).await?;

30 let fork = run.event_payload["pull_request"]["head"]["repo"]["fork"]

31 .as_bool()

32 .unwrap_or(false);

33 if fork {

34 secrets.clear();

35 }

36 let id = uuid::Uuid::now_v7();

37 let secret = crate::secrets::random_secret(32);

38 let token = format!("ci_{id}.{secret}");

39 let permissions = &context["permissions"];

40 job_token::ActiveModel {

41 id: Set(id),

42 job_id: Set(job.id),

43 repo_id: Set(job.repo_id),

44 actor_id: Set(run.trigger_user_id.unwrap_or(space.owner_id)),

45 token_hash: Set(crate::secrets::sha256_hex(secret.as_bytes())),

46 lease_token: Set(lease_token.clone()),

47 contents_read: Set(matches!(

48 permissions["contents"].as_str(),

49 Some("read" | "write")

50 )),

51 contents_write: Set(!fork && permissions["contents"].as_str() == Some("write")),

52 expires_at: Set(OffsetDateTime::now_utc()

53 + time::Duration::seconds(self.config().ci.job_timeout_max_secs as i64 + 600)),

54 }

55 .insert(self.db_write())

56 .await?;

57 secrets.insert("GITHUB_TOKEN".into(), token.clone());

58 let needs: BTreeMap<String, NeedResult> =

59 serde_json::from_value(context["needs_results"].clone()).unwrap_or_default();

60 let timeout = context["instance"]["timeout_seconds"]

61 .as_u64()

62 .unwrap_or(self.config().ci.job_timeout_max_secs)

63 .min(self.config().ci.job_timeout_max_secs)

64 .max(1);

65 let mut github = self.actions_github_context(run).await?;

66 github["job"] = context["job"].clone();

67 github["token"] = json!(token);

68 let mut env = BTreeMap::from([

69 ("CI".into(), "true".into()),

70 ("GITDATA_CI".into(), "true".into()),

71 ("GITDATA_RUN_ID".into(), run.id.to_string()),

72 ("GITDATA_JOB_ID".into(), job.id.to_string()),

73 ]);

74 if let Some(environment) = &job.environment {

75 env.insert("GITDATA_ENVIRONMENT".into(), environment.clone());

76 }

77 let actions = ActionsExecution {

78 workflow,

79 job: context["job"].as_str().unwrap_or(&job.job_key).into(),

80 matrix: context["instance"]["matrix"].clone(),

81 github,

82 event: run.event_payload.clone(),

83 needs,

84 inputs: serde_json::from_value(context["inputs"].clone()).unwrap_or_default(),

85 vars: self.actions_variables(job.repo_id).await?,

86 token,

87 runtime_url: format!("{}/", self.actions_runtime_url()?),

88 runtime_token: self.actions_runtime_token(job, run)?,

89 };

90 Ok(JobPayload {

91 job_id: job.id,

92 run_id: run.id,

93 repo_id: repo.id,

94 space_id: space.id,

95 run_attempt: job.attempt,

96 job_name: job.name.clone(),

97 clone_url: format!(

98 "{}/{}/{}.git",

99 self.config().git.base_url.trim_end_matches('/'),

100 space.lower_name,

101 repo.name

102 ),

103 commit_sha: run.commit_sha.clone(),

104 git_ref: run.git_ref.clone(),

105 branch: run.branch.clone(),

106 tag: run.tag.clone(),

107 base_sha: run.base_sha.clone(),

108 base_ref: run.base_ref.clone(),

109 pr_number: run.pr_number,

110 event: run.event.as_str().into(),

111 workflow_name: run.workflow_name.clone(),

112 workflow_file: run.workflow_file.clone(),

113 env,

114 secrets,

115 allow_git_token: false,

116 container: None,

117 timeout_secs: timeout,

118 steps: Vec::new(),

119 artifacts: Vec::new(),

120 needs_artifacts: Vec::new(),

121 lease: LeaseDto {

122 token: lease_token,

123 expires_at: job.lease_expires_at.unwrap_or(OffsetDateTime::now_utc()),

124 heartbeat_interval_secs: self.config().ci.heartbeat_interval_secs,

125 attempt: job.attempt,

126 },

127 actions: Some(actions),

128 })

129 }

130

131 async fn actions_job_secrets(

132 &self,

133 job: &workflow_job::Model,

134 run: &workflow_run::Model,

135 context: &Value,

136 ) -> CiResult<BTreeMap<String, String>> {

137 let mut contexts = Vec::new();

138 let mut cursor = context;

139 while cursor

140 .get("caller_context")

141 .is_some_and(|value| value.is_object())

142 {

143 contexts.push(cursor.clone());

144 cursor = &cursor["caller_context"];

145 if contexts.len() > 10 {

146 return Err(CiError::InvalidArgument(

147 "workflow call depth exceeds 10".into(),

148 ));

149 }

150 }

151 let mut secrets: BTreeMap<String, String> = self

152 .resolved_secrets(job.repo_id, job.space_id, None)

153 .await?

154 .into_iter()

155 .collect();

156 for current in contexts.into_iter().rev() {

157 let mapping = &current["secret_mapping"];

158 if mapping.as_str() == Some("inherit") {

159 continue;

160 }

161 if mapping.is_null() {

162 secrets.clear();

163 continue;

164 }

165 let needs: BTreeMap<String, NeedResult> =

166 serde_json::from_value(current["caller_needs"].clone()).unwrap_or_default();

167 let mut request = self

168 .actions_request(run, &current["caller_context"], &needs, "evaluate")

169 .await?;

170 request["value"] = mapping.clone();

171 request["secrets"] = json!(secrets);

172 secrets = serde_json::from_value(self.actions_engine(request).await?)?;

173 }

174 if let Some(environment) = &job.environment {

175 if let Some(environment) = self

176 .find_actions_environment(job.space_id, job.repo_id, environment)

177 .await?

178 {

179 let rows = models::ci::ci_secret::Entity::find()

180 .filter(models::ci::ci_secret::Column::EnvironmentId.eq(environment.id))

181 .all(self.db_write())

182 .await?;

183 for row in rows {

184 secrets.insert(

185 row.name,

186 crate::secrets::decrypt(self.encryption_key(), &row.value_ciphertext)?,

187 );

188 }

189 }

190 }

191 Ok(secrets)

192 }

193}

Added lib/ci/src/actions_runtime.rs +1042 −0 View lib/ci/src/actions_runtime.rs

@@ -0,0 +1,1042 @@

1use axum::{

2 Json, Router,

3 body::Bytes,

4 extract::{DefaultBodyLimit, Path, Query, State},

5 http::{HeaderMap, StatusCode},

6 response::{IntoResponse, Response},

7 routing::{get, post, put},

8};

9use base64::{Engine, engine::general_purpose::URL_SAFE_NO_PAD};

10use futures::StreamExt;

11use hmac::{Hmac, KeyInit, Mac};

12use models::ci::workflow_artifact::ArtifactState;

13use models::ci::{actions_cache, actions_identity, workflow_artifact, workflow_job, workflow_run};

14use sea_orm::{

15 ActiveModelTrait, ColumnTrait, ConnectionTrait, EntityTrait, QueryFilter, QueryOrder, Set,

16 TransactionTrait,

17};

18use serde::Deserialize;

19use serde_json::{Value, json};

20use sha2::{Digest, Sha256};

21use time::OffsetDateTime;

22use tokio::io::{AsyncSeekExt, AsyncWriteExt};

23use uuid::Uuid;

24

25use crate::{

26 CiService,

27 errors::{CiError, CiResult},

28};

29

30pub fn router() -> Router<CiService> {

31 Router::new()

32 .route(

33 "/twirp/github.actions.results.api.v1.ArtifactService/{method}",

34 post(artifact_rpc),

35 )

36 .route(

37 "/twirp/github.actions.results.api.v1.CacheService/{method}",

38 post(cache_rpc),

39 )

40 .route("/actions-runtime/uploads/{kind}/{id}", put(upload_blob))

41 .route("/actions-runtime/oidc", get(oidc_token))

42 .route("/.well-known/openid-configuration", get(oidc_configuration))

43 .route("/.well-known/jwks.json", get(oidc_jwks))

44 .route("/_apis/artifactcache/cache", get(legacy_cache_lookup))

45 .route("/_apis/artifactcache/caches", post(legacy_cache_reserve))

46 .route(

47 "/_apis/artifactcache/caches/{id}",

48 post(legacy_cache_finalize).patch(legacy_cache_upload),

49 )

50 .layer(DefaultBodyLimit::max(64 * 1024 * 1024))

51}

52

53pub struct RuntimeError(CiError);

54impl From<CiError> for RuntimeError {

55 fn from(error: CiError) -> Self {

56 Self(error)

57 }

58}

59impl From<storage::StorageError> for RuntimeError {

60 fn from(error: storage::StorageError) -> Self {

61 Self(error.into())

62 }

63}

64impl From<sea_orm::DbErr> for RuntimeError {

65 fn from(error: sea_orm::DbErr) -> Self {

66 Self(error.into())

67 }

68}

69impl IntoResponse for RuntimeError {

70 fn into_response(self) -> Response {

71 let (status, code) = match &self.0 {

72 CiError::Unauthorized => (StatusCode::UNAUTHORIZED, "unauthenticated"),

73 CiError::Forbidden => (StatusCode::FORBIDDEN, "permission_denied"),

74 CiError::NotFound => (StatusCode::NOT_FOUND, "not_found"),

75 CiError::Conflict(_) => (StatusCode::CONFLICT, "already_exists"),

76 CiError::InvalidArgument(_) => (StatusCode::BAD_REQUEST, "invalid_argument"),

77 _ => (StatusCode::INTERNAL_SERVER_ERROR, "internal"),

78 };

79 (status, Json(json!({"code":code,"msg":self.0.to_string()}))).into_response()

80 }

81}

82

83#[derive(Clone)]

84struct RuntimeContext {

85 job: workflow_job::Model,

86 run: workflow_run::Model,

87}

88

89impl CiService {

90 pub fn sign_actions_token(&self, payload: &Value) -> CiResult<String> {

91 let header = URL_SAFE_NO_PAD.encode(br#"{"alg":"HS256","typ":"JWT"}"#);

92 let data = URL_SAFE_NO_PAD.encode(serde_json::to_vec(payload)?);

93 let message = format!("{header}.{data}");

94 let mut mac = Hmac::<Sha256>::new_from_slice(self.encryption_key().as_bytes())

95 .map_err(|error| CiError::Internal(error.to_string()))?;

96 mac.update(message.as_bytes());

97 Ok(format!(

98 "{message}.{}",

99 URL_SAFE_NO_PAD.encode(mac.finalize().into_bytes())

100 ))

101 }

102

103 fn verify_actions_token(&self, token: &str) -> CiResult<Value> {

104 let parts: Vec<_> = token.split('.').collect();

105 if parts.len() != 3 {

106 return Err(CiError::Unauthorized);

107 }

108 let mut mac = Hmac::<Sha256>::new_from_slice(self.encryption_key().as_bytes())

109 .map_err(|error| CiError::Internal(error.to_string()))?;

110 mac.update(format!("{}.{}", parts[0], parts[1]).as_bytes());

111 let signature = URL_SAFE_NO_PAD

112 .decode(parts[2])

113 .map_err(|_| CiError::Unauthorized)?;

114 mac.verify_slice(&signature)

115 .map_err(|_| CiError::Unauthorized)?;

116 let data = URL_SAFE_NO_PAD

117 .decode(parts[1])

118 .map_err(|_| CiError::Unauthorized)?;

119 let value: Value = serde_json::from_slice(&data).map_err(|_| CiError::Unauthorized)?;

120 if value["exp"].as_i64().unwrap_or(0) <= OffsetDateTime::now_utc().unix_timestamp() {

121 return Err(CiError::Unauthorized);

122 }

123 Ok(value)

124 }

125

126 async fn runtime_context(&self, headers: &HeaderMap) -> CiResult<RuntimeContext> {

127 self.ensure_enabled()?;

128 let token = headers

129 .get("authorization")

130 .and_then(|value| value.to_str().ok())

131 .and_then(|value| value.strip_prefix("Bearer "))

132 .ok_or(CiError::Unauthorized)?;

133 let claims = self.verify_actions_token(token)?;

134 if claims["aud"] != "gitdata-actions-runtime" {

135 return Err(CiError::Unauthorized);

136 }

137 self.runtime_context_from_claims(&claims).await

138 }

139

140 async fn runtime_context_from_claims(&self, claims: &Value) -> CiResult<RuntimeContext> {

141 let id = claims["job_id"]

142 .as_str()

143 .and_then(|value| Uuid::parse_str(value).ok())

144 .ok_or(CiError::Unauthorized)?;

145 let job = workflow_job::Entity::find_by_id(id)

146 .one(self.db_write())

147 .await?

148 .ok_or(CiError::Unauthorized)?;

149 if !job.state.is_active() || job.lease_token.as_deref() != claims["lease_token"].as_str() {

150 return Err(CiError::Unauthorized);

151 }

152 let run = workflow_run::Entity::find_by_id(job.run_id)

153 .one(self.db_write())

154 .await?

155 .ok_or(CiError::Unauthorized)?;

156 Ok(RuntimeContext { job, run })

157 }

158

159 pub fn actions_runtime_url(&self) -> CiResult<String> {

160 let configured = &self.config().ci.runtime_base_url;

161 let base = if !configured.is_empty() {

162 configured.as_str()

163 } else if !self.config().webhook.public_base_url.is_empty() {

164 self.config().webhook.public_base_url.as_str()

165 } else {

166 self.config().git.base_url.as_str()

167 };

168 let url = reqwest::Url::parse(base).map_err(|error| {

169 CiError::InvalidArgument(format!("invalid CI runtime origin: {error}"))

170 })?;

171 if url.path() != "/" && !url.path().is_empty() {

172 return Err(CiError::InvalidArgument(

173 "ci.runtime_base_url must be a URL origin without a path prefix".into(),

174 ));

175 }

176 Ok(base.trim_end_matches('/').to_owned())

177 }

178

179 pub fn actions_runtime_token(

180 &self,

181 job: &workflow_job::Model,

182 run: &workflow_run::Model,

183 ) -> CiResult<String> {

184 self.sign_actions_token(&json!({"aud":"gitdata-actions-runtime","job_id":job.id,"run_id":run.id,"repo_id":job.repo_id,"lease_token":job.lease_token,

185 "scp":format!("Actions.Results:{}:{}",run.actions_id.unwrap_or(0),job.actions_id.unwrap_or(0)),"iat":OffsetDateTime::now_utc().unix_timestamp(),

186 "exp":OffsetDateTime::now_utc().unix_timestamp()+self.config().ci.job_timeout_max_secs as i64+600}))

187 }

188

189 fn signed_upload_url(

190 &self,

191 context: &RuntimeContext,

192 kind: &str,

193 id: &str,

194 ) -> CiResult<String> {

195 let token=self.sign_actions_token(&json!({"aud":"gitdata-actions-upload","job_id":context.job.id,"lease_token":context.job.lease_token,"kind":kind,"id":id,"exp":OffsetDateTime::now_utc().unix_timestamp()+3600}))?;

196 Ok(format!(

197 "{}/actions-runtime/uploads/{kind}/{id}?token={token}",

198 self.actions_runtime_url()?

199 ))

200 }

201

202 async fn numeric_actions_id(&self, entity_id: Uuid) -> CiResult<i64> {

203 actions_identity::Entity::find()

204 .filter(actions_identity::Column::EntityId.eq(entity_id))

205 .one(self.db_write())

206 .await?

207 .map(|identity| identity.id)

208 .ok_or(CiError::NotFound)

209 }

210

211 fn check_artifact_scope(&self, context: &RuntimeContext, body: &Value) -> CiResult<()> {

212 let run = field(body, "workflowRunBackendId", "workflow_run_backend_id");

213 if !run.is_empty() && run != context.run.actions_id.unwrap_or(0).to_string() {

214 return Err(CiError::Forbidden);

215 }

216 Ok(())

217 }

218

219 async fn artifact_call(

220 &self,

221 context: &RuntimeContext,

222 method: &str,

223 body: &Value,

224 ) -> CiResult<Value> {

225 self.check_artifact_scope(context, body)?;

226 let name = field(body, "name", "name");

227 match method {

228 "CreateArtifact" => {

229 if name.trim().is_empty() || name.len() > 255 {

230 return Err(CiError::InvalidArgument(

231 "artifact name must be 1-255 characters".into(),

232 ));

233 }

234 let txn = self.db_write().begin().await?;

235 txn.execute_unprepared(&format!(

236 "SELECT pg_advisory_xact_lock({})",

237 context.run.id.as_u128() as i64

238 ))

239 .await?;

240 let existing = workflow_artifact::Entity::find()

241 .filter(workflow_artifact::Column::RunId.eq(context.run.id))

242 .filter(workflow_artifact::Column::Name.eq(&name))

243 .one(&txn)

244 .await?;

245 if let Some(existing) = existing {

246 if existing.job_id == context.job.id

247 && existing.state == ArtifactState::Uploading

248 {

249 let url =

250 self.signed_upload_url(context, "artifact", &existing.id.to_string())?;

251 return Ok(json!({"ok":true,"signedUploadUrl":url}));

252 }

253 return Err(CiError::Conflict(

254 "artifact name already exists in this run".into(),

255 ));

256 }

257 let id = Uuid::now_v7();

258 let now = OffsetDateTime::now_utc();

259 let configured_expiry =

260 now + time::Duration::days(self.config().ci.artifact_retention_days.max(1));

261 let requested_expiry = body

262 .get("expiresAt")

263 .or_else(|| body.get("expires_at"))

264 .and_then(Value::as_str)

265 .and_then(|value| {

266 OffsetDateTime::parse(value, &time::format_description::well_known::Rfc3339)

267 .ok()

268 });

269 let expires = requested_expiry

270 .map(|value| value.min(configured_expiry))

271 .unwrap_or(configured_expiry);

272 workflow_artifact::ActiveModel {

273 id: Set(id),

274 run_id: Set(context.run.id),

275 job_id: Set(context.job.id),

276 name: Set(name),

277 object_key: Set(CiService::artifact_object_key(

278 context.run.id,

279 context.job.id,

280 id,

281 )),

282 size: Set(0),

283 sha256: Set(None),

284 mime_type: Set(body

285 .get("mimeType")

286 .or_else(|| body.get("mime_type"))

287 .and_then(|value| value.as_str().or_else(|| value["value"].as_str()))

288 .map(str::to_owned)),

289 state: Set(ArtifactState::Uploading),

290 expires_at: Set(Some(expires)),

291 created_at: Set(now),

292 updated_at: Set(now),

293 }

294 .insert(&txn)

295 .await?;

296 crate::actions::identity(&txn, id, "artifact").await?;

297 txn.commit().await?;

298 Ok(

299 json!({"ok":true,"signedUploadUrl":self.signed_upload_url(context,"artifact",&id.to_string())?}),

300 )

301 }

302 "FinalizeArtifact" => {

303 let row = workflow_artifact::Entity::find()

304 .filter(workflow_artifact::Column::RunId.eq(context.run.id))

305 .filter(workflow_artifact::Column::JobId.eq(context.job.id))

306 .filter(workflow_artifact::Column::Name.eq(name))

307 .one(self.db_write())

308 .await?

309 .ok_or(CiError::NotFound)?;

310 let declared = number(body.get("size"))?;

311 let (size, hash) = self

312 .uploaded_size_and_hash(&row.object_key, self.config().ci.artifact_max_bytes)

313 .await?;

314 if size != declared {

315 return Err(CiError::InvalidArgument(

316 "artifact size does not match upload".into(),

317 ));

318 }

319 let expected = body

320 .get("hash")

321 .and_then(|v| v.as_str().or_else(|| v["value"].as_str()))

322 .map(|v| v.trim_start_matches("sha256:"));

323 if expected.is_some_and(|value| value != hash) {

324 return Err(CiError::InvalidArgument(

325 "artifact digest does not match upload".into(),

326 ));

327 }

328 let id = self.numeric_actions_id(row.id).await?;

329 let mut active: workflow_artifact::ActiveModel = row.into();

330 active.size = Set(size as i64);

331 active.sha256 = Set(Some(hash));

332 active.state = Set(ArtifactState::Ready);

333 active.updated_at = Set(OffsetDateTime::now_utc());

334 active.update(self.db_write()).await?;

335 Ok(json!({"ok":true,"artifactId":id.to_string()}))

336 }

337 "ListArtifacts" => {

338 let rows = workflow_artifact::Entity::find()

339 .filter(workflow_artifact::Column::RunId.eq(context.run.id))

340 .filter(workflow_artifact::Column::State.eq(ArtifactState::Ready))

341 .all(self.db_write())

342 .await?;

343 let name_filter = field(body, "nameFilter", "name_filter");

344 let id_filter = body

345 .get("idFilter")

346 .or_else(|| body.get("id_filter"))

347 .and_then(|v| v.as_str().or_else(|| v["value"].as_str()));

348 let mut artifacts = Vec::new();

349 for row in rows {

350 if row

351 .expires_at

352 .is_some_and(|expiry| expiry <= OffsetDateTime::now_utc())

353 || (!name_filter.is_empty() && row.name != name_filter)

354 {

355 continue;

356 }

357 let id = self.numeric_actions_id(row.id).await?;

358 if id_filter.is_some_and(|filter| filter != id.to_string()) {

359 continue;

360 }

361 let job = workflow_job::Entity::find_by_id(row.job_id)

362 .one(self.db_write())

363 .await?

364 .ok_or(CiError::NotFound)?;

365 artifacts.push(json!({"workflowRunBackendId":context.run.actions_id.unwrap_or(0).to_string(),"workflowJobRunBackendId":job.actions_id.unwrap_or(0).to_string(),"databaseId":id.to_string(),"name":row.name,"size":row.size.to_string(),"createdAt":row.created_at.format(&time::format_description::well_known::Rfc3339).unwrap_or_default(),"hash":row.sha256.map(|hash|format!("sha256:{hash}")),"mimeType":row.mime_type}));

366 }

367 Ok(json!({"artifacts":artifacts}))

368 }

369 "GetSignedArtifactURL" | "DeleteArtifact" => {

370 let row = workflow_artifact::Entity::find()

371 .filter(workflow_artifact::Column::RunId.eq(context.run.id))

372 .filter(workflow_artifact::Column::Name.eq(&name))

373 .one(self.db_write())

374 .await?

375 .ok_or(CiError::NotFound)?;

376 if method == "DeleteArtifact" {

377 let id = self.numeric_actions_id(row.id).await?;

378 self.storage().client().delete(&row.object_key).await?;

379 workflow_artifact::Entity::delete_by_id(row.id)

380 .exec(self.db_write())

381 .await?;

382 Ok(json!({"ok":true,"artifactId":id.to_string()}))

383 } else {

384 if row.state != ArtifactState::Ready

385 || row

386 .expires_at

387 .is_some_and(|at| at <= OffsetDateTime::now_utc())

388 {

389 return Err(CiError::NotFound);

390 }

391 Ok(

392 json!({"signedUrl":self.storage().client().presign_get(&row.object_key,900)?}),

393 )

394 }

395 }

396 _ => Err(CiError::InvalidArgument(format!(

397 "unknown artifact method `{method}`"

398 ))),

399 }

400 }

401

402 async fn uploaded_size_and_hash(&self, key: &str, max: u64) -> CiResult<(u64, String)> {

403 let mut stream = self.storage().client().get_stream(key).await?;

404 let mut hash = Sha256::new();

405 let mut size = 0;

406 while let Some(chunk) = stream.next().await {

407 let chunk = chunk?;

408 size += chunk.len() as u64;

409 if size > max {

410 return Err(CiError::InvalidArgument(

411 "upload exceeds configured size limit".into(),

412 ));

413 }

414 hash.update(&chunk);

415 }

416 Ok((

417 size,

418 hash.finalize()

419 .iter()

420 .map(|byte| format!("{byte:02x}"))

421 .collect(),

422 ))

423 }

424

425 async fn cache_reserve(

426 &self,

427 context: &RuntimeContext,

428 body: &Value,

429 ) -> CiResult<actions_cache::Model> {

430 if context.run.event_payload["pull_request"]["head"]["repo"]["fork"]

431 .as_bool()

432 .unwrap_or(false)

433 {

434 return Err(CiError::Forbidden);

435 }

436 let key = field(body, "key", "key");

437 let version = field(body, "version", "version");

438 if key.is_empty() || key.len() > 512 || version.is_empty() || version.len() > 512 {

439 return Err(CiError::InvalidArgument(

440 "cache key and version must be 1-512 characters".into(),

441 ));

442 }

443 let txn = self.db_write().begin().await?;

444 txn.execute_unprepared(&format!(

445 "SELECT pg_advisory_xact_lock({})",

446 context.job.repo_id.as_u128() as i64

447 ))

448 .await?;

449 if let Some(row) = actions_cache::Entity::find()

450 .filter(actions_cache::Column::RepoId.eq(context.job.repo_id))

451 .filter(actions_cache::Column::GitRef.eq(&context.run.git_ref))

452 .filter(actions_cache::Column::Key.eq(&key))

453 .filter(actions_cache::Column::Version.eq(&version))

454 .one(&txn)

455 .await?

456 {

457 if row.job_id == context.job.id && !row.ready {

458 return Ok(row);

459 }

460 return Err(CiError::Conflict("cache entry already exists".into()));

461 }

462 let now = OffsetDateTime::now_utc();

463 let object_key = format!("ci/caches/{}/{}", context.job.repo_id, Uuid::now_v7());

464 let row = actions_cache::ActiveModel {

465 repo_id: Set(context.job.repo_id),

466 job_id: Set(context.job.id),

467 git_ref: Set(context.run.git_ref.clone()),

468 key: Set(key),

469 version: Set(version),

470 object_key: Set(object_key),

471 size: Set(0),

472 ready: Set(false),

473 created_at: Set(now),

474 last_accessed_at: Set(now),

475 expires_at: Set(

476 now + time::Duration::days(self.config().ci.cache_retention_days.max(1))

477 ),

478 ..Default::default()

479 }

480 .insert(&txn)

481 .await?;

482 txn.commit().await?;

483 Ok(row)

484 }

485

486 async fn cache_lookup(

487 &self,

488 context: &RuntimeContext,

489 key: &str,

490 restore_keys: &[String],

491 version: &str,

492 ) -> CiResult<Option<actions_cache::Model>> {

493 let repo = models::repo::repos::Entity::find_by_id(context.job.repo_id)

494 .one(self.db_write())

495 .await?

496 .ok_or(CiError::NotFound)?;

497 let mut refs = vec![context.run.git_ref.clone()];

498 if let Some(base) = &context.run.base_ref {

499 refs.push(format!("refs/heads/{base}"))

500 }

501 refs.push(format!("refs/heads/{}", repo.default_branch));

502 refs.dedup();

503 let rows = actions_cache::Entity::find()

504 .filter(actions_cache::Column::RepoId.eq(repo.id))

505 .filter(actions_cache::Column::Version.eq(version))

506 .filter(actions_cache::Column::Ready.eq(true))

507 .filter(actions_cache::Column::ExpiresAt.gt(OffsetDateTime::now_utc()))

508 .order_by_desc(actions_cache::Column::CreatedAt)

509 .all(self.db_write())

510 .await?;

511 for reference in refs {

512 let entries: Vec<_> = rows.iter().filter(|row| row.git_ref == reference).collect();

513 let matched = entries

514 .iter()

515 .find(|row| row.key == key)

516 .copied()

517 .or_else(|| entries.iter().find(|row| row.key.starts_with(key)).copied())

518 .or_else(|| {

519 restore_keys.iter().find_map(|prefix| {

520 entries

521 .iter()

522 .find(|row| row.key.starts_with(prefix))

523 .copied()

524 })

525 });

526 if let Some(row) = matched {

527 let mut active: actions_cache::ActiveModel = row.clone().into();

528 active.last_accessed_at = Set(OffsetDateTime::now_utc());

529 active.expires_at = Set(OffsetDateTime::now_utc()

530 + time::Duration::days(self.config().ci.cache_retention_days.max(1)));

531 return Ok(Some(active.update(self.db_write()).await?));

532 }

533 }

534 Ok(None)

535 }

536

537 async fn cache_finalize(

538 &self,

539 context: &RuntimeContext,

540 id: Option<i64>,

541 body: &Value,

542 ) -> CiResult<actions_cache::Model> {

543 let mut query = actions_cache::Entity::find()

544 .filter(actions_cache::Column::RepoId.eq(context.job.repo_id))

545 .filter(actions_cache::Column::JobId.eq(context.job.id));

546 if let Some(id) = id {

547 query = query.filter(actions_cache::Column::Id.eq(id))

548 } else {

549 query = query

550 .filter(actions_cache::Column::Key.eq(field(body, "key", "key")))

551 .filter(actions_cache::Column::Version.eq(field(body, "version", "version")))

552 }

553 let row = query.one(self.db_write()).await?.ok_or(CiError::NotFound)?;

554 let declared = number(

555 body.get("sizeBytes")

556 .or_else(|| body.get("size_bytes"))

557 .or_else(|| body.get("size")),

558 )?;

559 let (size, _) = self

560 .uploaded_size_and_hash(&row.object_key, self.config().ci.cache_max_bytes)

561 .await?;

562 if size != declared {

563 return Err(CiError::InvalidArgument(

564 "cache size does not match upload".into(),

565 ));

566 }

567 let mut active: actions_cache::ActiveModel = row.into();

568 active.size = Set(size as i64);

569 active.ready = Set(true);

570 active.last_accessed_at = Set(OffsetDateTime::now_utc());

571 Ok(active.update(self.db_write()).await?)

572 }

573

574 async fn assemble_blocks(&self, key: &str, blocks: &[String], max: u64) -> CiResult<()> {

575 let temporary = std::env::temp_dir().join(format!("gitdata-ci-upload-{}", Uuid::now_v7()));

576 let result: CiResult<()> = async {

577 let mut file = tokio::fs::OpenOptions::new()

578 .create_new(true)

579 .write(true)

580 .read(true)

581 .open(&temporary)

582 .await

583 .map_err(|error| CiError::Internal(error.to_string()))?;

584 let mut size = 0u64;

585 for block in blocks {

586 let data = self

587 .storage()

588 .client()

589 .get(&format!(

590 "{key}.blocks/{}",

591 crate::secrets::sha256_hex(block.as_bytes())

592 ))

593 .await?;

594 size += data.len() as u64;

595 if size > max {

596 return Err(CiError::InvalidArgument(

597 "upload exceeds configured size limit".into(),

598 ));

599 }

600 file.write_all(&data)

601 .await

602 .map_err(|error| CiError::Internal(error.to_string()))?;

603 }

604 file.seek(std::io::SeekFrom::Start(0))

605 .await

606 .map_err(|error| CiError::Internal(error.to_string()))?;

607 self.storage()

608 .client()

609 .put_stream(key, &mut file, "application/octet-stream")

610 .await?;

611 for block in blocks {

612 self.storage()

613 .client()

614 .delete(&format!(

615 "{key}.blocks/{}",

616 crate::secrets::sha256_hex(block.as_bytes())

617 ))

618 .await?;

619 }

620 Ok(())

621 }

622 .await;

623 let _ = tokio::fs::remove_file(temporary).await;

624 result

625 }

626}

627

628fn field(value: &Value, camel: &str, snake: &str) -> String {

629 value

630 .get(camel)

631 .or_else(|| value.get(snake))

632 .and_then(|v| v.as_str().or_else(|| v["value"].as_str()))

633 .unwrap_or_default()

634 .to_owned()

635}

636fn number(value: Option<&Value>) -> CiResult<u64> {

637 value

638 .and_then(|value| {

639 value

640 .as_u64()

641 .or_else(|| value.as_str().and_then(|text| text.parse().ok()))

642 })

643 .ok_or_else(|| CiError::InvalidArgument("expected a nonnegative size".into()))

644}

645

646async fn artifact_rpc(

647 State(service): State<CiService>,

648 Path(method): Path<String>,

649 headers: HeaderMap,

650 Json(body): Json<Value>,

651) -> Result<Json<Value>, RuntimeError> {

652 let context = service.runtime_context(&headers).await?;

653 Ok(Json(service.artifact_call(&context, &method, &body).await?))

654}

655

656#[derive(Deserialize)]

657struct OidcQuery {

658 audience: Option<String>,

659}

660

661async fn oidc_token(

662 State(service): State<CiService>,

663 headers: HeaderMap,

664 Query(query): Query<OidcQuery>,

665) -> Result<Json<Value>, RuntimeError> {

666 let context = service.runtime_context(&headers).await?;

667 if context

668 .job

669 .actions_context

670 .as_ref()

671 .and_then(|context| context["permissions"]["id-token"].as_str())

672 != Some("write")

673 {

674 return Err(CiError::Forbidden.into());

675 }

676 if service.config().ci.oidc_private_key.is_empty() {

677 return Err(CiError::InvalidArgument(

678 "ci.oidc_private_key is required to issue OIDC tokens".into(),

679 )

680 .into());

681 }

682 let github = service.actions_github_context(&context.run).await?;

683 let repository = github["repository"].as_str().unwrap_or_default();

684 let sub = if let Some(environment) = &context.job.environment {

685 format!(

686 "repo:{repository}:environment:{}",

687 environment.replace(':', "%3A")

688 )

689 } else if context.run.pr_number.is_some() {

690 format!("repo:{repository}:pull_request")

691 } else {

692 format!("repo:{repository}:ref:{}", context.run.git_ref)

693 };

694 let now = OffsetDateTime::now_utc().unix_timestamp();

695 let issuer = service.actions_runtime_url()?;

696 let mut claims = github;

697 claims["iss"] = json!(issuer);

698 claims["sub"] = json!(sub);

699 claims["aud"] = json!(query.audience.unwrap_or_else(|| format!(

700 "{}/{}",

701 issuer,

702 claims["repository_owner"].as_str().unwrap_or_default()

703 )));

704 claims["iat"] = json!(now);

705 claims["nbf"] = json!(now - 5);

706 claims["exp"] = json!(now + 300);

707 claims["jti"] = json!(Uuid::now_v7());

708 claims.as_object_mut().map(|value| value.remove("token"));

709 let value = service

710 .actions_engine(

711 json!({"op":"oidc_sign","token":service.config().ci.oidc_private_key,"value":claims}),

712 )

713 .await?;

714 Ok(Json(json!({"value":value})))

715}

716

717async fn oidc_configuration(State(service): State<CiService>) -> Result<Json<Value>, RuntimeError> {

718 let issuer = service.actions_runtime_url()?;

719 Ok(Json(

720 json!({"issuer":issuer,"jwks_uri":format!("{issuer}/.well-known/jwks.json"),"id_token_signing_alg_values_supported":["RS256"],"subject_types_supported":["public"],"response_types_supported":["id_token"]}),

721 ))

722}

723

724async fn oidc_jwks(State(service): State<CiService>) -> Result<Json<Value>, RuntimeError> {

725 if service.config().ci.oidc_private_key.is_empty() {

726 return Ok(Json(json!({"keys":[]})));

727 }

728 Ok(Json(

729 service

730 .actions_engine(json!({"op":"oidc_jwks","token":service.config().ci.oidc_private_key}))

731 .await?,

732 ))

733}

734

735async fn cache_rpc(

736 State(service): State<CiService>,

737 Path(method): Path<String>,

738 headers: HeaderMap,

739 Json(body): Json<Value>,

740) -> Result<Json<Value>, RuntimeError> {

741 let context = service.runtime_context(&headers).await?;

742 let result = match method.as_str() {

743 "CreateCacheEntry" => match service.cache_reserve(&context, &body).await {

744 Ok(row) => {

745 json!({"ok":true,"signedUploadUrl":service.signed_upload_url(&context,"cache",&row.id.to_string())?,"message":""})

746 }

747 Err(CiError::Conflict(message)) => {

748 json!({"ok":false,"signedUploadUrl":"","message":message})

749 }

750 Err(error) => return Err(error.into()),

751 },

752 "FinalizeCacheEntryUpload" => {

753 let row = service.cache_finalize(&context, None, &body).await?;

754 json!({"ok":true,"entryId":row.id.to_string(),"message":""})

755 }

756 "GetCacheEntryDownloadURL" => {

757 let restore = body

758 .get("restoreKeys")

759 .or_else(|| body.get("restore_keys"))

760 .map(|value| ci_protocol::actions::strings(Some(value)))

761 .unwrap_or_default();

762 match service

763 .cache_lookup(

764 &context,

765 &field(&body, "key", "key"),

766 &restore,

767 &field(&body, "version", "version"),

768 )

769 .await?

770 {

771 Some(row) => {

772 json!({"ok":true,"signedDownloadUrl":service.storage().client().presign_get(&row.object_key,900)? ,"matchedKey":row.key})

773 }

774 None => json!({"ok":false,"signedDownloadUrl":"","matchedKey":""}),

775 }

776 }

777 _ => {

778 return Err(

779 CiError::InvalidArgument(format!("unknown cache method `{method}`")).into(),

780 );

781 }

782 };

783 Ok(Json(result))

784}

785

786#[derive(Deserialize)]

787struct UploadQuery {

788 token: String,

789 comp: Option<String>,

790 blockid: Option<String>,

791}

792

793async fn upload_blob(

794 State(service): State<CiService>,

795 Path((kind, id)): Path<(String, String)>,

796 Query(query): Query<UploadQuery>,

797 body: Bytes,

798) -> Result<Response, RuntimeError> {

799 let claims = service.verify_actions_token(&query.token)?;

800 if claims["aud"] != "gitdata-actions-upload" || claims["kind"] != kind || claims["id"] != id {

801 return Err(CiError::Forbidden.into());

802 }

803 let context = service.runtime_context_from_claims(&claims).await?;

804 let (key, max) = if kind == "artifact" {

805 let id = Uuid::parse_str(&id).map_err(|_| CiError::NotFound)?;

806 let row = workflow_artifact::Entity::find_by_id(id)

807 .one(service.db_write())

808 .await?

809 .ok_or(CiError::NotFound)?;

810 if row.job_id != context.job.id || row.state != ArtifactState::Uploading {

811 return Err(CiError::Forbidden.into());

812 }

813 (row.object_key, service.config().ci.artifact_max_bytes)

814 } else if kind == "cache" {

815 let id = id.parse::<i64>().map_err(|_| CiError::NotFound)?;

816 let row = actions_cache::Entity::find_by_id(id)

817 .one(service.db_write())

818 .await?

819 .ok_or(CiError::NotFound)?;

820 if row.job_id != context.job.id || row.ready {

821 return Err(CiError::Forbidden.into());

822 }

823 (row.object_key, service.config().ci.cache_max_bytes)

824 } else {

825 return Err(CiError::NotFound.into());

826 };

827 match query.comp.as_deref() {

828 Some("block") => {

829 let block = query

830 .blockid

831 .ok_or_else(|| CiError::InvalidArgument("blockid is required".into()))?;

832 if block.len() > 256 {

833 return Err(CiError::InvalidArgument("blockid is too long".into()).into());

834 }

835 service

836 .storage()

837 .client()

838 .put(

839 &format!(

840 "{key}.blocks/{}",

841 crate::secrets::sha256_hex(block.as_bytes())

842 ),

843 &body,

844 )

845 .await

846 .map_err(CiError::from)?;

847 }

848 Some("blocklist") => {

849 let text = std::str::from_utf8(&body)

850 .map_err(|_| CiError::InvalidArgument("blocklist is not UTF-8".into()))?;

851 let mut blocks = Vec::new();

852 for part in text.split('<') {

853 if let Some(value) = part

854 .strip_prefix("Latest>")

855 .or_else(|| part.strip_prefix("Uncommitted>"))

856 .or_else(|| part.strip_prefix("Committed>"))

857 {

858 blocks.push(value.split('<').next().unwrap_or_default().to_owned());

859 }

860 }

861 if blocks.len() > 50000 {

862 return Err(CiError::InvalidArgument("too many upload blocks".into()).into());

863 }

864 service.assemble_blocks(&key, &blocks, max).await?;

865 }

866 None => {

867 if body.len() as u64 > max {

868 return Err(CiError::InvalidArgument(

869 "upload exceeds configured size limit".into(),

870 )

871 .into());

872 }

873 service

874 .storage()

875 .client()

876 .put(&key, &body)

877 .await

878 .map_err(CiError::from)?;

879 }

880 _ => return Err(CiError::InvalidArgument("unsupported upload operation".into()).into()),

881 }

882 Ok((

883 StatusCode::CREATED,

884 [

885 ("x-ms-request-id", Uuid::now_v7().to_string()),

886 ("x-ms-version", "2023-11-03".into()),

887 ("etag", format!("\"{}\"", Uuid::now_v7())),

888 ],

889 )

890 .into_response())

891}

892

893#[derive(Deserialize)]

894struct LegacyLookup {

895 keys: String,

896 version: String,

897}

898

899async fn legacy_cache_lookup(

900 State(service): State<CiService>,

901 headers: HeaderMap,

902 Query(query): Query<LegacyLookup>,

903) -> Result<Response, RuntimeError> {

904 let context = service.runtime_context(&headers).await?;

905 let keys: Vec<String> = query.keys.split(',').map(str::to_owned).collect();

906 match service.cache_lookup(&context,keys.first().map(String::as_str).unwrap_or_default(),keys.get(1..).unwrap_or_default(),&query.version).await?{

907 Some(row)=>Ok(Json(json!({"cacheKey":row.key,"scope":row.git_ref,"creationTime":row.created_at.format(&time::format_description::well_known::Rfc3339).unwrap_or_default(),"archiveLocation":service.storage().client().presign_get(&row.object_key,900)?})).into_response()),

908 None=>Ok(StatusCode::NO_CONTENT.into_response()),

909 }

910}

911

912async fn legacy_cache_reserve(

913 State(service): State<CiService>,

914 headers: HeaderMap,

915 Json(body): Json<Value>,

916) -> Result<Json<Value>, RuntimeError> {

917 let context = service.runtime_context(&headers).await?;

918 let row = service.cache_reserve(&context, &body).await?;

919 Ok(Json(json!({"cacheId":row.id})))

920}

921

922async fn legacy_cache_upload(

923 State(service): State<CiService>,

924 Path(id): Path<i64>,

925 headers: HeaderMap,

926 body: Bytes,

927) -> Result<StatusCode, RuntimeError> {

928 let context = service.runtime_context(&headers).await?;

929 let row = actions_cache::Entity::find_by_id(id)

930 .one(service.db_write())

931 .await

932 .map_err(CiError::from)?

933 .ok_or(CiError::NotFound)?;

934 if row.job_id != context.job.id || row.ready {

935 return Err(CiError::Forbidden.into());

936 }

937 let range = headers

938 .get("content-range")

939 .and_then(|value| value.to_str().ok())

940 .ok_or_else(|| CiError::InvalidArgument("content-range is required".into()))?;

941 let span = range

942 .strip_prefix("bytes ")

943 .and_then(|value| value.split('/').next())

944 .and_then(|value| value.split_once('-'))

945 .ok_or_else(|| CiError::InvalidArgument("invalid content-range".into()))?;

946 let start = span

947 .0

948 .parse::<u64>()

949 .map_err(|_| CiError::InvalidArgument("invalid content-range".into()))?;

950 let end = span

951 .1

952 .parse::<u64>()

953 .map_err(|_| CiError::InvalidArgument("invalid content-range".into()))?;

954 if end < start

955 || end - start + 1 != body.len() as u64

956 || end >= service.config().ci.cache_max_bytes

957 {

958 return Err(CiError::InvalidArgument("cache upload range is invalid".into()).into());

959 }

960 service

961 .storage()

962 .client()

963 .put(&format!("{}.ranges/{start:020}", row.object_key), &body)

964 .await

965 .map_err(CiError::from)?;

966 Ok(StatusCode::NO_CONTENT)

967}

968

969async fn legacy_cache_finalize(

970 State(service): State<CiService>,

971 Path(id): Path<i64>,

972 headers: HeaderMap,

973 Json(body): Json<Value>,

974) -> Result<StatusCode, RuntimeError> {

975 let context = service.runtime_context(&headers).await?;

976 let row = actions_cache::Entity::find_by_id(id)

977 .one(service.db_write())

978 .await

979 .map_err(CiError::from)?

980 .ok_or(CiError::NotFound)?;

981 if row.job_id != context.job.id {

982 return Err(CiError::Forbidden.into());

983 }

984 let mut ranges = service

985 .storage()

986 .client()

987 .list(&format!("{}.ranges/", row.object_key))

988 .await

989 .map_err(CiError::from)?;

990 ranges.sort_by(|a, b| a.key.cmp(&b.key));

991 let temporary = std::env::temp_dir().join(format!("gitdata-cache-upload-{}", Uuid::now_v7()));

992 let assembled: CiResult<()> = async {

993 let mut file = tokio::fs::OpenOptions::new()

994 .create_new(true)

995 .write(true)

996 .read(true)

997 .open(&temporary)

998 .await

999 .map_err(|error| CiError::Internal(error.to_string()))?;

1000 let mut expected = 0;

1001 for range in &ranges {

1002 let offset = range

1003 .key

1004 .rsplit('/')

1005 .next()

1006 .and_then(|value| value.parse::<u64>().ok())

1007 .ok_or_else(|| CiError::InvalidArgument("invalid cache segment".into()))?;

1008 if offset != expected {

1009 return Err(CiError::InvalidArgument(

1010 "cache segments overlap or have gaps".into(),

1011 ));

1012 }

1013 let data = service.storage().client().get(&range.key).await?;

1014 expected += data.len() as u64;

1015 file.write_all(&data)

1016 .await

1017 .map_err(|error| CiError::Internal(error.to_string()))?;

1018 }

1019 file.seek(std::io::SeekFrom::Start(0))

1020 .await

1021 .map_err(|error| CiError::Internal(error.to_string()))?;

1022 service

1023 .storage()

1024 .client()

1025 .put_stream(&row.object_key, &mut file, "application/octet-stream")

1026 .await?;

1027 Ok(())

1028 }

1029 .await;

1030 let _ = tokio::fs::remove_file(&temporary).await;

1031 assembled?;

1032 service.cache_finalize(&context, Some(id), &body).await?;

1033 for range in ranges {

1034 service

1035 .storage()

1036 .client()

1037 .delete(&range.key)

1038 .await

1039 .map_err(CiError::from)?;

1040 }

1041 Ok(StatusCode::NO_CONTENT)

1042}

Added lib/ci/src/actions_scheduler.rs +908 −0 View lib/ci/src/actions_scheduler.rs

@@ -0,0 +1,908 @@

1use std::collections::BTreeMap;

2

3use ci_protocol::actions::{ActionsWorkflow, EngineExpansion, NeedResult, strings};

4use models::ci::workflow_job::WorkflowJobState as JobState;

5use models::ci::{workflow_job, workflow_run};

6use sea_orm::{

7 ActiveModelTrait, ColumnTrait, EntityTrait, QueryFilter, QuerySelect, Set, TransactionTrait,

8};

9use serde_json::{Value, json};

10use time::OffsetDateTime;

11use uuid::Uuid;

12

13use crate::{

14 CiService,

15 errors::{CiError, CiResult},

16};

17

18impl CiService {

19 pub async fn claim_actions_job(

20 &self,

21 runner: &models::ci::runner::Model,

22 job: &workflow_job::Model,

23 labels: &[String],

24 ) -> CiResult<bool> {

25 use sea_orm::{ConnectionTrait, PaginatorTrait};

26 let capabilities: Vec<String> = runner

27 .capabilities

28 .as_ref()

29 .and_then(|value| serde_json::from_value(value.clone()).ok())

30 .unwrap_or_default();

31 let context = job.actions_context.as_ref().unwrap_or(&Value::Null);

32 if let Some(group) = context["instance"]["group"].as_str() {

33 if !capabilities

34 .iter()

35 .any(|cap| cap == &format!("runner-group:{group}"))

36 {

37 return Ok(false);

38 }

39 }

40 let definition =

41 &context["workflow"]["document"]["jobs"][context["job"].as_str().unwrap_or_default()];

42 if (!definition["container"].is_null()

43 || definition["services"]

44 .as_object()

45 .is_some_and(|value| !value.is_empty())

46 || definition["steps"].as_array().is_some_and(|steps| {

47 steps.iter().any(|step| {

48 step["uses"]

49 .as_str()

50 .is_some_and(|value| value.starts_with("docker://"))

51 })

52 }))

53 && !capabilities.iter().any(|value| value == "docker")

54 {

55 return Ok(false);

56 }

57 if let Some(environment) = job.environment.as_deref() {

58 if let Some(environment) = self

59 .find_actions_environment(job.space_id, job.repo_id, environment)

60 .await?

61 {

62 let required: Vec<String> =

63 serde_json::from_value(environment.required_labels).unwrap_or_default();

64 if required.iter().any(|label| !labels.contains(label)) {

65 return Ok(false);

66 }

67 }

68 }

69 let txn = self.db_write().begin().await?;

70 for lock in [runner.id.as_u128() as i64, job.repo_id.as_u128() as i64] {

71 txn.execute_unprepared(&format!("SELECT pg_advisory_xact_lock({lock})"))

72 .await?;

73 }

74 let current = workflow_job::Entity::find_by_id(job.id)

75 .lock_exclusive()

76 .one(&txn)

77 .await?

78 .ok_or(CiError::NotFound)?;

79 if current.state != JobState::Queued {

80 return Ok(false);

81 }

82 let active = workflow_job::Entity::find()

83 .filter(workflow_job::Column::RunnerId.eq(runner.id))

84 .filter(workflow_job::Column::State.is_in([JobState::Claimed, JobState::Running]))

85 .count(&txn)

86 .await?;

87 if active >= runner.capacity.max(1) as u64 || !self.actions_can_claim(&current).await? {

88 return Ok(false);

89 }

90 let now = OffsetDateTime::now_utc();

91 let mut active: workflow_job::ActiveModel = current.into();

92 active.state = Set(JobState::Claimed);

93 active.runner_id = Set(Some(runner.id));

94 active.lease_token = Set(Some(crate::secrets::random_secret(24)));

95 active.lease_expires_at = Set(Some(

96 now + time::Duration::seconds(self.config().ci.lease_secs as i64),

97 ));

98 active.attempt = Set(job.attempt + 1);

99 active.updated_at = Set(now);

100 active.update(&txn).await?;

101 txn.commit().await?;

102 Ok(true)

103 }

104

105 pub async fn actions_can_claim(&self, job: &workflow_job::Model) -> CiResult<bool> {

106 use sea_orm::PaginatorTrait;

107 let context = job.actions_context.as_ref().unwrap_or(&Value::Null);

108 if context["cancel_requested"].as_bool().unwrap_or(false) {

109 return Ok(false);

110 }

111 if let Some(parent) = job.logical_job_id {

112 let active = workflow_job::Entity::find()

113 .filter(workflow_job::Column::LogicalJobId.eq(parent))

114 .filter(workflow_job::Column::State.is_in([JobState::Claimed, JobState::Running]))

115 .filter(workflow_job::Column::IsTemplate.eq(false))

116 .count(self.db_write())

117 .await?;

118 if active

119 >= context["instance"]["max_parallel"]

120 .as_u64()

121 .unwrap_or(1)

122 .max(1)

123 {

124 return Ok(false);

125 }

126 }

127 let run = workflow_run::Entity::find_by_id(job.run_id)

128 .one(self.db_write())

129 .await?

130 .ok_or(CiError::NotFound)?;

131 if let Some(group) = &run.concurrency_group {

132 let active = workflow_run::Entity::find()

133 .filter(workflow_run::Column::RepoId.eq(run.repo_id))

134 .filter(workflow_run::Column::ConcurrencyGroup.eq(group))

135 .filter(workflow_run::Column::Id.ne(run.id))

136 .filter(workflow_run::Column::State.is_in([

137 models::ci::workflow_run::WorkflowRunState::Running,

138 models::ci::workflow_run::WorkflowRunState::CancelRequested,

139 ]))

140 .count(self.db_write())

141 .await?;

142 if active > 0 {

143 return Ok(false);

144 }

145 }

146 let concurrency = &context["instance"]["concurrency"];

147 let group = concurrency

148 .as_str()

149 .or_else(|| concurrency["group"].as_str());

150 if let Some(group) = group {

151 let active = workflow_job::Entity::find()

152 .filter(workflow_job::Column::RepoId.eq(job.repo_id))

153 .filter(workflow_job::Column::State.is_in([JobState::Claimed, JobState::Running]))

154 .filter(workflow_job::Column::IsTemplate.eq(false))

155 .all(self.db_write())

156 .await?;

157 if active.iter().any(|other| {

158 other.actions_context.as_ref().is_some_and(|ctx| {

159 ctx["instance"]["concurrency"]

160 .as_str()

161 .or_else(|| ctx["instance"]["concurrency"]["group"].as_str())

162 .is_some_and(|other| other.eq_ignore_ascii_case(group))

163 })

164 }) {

165 return Ok(false);

166 }

167 }

168 self.actions_environment_available(job).await

169 }

170

171 pub async fn actions_github_context(&self, run: &workflow_run::Model) -> CiResult<Value> {

172 let repo = models::repo::repos::Entity::find_by_id(run.repo_id)

173 .one(self.db_read())

174 .await?

175 .ok_or(CiError::NotFound)?;

176 let space = self.space_of_repo(&repo).await?;

177 let actor = if let Some(id) = run.trigger_user_id {

178 models::users::user::Entity::find_by_id(id)

179 .one(self.db_read())

180 .await?

181 .map(|user| user.username)

182 .unwrap_or_else(|| id.to_string())

183 } else {

184 space.lower_name.clone()

185 };

186 Ok(

187 json!({"workflow":run.workflow_name,"workflow_ref":format!("{}/{}@{}",space.lower_name,run.workflow_file,run.git_ref),

188 "run_id":run.actions_id.unwrap_or(0).to_string(),"run_number":run.run_number.unwrap_or(1).to_string(),"run_attempt":(run.attempt+1).to_string(),

189 "repository":format!("{}/{}",space.lower_name,repo.name),"repository_owner":space.lower_name,"repository_id":repo.id.to_string(),

190 "actor":actor,"event_name":run.event.as_str(),"sha":run.commit_sha,"ref":run.git_ref,

191 "head_ref":run.branch.clone().unwrap_or_default(),"base_ref":run.base_ref.clone().unwrap_or_default(),

192 "server_url":self.config().git.base_url.trim_end_matches('/'),"api_url":format!("{}/api/v1",self.config().git.base_url.trim_end_matches('/')),

193 "graphql_url":"","retention_days":self.config().ci.artifact_retention_days.to_string()}),

194 )

195 }

196

197 pub async fn actions_variables(&self, repo_id: Uuid) -> CiResult<BTreeMap<String, String>> {

198 let rows = models::ci::variable::Entity::find()

199 .filter(models::ci::variable::Column::RepoId.eq(repo_id))

200 .all(self.db_read())

201 .await?;

202 Ok(rows.into_iter().map(|row| (row.name, row.value)).collect())

203 }

204

205 pub async fn actions_request(

206 &self,

207 run: &workflow_run::Model,

208 context: &Value,

209 needs: &BTreeMap<String, NeedResult>,

210 op: &str,

211 ) -> CiResult<Value> {

212 let workflow: ActionsWorkflow = serde_json::from_value(context["workflow"].clone())?;

213 Ok(

214 json!({"op":op,"source":workflow.source,"file":workflow.file,"job":context["job"],

215 "inputs":context.get("inputs").cloned().unwrap_or(json!({})),"matrix":context.get("instance").and_then(|i|i.get("matrix")).cloned().unwrap_or(json!({})),

216 "needs":needs,"github":self.actions_github_context(run).await?,"event":run.event_payload,

217 "vars":self.actions_variables(run.repo_id).await?}),

218 )

219 }

220

221 pub fn actions_needs(

222 job: &workflow_job::Model,

223 jobs: &[workflow_job::Model],

224 ) -> BTreeMap<String, NeedResult> {

225 let scope = job

226 .actions_context

227 .as_ref()

228 .and_then(|v| v["scope"].as_str())

229 .unwrap_or_default();

230 strings(Some(&job.needs))

231 .into_iter()

232 .filter_map(|key| {

233 let row = jobs

234 .iter()

235 .find(|row| row.job_key == key && row.is_template)?;

236 Some((

237 key.strip_prefix(scope).unwrap_or(&key).to_owned(),

238 NeedResult {

239 result: actions_result(row).into(),

240 outputs: row

241 .outputs

242 .as_ref()

243 .and_then(|v| serde_json::from_value(v.clone()).ok())

244 .unwrap_or_default(),

245 },

246 ))

247 })

248 .collect()

249 }

250

251 pub async fn advance_actions_run(&self, run: &workflow_run::Model) -> CiResult<u64> {

252 self.initialize_actions_metadata(run).await?;

253 let jobs = workflow_job::Entity::find()

254 .filter(workflow_job::Column::RunId.eq(run.id))

255 .all(self.db_write())

256 .await?;

257 let mut changed = 0;

258 for template in jobs

259 .iter()

260 .filter(|job| job.is_template && !job.state.is_terminal())

261 {

262 if template.state == JobState::Running {

263 if self

264 .aggregate_actions_template(run, template, &jobs)

265 .await?

266 {

267 changed += 1

268 }

269 continue;

270 }

271 if template.state != JobState::Pending {

272 continue;

273 }

274 let dependencies = strings(Some(&template.needs));

275 if dependencies.iter().any(|key| {

276 !jobs

277 .iter()

278 .any(|job| job.is_template && job.job_key == *key && job.state.is_terminal())

279 }) {

280 continue;

281 }

282 match self.expand_actions_template(run, template, &jobs).await {

283 Ok(count) => changed += count,

284 Err(error) => {

285 let mut active: workflow_job::ActiveModel = template.clone().into();

286 active.state = Set(JobState::Failure);

287 active.outcome = Set(Some("failure".into()));

288 active.error = Set(Some(error.to_string()));

289 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

290 active.updated_at = Set(OffsetDateTime::now_utc());

291 active.update(self.db_write()).await?;

292 tracing::warn!(job=%template.id,%error,"Actions job planning failed");

293 changed += 1;

294 }

295 }

296 }

297 for failed in jobs.iter().filter(|job| {

298 !job.is_template

299 && job.state.is_terminal()

300 && !matches!(job.state, JobState::Success | JobState::Skipped)

301 }) {

302 let context = failed.actions_context.as_ref().unwrap_or(&Value::Null);

303 if !context["instance"]["fail_fast"].as_bool().unwrap_or(true)

304 || context["instance"]["continue_on_error"]

305 .as_bool()

306 .unwrap_or(false)

307 {

308 continue;

309 }

310 for sibling in jobs.iter().filter(|job| {

311 job.logical_job_id == failed.logical_job_id

312 && job.id != failed.id

313 && !job.state.is_terminal()

314 }) {

315 let mut active: workflow_job::ActiveModel = sibling.clone().into();

316 if sibling.state.is_active() {

317 let mut data = sibling.actions_context.clone().unwrap_or(json!({}));

318 data["cancel_requested"] = json!(true);

319 active.actions_context = Set(Some(data));

320 } else {

321 active.state = Set(JobState::Cancelled);

322 active.outcome = Set(Some("cancelled".into()));

323 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

324 }

325 active.updated_at = Set(OffsetDateTime::now_utc());

326 active.update(self.db_write()).await?;

327 changed += 1;

328 }

329 }

330 Ok(changed)

331 }

332

333 async fn initialize_actions_metadata(&self, run: &workflow_run::Model) -> CiResult<()> {

334 if run.definition["_actions_initialized"]

335 .as_bool()

336 .unwrap_or(false)

337 {

338 return Ok(());

339 }

340 let workflow = ci_protocol::actions::from_snapshot(&run.definition)

341 .ok_or_else(|| CiError::Internal("Actions snapshot missing".into()))?;

342 let key = workflow

343 .projection()

344 .jobs

345 .keys()

346 .next()

347 .cloned()

348 .ok_or_else(|| CiError::InvalidArgument("workflow has no jobs".into()))?;

349 let context = json!({"workflow":workflow,"job":key,"inputs":run.event_payload.get("_gitdata_inputs").or_else(||run.event_payload.get("inputs")).cloned().unwrap_or(json!({}))});

350 let mut request = self

351 .actions_request(run, &context, &BTreeMap::new(), "evaluate")

352 .await?;

353 request["value"] = json!({"concurrency":workflow.document["concurrency"],"run-name":workflow.document["run-name"]});

354 let evaluated = self.actions_engine(request).await?;

355 let concurrency = &evaluated["concurrency"];

356 let group = concurrency

357 .as_str()

358 .or_else(|| concurrency["group"].as_str())

359 .filter(|group| !group.is_empty())

360 .map(str::to_lowercase);

361 let mut definition = run.definition.clone();

362 definition["_actions_initialized"] = json!(true);

363 if let Some(name) = evaluated["run-name"].as_str() {

364 definition["_display_name"] = json!(name);

365 }

366 if group.is_some() {

367 definition["concurrency"] = json!({"group":group,"cancel_in_progress":concurrency["cancel-in-progress"].as_bool().unwrap_or(false)});

368 }

369 let mut active: workflow_run::ActiveModel = run.clone().into();

370 active.definition = Set(definition);

371 active.concurrency_group = Set(group);

372 active.update(self.db_write()).await?;

373 Ok(())

374 }

375

376 pub async fn enforce_actions_concurrency(&self) -> CiResult<u64> {

377 use models::ci::workflow_run::WorkflowRunState as RunState;

378 let runs = workflow_run::Entity::find()

379 .filter(workflow_run::Column::State.is_in([

380 RunState::Queued,

381 RunState::Running,

382 RunState::CancelRequested,

383 ]))

384 .all(self.db_write())

385 .await?;

386 let mut groups = BTreeMap::<(Uuid, String), Vec<workflow_run::Model>>::new();

387 let mut changed = 0;

388 for run in runs {

389 if ci_protocol::actions::from_snapshot(&run.definition).is_some() {

390 if let Some(group) = &run.concurrency_group {

391 groups

392 .entry((run.repo_id, group.to_lowercase()))

393 .or_default()

394 .push(run);

395 }

396 }

397 }

398 for (_, mut runs) in groups {

399 runs.sort_by_key(|run| (run.created_at, run.id));

400 let pending: Vec<_> = runs

401 .iter()

402 .filter(|run| run.state == RunState::Queued)

403 .collect();

404 if let Some(latest) = pending.last() {

405 for old in pending.iter().take(pending.len().saturating_sub(1)) {

406 self.mark_run_cancel_requested(old.id).await?;

407 changed += 1;

408 }

409 if latest.definition["concurrency"]["cancel_in_progress"]

410 .as_bool()

411 .unwrap_or(false)

412 {

413 for active in runs.iter().filter(|run| run.state == RunState::Running) {

414 self.mark_run_cancel_requested(active.id).await?;

415 changed += 1;

416 }

417 }

418 }

419 }

420 let jobs = workflow_job::Entity::find()

421 .filter(workflow_job::Column::IsTemplate.eq(false))

422 .filter(workflow_job::Column::State.is_in([

423 JobState::Queued,

424 JobState::Claimed,

425 JobState::Running,

426 ]))

427 .all(self.db_write())

428 .await?;

429 let mut groups = BTreeMap::<(Uuid, String), Vec<workflow_job::Model>>::new();

430 for job in jobs {

431 if let Some(concurrency) = job

432 .actions_context

433 .as_ref()

434 .map(|ctx| &ctx["instance"]["concurrency"])

435 {

436 if let Some(group) = concurrency

437 .as_str()

438 .or_else(|| concurrency["group"].as_str())

439 {

440 groups

441 .entry((job.repo_id, group.to_lowercase()))

442 .or_default()

443 .push(job);

444 }

445 }

446 }

447 for (_, mut jobs) in groups {

448 jobs.sort_by_key(|job| (job.created_at, job.id));

449 let pending: Vec<_> = jobs

450 .iter()

451 .filter(|job| job.state == JobState::Queued)

452 .collect();

453 if let Some(latest) = pending.last() {

454 for old in pending.iter().take(pending.len().saturating_sub(1)) {

455 let mut active: workflow_job::ActiveModel = (*old).clone().into();

456 active.state = Set(JobState::Cancelled);

457 active.outcome = Set(Some("cancelled".into()));

458 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

459 active.update(self.db_write()).await?;

460 changed += 1;

461 }

462 if latest.actions_context.as_ref().is_some_and(|ctx| {

463 ctx["instance"]["concurrency"]["cancel-in-progress"]

464 .as_bool()

465 .unwrap_or(false)

466 }) {

467 for old in jobs.iter().filter(|job| job.state.is_active()) {

468 let mut context = old.actions_context.clone().unwrap_or(json!({}));

469 context["cancel_requested"] = json!(true);

470 let mut active: workflow_job::ActiveModel = old.clone().into();

471 active.actions_context = Set(Some(context));

472 active.update(self.db_write()).await?;

473 changed += 1;

474 }

475 }

476 }

477 }

478 Ok(changed)

479 }

480

481 async fn expand_actions_template(

482 &self,

483 run: &workflow_run::Model,

484 template: &workflow_job::Model,

485 jobs: &[workflow_job::Model],

486 ) -> CiResult<u64> {

487 let context = template

488 .actions_context

489 .clone()

490 .ok_or_else(|| CiError::Internal("Actions context missing".into()))?;

491 let needs = Self::actions_needs(template, jobs);

492 let request = self

493 .actions_request(run, &context, &needs, "expand")

494 .await?;

495 let expansion: EngineExpansion =

496 serde_json::from_value(self.actions_engine(request).await?)?;

497 let workflow: ActionsWorkflow = serde_json::from_value(context["workflow"].clone())?;

498 let mut resolved = Vec::new();

499 for instance in expansion.instances {

500 let call = if instance.uses.is_empty() {

501 None

502 } else {

503 Some(

504 self.resolve_actions_call(run, &workflow, &instance.uses)

505 .await?,

506 )

507 };

508 resolved.push((instance, call));

509 }

510 let txn = self.db_write().begin().await?;

511 let current = workflow_job::Entity::find_by_id(template.id)

512 .lock_exclusive()

513 .one(&txn)

514 .await?

515 .ok_or(CiError::NotFound)?;

516 if current.state != JobState::Pending {

517 return Ok(0);

518 }

519 let now = OffsetDateTime::now_utc();

520 let mut active: workflow_job::ActiveModel = current.into();

521 if expansion.skipped || resolved.is_empty() {

522 active.state = Set(JobState::Skipped);

523 active.finished_at = Set(Some(now));

524 active.outcome = Set(Some("skipped".into()));

525 } else {

526 active.state = Set(JobState::Running);

527 active.started_at = Set(Some(now));

528 }

529 let mut updated_context = context.clone();

530 updated_context["expanded"] = json!(true);

531 active.actions_context = Set(Some(updated_context));

532 active.updated_at = Set(now);

533 active.update(&txn).await?;

534 for (instance, call) in resolved {

535 let id = Uuid::now_v7();

536 let key = format!("{}[{}]", template.job_key, instance.index);

537 let mut child_context = context.clone();

538 child_context["instance"] = serde_json::to_value(&instance)?;

539 child_context["needs_results"] = serde_json::to_value(&needs)?;

540 child_context["permissions"] = effective_permissions(

541 &workflow.document,

542 context.get("permissions"),

543 &instance.permissions,

544 );

545 child_context["kind"] = json!(if call.is_some() {

546 "call_instance"

547 } else {

548 "instance"

549 });

550 if let Some((called, sha)) = &call {

551 child_context["called_workflow"] = serde_json::to_value(called)?;

552 child_context["resolved_sha"] = json!(sha);

553 }

554 workflow_job::ActiveModel {

555 id: Set(id),

556 run_id: Set(run.id),

557 repo_id: Set(template.repo_id),

558 space_id: Set(template.space_id),

559 name: Set(instance.name.clone()),

560 job_key: Set(key.clone()),

561 definition: Set(template.definition.clone()),

562 actions_context: Set(Some(child_context.clone())),

563 outputs: Set(None),

564 outcome: Set(None),

565 logical_job_id: Set(Some(template.id)),

566 is_template: Set(call.is_some()),

567 actions_id: Set(Some(crate::actions::identity(&txn, id, "job").await?)),

568 runs_on: Set(json!(instance.runs_on)),

569 environment: Set(

570 (!instance.environment.is_empty()).then_some(instance.environment.clone())

571 ),

572 needs: Set(template.needs.clone()),

573 state: Set(if call.is_some() {

574 JobState::Running

575 } else {

576 JobState::Queued

577 }),

578 runner_id: Set(None),

579 lease_token: Set(None),

580 lease_expires_at: Set(None),

581 attempt: Set(0),

582 exit_code: Set(None),

583 error: Set(None),

584 started_at: Set(call.as_ref().map(|_| now)),

585 finished_at: Set(None),

586 created_at: Set(now),

587 updated_at: Set(now),

588 }

589 .insert(&txn)

590 .await?;

591 if let Some((called, _)) = call {

592 if called.triggers().get("workflow_call").is_none() {

593 return Err(CiError::InvalidArgument(format!(

594 "{} does not declare workflow_call",

595 called.file

596 )));

597 }

598 let inputs = validate_call_inputs(&called, instance.inputs.clone())?;

599 let scope = format!("{key}/");

600 let projection = called.projection();

601 for (job_key, job) in projection.jobs {

602 let mut next_context = json!({"workflow":called,"job":job_key,"scope":scope,"inputs":inputs,"kind":"template","expanded":false,"permissions":child_context["permissions"],"caller_context":context,"secret_mapping":workflow.document["jobs"][context["job"].as_str().unwrap_or_default()]["secrets"]});

603 next_context["caller_needs"] = serde_json::to_value(&needs)?;

604 workflow_job::ActiveModel {

605 id: Set(Uuid::now_v7()),

606 run_id: Set(run.id),

607 repo_id: Set(template.repo_id),

608 space_id: Set(template.space_id),

609 name: Set(job_key.clone()),

610 job_key: Set(format!("{scope}{job_key}")),

611 definition: Set(serde_json::to_value(&job)?),

612 actions_context: Set(Some(next_context)),

613 is_template: Set(true),

614 logical_job_id: Set(Some(id)),

615 runs_on: Set(json!(job.runs_on)),

616 environment: Set(None),

617 needs: Set(json!(

618 job.needs

619 .iter()

620 .map(|name| format!("{scope}{name}"))

621 .collect::<Vec<_>>()

622 )),

623 state: Set(JobState::Pending),

624 attempt: Set(0),

625 created_at: Set(now),

626 updated_at: Set(now),

627 ..Default::default()

628 }

629 .insert(&txn)

630 .await?;

631 }

632 }

633 }

634 txn.commit().await?;

635 Ok(1)

636 }

637

638 async fn aggregate_actions_template(

639 &self,

640 run: &workflow_run::Model,

641 template: &workflow_job::Model,

642 jobs: &[workflow_job::Model],

643 ) -> CiResult<bool> {

644 let children: Vec<_> = jobs

645 .iter()

646 .filter(|job| job.logical_job_id == Some(template.id))

647 .collect();

648 if children.is_empty() || children.iter().any(|job| !job.state.is_terminal()) {

649 return Ok(false);

650 }

651 let failed = children.iter().any(|job| actions_result(job) == "failure");

652 let cancelled = children

653 .iter()

654 .all(|job| actions_result(job) == "cancelled");

655 let state = if failed {

656 JobState::Failure

657 } else if cancelled {

658 JobState::Cancelled

659 } else {

660 JobState::Success

661 };

662 let mut outputs = BTreeMap::<String, String>::new();

663 let context = template.actions_context.as_ref().unwrap_or(&Value::Null);

664 if context["kind"] == "call_instance" {

665 let called: ActionsWorkflow =

666 serde_json::from_value(context["called_workflow"].clone())?;

667 let mut eval_context = context.clone();

668 eval_context["workflow"] = serde_json::to_value(&called)?;

669 eval_context["job"] = json!(

670 called

671 .projection()

672 .jobs

673 .keys()

674 .next()

675 .cloned()

676 .unwrap_or_default()

677 );

678 let mut request = self

679 .actions_request(run, &eval_context, &BTreeMap::new(), "evaluate")

680 .await?;

681 request["jobs"]=json!(children.iter().filter_map(|job|job.actions_context.as_ref().and_then(|ctx|ctx["job"].as_str()).map(|key|(key.to_owned(),json!({"outputs":job.outputs.clone().unwrap_or(json!({})),"result":actions_result(job)})))).collect::<BTreeMap<_,_>>());

682 request["value"] = called.document["on"]["workflow_call"]["outputs"].clone();

683 let evaluated = self.actions_engine(request).await?;

684 for (key, value) in evaluated.as_object().into_iter().flatten() {

685 if let Some(value) = value["value"].as_str() {

686 outputs.insert(key.clone(), value.into());

687 }

688 }

689 } else {

690 let mut ordered = children;

691 ordered.sort_by_key(|job| (job.finished_at, job.id));

692 for child in ordered.iter().filter(|job| job.state == JobState::Success) {

693 let values: BTreeMap<String, String> = child

694 .outputs

695 .as_ref()

696 .and_then(|v| serde_json::from_value(v.clone()).ok())

697 .unwrap_or_default();

698 for (key, value) in values {

699 if !value.is_empty() {

700 outputs.insert(key, value);

701 }

702 }

703 }

704 }

705 let mut active: workflow_job::ActiveModel = template.clone().into();

706 active.state = Set(state);

707 active.outcome = Set(Some(state.as_str().into()));

708 active.outputs = Set(Some(json!(outputs)));

709 active.finished_at = Set(Some(OffsetDateTime::now_utc()));

710 active.updated_at = Set(OffsetDateTime::now_utc());

711 active.update(self.db_write()).await?;

712 Ok(true)

713 }

714

715 async fn resolve_actions_call(

716 &self,

717 run: &workflow_run::Model,

718 _caller: &ActionsWorkflow,

719 reference: &str,

720 ) -> CiResult<(ActionsWorkflow, String)> {

721 if reference.starts_with("./") {

722 let repo = models::repo::repos::Entity::find_by_id(run.repo_id)

723 .one(self.db_read())

724 .await?

725 .ok_or(CiError::NotFound)?;

726 let space = self.space_of_repo(&repo).await?;

727 let loaded = self

728 .load_workflow_by_file(

729 &space,

730 &repo.name,

731 &run.commit_sha,

732 reference.trim_start_matches("./"),

733 )

734 .await?;

735 return Ok((

736 loaded.actions.ok_or_else(|| {

737 CiError::InvalidArgument("called workflow has no Actions definition".into())

738 })?,

739 run.commit_sha.clone(),

740 ));

741 }

742 let (path, revision) = reference.rsplit_once('@').ok_or_else(|| {

743 CiError::InvalidArgument("reusable workflow reference must include @ref".into())

744 })?;

745 let parts: Vec<_> = path.split('/').collect();

746 if parts.len() != 5

747 || parts[2] != ".github"

748 || parts[3] != "workflows"

749 || parts.iter().any(|part| part.is_empty() || *part == "..")

750 {

751 return Err(CiError::InvalidArgument(

752 "expected owner/repo/.github/workflows/file@ref".into(),

753 ));

754 }

755 let remote = format!("https://github.com/{}/{}.git", parts[0], parts[1]);

756 let output = tokio::time::timeout(

757 std::time::Duration::from_secs(30),

758 tokio::process::Command::new("git")

759 .arg("ls-remote")

760 .arg(&remote)

761 .arg(revision)

762 .arg(format!("refs/heads/{revision}"))

763 .arg(format!("refs/tags/{revision}"))

764 .arg(format!("refs/tags/{revision}^{{}}"))

765 .output(),

766 )

767 .await

768 .map_err(|_| CiError::InvalidArgument("reusable workflow resolution timed out".into()))?

769 .map_err(|error| CiError::Internal(error.to_string()))?;

770 let text = String::from_utf8_lossy(&output.stdout);

771 let sha = if revision.len() == 40 && revision.chars().all(|ch| ch.is_ascii_hexdigit()) {

772 revision.to_owned()

773 } else {

774 text.lines()

775 .last()

776 .and_then(|line| line.split_whitespace().next())

777 .ok_or_else(|| {

778 CiError::InvalidArgument(format!("workflow ref `{reference}` was not found"))

779 })?

780 .to_owned()

781 };

782 let url = format!(

783 "https://raw.githubusercontent.com/{}/{}/{}/.github/workflows/{}",

784 parts[0], parts[1], sha, parts[4]

785 );

786 let source = reqwest::Client::builder()

787 .timeout(std::time::Duration::from_secs(30))

788 .build()

789 .map_err(|error| CiError::Internal(error.to_string()))?

790 .get(url)

791 .send()

792 .await

793 .map_err(|error| CiError::InvalidArgument(error.to_string()))?

794 .error_for_status()

795 .map_err(|error| CiError::InvalidArgument(error.to_string()))?

796 .text()

797 .await

798 .map_err(|error| CiError::InvalidArgument(error.to_string()))?;

799 if source.len() > 256 * 1024 {

800 return Err(CiError::InvalidArgument(

801 "called workflow exceeds size limit".into(),

802 ));

803 }

804 Ok((

805 self.parse_actions_workflow(&format!(".github/workflows/{}", parts[4]), &source)

806 .await?,

807 sha,

808 ))

809 }

810}

811

812pub fn actions_result(job: &workflow_job::Model) -> &str {

813 if job.actions_context.as_ref().is_some_and(|ctx| {

814 ctx["instance"]["continue_on_error"]

815 .as_bool()

816 .unwrap_or(false)

817 }) && job.state == JobState::Failure

818 {

819 return "success";

820 }

821 match job.state {

822 JobState::Success => "success",

823 JobState::Skipped => "skipped",

824 JobState::Cancelled => "cancelled",

825 JobState::Failure | JobState::TimedOut => "failure",

826 _ => "pending",

827 }

828}

829

830fn permission_map(value: &Value) -> Value {

831 match value.as_str() {

832 Some("read-all") => json!({"contents":"read","id-token":"none"}),

833 Some("write-all") => json!({"contents":"write","id-token":"write"}),

834 _ => value.clone(),

835 }

836}

837

838fn effective_permissions(workflow: &Value, ceiling: Option<&Value>, job: &Value) -> Value {

839 let mut value = permission_map(if !job.is_null() {

840 job

841 } else {

842 workflow.get("permissions").unwrap_or(&Value::Null)

843 });

844 if value.is_null() {

845 value = json!({"contents":"read"})

846 }

847 if let Some(ceiling) = ceiling.filter(|v| !v.is_null()) {

848 for name in ["contents", "id-token"] {

849 let rank = |v: Option<&str>| match v {

850 Some("write") => 2,

851 Some("read") => 1,

852 _ => 0,

853 };

854 if rank(value[name].as_str()) > rank(ceiling[name].as_str()) {

855 value[name] = ceiling[name].clone();

856 }

857 }

858 }

859 value

860}

861

862fn validate_call_inputs(workflow: &ActionsWorkflow, provided: Value) -> CiResult<Value> {

863 let mut values = provided.as_object().cloned().unwrap_or_default();

864 let definitions = workflow.document["on"]["workflow_call"]["inputs"]

865 .as_object()

866 .cloned()

867 .unwrap_or_default();

868 for name in values.keys() {

869 if !definitions.contains_key(name) {

870 return Err(CiError::InvalidArgument(format!(

871 "unknown workflow_call input `{name}`"

872 )));

873 }

874 }

875 for (name, definition) in definitions {

876 if !values.contains_key(&name) {

877 if definition["required"].as_bool().unwrap_or(false)

878 && definition.get("default").is_none()

879 {

880 return Err(CiError::InvalidArgument(format!(

881 "required workflow_call input `{name}` is missing"

882 )));

883 }

884 values.insert(

885 name.clone(),

886 definition.get("default").cloned().unwrap_or_else(|| {

887 match definition["type"].as_str() {

888 Some("boolean") => json!(false),

889 Some("number") => json!(0),

890 _ => json!(""),

891 }

892 }),

893 );

894 }

895 let value = &values[&name];

896 let valid = match definition["type"].as_str() {

897 Some("boolean") => value.is_boolean(),

898 Some("number") => value.is_number(),

899 _ => value.is_string(),

900 };

901 if !valid {

902 return Err(CiError::InvalidArgument(format!(

903 "workflow_call input `{name}` has the wrong type"

904 )));

905 }

906 }

907 Ok(Value::Object(values))

908}

Modified lib/ci/src/environments.rs +4 −4 View lib/ci/src/environments.rs

@@ -290,10 +290,8 @@

290290 merged.insert(row.name, value);

291291 }

292292 if let Some(environment) = environment {

293 let env_row = ci_environment::Entity::find()

294 .filter(ci_environment::Column::SpaceId.eq(space_id))

295 .filter(ci_environment::Column::Name.eq(environment.to_string()))

296 .one(self.db_read())

293 let env_row = self

294 .find_actions_environment(space_id, repo_id, environment)

297295 .await?;

298296 if let Some(env_row) = env_row {

299297 let env_secrets = ci_secret::Entity::find()

@@ -379,6 +377,8 @@

379377 }

380378 None => {

381379 ci_environment::ActiveModel {

380 repo_id: Set(None),

381 rules: Set(None),

382382 id: Set(Uuid::now_v7()),

383383 space_id: Set(space_id),

384384 name: Set(name.to_string()),

Modified lib/ci/src/lib.rs +6 −0 View lib/ci/src/lib.rs

@@ -1,3 +1,9 @@

1pub mod actions;

2pub mod actions_environments;

3pub mod actions_events;

4mod actions_payload;

5pub mod actions_runtime;

6mod actions_scheduler;

17pub mod environments;

28pub mod errors;

39pub mod events;

Modified lib/ci/src/openapi.rs +9 −0 View lib/ci/src/openapi.rs

@@ -6,6 +6,15 @@

66#[openapi(

77 info(title = "gitdata pipelines api", version = "1.0"),

88 paths(

9 crate::handlers::actions::dispatch,

10 crate::handlers::actions::environments,

11 crate::handlers::actions::upsert_environment,

12 crate::handlers::actions::approvals,

13 crate::handlers::actions::review,

14 crate::handlers::actions::variables,

15 crate::handlers::actions::upsert_variable,

16 crate::handlers::actions::delete_variable,

17 crate::handlers::actions::download_actions_artifact,

918 crate::handlers::runs::list_runs,

1019 crate::handlers::runs::get_run,

1120 crate::handlers::runs::cancel_run,

Modified lib/ci/src/routes.rs +34 −0 View lib/ci/src/routes.rs

@@ -7,6 +7,39 @@

77pub fn router() -> Router<CiService> {

88 Router::new()

99 .route(

10 "/repos/{space_name}/{repo_name}/pipelines/dispatches",

11 post(crate::handlers::actions::dispatch),

12 )

13 .route(

14 "/repos/{space_name}/{repo_name}/environments",

15 get(crate::handlers::actions::environments),

16 )

17 .route(

18 "/repos/{space_name}/{repo_name}/environments/{name}",

19 put(crate::handlers::actions::upsert_environment),

20 )

21 .route(

22 "/repos/{space_name}/{repo_name}/pipelines/{run_id}/approvals",

23 get(crate::handlers::actions::approvals),

24 )

25 .route(

26 "/repos/{space_name}/{repo_name}/pipelines/{run_id}/approvals/{review_id}",

27 post(crate::handlers::actions::review),

28 )

29 .route(

30 "/repos/{space_name}/{repo_name}/variables",

31 get(crate::handlers::actions::variables),

32 )

33 .route(

34 "/repos/{space_name}/{repo_name}/variables/{name}",

35 put(crate::handlers::actions::upsert_variable)

36 .delete(crate::handlers::actions::delete_variable),

37 )

38 .route(

39 "/repos/{space_name}/{repo_name}/pipelines/actions-artifacts/{artifact_id}/download",

40 get(crate::handlers::actions::download_actions_artifact),

41 )

42 .route(

1043 "/repos/{space_name}/{repo_name}/pipelines",

1144 get(runs::list_runs),

1245 )

@@ -71,6 +104,7 @@

71104 .route("/runner/jobs/claim", post(runner_protocol::claim))

72105 .route("/runner/jobs/{job_id}/start", post(runner_protocol::start))

73106 .route("/runner/jobs/{job_id}/renew", post(runner_protocol::renew))

107 .route("/runner/jobs/{job_id}/events", post(runner_protocol::event))

74108 .route(

75109 "/runner/jobs/{job_id}/steps/{index}",

76110 post(runner_protocol::report_step),

Modified lib/ci/src/runners.rs +48 −3 View lib/ci/src/runners.rs

@@ -115,6 +115,44 @@

115115 capacity: i32,

116116 version: Option<String>,

117117 ) -> CiResult<RegisterResponse> {

118 self.register_runner_inner(token, name, labels, capacity, version)

119 .await

120 }

121

122 pub async fn configure_runner_protocol(

123 &self,

124 runner_id: Uuid,

125 versions: &[u32],

126 capabilities: &[String],

127 ) -> CiResult<u32> {

128 let version = if versions.contains(&2) {

129 2

130 } else if versions.is_empty() || versions.contains(&1) {

131 1

132 } else {

133 return Err(CiError::InvalidArgument(

134 "no common runner protocol version".into(),

135 ));

136 };

137 let row = runner::Entity::find_by_id(runner_id)

138 .one(self.db_write())

139 .await?

140 .ok_or(CiError::NotFound)?;

141 let mut active: runner::ActiveModel = row.into();

142 active.protocol_version = Set(Some(version as i32));

143 active.capabilities = Set(Some(serde_json::to_value(capabilities)?));

144 active.update(self.db_write()).await?;

145 Ok(version)

146 }

147

148 async fn register_runner_inner(

149 &self,

150 token: &str,

151 name: &str,

152 labels: Vec<String>,

153 capacity: i32,

154 version: Option<String>,

155 ) -> CiResult<RegisterResponse> {

118156 self.ensure_enabled()?;

119157 if name.trim().is_empty() || name.len() > 200 {

120158 return Err(CiError::InvalidArgument(

@@ -154,6 +192,7 @@

154192 last_seen_at: Set(Some(now)),

155193 created_at: Set(now),

156194 updated_at: Set(now),

195 ..Default::default()

157196 }

158197 .insert(self.db_write())

159198 .await?;

@@ -190,7 +229,7 @@

190229 credential,

191230 heartbeat_interval_secs: self.config().ci.heartbeat_interval_secs,

192231 poll_timeout_secs: self.config().ci.long_poll_timeout_secs,

193 protocol_version: ci_protocol::PROTOCOL_VERSION,

232 protocol_version: 1,

194233 })

195234 }

196235

@@ -259,13 +298,18 @@

259298 lease_token: String,

260299 ) -> CiResult<ci_protocol::runner::JobPayload> {

261300 let job = workflow_job::Entity::find_by_id(job_id)

262 .one(self.db_read())

301 .one(self.db_write())

263302 .await?

264303 .ok_or(CiError::NotFound)?;

265304 let run = workflow_run::Entity::find_by_id(job.run_id)

266 .one(self.db_read())

305 .one(self.db_write())

267306 .await?

268307 .ok_or(CiError::NotFound)?;

308 if job.actions_context.is_some() {

309 return self

310 .build_actions_job_payload(&job, &run, lease_token)

311 .await;

312 }

269313 let repo = self.repo_by_id(job.repo_id).await?;

270314 let space = self.space_of_repo(&repo).await?;

271315 let definition: serde_json::Value = job.definition.clone();

@@ -409,6 +453,7 @@

409453 heartbeat_interval_secs: self.config().ci.heartbeat_interval_secs,

410454 attempt: job.attempt,

411455 },

456 actions: None,

412457 };

413458 Ok(payload)

414459 }

Modified lib/ci/src/runs.rs +114 −28 View lib/ci/src/runs.rs

@@ -1,6 +1,6 @@

11use std::collections::BTreeMap;

22

3use ci_protocol::dsl::{Job, WorkflowDefinition};

3use ci_protocol::dsl::Job;

44use ci_protocol::events::{CiEvent, PullRequestEvent, PushEvent};

55use models::ci::workflow_job::{self, WorkflowJobState};

66use models::ci::workflow_run::{self, WorkflowEventKind, WorkflowRunState};

@@ -16,7 +16,7 @@

1616

1717use crate::errors::{CiError, CiResult};

1818use crate::service::CiService;

19use crate::workflows::{LoadedWorkflow, pull_request_trigger_matches, push_trigger_matches};

19use crate::workflows::LoadedWorkflow;

2020

2121#[derive(Debug, Clone)]

2222pub struct RunSeed {

@@ -67,6 +67,7 @@

6767 "old_oid": update.old_oid,

6868 "new_oid": update.new_oid,

6969 "actor_id": event.actor_id,

70 "event_id": event.id,

7071 }),

7172 commit_sha: update.new_oid.clone(),

7273 git_ref: update.name.clone(),

@@ -85,13 +86,15 @@

8586 RunSeed {

8687 event: WorkflowEventKind::PullRequest,

8788 event_payload: serde_json::json!({

88 "action": event.action.as_str(),

89 "action": if event.action==ci_protocol::events::PullRequestAction::Merged{"closed"}else{event.action.as_str()},

8990 "number": event.number,

9091 "head_ref": event.head_ref,

9192 "base_ref": event.base_ref,

9293 "head_sha": event.head_sha,

9394 "base_sha": event.base_sha,

9495 "actor_id": event.actor_id,

96 "event_id": event.id,

97 "pull_request":{"number":event.number,"merged":event.action==ci_protocol::events::PullRequestAction::Merged,"head":{"sha":event.head_sha,"ref":event.head_ref,"repo":{"fork":event.head_repo_id.is_some_and(|id|id!=event.repo_id)}},"base":{"sha":event.base_sha,"ref":event.base_ref}},

9598 }),

9699 commit_sha: event.head_sha.clone(),

97100 git_ref: format!("refs/pull/{}/head", event.number),

@@ -111,9 +114,13 @@

111114 if !self.enabled() {

112115 return Ok(Vec::new());

113116 }

117 if let CiEvent::Trigger(event) = event {

118 return self.handle_trigger_event(event).await;

119 }

114120 let (repo_id, space_id, seeds) = match event {

115121 CiEvent::Push(push) => (push.repo_id, push.space_id, push_seeds(push)),

116122 CiEvent::PullRequest(pr) => (pr.repo_id, pr.space_id, vec![pull_request_seed(pr)]),

123 CiEvent::Trigger(_) => unreachable!(),

117124 };

118125 let repo = repos::Entity::find_by_id(repo_id)

119126 .one(self.db_read())

@@ -129,8 +136,30 @@

129136 let workflows = self

130137 .load_valid_workflows(&space, &repo.name, &seed.commit_sha)

131138 .await?;

139 if seed.branch.as_deref() == Some(repo.default_branch.as_str())

140 && seed.event == WorkflowEventKind::Push

141 {

142 self.sync_actions_schedules(repo.id, &workflows).await?;

143 }

144 let paths = if workflows.iter().any(|workflow| {

145 workflow.actions.as_ref().is_some_and(|actions| {

146 actions.document["on"][seed.event.as_str()]

147 .get("paths")

148 .is_some()

149 || actions.document["on"][seed.event.as_str()]

150 .get("paths-ignore")

151 .is_some()

152 })

153 }) {

154 self.actions_changed_paths(repo.id, &seed).await?

155 } else {

156 Vec::new()

157 };

132158 for workflow in workflows {

133 if !matches_trigger(&workflow, event, &seed) {

159 if !self

160 .actions_event_matches(&workflow, &seed, paths.clone())

161 .await?

162 {

134163 continue;

135164 }

136165 let run_id = self.create_run(&repo, &space, &seed, &workflow).await?;

@@ -149,7 +178,25 @@

149178 ) -> CiResult<Uuid> {

150179 let run_id = Uuid::now_v7();

151180 let now = OffsetDateTime::now_utc();

152 let definition = serde_json::to_value(&workflow.definition)?;

181 let mut definition = serde_json::to_value(&workflow.definition)?;

182 if let Some(actions) = &workflow.actions {

183 definition[ci_protocol::actions::SNAPSHOT_KEY] = serde_json::to_value(actions)?;

184 }

185 let event_key = seed

186 .event_payload

187 .get("event_id")

188 .map(|id| format!("{id}:{}:{}", workflow.file, seed.git_ref));

189 if let Some(key) = &event_key {

190 if let Some(existing) = workflow_run::Entity::find()

191 .filter(workflow_run::Column::EventKey.eq(key))

192 .one(self.db_write())

193 .await?

194 {

195 self.emit_actions_workflow_run(&existing, "requested")

196 .await?;

197 return Ok(existing.id);

198 }

199 }

153200 let concurrency_group = workflow

154201 .definition

155202 .concurrency

@@ -157,6 +204,32 @@

157204 .map(|c| expand_concurrency_group(&c.group, seed));

158205

159206 let txn = self.db_write().begin().await?;

207 use sea_orm::ConnectionTrait;

208 txn.execute_unprepared(&format!(

209 "SELECT pg_advisory_xact_lock({})",

210 repo.id.as_u128() as i64

211 ))

212 .await?;

213 if let Some(key) = &event_key {

214 if let Some(existing) = workflow_run::Entity::find()

215 .filter(workflow_run::Column::EventKey.eq(key))

216 .one(&txn)

217 .await?

218 {

219 return Ok(existing.id);

220 }

221 }

222 let actions_id = if workflow.actions.is_some() {

223 Some(crate::actions::identity(&txn, run_id, "run").await?)

224 } else {

225 None

226 };

227 let run_number = if workflow.actions.is_some() {

228 let row=txn.query_one_raw(sea_orm::Statement::from_sql_and_values(sea_orm::DatabaseBackend::Postgres,"INSERT INTO ci_workflow_counter (repo_id,workflow_file,last_number) VALUES ($1,$2,1) ON CONFLICT (repo_id,workflow_file) DO UPDATE SET last_number=ci_workflow_counter.last_number+1 RETURNING last_number",[repo.id.into(),workflow.file.clone().into()])).await?.ok_or_else(||CiError::Internal("workflow counter returned no row".into()))?;

229 Some(row.try_get("", "last_number")?)

230 } else {

231 None

232 };

160233 workflow_run::ActiveModel {

161234 id: Set(run_id),

162235 repo_id: Set(repo.id),

@@ -166,6 +239,9 @@

166239 event: Set(seed.event),

167240 event_payload: Set(seed.event_payload.clone()),

168241 definition: Set(definition.clone()),

242 actions_id: Set(actions_id),

243 run_number: Set(run_number),

244 event_key: Set(event_key),

169245 commit_sha: Set(seed.commit_sha.clone()),

170246 git_ref: Set(seed.git_ref.clone()),

171247 branch: Set(seed.branch.clone()),

@@ -181,6 +257,7 @@

181257 finished_at: Set(None),

182258 created_at: Set(now),

183259 updated_at: Set(now),

260 ..Default::default()

184261 }

185262 .insert(&txn)

186263 .await?;

@@ -193,7 +270,9 @@

193270 Some(condition) => crate::share::conditions::condition_matches(condition, &facts),

194271 None => true,

195272 };

196 let state = if condition_ok {

273 let state = if workflow.actions.is_some() {

274 WorkflowJobState::Pending

275 } else if condition_ok {

197276 if job.needs.is_empty() {

198277 WorkflowJobState::Queued

199278 } else {

@@ -210,6 +289,8 @@

210289 name: Set(job_key.clone()),

211290 job_key: Set(job_key.clone()),

212291 definition: Set(serde_json::to_value(job)?),

292 is_template: Set(workflow.actions.is_some()),

293 actions_context: Set(workflow.actions.as_ref().map(|actions| serde_json::json!({"workflow":actions,"job":job_key,"scope":"","inputs":seed.event_payload.get("_gitdata_inputs").or_else(||seed.event_payload.get("inputs")).cloned().unwrap_or(serde_json::json!({})),"kind":"template","expanded":false}))),

213294 runs_on: Set(serde_json::to_value(&job.runs_on)?),

214295 environment: Set(environment),

215296 needs: Set(serde_json::to_value(&job.needs)?),

@@ -224,11 +305,15 @@

224305 finished_at: Set(None),

225306 created_at: Set(now),

226307 updated_at: Set(now),

308 ..Default::default()

227309 }

228310 .insert(&txn)

229311 .await?;

230312

231313 for (index, step) in job.steps.iter().enumerate() {

314 if workflow.actions.is_some() {

315 break;

316 }

232317 workflow_step::ActiveModel {

233318 id: Set(Uuid::now_v7()),

234319 job_id: Set(job_id),

@@ -246,6 +331,7 @@

246331 finished_at: Set(None),

247332 created_at: Set(now),

248333 updated_at: Set(now),

334 ..Default::default()

249335 }

250336 .insert(&txn)

251337 .await?;

@@ -253,6 +339,14 @@

253339 }

254340

255341 txn.commit().await?;

342 if workflow.actions.is_some() {

343 if let Some(run) = workflow_run::Entity::find_by_id(run_id)

344 .one(self.db_write())

345 .await?

346 {

347 self.emit_actions_workflow_run(&run, "requested").await?;

348 }

349 }

256350 tracing::info!(

257351 run_id = %run_id,

258352 repo_id = %repo.id,

@@ -281,15 +375,27 @@

281375 .await?

282376 .ok_or(CiError::NotFound)?;

283377 let space = self.space_of_repo(&repo).await?;

284 let definition: WorkflowDefinition = serde_json::from_value(run.definition.clone())?;

378 let actions = ci_protocol::actions::from_snapshot(&run.definition);

379 if actions.is_none() {

380 return Err(CiError::InvalidArgument("historical run has no GitHub Actions definition; trigger a workflow under .github/workflows/".into()));

381 }

382 let definition = actions

383 .as_ref()

384 .expect("Actions snapshot checked")

385 .projection();

285386 let workflow = LoadedWorkflow {

286387 file: run.workflow_file.clone(),

287388 name: run.workflow_name.clone(),

288389 definition,

390 actions,

289391 };

290392 let seed = RunSeed {

291393 event: run.event,

292 event_payload: run.event_payload.clone(),

394 event_payload: {

395 let mut value = run.event_payload.clone();

396 value.as_object_mut().map(|v| v.remove("event_id"));

397 value

398 },

293399 commit_sha: run.commit_sha.clone(),

294400 git_ref: run.git_ref.clone(),

295401 branch: run.branch.clone(),

@@ -300,26 +406,6 @@

300406 trigger_user_id: Some(caller),

301407 };

302408 self.create_run(&repo, &space, &seed, &workflow).await

303 }

304}

305

306fn matches_trigger(workflow: &LoadedWorkflow, event: &CiEvent, seed: &RunSeed) -> bool {

307 let triggers = &workflow.definition.on;

308 match event {

309 CiEvent::Push(_) => match triggers.push.as_ref() {

310 Some(trigger) => {

311 push_trigger_matches(trigger, seed.branch.as_deref(), seed.tag.as_deref())

312 }

313 None => false,

314 },

315 CiEvent::PullRequest(pr) => match triggers.pull_request.as_ref() {

316 Some(trigger) => pull_request_trigger_matches(

317 trigger,

318 seed.base_ref.as_deref().unwrap_or_default(),

319 pr.action,

320 ),

321 None => false,

322 },

323409 }

324410}

325411

Modified lib/ci/src/scheduler.rs +107 −16 View lib/ci/src/scheduler.rs

@@ -53,7 +53,11 @@

5353 .await?;

5454 let mut moved = 0u64;

5555 for run in runs {

56 moved += self.advance_run(run.id).await?;

56 moved += if ci_protocol::actions::from_snapshot(&run.definition).is_some() {

57 self.advance_actions_run(&run).await?

58 } else {

59 self.advance_run(run.id).await?

60 };

5761 }

5862 Ok(moved)

5963 }

@@ -143,6 +147,7 @@

143147 }

144148

145149 pub async fn enforce_concurrency(&self) -> CiResult<u64> {

150 let actions_changed = self.enforce_actions_concurrency().await?;

146151 let groups: Vec<GroupRow> = GroupRow::find_by_statement(Statement::from_string(

147152 DatabaseBackend::Postgres,

148153 "SELECT DISTINCT repo_id, concurrency_group FROM workflow_run \

@@ -153,7 +158,7 @@

153158 .all(self.db_read())

154159 .await?;

155160

156 let mut affected = 0u64;

161 let mut affected = actions_changed;

157162 for group in groups {

158163 let runs = workflow_run::Entity::find()

159164 .filter(workflow_run::Column::RepoId.eq(group.repo_id))

@@ -169,6 +174,12 @@

169174 if runs.len() < 2 {

170175 continue;

171176 }

177 if runs

178 .iter()

179 .any(|run| ci_protocol::actions::from_snapshot(&run.definition).is_some())

180 {

181 continue;

182 }

172183 let oldest = &runs[0];

173184 for run in runs.iter().skip(1) {

174185 let cancel_in_progress = run

@@ -241,6 +252,20 @@

241252 .await?;

242253 let now = OffsetDateTime::now_utc();

243254 for job in jobs {

255 if job.is_template {

256 continue;

257 }

258 if job.actions_context.is_some() {

259 if runner.protocol_version.unwrap_or(1) < 2

260 || !runner

261 .capabilities

262 .as_ref()

263 .and_then(|v| v.as_array())

264 .is_some_and(|v| v.iter().any(|c| c.as_str() == Some("actions-v2")))

265 {

266 continue;

267 }

268 }

244269 let job_labels: Vec<String> =

245270 serde_json::from_value(job.runs_on.clone()).unwrap_or_default();

246271 if !labels.to_vec().contains_all(&job_labels) {

@@ -251,16 +276,26 @@

251276 {

252277 continue;

253278 }

254 if let Some(environment) = &job.environment

279 if job.actions_context.is_some() {

280 if self.claim_actions_job(&runner, &job, labels).await? {

281 self.mark_run_running(job.run_id, runner_id).await?;

282 self.start_deployment(&job).await?;

283 return Ok(Some(job.id));

284 }

285 continue;

286 }

287 if job.actions_context.is_none()

288 && let Some(environment) = &job.environment

255289 && !self

256290 .runner_matches_environment(job.space_id, environment, labels)

257291 .await?

258292 {

259293 continue;

260294 }

261 if !self

262 .environment_available(job.space_id, job.environment.as_ref())

263 .await?

295 if job.actions_context.is_none()

296 && !self

297 .environment_available(job.space_id, job.environment.as_ref())

298 .await?

264299 {

265300 continue;

266301 }

@@ -411,7 +446,7 @@

411446

412447 async fn mark_run_running(&self, run_id: Uuid, runner_id: Uuid) -> CiResult<()> {

413448 let now = OffsetDateTime::now_utc();

414 workflow_run::Entity::update_many()

449 let updated = workflow_run::Entity::update_many()

415450 .col_expr(

416451 workflow_run::Column::State,

417452 Expr::value(WorkflowRunState::Running),

@@ -423,6 +458,14 @@

423458 .filter(workflow_run::Column::State.eq(WorkflowRunState::Queued))

424459 .exec(self.db_write())

425460 .await?;

461 if updated.rows_affected > 0 {

462 if let Some(run) = workflow_run::Entity::find_by_id(run_id)

463 .one(self.db_write())

464 .await?

465 {

466 self.emit_actions_workflow_run(&run, "in_progress").await?;

467 }

468 }

426469 Ok(())

427470 }

428471

@@ -559,6 +602,22 @@

559602 .exec(self.db_write())

560603 .await?;

561604

605 workflow_job::Entity::update_many()

606 .col_expr(

607 workflow_job::Column::State,

608 Expr::value(WorkflowJobState::Cancelled),

609 )

610 .col_expr(

611 workflow_job::Column::Outcome,

612 Expr::value(Some("cancelled".to_owned())),

613 )

614 .col_expr(workflow_job::Column::FinishedAt, Expr::value(Some(now)))

615 .filter(workflow_job::Column::RunId.eq(run.id))

616 .filter(workflow_job::Column::IsTemplate.eq(true))

617 .filter(workflow_job::Column::State.eq(WorkflowJobState::Running))

618 .exec(self.db_write())

619 .await?;

620

562621 let stale = now - time::Duration::seconds(grace.max(1));

563622 let forced = workflow_job::Entity::update_many()

564623 .col_expr(

@@ -611,14 +670,20 @@

611670 if jobs.is_empty() || !jobs.iter().all(|j| j.state.is_terminal()) {

612671 continue;

613672 }

614 let failed = jobs.iter().any(|j| {

615 matches!(

616 j.state,

617 WorkflowJobState::Failure

618 | WorkflowJobState::TimedOut

619 | WorkflowJobState::Cancelled

620 )

621 });

673 let failed = jobs

674 .iter()

675 .filter(|job| {

676 ci_protocol::actions::from_snapshot(&run.definition).is_none()

677 || (job.is_template && job.logical_job_id.is_none())

678 })

679 .any(|j| {

680 matches!(

681 j.state,

682 WorkflowJobState::Failure

683 | WorkflowJobState::TimedOut

684 | WorkflowJobState::Cancelled

685 )

686 });

622687 let state = if failed {

623688 WorkflowRunState::Failure

624689 } else {

@@ -653,7 +718,7 @@

653718 return Ok(());

654719 }

655720 let run = workflow_run::Entity::find_by_id(run_id)

656 .one(self.db_read())

721 .one(self.db_write())

657722 .await?

658723 .ok_or(CiError::NotFound)?;

659724 self.publish_commit_status(&run, state).await?;

@@ -668,6 +733,7 @@

668733 )

669734 .await;

670735 tracing::info!(run_id = %run_id, state = ?state, "workflow run finished");

736 self.emit_actions_workflow_run(&run, "completed").await?;

671737 Ok(())

672738 }

673739

@@ -718,6 +784,12 @@

718784 && run.state == WorkflowRunState::CancelRequested

719785 {

720786 out.push(job.id);

787 } else if job

788 .actions_context

789 .as_ref()

790 .is_some_and(|context| context["cancel_requested"].as_bool().unwrap_or(false))

791 {

792 out.push(job.id);

721793 }

722794 }

723795 Ok(out)

@@ -840,6 +912,9 @@

840912 if job.state.is_terminal() {

841913 return Ok(job.run_id);

842914 }

915 if finish.lease_token.is_some() && finish.lease_token != job.lease_token {

916 return Err(CiError::Conflict("job lease was superseded".into()));

917 }

843918 let now = OffsetDateTime::now_utc();

844919 let state = match finish.state {

845920 JobFinishState::Success => WorkflowJobState::Success,

@@ -850,6 +925,14 @@

850925 let updated = workflow_job::Entity::update_many()

851926 .col_expr(workflow_job::Column::State, Expr::value(state))

852927 .col_expr(

928 workflow_job::Column::Outputs,

929 Expr::value(Some(serde_json::to_value(&finish.outputs)?)),

930 )

931 .col_expr(

932 workflow_job::Column::Outcome,

933 Expr::value(Some(state.as_str().to_owned())),

934 )

935 .col_expr(

853936 workflow_job::Column::ExitCode,

854937 Expr::value(finish.exit_code),

855938 )

@@ -869,6 +952,14 @@

869952 .col_expr(workflow_job::Column::UpdatedAt, Expr::value(now))

870953 .filter(workflow_job::Column::Id.eq(job_id))

871954 .filter(workflow_job::Column::RunnerId.eq(runner_id))

955 .filter(

956 sea_orm::Condition::all().add_option(

957 finish

958 .lease_token

959 .as_ref()

960 .map(|token| workflow_job::Column::LeaseToken.eq(token.clone())),

961 ),

962 )

872963 .exec(self.db_write())

873964 .await?;

874965 if updated.rows_affected == 0 {

Modified lib/ci/src/storage.rs +1 −0 View lib/ci/src/storage.rs

@@ -52,6 +52,7 @@

5252 object_key: Set(key.clone()),

5353 size: Set(size),

5454 sha256: Set(sha256),

55 mime_type: Set(None),

5556 state: Set(ArtifactState::Uploading),

5657 expires_at: Set(Some(expires_at)),

5758 created_at: Set(now),

Modified lib/ci/src/triggers.rs +131 −4 View lib/ci/src/triggers.rs

@@ -1,6 +1,6 @@

1use models::ci::workflow_job;

21use models::ci::workflow_run::{self, WorkflowEventKind, WorkflowRunState};

32use models::ci::workflow_step;

3use models::ci::{actions_identity, workflow_job};

44use models::repo::repos;

55use models::space::spaces;

66use sea_orm::{ColumnTrait, EntityTrait, PaginatorTrait, QueryFilter, QueryOrder, QuerySelect};

@@ -17,6 +17,9 @@

1717 pub repo_id: Uuid,

1818 pub workflow_file: String,

1919 pub workflow_name: String,

20 pub display_name: Option<String>,

21 pub actions_id: Option<i64>,

22 pub run_number: Option<i64>,

2023 pub event: String,

2124 pub commit_sha: String,

2225 pub git_ref: String,

@@ -40,6 +43,12 @@

4043 pub exit_code: Option<i32>,

4144 pub started_at: Option<OffsetDateTime>,

4245 pub finished_at: Option<OffsetDateTime>,

46 pub stage: String,

47 pub outcome: Option<String>,

48 pub conclusion: Option<String>,

49 pub outputs: serde_json::Value,

50 pub annotations: serde_json::Value,

51 pub summary: Option<String>,

4352}

4453

4554#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]

@@ -57,6 +66,12 @@

5766 pub started_at: Option<OffsetDateTime>,

5867 pub finished_at: Option<OffsetDateTime>,

5968 pub steps: Vec<StepDto>,

69 pub job_key: String,

70 pub is_template: bool,

71 pub logical_job_id: Option<Uuid>,

72 pub matrix: serde_json::Value,

73 pub outputs: serde_json::Value,

74 pub outcome: Option<String>,

6075}

6176

6277#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]

@@ -69,6 +84,7 @@

6984#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]

7085pub struct ArtifactDto {

7186 pub id: Uuid,

87 pub actions_id: Option<i64>,

7288 pub job_id: Uuid,

7389 pub name: String,

7490 pub size: i64,

@@ -88,10 +104,30 @@

88104 pub offset: Option<u64>,

89105}

90106

107#[derive(Debug, Clone, serde::Deserialize, serde::Serialize, utoipa::ToSchema)]

108#[serde(untagged)]

109pub enum TriggerInputValue {

110 Boolean(bool),

111 Number(f64),

112 String(String),

113}

114

115impl From<TriggerInputValue> for serde_json::Value {

116 fn from(value: TriggerInputValue) -> Self {

117 match value {

118 TriggerInputValue::Boolean(value) => serde_json::json!(value),

119 TriggerInputValue::Number(value) => serde_json::json!(value),

120 TriggerInputValue::String(value) => serde_json::json!(value),

121 }

122 }

123}

124

91125#[derive(Debug, Clone, serde::Deserialize, utoipa::ToSchema)]

92126pub struct TriggerRunCmd {

93127 pub workflow: String,

94128 pub git_ref: String,

129 #[serde(default)]

130 pub inputs: std::collections::BTreeMap<String, TriggerInputValue>,

95131}

96132

97133#[derive(Debug, Clone, serde::Serialize, utoipa::ToSchema)]

@@ -130,6 +166,9 @@

130166 repo_id: m.repo_id,

131167 workflow_file: m.workflow_file,

132168 workflow_name: m.workflow_name,

169 display_name: m.definition["_display_name"].as_str().map(str::to_owned),

170 actions_id: m.actions_id,

171 run_number: m.run_number,

133172 event: m.event.as_str().to_string(),

134173 commit_sha: m.commit_sha,

135174 git_ref: m.git_ref,

@@ -211,6 +250,17 @@

211250 job_dtos.push(JobDto {

212251 id: job.id,

213252 name: job.name.clone(),

253 job_key: job.job_key.clone(),

254 is_template: job.is_template,

255 logical_job_id: job.logical_job_id,

256 matrix: job

257 .actions_context

258 .as_ref()

259 .and_then(|ctx| ctx["instance"].get("matrix"))

260 .cloned()

261 .unwrap_or(serde_json::json!({})),

262 outputs: job.outputs.clone().unwrap_or(serde_json::json!({})),

263 outcome: job.outcome.clone(),

214264 state: job_state_str(job.state),

215265 runs_on: serde_json::from_value(job.runs_on.clone()).unwrap_or_default(),

216266 environment: job.environment.clone(),

@@ -230,16 +280,31 @@

230280 exit_code: step.exit_code,

231281 started_at: step.started_at,

232282 finished_at: step.finished_at,

283 stage: step.stage.unwrap_or_else(|| "main".into()),

284 outcome: step.outcome,

285 conclusion: step.conclusion,

286 outputs: step.outputs.unwrap_or(serde_json::json!({})),

287 annotations: step.annotations.unwrap_or(serde_json::json!([])),

288 summary: step.summary,

233289 })

234290 .collect(),

235291 });

236292 }

237 let artifacts = self

238 .artifacts_for_run(run_id)

239 .await?

293 let artifacts = self.artifacts_for_run(run_id).await?;

294 let artifact_ids: Vec<Uuid> = artifacts.iter().map(|artifact| artifact.id).collect();

295 let identities = actions_identity::Entity::find()

296 .filter(actions_identity::Column::EntityId.is_in(artifact_ids))

297 .all(self.db_read())

298 .await?;

299 let identity_by_entity: std::collections::BTreeMap<Uuid, i64> = identities

240300 .into_iter()

301 .map(|identity| (identity.entity_id, identity.id))

302 .collect();

303 let artifacts = artifacts

304 .into_iter()

241305 .map(|a| ArtifactDto {

242306 id: a.id,

307 actions_id: identity_by_entity.get(&a.id).copied(),

243308 job_id: a.job_id,

244309 name: a.name,

245310 size: a.size,

@@ -287,6 +352,18 @@

287352 "workflow does not declare `workflow_dispatch`".to_string(),

288353 ));

289354 }

355 let provided = cmd

356 .inputs

357 .iter()

358 .map(|(name, value)| (name.clone(), value.clone().into()))

359 .collect::<std::collections::BTreeMap<_, _>>();

360 let inputs = validate_dispatch_inputs(

361 workflow

362 .actions

363 .as_ref()

364 .ok_or_else(|| CiError::InvalidArgument("Actions definition missing".into()))?,

365 &provided,

366 )?;

290367 let (branch, tag) = classify_ref(&cmd.git_ref);

291368 let seed = RunSeed {

292369 event: WorkflowEventKind::WorkflowDispatch,

@@ -294,6 +371,9 @@

294371 "ref": cmd.git_ref,

295372 "sha": sha,

296373 "actor_id": caller,

374 "inputs": inputs.as_object().map(|values|values.iter().map(|(name,value)|(name.clone(),value.as_str().map(str::to_owned).unwrap_or_else(||value.to_string()))).collect::<std::collections::BTreeMap<_,_>>()).unwrap_or_default(),

375 "_gitdata_inputs": inputs,

376 "event_id": Uuid::now_v7(),

297377 }),

298378 commit_sha: sha,

299379 git_ref: cmd.git_ref.clone(),

@@ -308,6 +388,53 @@

308388 }

309389}

310390

391fn validate_dispatch_inputs(

392 workflow: &ci_protocol::actions::ActionsWorkflow,

393 provided: &std::collections::BTreeMap<String, serde_json::Value>,

394) -> CiResult<serde_json::Value> {

395 let definitions = workflow.inputs();

396 let mut inputs = serde_json::Map::new();

397 for name in provided.keys() {

398 if !definitions.contains_key(name) {

399 return Err(CiError::InvalidArgument(format!(

400 "unknown workflow_dispatch input `{name}`"

401 )));

402 }

403 }

404 for (name, definition) in definitions {

405 let value = provided.get(&name).cloned().or(definition.default.clone());

406 if definition.required

407 && value.as_ref().is_none_or(|value| {

408 value.is_null() || (value.is_string() && value.as_str() == Some(""))

409 })

410 {

411 return Err(CiError::InvalidArgument(format!(

412 "required input `{name}` is missing"

413 )));

414 }

415 let value = value.unwrap_or_else(|| match definition.kind.as_deref() {

416 Some("boolean") => serde_json::json!(false),

417 Some("number") => serde_json::json!(0),

418 _ => serde_json::json!(""),

419 });

420 let valid = match definition.kind.as_deref() {

421 Some("boolean") => value.is_boolean(),

422 Some("number") => value.is_number(),

423 Some("choice") => value

424 .as_str()

425 .is_some_and(|value| definition.options.iter().any(|option| option == value)),

426 _ => value.is_string(),

427 };

428 if !valid {

429 return Err(CiError::InvalidArgument(format!(

430 "input `{name}` has the wrong type or choice"

431 )));

432 }

433 inputs.insert(name, value);

434 }

435 Ok(serde_json::Value::Object(inputs))

436}

437

311438#[must_use]

312439pub fn classify_ref(git_ref: &str) -> (Option<String>, Option<String>) {

313440 if let Some(tag) = git_ref.strip_prefix("refs/tags/") {

Modified lib/ci/src/workflows.rs +96 −64 View lib/ci/src/workflows.rs

@@ -1,6 +1,7 @@

1use ci_protocol::actions::{ACTIONS_WORKFLOW_DIR, ActionsWorkflow};

12use ci_protocol::dsl::{

23 PullRequestTrigger, PushTrigger, TriggerSummary, WORKFLOW_DIR, WorkflowDefinition,

3 WorkflowSummary, parse_workflow_str,

4 WorkflowSummary,

45};

56use git::service::GitService;

67use models::space::spaces;

@@ -13,6 +14,7 @@

1314 pub file: String,

1415 pub name: String,

1516 pub definition: WorkflowDefinition,

17 pub actions: Option<ActionsWorkflow>,

1618}

1719

1820const MAX_WORKFLOW_BYTES: u64 = 256 * 1024;

@@ -22,8 +24,6 @@

2224 GitService::new(self.state.clone())

2325 }

2426

25 /// Lists the `.workflows/` directory at `revision`, returning every

26 /// YAML file with its parse outcome.

2727 pub async fn list_workflows(

2828 &self,

2929 space: &spaces::Model,

@@ -45,17 +45,25 @@

4545 valid: false,

4646 errors: vec![error.to_string()],

4747 triggers: TriggerSummary::default(),

48 inputs: Default::default(),

49 engine: "actions".into(),

4850 });

4951 continue;

5052 }

5153 };

52 match parse_workflow_str(&source) {

53 Ok(definition) => out.push(WorkflowSummary {

54 match self.parse_loaded_workflow(&entry, &source).await {

55 Ok(workflow) => out.push(WorkflowSummary {

5456 file: entry.clone(),

55 name: definition.name.clone().unwrap_or_else(|| file_stem(&entry)),

57 name: workflow.name,

5658 valid: true,

5759 errors: Vec::new(),

58 triggers: TriggerSummary::from_triggers(&definition.on),

60 triggers: TriggerSummary::from_triggers(&workflow.definition.on),

61 inputs: workflow

62 .actions

63 .as_ref()

64 .map(ActionsWorkflow::inputs)

65 .unwrap_or_default(),

66 engine: "actions".into(),

5967 }),

6068 Err(error) => out.push(WorkflowSummary {

6169 file: entry.clone(),

@@ -63,6 +71,8 @@

6371 valid: false,

6472 errors: vec![error.to_string()],

6573 triggers: TriggerSummary::default(),

74 inputs: Default::default(),

75 engine: "actions".into(),

6676 }),

6777 }

6878 }

@@ -88,12 +98,8 @@

8898 continue;

8999 }

90100 };

91 match parse_workflow_str(&source) {

92 Ok(definition) => out.push(LoadedWorkflow {

93 name: definition.name.clone().unwrap_or_else(|| file_stem(&entry)),

94 file: entry,

95 definition,

96 }),

101 match self.parse_loaded_workflow(&entry, &source).await {

102 Ok(workflow) => out.push(workflow),

97103 Err(error) => {

98104 tracing::warn!(file = %entry, error = %error, "workflow parse failed; skipping");

99105 }

@@ -109,19 +115,33 @@

109115 revision: &str,

110116 file: &str,

111117 ) -> CiResult<LoadedWorkflow> {

112 let normalized = normalize_workflow_file(file)?;

118 let mut normalized = normalize_workflow_file(file)?;

119 if !file.trim().starts_with(&format!("{WORKFLOW_DIR}/"))

120 && !normalized.starts_with(ACTIONS_WORKFLOW_DIR)

121 {

122 let candidate = format!("{ACTIONS_WORKFLOW_DIR}/{normalized}");

123 if self

124 .workflow_entries(space, repo_name, revision)

125 .await?

126 .contains(&candidate)

127 {

128 normalized = candidate;

129 }

130 }

113131 let source = self

114132 .read_workflow_file(space, repo_name, revision, &normalized)

115133 .await?;

116 let definition = parse_workflow_str(&source)

117 .map_err(|e| CiError::InvalidArgument(format!("workflow `{normalized}`: {e}")))?;

134 self.parse_loaded_workflow(&normalized, &source).await

135 }

136

137 async fn parse_loaded_workflow(&self, file: &str, source: &str) -> CiResult<LoadedWorkflow> {

138 let actions = Some(self.parse_actions_workflow(file, source).await?);

139 let definition = actions.as_ref().unwrap().projection();

118140 Ok(LoadedWorkflow {

119 name: definition

120 .name

121 .clone()

122 .unwrap_or_else(|| file_stem(&normalized)),

123 file: normalized,

141 file: file.to_owned(),

142 name: definition.name.clone().unwrap_or_else(|| file_stem(file)),

124143 definition,

144 actions,

125145 })

126146 }

127147

@@ -132,46 +152,48 @@

132152 revision: &str,

133153 ) -> CiResult<Vec<String>> {

134154 let service = self.git_service();

135 let entries = service

136 .tree_read(

137 space.lower_name.clone(),

138 repo_name.to_string(),

139 revision.to_string(),

140 std::path::PathBuf::from(WORKFLOW_DIR),

141 )

142 .await;

143 let entries = match entries {

144 Ok(entries) => entries,

145 Err(git::share::errors::GitError::NotFound) => return Ok(Vec::new()),

146 Err(error) => return Err(error.into()),

147 };

148 // `ls-tree <rev> -- .workflows/` returns repo-relative names

149 // (`.workflows/ci.yml`), while `ls-tree <rev>:.workflows` returns bare

150 // names; normalize both to the bare file name the rest of this module

151 // expects.

152 let prefix = format!("{WORKFLOW_DIR}/");

153 let mut out: Vec<String> = entries

154 .into_iter()

155 .filter(|entry| {

156 entry.mode == gitbaby::commit::types::TreeEntryMode::Blob

157 || entry.mode == gitbaby::commit::types::TreeEntryMode::Exec

158 })

159 .map(|entry| {

160 entry

161 .name

162 .strip_prefix(prefix.as_str())

163 .unwrap_or(entry.name.as_str())

164 .to_string()

165 })

166 .filter(|name| {

167 std::path::Path::new(name).extension().is_some_and(|ext| {

168 ext.eq_ignore_ascii_case("yml") || ext.eq_ignore_ascii_case("yaml")

155 let mut selected = std::collections::BTreeMap::new();

156 for directory in [ACTIONS_WORKFLOW_DIR] {

157 let entries = service

158 .tree_read(

159 space.lower_name.clone(),

160 repo_name.to_string(),

161 revision.to_string(),

162 std::path::PathBuf::from(directory),

163 )

164 .await;

165 let entries = match entries {

166 Ok(entries) => entries,

167 Err(git::share::errors::GitError::NotFound) => continue,

168 Err(error) => return Err(error.into()),

169 };

170 let prefix = format!("{directory}/");

171 let mut out: Vec<String> = entries

172 .into_iter()

173 .filter(|entry| {

174 entry.mode == gitbaby::commit::types::TreeEntryMode::Blob

175 || entry.mode == gitbaby::commit::types::TreeEntryMode::Exec

176 })

177 .map(|entry| {

178 entry

179 .name

180 .strip_prefix(prefix.as_str())

181 .unwrap_or(entry.name.as_str())

182 .to_string()

183 })

184 .filter(|name| {

185 std::path::Path::new(name).extension().is_some_and(|ext| {

186 ext.eq_ignore_ascii_case("yml") || ext.eq_ignore_ascii_case("yaml")

187 })

169188 })

170 })

171 .collect();

172 out.sort();

173 out.dedup();

174 Ok(out)

189 .collect();

190 out.sort();

191 out.dedup();

192 for name in out {

193 selected.insert(name.clone(), format!("{directory}/{name}"));

194 }

195 }

196 Ok(selected.into_values().collect())

175197 }

176198

177199 async fn read_workflow_file(

@@ -182,7 +204,11 @@

182204 file: &str,

183205 ) -> CiResult<String> {

184206 let service = self.git_service();

185 let path = format!("{WORKFLOW_DIR}/{file}");

207 let path = if file.starts_with(&format!("{ACTIONS_WORKFLOW_DIR}/")) {

208 file.to_owned()

209 } else {

210 format!("{WORKFLOW_DIR}/{file}")

211 };

186212 let entry = service

187213 .tree_resolve_path(

188214 space.lower_name.clone(),

@@ -219,8 +245,10 @@

219245

220246pub fn normalize_workflow_file(file: &str) -> CiResult<String> {

221247 let trimmed = file.trim();

248 let actions = trimmed.starts_with(&format!("{ACTIONS_WORKFLOW_DIR}/"));

222249 let name = trimmed

223 .strip_prefix(&format!("{WORKFLOW_DIR}/"))

250 .strip_prefix(&format!("{ACTIONS_WORKFLOW_DIR}/"))

251 .or_else(|| trimmed.strip_prefix(&format!("{WORKFLOW_DIR}/")))

224252 .unwrap_or(trimmed);

225253 if name.is_empty()

226254 || name.contains('/')

@@ -229,7 +257,7 @@

229257 || name.len() > 200

230258 {

231259 return Err(CiError::InvalidArgument(

232 "workflow file must be a bare file name under .workflows/".to_string(),

260 "workflow file must be under .github/workflows/".to_string(),

233261 ));

234262 }

235263 let ext_ok = std::path::Path::new(name)

@@ -240,7 +268,11 @@

240268 "workflow file must end with .yml or .yaml".to_string(),

241269 ));

242270 }

243 Ok(name.to_string())

271 Ok(if actions {

272 format!("{ACTIONS_WORKFLOW_DIR}/{name}")

273 } else {

274 name.to_string()

275 })

244276}

245277

246278#[must_use]

Modified lib/ci/Cargo.toml +4 −0 View lib/ci/Cargo.toml

@@ -33,4 +33,8 @@

3333time = { workspace = true }

3434thiserror = { workspace = true }

3535async-trait = { workspace = true }

36futures = { workspace = true }

3637

38[dev-dependencies]

39runner = { path = "../../app/runner" }

40anyhow = { workspace = true }

Modified lib/webapi/src/lib.rs +1 −1 View lib/webapi/src/lib.rs

@@ -103,5 +103,5 @@

103103 .layer(metrics::metrics_layer())

104104 .layer(AppTraceLayer::new());

105105

106 Ok(router)

106 Ok(router.merge(ci::actions_runtime::router().with_state(ci)))

107107}

Modified Cargo.lock +3 −0 View Cargo.lock

@@ -1300,6 +1300,7 @@

13001300version = "1.0.0"

13011301dependencies = [

13021302 "aes-gcm 0.10.3",

1303 "anyhow",

13031304 "async-trait",

13041305 "auth",

13051306 "axum",

@@ -1307,6 +1308,7 @@

13071308 "ci-protocol",

13081309 "config",

13091310 "export",

1311 "futures",

13101312 "git",

13111313 "gitbaby",

13121314 "globset",

@@ -1314,6 +1316,7 @@

13141316 "models",

13151317 "rand 0.8.7",

13161318 "reqwest 0.12.28",

1319 "runner",

13171320 "sea-orm",

13181321 "serde",

13191322 "serde_json",