fix(search): restore code indexing and make hits readable

repo_root was used as a drive-relative path on Windows, so git could read refs but not objects (fatal: not a tree object) and every index job failed; anchor the configured git roots to absolute paths. ensure_index treated Meilisearch's 202 duplicate-index response as success and then failed on the index_already_exists task, so no index was ever populated: probe index existence first, tolerate the duplicate task, and cache configured indexes per process. Symbol rows now only surface definitions with a capped reference set, results interleave per kind instead of being sorted by title (which let symbol noise push code hits off the page), and snippets are cut from the matched line.

ZhenYi committed 1 month ago

parent 5ff6be8 commit c9a5cc1 Browse files

lib/config/src/git.rs +31 −2 View

@@ -1,6 +1,33 @@

11use serde::Deserialize;

2use serde::de::Deserializer;

3use std::path::Path;

24use std::time::Duration;

6/// Windows 上 `/opt/gitdata/repos` 会退化成盘符相对路径（没有盘符前缀），git 读得到

7/// refs 却读不到 object，表现为 `fatal: not a tree object`。这里统一锚定成带盘符的

8/// 绝对路径；POSIX 平台本来就是绝对路径，保持原样。

9pub fn anchor_local_path(raw: &str) -> String {

10 let trimmed = raw.trim();

11 if trimmed.is_empty() {

12 return raw.to_string();

13 }

14 let path = Path::new(trimmed);

15 if path.is_absolute() {

16 return trimmed.to_string();

17 }

18 match std::path::absolute(path) {

19 Ok(anchored) => anchored.to_string_lossy().into_owned(),

20 Err(_) => trimmed.to_string(),

21 }

22}

24fn deserialize_anchored_path<'de, D>(deserializer: D) -> Result<String, D::Error>

25where

26 D: Deserializer<'de>,

27{

28 String::deserialize(deserializer).map(|value| anchor_local_path(&value))

29}

431mod duration_secs {

532 use serde::{Deserialize, Deserializer};

633 use std::time::Duration;

@@ -28,7 +55,9 @@

2855#[derive(Clone, Debug, Deserialize)]

2956#[serde(default)]

3057pub struct GitConfig {

58 #[serde(deserialize_with = "deserialize_anchored_path")]

3159 pub repo_root: String,

60 #[serde(deserialize_with = "deserialize_anchored_path")]

3261 pub objects_root: String,

3362 pub max_blob_size: u64,

3463 pub git_command_timeout_secs: u64,

@@ -47,8 +76,8 @@

4776impl Default for GitConfig {

4877 fn default() -> Self {

4978 Self {

50 repo_root: "/opt/gitdata/repos".to_string(),

51 objects_root: "/opt/gitdata/objects".to_string(),

79 repo_root: anchor_local_path("/opt/gitdata/repos"),

80 objects_root: anchor_local_path("/opt/gitdata/objects"),

5281 max_blob_size: 100 * 1024 * 1024,

5382 git_command_timeout_secs: 300,

5483 git_rpc_timeout_secs: 3600,

lib/search/src/service.rs +253 −64 View

@@ -16,6 +16,7 @@

1616use sea_orm::{ColumnTrait, EntityTrait, QueryFilter, QueryOrder};

1717use serde::{Deserialize, Serialize};

1818use serde_json::{Value, json};

19use tracing::warn;

1920use utoipa::{IntoParams, ToSchema};

2021use uuid::Uuid;

2122

@@ -27,6 +28,8 @@

2728 Disabled,

2829 #[error("invalid search query: {0}")]

2930 Invalid(String),

31 #[error("search index task failed [{code}]: {message}")]

32 Task { code: String, message: String },

3033 #[error("search service request failed: {0}")]

3134 Http(#[from] reqwest::Error),

3235 #[error("search service returned {status}: {body}")]

@@ -43,6 +46,10 @@

4346 TimedOut,

4447}

4548

49fn is_duplicate_index(error: &SearchError) -> bool {

50 matches!(error, SearchError::Task { code, .. } if code == "index_already_exists")

51}

4653#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, ToSchema)]

4754#[serde(rename_all = "snake_case")]

4855pub enum SearchKind {

@@ -159,20 +166,100 @@

159166 definition: Option<bool>,

160167 title: String,

161168 content: String,

169 #[serde(default, skip_serializing, rename = "_matchesPosition")]

170 matches: HashMap<String, Vec<MatchPosition>>,

171}

172

173#[derive(Debug, Clone, Deserialize)]

174struct MatchPosition {

175 #[serde(default)]

176 start: usize,

177 #[serde(default)]

178 length: usize,

179}

180

181fn char_floor(text: &str, index: usize) -> usize {

182 let mut value = index.min(text.len());

183 while value > 0 && !text.is_char_boundary(value) {

184 value -= 1;

185 }

186 value

187}

188

189fn char_ceil(text: &str, index: usize) -> usize {

190 let mut value = index.min(text.len());

191 while value < text.len() && !text.is_char_boundary(value) {

192 value += 1;

193 }

194 value

195}

196

197// 片段从命中的那一行开始，前端只渲染一行，命中必须出现在开头。

198fn window_snippet(text: &str, start: usize, length: usize) -> String {

199 if start >= text.len() {

200 return compact_snippet(text);

201 }

202 let hit_start = char_floor(text, start);

203 let hit_end = char_ceil(text, (start + length).min(text.len()));

204 let from = text[..hit_start].rfind('\n').map_or(0, |offset| offset + 1);

205 let mut until = char_ceil(text, (hit_end + 300).min(text.len()));

206 if let Some(offset) = text[until..].find('\n') {

207 until = char_ceil(text, until + offset);

208 }

209 let normalized = text[from..until]

210 .split_whitespace()

211 .collect::<Vec<_>>()

212 .join(" ");

213 let prefix = if from > 0 { "…" } else { "" };

214 let suffix = if until < text.len() { "…" } else { "" };

215 format!("{prefix}{normalized}{suffix}")

216}

217

218fn symbol_label(kind: &str) -> &'static str {

219 match kind {

220 "function_item" | "function_declaration" | "function_definition" => "function",

221 "method_definition" | "method_declaration" => "method",

222 "struct_item" | "struct_specifier" => "struct",

223 "enum_item" | "enum_specifier" => "enum",

224 "trait_item" => "trait",

225 "impl_item" => "impl",

226 "type_item" | "type_declaration" => "type alias",

227 "class_definition" | "class_declaration" => "class",

228 "interface_declaration" => "interface",

229 "module" | "namespace_definition" => "module",

230 "reference" => "reference",

231 _ => "symbol",

232 }

162233}

163234

235fn matched_snippet(document: &SearchDocument) -> String {

236 let matched = document

237 .matches

238 .get("content")

239 .and_then(|hits| hits.iter().min_by_key(|hit| hit.start));

240 match matched {

241 Some(matched) => window_snippet(&document.content, matched.start, matched.length),

242 None => compact_snippet(&document.content),

243 }

244}

245

164246impl SearchDocument {

165247 fn hit(self) -> SearchHit {

248 let kind = match self.kind.as_str() {

249 "symbol" => SearchKind::Symbol,

250 "issue" => SearchKind::Issue,

251 "pull" => SearchKind::Pull,

252 _ => SearchKind::Code,

253 };

254 let snippet = match (&kind, self.symbol_kind.as_deref()) {

255 (SearchKind::Symbol, Some(symbol_kind)) => symbol_label(symbol_kind).to_string(),

256 _ => matched_snippet(&self),

257 };

166258 SearchHit {

167 kind: match self.kind.as_str() {

168 "symbol" => SearchKind::Symbol,

169 "issue" => SearchKind::Issue,

170 "pull" => SearchKind::Pull,

171 _ => SearchKind::Code,

172 },

259 kind,

173260 id: self.id,

174261 title: self.title,

175 snippet: compact_snippet(&self.content),

262 snippet,

176263 space_name: self.space_name,

177264 repo_name: self.repo_name,

178265 board_id: self.board_id.and_then(|id| Uuid::parse_str(&id).ok()),

@@ -197,6 +284,7 @@

197284 base: String,

198285 api_key: String,

199286 timeout: Duration,

287 configured: std::sync::Arc<tokio::sync::Mutex<HashSet<String>>>,

200288}

201289

202290impl MeiliClient {

@@ -209,6 +297,7 @@

209297 base: config.url.trim_end_matches('/').to_string(),

210298 api_key: config.api_key.clone(),

211299 timeout: Duration::from_secs(config.task_timeout_secs),

300 configured: std::sync::Arc::new(tokio::sync::Mutex::new(HashSet::new())),

212301 })

213302 }

214303

@@ -228,19 +317,44 @@

228317 Ok(serde_json::from_str(&body)?)

229318 }

230319

231 async fn ensure_index(&self, index: &str, searchable: &[&str]) -> Result<(), SearchError> {

320 async fn index_exists(&self, index: &str) -> Result<bool, SearchError> {

232321 let response = self

233 .request(reqwest::Method::POST, "/indexes")

234 .json(&json!({ "uid": index, "primaryKey": "id" }))

322 .request(reqwest::Method::GET, &format!("/indexes/{index}"))

235323 .send()

236324 .await?;

237 if response.status().is_success() {

238 let task: Value = response.json().await?;

239 self.wait_task(&task).await?;

240 } else if response.status() != StatusCode::CONFLICT {

325 match response.status() {

326 StatusCode::OK => Ok(true),

327 StatusCode::NOT_FOUND => Ok(false),

328 status => {

329 let body = response.text().await?;

330 Err(SearchError::Remote { status, body })

331 }

332 }

333 }

334

335 async fn ensure_index(&self, index: &str, searchable: &[&str]) -> Result<(), SearchError> {

336 let mut configured = self.configured.lock().await;

337 if configured.contains(index) {

338 return Ok(());

339 }

340 if !self.index_exists(index).await? {

341 let response = self

342 .request(reqwest::Method::POST, "/indexes")

343 .json(&json!({ "uid": index, "primaryKey": "id" }))

344 .send()

345 .await?;

241346 let status = response.status();

242 let body = response.text().await?;

243 return Err(SearchError::Remote { status, body });

347 if status.is_success() {

348 let task: Value = response.json().await?;

349 if let Err(error) = self.wait_task(&task).await

350 && !is_duplicate_index(&error)

351 {

352 return Err(error);

353 }

354 } else if status != StatusCode::CONFLICT {

355 let body = response.text().await?;

356 return Err(SearchError::Remote { status, body });

357 }

244358 }

245359 let task = self

246360 .json(

@@ -252,7 +366,9 @@

252366 })),

253367 )

254368 .await?;

255 self.wait_task(&task).await

369 self.wait_task(&task).await?;

370 configured.insert(index.to_string());

371 Ok(())

256372 }

257373

258374 async fn add_documents(

@@ -315,6 +431,7 @@

315431 "limit": limit,

316432 "offset": offset,

317433 "attributesToRetrieve": ["*"],

434 "showMatchesPosition": true,

318435 })),

319436 )

320437 .await?;

@@ -342,15 +459,18 @@

342459 match state.get("status").and_then(Value::as_str) {

343460 Some("succeeded") => return Ok(()),

344461 Some("failed") | Some("canceled") => {

345 return Err(SearchError::Invalid(

346 state

347 .get("error")

348 .and_then(Value::as_object)

349 .and_then(|error| error.get("message"))

350 .and_then(Value::as_str)

351 .unwrap_or("Meilisearch task failed")

352 .to_string(),

353 ));

462 let error = state.get("error").and_then(Value::as_object);

463 let code = error

464 .and_then(|error| error.get("code"))

465 .and_then(Value::as_str)

466 .unwrap_or("task_failed")

467 .to_string();

468 let message = error

469 .and_then(|error| error.get("message"))

470 .and_then(Value::as_str)

471 .unwrap_or("Meilisearch task failed")

472 .to_string();

473 return Err(SearchError::Task { code, message });

354474 }

355475 _ if std::time::Instant::now() >= deadline => return Err(SearchError::TimedOut),

356476 _ => tokio::time::sleep(Duration::from_millis(100)).await,

@@ -431,6 +551,17 @@

431551 caller: Option<Uuid>,

432552 query: SearchQuery,

433553 ) -> Result<SearchPage, SearchError> {

554 self.search_scoped(caller, query, true).await

555 }

556

557 // definitions_only 决定符号检索是否只返回定义：符号索引同时登记了标识符引用，

558 // 不加这层过滤时引用数量会把定义和代码命中完全淹没。

559 async fn search_scoped(

560 &self,

561 caller: Option<Uuid>,

562 query: SearchQuery,

563 definitions_only: bool,

564 ) -> Result<SearchPage, SearchError> {

434565 let q = query.q.trim();

435566 if q.is_empty() {

436567 return Err(SearchError::Invalid("q must not be empty".to_string()));

@@ -445,7 +576,7 @@

445576 .collect();

446577 let visible_spaces = self.visible_space_ids(caller).await?;

447578 let meili = self.meili()?;

448 let mut items = Vec::new();

579 let mut buckets: Vec<Vec<SearchHit>> = Vec::new();

449580 let mut total = 0usize;

450581 for kind in kinds {

451582 if matches!(kind, SearchKind::Code | SearchKind::Symbol) && visible_repo_ids.is_empty()

@@ -460,23 +591,24 @@

460591 }

461592 let filter = build_filter(

462593 kind,

463 &visible_repo_ids,

464 &visible_spaces,

465 query.space_name.as_deref(),

466 query.repo_name.as_deref(),

467 query.language.as_deref(),

468 query.state.as_deref(),

594 &FilterOptions {

595 definitions_only,

596 repo_ids: &visible_repo_ids,

597 space_ids: &visible_spaces,

598 space_name: query.space_name.as_deref(),

599 repo_name: query.repo_name.as_deref(),

600 language: query.language.as_deref(),

601 state: query.state.as_deref(),

602 },

469603 );

470604 let (hits, count) = meili

471605 .search(&self.index(kind), q, filter, limit, offset)

472606 .await?;

473607 total += count;

474 items.extend(hits.into_iter().map(SearchDocument::hit));

608 buckets.push(hits.into_iter().map(SearchDocument::hit).collect());

475609 }

476 items.sort_by(|a, b| a.title.cmp(&b.title));

477 items.truncate(limit);

478610 Ok(SearchPage {

479 items,

611 items: merge_buckets(buckets, limit),

480612 total,

481613 limit,

482614 offset,

@@ -517,13 +649,27 @@

517649 limit: Option<usize>,

518650 offset: Option<usize>,

519651 ) -> Result<SearchPage, SearchError> {

520 self.symbols(caller, space_name, repo_name, symbol_name, limit, offset)

521 .await

652 self.search_scoped(

653 caller,

654 SearchQuery {

655 q: symbol_name,

656 kinds: Some("symbol".to_string()),

657 space_name: Some(space_name),

658 repo_name: Some(repo_name),

659 language: None,

660 state: None,

661 limit,

662 offset,

663 },

664 false,

665 )

666 .await

522667 }

523668

524669 pub async fn index_repo(&self, repo_id: Uuid, repo_root: &Path) -> Result<(), SearchError> {

525670 self.ensure_indexes().await?;

526671 let Some(context) = self.repo_context(repo_id).await? else {

672 warn!(%repo_id, "search code index skipped: repository or space not found");

527673 return Ok(());

528674 };

529675 let repo_dir = repo_root.join(repo_id.to_string());

@@ -537,9 +683,13 @@

537683 .await

538684 {

539685 Ok(value) => String::from_utf8_lossy(&value).trim().to_string(),

540 Err(_) => return Ok(()),

686 Err(error) => {

687 warn!(%repo_id, error = %error, "search code index skipped: default branch not found");

688 return Ok(());

689 }

541690 };

542691 if sha.is_empty() {

692 warn!(%repo_id, "search code index skipped: default branch is empty");

543693 return Ok(());

544694 }

545695 let meili = self.meili()?;

@@ -552,7 +702,8 @@

552702 .await?;

553703 let tree = git_output(&repo_dir, &["ls-tree", "-r", "-z", "--long", &sha]).await?;

554704 let mut code_documents = Vec::new();

555 let mut symbol_documents = Vec::new();

705 let mut definition_documents = Vec::new();

706 let mut reference_documents = Vec::new();

556707 for (path, size) in tree_entries(&tree) {

557708 let Some((language, _)) = language_for_path(&path) else {

558709 continue;

@@ -589,7 +740,7 @@

589740 });

590741 }

591742 for (ordinal, symbol) in symbols_for(&path, &text).into_iter().enumerate() {

592 symbol_documents.push(SearchDocument {

743 let document = SearchDocument {

593744 id: Uuid::now_v7().to_string(),

594745 kind: SearchKind::Symbol.as_str().to_string(),

595746 repo_id: Some(repo_id.to_string()),

@@ -608,9 +759,16 @@

608759 title: symbol.name,

609760 content: format!("{}:{}", path, symbol.line),

610761 ..Default::default()

611 });

762 };

763 if symbol.is_definition {

764 definition_documents.push(document);

765 } else if reference_documents.len() < MAX_REFERENCE_DOCUMENTS {

766 reference_documents.push(document);

767 }

612768 }

613769 }

770 let mut symbol_documents = definition_documents;

771 symbol_documents.extend(reference_documents);

614772 for batch in code_documents.chunks(self.config.batch_size) {

615773 meili

616774 .add_documents(&self.index(SearchKind::Code), batch)

@@ -892,35 +1050,62 @@

8921050 }

8931051}

8941052

895fn build_filter(

896 kind: SearchKind,

897 repo_ids: &[String],

898 space_ids: &[String],

899 space_name: Option<&str>,

900 repo_name: Option<&str>,

901 language: Option<&str>,

902 state: Option<&str>,

903) -> Option<String> {

1053/// 各类型按相关度轮流出结果，避免符号命中把代码命中挤出结果页。

1054fn merge_buckets(buckets: Vec<Vec<SearchHit>>, limit: usize) -> Vec<SearchHit> {

1055 let mut merged: Vec<SearchHit> = Vec::with_capacity(limit);

1056 let mut rank = 0usize;

1057 while merged.len() < limit {

1058 let mut progressed = false;

1059 for bucket in &buckets {

1060 let Some(hit) = bucket.get(rank) else {

1061 continue;

1062 };

1063 merged.push(hit.clone());

1064 progressed = true;

1065 if merged.len() == limit {

1066 break;

1067 }

1068 }

1069 if !progressed {

1070 break;

1071 }

1072 rank += 1;

1073 }

1074 merged

1075}

1076

1077struct FilterOptions<'a> {

1078 definitions_only: bool,

1079 repo_ids: &'a [String],

1080 space_ids: &'a [String],

1081 space_name: Option<&'a str>,

1082 repo_name: Option<&'a str>,

1083 language: Option<&'a str>,

1084 state: Option<&'a str>,

1085}

1086

1087fn build_filter(kind: SearchKind, options: &FilterOptions<'_>) -> Option<String> {

9041088 let mut filters = Vec::new();

9051089 if matches!(

9061090 kind,

9071091 SearchKind::Code | SearchKind::Symbol | SearchKind::Pull

9081092 ) {

909 filters.push(in_filter("repo_id", repo_ids));

1093 filters.push(in_filter("repo_id", options.repo_ids));

9101094 } else {

911 filters.push(in_filter("space_id", space_ids));

912 }

913 if let Some(value) = space_name.filter(|value| !value.trim().is_empty()) {

914 filters.push(eq_filter("space_name", value));

915 }

916 if let Some(value) = repo_name.filter(|value| !value.trim().is_empty()) {

917 filters.push(eq_filter("repo_name", value));

1095 filters.push(in_filter("space_id", options.space_ids));

9181096 }

919 if let Some(value) = language.filter(|value| !value.trim().is_empty()) {

920 filters.push(eq_filter("language", value));

1097 if kind == SearchKind::Symbol && options.definitions_only {

1098 filters.push("definition = true".to_string());

9211099 }

922 if let Some(value) = state.filter(|value| !value.trim().is_empty()) {

923 filters.push(eq_filter("state", value));

1100 for (field, value) in [

1101 ("space_name", options.space_name),

1102 ("repo_name", options.repo_name),

1103 ("language", options.language),

1104 ("state", options.state),

1105 ] {

1106 if let Some(value) = value.filter(|value| !value.trim().is_empty()) {

1107 filters.push(eq_filter(field, value));

1108 }

9241109 }

9251110 Some(filters.join(" AND "))

9261111}

@@ -958,6 +1143,10 @@

9581143 }

9591144}

9601145

1146/// 符号索引里“引用”条目的上限：一个文件的标识符出现次数可以非常多，不设上限时

1147/// 大仓库会把索引撑爆并把定义挤出结果。

1148const MAX_REFERENCE_DOCUMENTS: usize = 4_000;

1149

9611150fn tree_entries(value: &[u8]) -> Vec<(String, u64)> {

9621151 value

9631152 .split(|byte| *byte == 0)

lib/webapi/src/search.rs +1 −1 View

@@ -45,7 +45,7 @@

4545 "invalid search request",

4646 "invalid_search_query",

4747 ),

48 SearchError::Remote { .. } | SearchError::TimedOut => (

48 SearchError::Task { .. } | SearchError::Remote { .. } | SearchError::TimedOut => (

4949 StatusCode::SERVICE_UNAVAILABLE,

5050 "search is temporarily unavailable",

5151 "search_unavailable",