feat(search): index every text language and cap index growth
the code index only accepted rs/js/ts/jsx/tsx/py/go/java/c/h/cpp, so any other file type was silently unsearchable. language_label now maps ~70 extensions plus name-only files (Dockerfile, Makefile, Gemfile, ...) while tree-sitter symbol parsing stays limited to the languages it supports. lock files, minified bundles, sourcemaps and .d.ts are skipped, and per-repository document caps keep a single large repo from flooding Meilisearch.
ZhenYi committed
Modified lib/search/src/parser.rs lib/search/src/parser.rs +138 −18 View lib/search/src/parser.rs
@@ -9,31 +9,151 @@
99 pub is_definition: bool,
1010}
1111
12pub fn language_for_path(path: &str) -> Option<(&'static str, Language)> {
13 let name = path.rsplit('/').next().unwrap_or(path).to_ascii_lowercase();
14 let extension = name.rsplit_once('.').map(|(_, ext)| ext)?;
15 let language = match extension {
16 "rs" => ("Rust", tree_sitter_rust::LANGUAGE.into()),
17 "js" | "mjs" | "cjs" | "jsx" => ("JavaScript", tree_sitter_javascript::LANGUAGE.into()),
18 "ts" => (
19 "TypeScript",
20 tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
21 ),
22 "tsx" => ("TypeScript", tree_sitter_typescript::LANGUAGE_TSX.into()),
23 "py" => ("Python", tree_sitter_python::LANGUAGE.into()),
24 "go" => ("Go", tree_sitter_go::LANGUAGE.into()),
25 "java" => ("Java", tree_sitter_java::LANGUAGE.into()),
26 "c" | "h" => ("C", tree_sitter_c::LANGUAGE.into()),
27 "cc" | "cp" | "cpp" | "cxx" | "c++" | "hh" | "hpp" | "hxx" => {
28 ("C++", tree_sitter_cpp::LANGUAGE.into())
12fn file_name(path: &str) -> String {
13 path.rsplit(['/', '\\'])
14 .next()
15 .unwrap_or(path)
16 .to_ascii_lowercase()
17}
18
19/// 锁文件、压缩产物、sourcemap 参与检索只会制造噪音。
20fn is_generated_file(name: &str) -> bool {
21 matches!(
22 name,
23 "package-lock.json"
24 | "npm-shrinkwrap.json"
25 | "pnpm-lock.yaml"
26 | "yarn.lock"
27 | "composer.lock"
28 | "podfile.lock"
29 | "gemfile.lock"
30 ) || name.ends_with(".min.js")
31 || name.ends_with(".min.css")
32 || name.ends_with(".map")
33 || name.ends_with(".d.ts")
34}
35
36/// 代码检索覆盖的语言：按文件名给出展示与过滤用的语言名。
37/// 未收录的后缀视为不可索引，避免把二进制或生成产物灌进索引。
38pub fn language_label(path: &str) -> Option<&'static str> {
39 let name = file_name(path);
40 if is_generated_file(&name) {
41 return None;
42 }
43 if let Some((_, extension)) = name.rsplit_once('.') {
44 return Some(match extension {
45 "rs" => "Rust",
46 "js" | "mjs" | "cjs" | "jsx" | "flow" => "JavaScript",
47 "ts" => "TypeScript",
48 "tsx" => "TypeScript (React)",
49 "py" | "pyi" | "pyw" => "Python",
50 "go" => "Go",
51 "java" => "Java",
52 "kt" | "kts" => "Kotlin",
53 "swift" => "Swift",
54 "scala" | "sc" => "Scala",
55 "c" | "h" => "C",
56 "cc" | "cp" | "cpp" | "cxx" | "c++" | "hh" | "hpp" | "hxx" => "C++",
57 "cs" | "csx" => "C#",
58 "rb" | "rake" | "gemspec" => "Ruby",
59 "php" | "phtml" => "PHP",
60 "pl" | "pm" => "Perl",
61 "lua" => "Lua",
62 "r" => "R",
63 "m" => "Objective-C",
64 "mm" => "Objective-C++",
65 "sh" | "bash" | "zsh" | "ksh" | "cli" => "Shell",
66 "fish" => "fish",
67 "ps1" | "psm1" | "psd1" => "PowerShell",
68 "bat" | "cmd" => "Batch",
69 "sql" => "SQL",
70 "html" | "htm" | "xhtml" => "HTML",
71 "vue" => "Vue",
72 "svelte" => "Svelte",
73 "astro" => "Astro",
74 "css" => "CSS",
75 "scss" => "SCSS",
76 "sass" => "Sass",
77 "less" => "Less",
78 "styl" => "Stylus",
79 "xml" | "xsd" | "xsl" | "svg" => "XML",
80 "json" | "jsonc" | "json5" => "JSON",
81 "yaml" | "yml" => "YAML",
82 "toml" => "TOML",
83 "ini" | "cfg" | "conf" | "properties" | "env" => "INI",
84 "md" | "markdown" | "mdx" => "Markdown",
85 "rst" => "reStructuredText",
86 "adoc" | "asciidoc" => "AsciiDoc",
87 "txt" | "text" => "Text",
88 "tex" | "cls" | "sty" => "LaTeX",
89 "proto" => "Protocol Buffer",
90 "thrift" => "Thrift",
91 "graphql" | "gql" => "GraphQL",
92 "tf" | "tfvars" | "hcl" => "Terraform",
93 "nix" => "Nix",
94 "cmake" => "CMake",
95 "mk" | "am" => "Makefile",
96 "gradle" => "Gradle",
97 "groovy" | "gvy" | "gy" => "Groovy",
98 "ex" | "exs" | "eex" => "Elixir",
99 "erl" | "hrl" => "Erlang",
100 "hs" | "lhs" => "Haskell",
101 "ml" | "mli" => "OCaml",
102 "clj" | "cljs" | "cljc" | "edn" => "Clojure",
103 "nim" => "Nim",
104 "zig" => "Zig",
105 "jl" => "Julia",
106 "dart" => "Dart",
107 "v" => "V",
108 "sol" => "Solidity",
109 "vim" => "Vim Script",
110 "el" => "Emacs Lisp",
111 "bzl" | "bazel" => "Starlark",
112 "bicep" => "Bicep",
113 "mustache" | "hbs" => "Handlebars",
114 "ejs" => "EJS",
115 "twig" => "Twig",
116 "liquid" => "Liquid",
117 "patch" | "diff" => "Patch",
118 _ => return special_file_label(&name),
119 });
120 }
121 special_file_label(&name)
122}
123
124fn special_file_label(name: &str) -> Option<&'static str> {
125 Some(match name {
126 "dockerfile" | "containerfile" => "Dockerfile",
127 "makefile" | "gnumakefile" => "Makefile",
128 "justfile" => "Just",
129 "cmakelists.txt" => "CMake",
130 "gemfile" | "rakefile" | "brewfile" | "vagrantfile" => "Ruby",
131 "license" | "license-apache" | "license-mit" | "license-cc0" | "readme" | "changelog" => {
132 "Text"
29133 }
30134 _ => return None,
135 })
136}
137
138/// 只有这几种语言接 tree-sitter 符号解析，其余语言仍可按内容检索。
139fn tree_sitter_language(label: &str) -> Option<Language> {
140 let language = match label {
141 "Rust" => tree_sitter_rust::LANGUAGE.into(),
142 "JavaScript" => tree_sitter_javascript::LANGUAGE.into(),
143 "TypeScript" => tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
144 "TypeScript (React)" => tree_sitter_typescript::LANGUAGE_TSX.into(),
145 "Python" => tree_sitter_python::LANGUAGE.into(),
146 "Go" => tree_sitter_go::LANGUAGE.into(),
147 "Java" => tree_sitter_java::LANGUAGE.into(),
148 "C" => tree_sitter_c::LANGUAGE.into(),
149 "C++" => tree_sitter_cpp::LANGUAGE.into(),
150 _ => return None,
31151 };
32152 Some(language)
33153}
34154
35155pub fn symbols_for(path: &str, text: &str) -> Vec<ParsedSymbol> {
36 let Some((_, language)) = language_for_path(path) else {
156 let Some(language) = language_label(path).and_then(tree_sitter_language) else {
37157 return Vec::new();
38158 };
39159 let mut parser = Parser::new();
Modified lib/search/src/service.rs lib/search/src/service.rs +17 −2 View lib/search/src/service.rs
@@ -20,7 +20,7 @@
2020use utoipa::{IntoParams, ToSchema};
2121use uuid::Uuid;
2222
23use crate::parser::{language_for_path, symbols_for};
23use crate::parser::{language_label, symbols_for};
2424
2525#[derive(Debug, thiserror::Error)]
2626pub enum SearchError {
@@ -704,8 +704,9 @@
704704 let mut code_documents = Vec::new();
705705 let mut definition_documents = Vec::new();
706706 let mut reference_documents = Vec::new();
707 let mut capped = false;
707708 for (path, size) in tree_entries(&tree) {
708 let Some((language, _)) = language_for_path(&path) else {
709 let Some(language) = language_label(&path) else {
709710 continue;
710711 };
711712 if size > self.config.max_file_bytes as u64 {
@@ -722,6 +723,10 @@
722723 continue;
723724 };
724725 for (start, end, chunk) in text_chunks(&text, self.config.chunk_lines) {
726 if code_documents.len() >= MAX_CODE_DOCUMENTS {
727 capped = true;
728 break;
729 }
725730 code_documents.push(SearchDocument {
726731 id: Uuid::now_v7().to_string(),
727732 kind: SearchKind::Code.as_str().to_string(),
@@ -767,6 +772,13 @@
767772 }
768773 }
769774 }
775 if capped {
776 warn!(
777 %repo_id,
778 limit = MAX_CODE_DOCUMENTS,
779 "search code index reached document cap"
780 );
781 }
770782 let mut symbol_documents = definition_documents;
771783 symbol_documents.extend(reference_documents);
772784 for batch in code_documents.chunks(self.config.batch_size) {
@@ -1147,6 +1159,9 @@
11471159/// 大仓库会把索引撑爆并把定义挤出结果。
11481160const MAX_REFERENCE_DOCUMENTS: usize = 4_000;
11491161
1162/// 单个仓库代码分块的上限，防止超大仓库一次索引把检索服务打满。
1163const MAX_CODE_DOCUMENTS: usize = 20_000;
1164
11501165fn tree_entries(value: &[u8]) -> Vec<(String, u64)> {
11511166 value
11521167 .split(|byte| *byte == 0)