Skip to content
zhenyi

gitdata-core

0
Clone

HTTPS

SSH

SSH needs a key in your settings.

Download ZIP

refactor: introduce resolver and embed handling for improved media processing

ZhenYi committed
parent e891f19 commit 6962e15 Browse files

20 files changed

+1386 −909

File tree 20
Modified lib/chat/src/services/messages.rs View lib/chat/src/services/messages.rs
@@ -139,37 +139,7 @@
139139 pub reactions: Vec<ReactionAggDto>,
140140}
141141
142#[derive(Debug, Clone, Serialize, Deserialize, ToSchema)]
143pub struct MessageEmbedDto {
144 #[serde(rename = "type")]
145 pub embed_type: String,
146 pub url: String,
147 #[serde(default, skip_serializing_if = "Option::is_none")]
148 pub platform: Option<String>,
149 #[serde(default, skip_serializing_if = "Option::is_none")]
150 pub platform_name: Option<String>,
151 #[serde(default, skip_serializing_if = "Option::is_none")]
152 pub category: Option<String>,
153 #[serde(default, skip_serializing_if = "Option::is_none")]
154 pub kind: Option<String>,
155 #[serde(default, skip_serializing_if = "Option::is_none")]
156 pub aspect: Option<String>,
157 /// 平台声明的固定像素高；前端据它给 iframe 定高（重量级见 `social::render`）。
158 #[serde(default, skip_serializing_if = "Option::is_none")]
159 pub height: Option<u32>,
160 #[serde(default, skip_serializing_if = "Option::is_none")]
161 pub src: Option<String>,
162 #[serde(default, skip_serializing_if = "Option::is_none")]
163 pub title: Option<String>,
164 #[serde(default, skip_serializing_if = "Option::is_none")]
165 pub description: Option<String>,
166 #[serde(default, skip_serializing_if = "Option::is_none")]
167 pub image_url: Option<String>,
168 #[serde(default, skip_serializing_if = "Option::is_none")]
169 pub site_name: Option<String>,
170 #[serde(default, skip_serializing_if = "Option::is_none")]
171 pub html: Option<String>,
172}
142pub use social::MessageEmbed as MessageEmbedDto;
173143
174144#[derive(Debug, Clone, Serialize, ToSchema)]
175145pub struct MessagePage {
Modified lib/chat/src/services/mod.rs View lib/chat/src/services/mod.rs
@@ -4,7 +4,6 @@
44pub mod drafts;
55pub mod emojis;
66pub mod messages;
7pub mod oembed;
87pub mod outbound;
98pub mod presence;
109pub mod reactions;
Deleted lib/chat/src/services/oembed.rs
@@ -1,124 +0,0 @@
1//! oEmbed 抓取：把平台声明的端点返回的 JSON 收敛成 [`social::Enrichment`]。
2//!
3//! 第三方返回的 HTML 不会被注入消息流：只从 `html` 里抠出 iframe 的 `src`，而且
4//! `social` 侧还会再校验一次，只有落在该平台自己声明的域名里才会被采用。抓取失败
5//! 一律返回 `None`，交给 OpenGraph 兜底，也不会写进 `chat_link_unfurl` 缓存。
6
7use social::Enrichment;
8
9use crate::services::outbound::{fetch_following, validate_outbound_url};
10
11const MAX_BYTES: usize = 262_144;
12
13/// 抓取 URL 所属平台的 oEmbed 数据；平台没声明端点或抓取失败时返回 `None`。
14pub async fn fetch(url: &str) -> Option<Enrichment> {
15 let endpoint = social::oembed_endpoint(url)?;
16 let body = fetch_following(
17 &endpoint.endpoint,
18 "application/json",
19 Some("json"),
20 MAX_BYTES,
21 )
22 .await
23 .ok()?
24 .body;
25 parse(&body)
26}
27
28/// 抓取页面自己声明的 oEmbed 端点。
29///
30/// 比 [`fetch`] 覆盖面大得多：注册表只硬编码了十几个平台，而页面里的
31/// `<link rel="alternate" type="application/json+oembed">` 是平台自己发布的，
32/// 新增平台不需要跟着改代码。
33pub async fn fetch_endpoint(endpoint: &str) -> Option<Enrichment> {
34 if !endpoint.starts_with("http://") && !endpoint.starts_with("https://") {
35 return None;
36 }
37 if validate_outbound_url(endpoint).is_err() {
38 return None;
39 }
40 let body = fetch_following(endpoint, "application/json", Some("json"), MAX_BYTES)
41 .await
42 .ok()?
43 .body;
44 parse(&body)
45}
46
47fn string_field(value: &serde_json::Value, key: &str) -> Option<String> {
48 let text = value.get(key)?.as_str()?.trim();
49 (!text.is_empty()).then(|| text.to_string())
50}
51
52/// 从 oEmbed 的 `html` 字段里取出 iframe 的 `src`。
53fn iframe_src(html: &str) -> Option<String> {
54 let start = html.to_ascii_lowercase().find("<iframe")?;
55 let tail = &html[start..];
56 let index = tail.to_ascii_lowercase().find("src=")? + 4;
57 let after = tail[index..].trim_start();
58 let quote = after.chars().next()?;
59 if quote != '"' && quote != '\'' {
60 return None;
61 }
62 let rest = &after[1..];
63 let end = rest.find(quote)?;
64 let src = rest[..end].trim();
65 (!src.is_empty()).then(|| src.to_string())
66}
67
68/// 解析 oEmbed 响应体；没有任何可用字段时返回 `None`。
69pub fn parse(body: &str) -> Option<Enrichment> {
70 let value: serde_json::Value = serde_json::from_str(body).ok()?;
71 if !value.is_object() {
72 return None;
73 }
74 let enrichment = Enrichment {
75 title: string_field(&value, "title"),
76 description: string_field(&value, "description")
77 .or_else(|| string_field(&value, "author_name")),
78 image_url: string_field(&value, "thumbnail_url"),
79 frame_src: string_field(&value, "html").as_deref().and_then(iframe_src),
80 };
81 (enrichment.title.is_some()
82 || enrichment.description.is_some()
83 || enrichment.image_url.is_some()
84 || enrichment.frame_src.is_some())
85 .then_some(enrichment)
86}
87
88#[cfg(test)]
89mod tests {
90 use super::*;
91
92 #[test]
93 fn parses_the_youtube_payload() {
94 let body = r#"{"title":"Rick Astley - Never Gonna Give You Up (Official Video)","author_name":"Rick Astley","author_url":"https://www.youtube.com/@RickAstleyYT","thumbnail_url":"https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg","type":"video","version":"1.0"}"#;
95 let enrichment = parse(body).unwrap();
96 assert_eq!(
97 enrichment.title.as_deref(),
98 Some("Rick Astley - Never Gonna Give You Up (Official Video)")
99 );
100 assert_eq!(enrichment.description.as_deref(), Some("Rick Astley"));
101 assert_eq!(
102 enrichment.image_url.as_deref(),
103 Some("https://i.ytimg.com/vi/dQw4w9WgXcQ/hqdefault.jpg")
104 );
105 assert!(enrichment.frame_src.is_none());
106 }
107
108 #[test]
109 fn pulls_the_frame_out_of_the_spotify_payload() {
110 let body = r#"{"title":"Never Gonna Give You Up","html":"<iframe style=\"border-radius: 12px\" width=\"100%\" height=\"152\" title=\"Spotify Embed: Never Gonna Give You Up\" frameborder=\"0\" allowfullscreen allow=\"autoplay; clipboard-write; encrypted-media; fullscreen; picture-in-picture\" loading=\"lazy\" src=\"https://open.spotify.com/embed/track/4cOdK2wGLETKBW3PvgPWqT?utm_source=oembed\"></iframe>"}"#;
111 let enrichment = parse(body).unwrap();
112 assert_eq!(
113 enrichment.frame_src.as_deref(),
114 Some("https://open.spotify.com/embed/track/4cOdK2wGLETKBW3PvgPWqT?utm_source=oembed")
115 );
116 }
117
118 #[test]
119 fn rejects_bodies_without_useful_fields() {
120 assert!(parse("<html>nope</html>").is_none());
121 assert!(parse("{}").is_none());
122 assert!(parse(r#"{"type":"video","html":"<blockquote>quote</blockquote>"}"#).is_none());
123 }
124}
Modified lib/chat/src/services/outbound.rs View lib/chat/src/services/outbound.rs
@@ -14,10 +14,6 @@
1414
1515const MAX_DELIVERY_ATTEMPTS: i32 = 5;
1616
17/// 多数站点（知乎、微博、CSDN）会对默认爬虫 UA 直接 403，浏览器 UA 才有正常 HTML。
18const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) \
19 Chrome/131.0.0.0 Safari/537.36";
20
2117#[derive(Debug, Clone, Serialize, ToSchema)]
2218pub struct OutboundWebhookDto {
2319 pub id: Uuid,
@@ -126,142 +122,6 @@
126122 }
127123 }
128124 }
129}
130
131/// 抓取结果：正文 + 最终落地的 URL。
132///
133/// 短链（`b23.tv` / `t.co` / `xhslink.com`）只有跟随重定向才能拿到真实内容，
134/// 而落地地址又决定这条链接该按哪个平台渲染，所以两者一起返回。
135pub(crate) struct Fetched {
136 pub body: String,
137 pub final_url: String,
138}
139
140/// 从 `content-type` 里摘 charset；缺失时按 `None`（交给调用方嗅探）。
141fn charset_of(content_type: &str) -> Option<String> {
142 let lower = content_type.to_ascii_lowercase();
143 let start = lower.find("charset=")? + "charset=".len();
144 let rest = &content_type[start..];
145 let value = rest
146 .split(|c: char| c == ';' || c == ',' || c.is_whitespace())
147 .next()
148 .unwrap_or("")
149 .trim_matches(|c: char| c == '"' || c == '\'');
150 (!value.is_empty()).then(|| value.to_string())
151}
152
153/// 按声明编码解码；没声明时先看 UTF-8 能否直通，否则回退 GB18030（中文站常见）。
154pub(crate) fn decode_body(body: &[u8], content_type: &str) -> String {
155 if let Some(label) = charset_of(content_type)
156 && let Some(encoding) = encoding_rs::Encoding::for_label(label.as_bytes())
157 {
158 let (text, _, _) = encoding.decode(body);
159 return text.into_owned();
160 }
161 match std::str::from_utf8(body) {
162 Ok(text) => text.to_string(),
163 Err(_) => {
164 let (text, _, _) = encoding_rs::GB18030.decode(body);
165 text.into_owned()
166 }
167 }
168}
169
170/// SSRF 防护后的 GET：只允许公网地址、限时限量，跟随重定向并对每一跳重做校验。
171///
172/// `expect_content_type` 非空时，响应 `content-type` 必须包含该片段（读 body 之前检查）。
173/// 重定向是手工跟的：`reqwest` 自带的跳转策略不会拦截落到内网地址的 `Location`，
174/// 所以每一跳都要重新过一遍 [`validate_outbound_url`] 与 DNS 归属检查。
175pub(crate) async fn fetch_following(
176 url: &str,
177 accept: &str,
178 expect_content_type: Option<&str>,
179 max_bytes: usize,
180) -> Result<Fetched, String> {
181 fetch(url, accept, expect_content_type, max_bytes).await
182}
183
184const MAX_REDIRECTS: usize = 6;
185
186async fn fetch(
187 url: &str,
188 accept: &str,
189 expect_content_type: Option<&str>,
190 max_bytes: usize,
191) -> Result<Fetched, String> {
192 let mut current = url.to_string();
193 for _ in 0..=MAX_REDIRECTS {
194 let uri = validate_outbound_url(&current).map_err(|e| e.to_string())?;
195 let host = uri.host().unwrap_or("").to_string();
196 let port = uri.port_u16();
197 let addrs = tokio::net::lookup_host((host.as_str(), port.unwrap_or(443)))
198 .await
199 .map_err(|e| format!("dns resolution failed: {e}"))?
200 .collect::<Vec<_>>();
201 if addrs.is_empty() {
202 return Err("dns resolution returned no address".to_string());
203 }
204 if addrs.iter().any(|addr| !is_public_ip(&addr.ip())) {
205 return Err("url resolves to a private or loopback address".to_string());
206 }
207 let mut client_builder = reqwest::Client::builder()
208 .timeout(std::time::Duration::from_secs(10))
209 .redirect(reqwest::redirect::Policy::none())
210 .user_agent(USER_AGENT);
211 for addr in &addrs {
212 client_builder = client_builder.resolve(&host, *addr);
213 }
214 let client = client_builder.build().map_err(|e| e.to_string())?;
215 let response = client
216 .get(&current)
217 .header("accept", accept)
218 .send()
219 .await
220 .map_err(|e| format!("request failed: {e}"))?;
221
222 if response.status().is_redirection() {
223 let location = response
224 .headers()
225 .get("location")
226 .and_then(|value| value.to_str().ok())
227 .ok_or_else(|| "redirect without a location header".to_string())?;
228 let base = url::Url::parse(&current).map_err(|e| e.to_string())?;
229 current = base
230 .join(location)
231 .map_err(|e| format!("invalid redirect target: {e}"))?
232 .to_string();
233 continue;
234 }
235 if !response.status().is_success() {
236 return Err(format!("endpoint returned {}", response.status().as_u16()));
237 }
238 let content_type = response
239 .headers()
240 .get("content-type")
241 .and_then(|value| value.to_str().ok())
242 .unwrap_or("")
243 .to_string();
244 let lowered = content_type.to_ascii_lowercase();
245 if let Some(expected) = expect_content_type
246 && !lowered.is_empty()
247 && !lowered.contains(expected)
248 {
249 return Err(format!("unexpected content-type {lowered}"));
250 }
251 let mut body: Vec<u8> = Vec::new();
252 let mut stream = response;
253 while let Some(chunk) = stream.chunk().await.map_err(|e| e.to_string())? {
254 body.extend_from_slice(&chunk);
255 if body.len() >= max_bytes {
256 break;
257 }
258 }
259 return Ok(Fetched {
260 body: decode_body(&body, &content_type),
261 final_url: current,
262 });
263 }
264 Err("too many redirects".to_string())
265125}
266126
267127async fn require_room_manager(
Modified lib/chat/src/services/unfurl.rs View lib/chat/src/services/unfurl.rs
@@ -1,29 +1,12 @@
1use models::chat::link_unfurls::{self, UnfurlStatus};
21use models::chat::messages;
32use sea_orm::{ActiveModelTrait, EntityTrait, Set};
4use sha2::{Digest, Sha256};
5use time::OffsetDateTime;
63use uuid::Uuid;
74
85use crate::services::service::ChatService;
9use crate::services::{oembed, outbound};
106use crate::share::notify;
117use crate::share::realtime::RoomEvent;
128
139const MAX_URLS_PER_MESSAGE: usize = 3;
14const MAX_FETCH_BYTES: usize = 1_048_576;
15const CACHE_FRESH_DAYS: i64 = 7;
16const FAILED_RETRY_MINUTES: i64 = 60;
17
18fn url_hash(url: &str) -> String {
19 let mut hasher = Sha256::new();
20 hasher.update(url.as_bytes());
21 hasher
22 .finalize()
23 .iter()
24 .map(|b| format!("{b:02x}"))
25 .collect()
26}
2710
2811/// 裁掉句尾标点，但保留括号配对时的右括号。
2912///
@@ -74,398 +57,12 @@
7457 }
7558 if urls.len() >= MAX_URLS_PER_MESSAGE {
7659 break;
77 }
78 }
79 urls
80}
81
82fn meta_content(head: &str, property: &str) -> Option<String> {
83 let lower = head.to_ascii_lowercase();
84 let mut search_from = 0;
85 loop {
86 let idx = lower[search_from..].find("<meta")? + search_from;
87 let end = lower[idx..].find('>')? + idx;
88 let tag = &head[idx..=end];
89 let tag_lower = &lower[idx..=end];
90 search_from = end + 1;
91 let matches = tag_lower.contains(&format!("property=\"{property}\""))
92 || tag_lower.contains(&format!("property='{property}'"))
93 || tag_lower.contains(&format!("name=\"{property}\""))
94 || tag_lower.contains(&format!("name='{property}'"));
95 if !matches {
96 continue;
97 }
98 for attr in ["content=\"", "content='"] {
99 if let Some(start) = tag.find(attr) {
100 let value_start = start + attr.len();
101 let quote = attr.chars().last()?;
102 if let Some(value_end) = tag[value_start..].find(quote) {
103 let value = tag[value_start..value_start + value_end].trim();
104 if !value.is_empty() {
105 return Some(decode_entities(value).chars().take(300).collect());
106 }
107 }
108 }
109 }
110 }
111}
112
113/// 依次尝试多个 `meta` 键，返回第一个非空值。
114///
115/// 只认 `og:` 会让大量站点拿到空卡片：很多新闻/电商站点只发 `twitter:` 卡片，
116/// 或者只留一个裸 `description`。
117fn meta_any(head: &str, properties: &[&str]) -> Option<String> {
118 properties
119 .iter()
120 .find_map(|property| meta_content(head, property))
121}
122
123fn page_title(head: &str) -> Option<String> {
124 let lower = head.to_ascii_lowercase();
125 let start = lower.find("<title")?;
126 let content_start = lower[start..].find('>')? + start + 1;
127 let end = lower[content_start..].find("</title>")? + content_start;
128 let title = head[content_start..end].trim();
129 (!title.is_empty()).then(|| decode_entities(title).chars().take(200).collect())
130}
131
132/// 从 `<link rel="alternate" type="application/json+oembed" href="...">` 取出站点
133/// 自己声明的 oEmbed 端点。
134///
135/// 注册表里硬编码的端点只有十几个；页面自声明的端点覆盖面大得多，而且平台改了
136/// 端点也不用跟着发版。
137fn link_oembed(head: &str) -> Option<String> {
138 let lower = head.to_ascii_lowercase();
139 let mut search_from = 0;
140 loop {
141 let idx = lower[search_from..].find("<link")? + search_from;
142 let end = lower[idx..].find('>')? + idx;
143 let tag = &head[idx..=end];
144 let tag_lower = &lower[idx..=end];
145 search_from = end + 1;
146 if !tag_lower.contains("oembed") {
147 continue;
148 }
149 // 只认 JSON 形态：XML 端点的解析成本远大于收益。
150 if !tag_lower.contains("json") {
151 continue;
152 }
153 for attr in ["href=\"", "href='"] {
154 if let Some(start) = tag.find(attr) {
155 let value_start = start + attr.len();
156 let quote = attr.chars().last()?;
157 if let Some(value_end) = tag[value_start..].find(quote) {
158 let value = tag[value_start..value_start + value_end].trim();
159 if !value.is_empty() {
160 return Some(decode_entities(value));
161 }
162 }
163 }
164 }
165 }
166}
167
168/// JSON-LD 里能当卡片用的字段（`headline` / `name` / `description` / `image`）。
169///
170/// `image` 可能是字符串、`{"url": ...}`，或者这两者的数组。
171fn json_ld_value(block: &serde_json::Value, key: &str) -> Option<String> {
172 let node = block.get(key)?;
173 let text = match node {
174 serde_json::Value::String(value) => value.clone(),
175 serde_json::Value::Object(map) => map.get("url")?.as_str()?.to_string(),
176 serde_json::Value::Array(items) => items.first().and_then(|item| match item {
177 serde_json::Value::String(value) => Some(value.clone()),
178 serde_json::Value::Object(map) => map.get("url")?.as_str().map(str::to_string),
179 _ => None,
180 })?,
181 _ => return None,
182 };
183 let trimmed = text.trim();
184 (!trimmed.is_empty()).then(|| decode_entities(trimmed))
185}
186
187/// 扫描页面里的 `application/ld+json`，取第一个带可用字段的对象。
188///
189/// 不少站点（`@type: NewsArticle` 之类）不发 OpenGraph 却一定有结构化数据。
190fn json_ld_meta(head: &str) -> (Option<String>, Option<String>, Option<String>) {
191 let lower = head.to_ascii_lowercase();
192 let mut search_from = 0;
193 while let Some(idx) = lower[search_from..]
194 .find("<script")
195 .map(|i| i + search_from)
196 {
197 let Some(end) = lower[idx..].find('>').map(|i| i + idx) else {
198 break;
199 };
200 let open_tag = &lower[idx..=end];
201 search_from = end + 1;
202 if !open_tag.contains("application/ld+json") {
203 continue;
204 }
205 let Some(close) = lower[search_from..]
206 .find("</script>")
207 .map(|i| i + search_from)
208 else {
209 break;
210 };
211 let raw = &head[search_from..close];
212 search_from = close + "</script>".len();
213 let Ok(value) = serde_json::from_str::<serde_json::Value>(raw.trim()) else {
214 continue;
215 };
216 // `@graph` 是常见的外层容器。
217 let candidates: Vec<&serde_json::Value> = match &value {
218 serde_json::Value::Array(items) => items.iter().collect(),
219 serde_json::Value::Object(map) => match map.get("@graph") {
220 Some(serde_json::Value::Array(items)) => items.iter().collect(),
221 _ => vec![&value],
222 },
223 _ => continue,
224 };
225 for node in candidates {
226 let title = json_ld_value(node, "headline").or_else(|| json_ld_value(node, "name"));
227 let description = json_ld_value(node, "description");
228 let image = json_ld_value(node, "image");
229 if title.is_some() || description.is_some() || image.is_some() {
230 return (title, description, image);
231 }
232 }
233 }
234 (None, None, None)
235}
236
237/// 把 `//host/path` 与 `/path` 补成绝对 URL。
238///
239/// 站点常常发协议相对地址（`//n.sinaimg.cn/...`），直接塞进 `<img src>` 会让
240/// 前端按当前页面解析，图片永远加载不出来。
241fn absolute_url(value: &str, base: &str) -> Option<String> {
242 let trimmed = value.trim();
243 if trimmed.is_empty() || trimmed.len() > 2048 {
244 return None;
245 }
246 if let Ok(parsed) = url::Url::parse(trimmed) {
247 return matches!(parsed.scheme(), "http" | "https").then(|| parsed.to_string());
248 }
249 let base = url::Url::parse(base).ok()?;
250 let joined = base.join(trimmed).ok()?;
251 matches!(joined.scheme(), "http" | "https").then(|| joined.to_string())
252}
253
254fn decode_entities(value: &str) -> String {
255 value
256 .replace("&amp;", "&")
257 .replace("&lt;", "<")
258 .replace("&gt;", ">")
259 .replace("&quot;", "\"")
260 .replace("&#39;", "'")
261 .replace("&apos;", "'")
262 .replace("&nbsp;", " ")
263}
264
265struct FetchedMeta {
266 title: Option<String>,
267 description: Option<String>,
268 image_url: Option<String>,
269 site_name: Option<String>,
270 /// 跟随重定向后的落地地址。
271 final_url: Option<String>,
272 /// 页面自声明的 oEmbed 端点。
273 oembed: Option<String>,
274}
275
276/// 写进 `chat_link_unfurl` 的卡片字段。
277#[derive(Default)]
278struct CardMeta {
279 final_url: Option<String>,
280 oembed: Option<String>,
281 title: Option<String>,
282 description: Option<String>,
283 image_url: Option<String>,
284 site_name: Option<String>,
285}
286
287/// 抓取并解析卡片元数据，同时回报最终落地 URL 与页面自声明的 oEmbed 端点。
288async fn fetch_meta(url: &str) -> Result<FetchedMeta, String> {
289 let fetched =
290 outbound::fetch_following(url, "text/html", Some("text/html"), MAX_FETCH_BYTES).await?;
291 let text = fetched.body;
292 let head = match text.find("</head>") {
293 Some(idx) => &text[..idx],
294 None => safe_prefix(&text, MAX_FETCH_BYTES),
295 };
296 let (ld_title, ld_description, ld_image) = json_ld_meta(head);
297 let base = fetched.final_url.as_str();
298
299 Ok(FetchedMeta {
300 title: meta_any(head, &["og:title", "twitter:title"])
301 .or(ld_title)
302 .or_else(|| page_title(head)),
303 description: meta_any(
304 head,
305 &["og:description", "twitter:description", "description"],
306 )
307 .or(ld_description),
308 image_url: meta_any(head, &["og:image", "og:image:url", "twitter:image"])
309 .or(ld_image)
310 .and_then(|value| absolute_url(&value, base)),
311 site_name: meta_any(head, &["og:site_name", "twitter:site", "application-name"]),
312 final_url: Some(fetched.final_url.clone()),
313 oembed: link_oembed(head),
314 })
315}
316
317/// 按字节上限截断，且保证不切断多字节字符。
318fn safe_prefix(text: &str, max_bytes: usize) -> &str {
319 let mut end = text.len().min(max_bytes);
320 while !text.is_char_boundary(end) {
321 end -= 1;
322 }
323 &text[..end]
324}
325
326fn link_embed_json(url: &str, row: &link_unfurls::Model) -> serde_json::Value {
327 serde_json::json!({
328 "type": "link",
329 "url": url,
330 "title": row.title,
331 "description": row.description,
332 "image_url": row.image_url,
333 "site_name": row.site_name,
334 })
335}
336
337/// 数据库行里已抓到的字段转成 [`social::Enrichment`]。
338fn row_enrichment(row: &link_unfurls::Model) -> Option<social::Enrichment> {
339 let enrichment = social::Enrichment {
340 title: row.title.clone(),
341 description: row.description.clone(),
342 image_url: row.image_url.clone(),
343 frame_src: None,
344 };
345 (enrichment.title.is_some()
346 || enrichment.description.is_some()
347 || enrichment.image_url.is_some())
348 .then_some(enrichment)
349}
350
351impl ChatService {
352 /// 带缓存的 OpenGraph 抓取：命中新鲜缓存直接返回，抓取失败记 Failed 并返回 `None`。
353 async fn unfurl_row(&self, url: &str) -> Option<link_unfurls::Model> {
354 let hash = url_hash(url);
355 let now = OffsetDateTime::now_utc();
356 let cached = link_unfurls::Entity::find_by_id(hash.clone())
357 .one(self.db().read())
358 .await
359 .ok()
360 .flatten();
361 if let Some(row) = &cached {
362 let fresh = row.status == UnfurlStatus::Ready
363 && row.fetched_at > now - time::Duration::days(CACHE_FRESH_DAYS);
364 let retryable_failed = row.status == UnfurlStatus::Failed
365 && row.fetched_at < now - time::Duration::minutes(FAILED_RETRY_MINUTES);
366 if fresh {
367 return Some(row.clone());
368 }
369 if row.status == UnfurlStatus::Failed && !retryable_failed {
370 return None;
371 }
372 }
373
374 let meta = match fetch_meta(url).await {
375 Ok(meta) => meta,
376 Err(_) => {
377 self.upsert_unfurl_row(&hash, url, None, now).await;
378 return None;
379 }
380 };
381 // 自声明 oEmbed 的页面可能一个 OG 标签都没有，但仍然能出完整卡片。
382 if meta.title.is_none()
383 && meta.description.is_none()
384 && meta.image_url.is_none()
385 && meta.oembed.is_none()
386 {
387 self.upsert_unfurl_row(&hash, url, None, now).await;
388 return None;
389 }
390 self.upsert_unfurl_row(
391 &hash,
392 url,
393 Some(CardMeta {
394 final_url: meta.final_url.clone(),
395 oembed: meta.oembed.clone(),
396 title: meta.title.clone(),
397 description: meta.description.clone(),
398 image_url: meta.image_url.clone(),
399 site_name: meta.site_name.clone(),
400 }),
401 now,
402 )
403 .await
404 }
405
406 /// 卡片类平台联网补全：注册表 oEmbed → 页面自声明 oEmbed → OpenGraph。
407 async fn enrich_card(&self, url: &str) -> Option<social::Enrichment> {
408 if let Some(enrichment) = oembed::fetch(url).await {
409 return Some(enrichment);
410 }
411 let row = self.unfurl_row(url).await?;
412 if row_enrichment(&row).is_none()
413 && let Some(endpoint) = row.oembed.as_deref()
414 && let Some(from_oembed) = oembed::fetch_endpoint(endpoint).await
415 {
416 return Some(from_oembed);
417 }
418 row_enrichment(&row)
419 }
420
421 async fn upsert_unfurl_row(
422 &self,
423 hash: &str,
424 url: &str,
425 meta: Option<CardMeta>,
426 now: OffsetDateTime,
427 ) -> Option<link_unfurls::Model> {
428 let status = match &meta {
429 Some(_) => UnfurlStatus::Ready,
430 None => UnfurlStatus::Failed,
431 };
432 let meta = meta.unwrap_or_default();
433 let url = url.chars().take(2048).collect::<String>();
434 let row = link_unfurls::ActiveModel {
435 url_hash: Set(hash.to_string()),
436 url: Set(url.clone()),
437 final_url: Set(meta.final_url.clone()),
438 oembed: Set(meta.oembed.clone()),
439 title: Set(meta.title.clone()),
440 description: Set(meta.description.clone()),
441 image_url: Set(meta.image_url.clone()),
442 site_name: Set(meta.site_name.clone()),
443 status: Set(status),
444 fetched_at: Set(now),
445 };
446 match row.insert(self.db().write()).await {
447 Ok(inserted) => Some(inserted),
448 Err(_) => {
449 let existing = link_unfurls::Entity::find_by_id(hash.to_string())
450 .one(self.db().write())
451 .await
452 .ok()
453 .flatten()?;
454 let mut am: link_unfurls::ActiveModel = existing.into();
455 am.url = Set(url);
456 am.final_url = Set(meta.final_url);
457 am.oembed = Set(meta.oembed);
458 am.title = Set(meta.title);
459 am.description = Set(meta.description);
460 am.image_url = Set(meta.image_url);
461 am.site_name = Set(meta.site_name);
462 am.status = Set(status);
463 am.fetched_at = Set(now);
464 am.update(self.db().write()).await.ok()
465 }
46660 }
46761 }
62 urls
63}
46864
65impl ChatService {
46966 pub async fn unfurl_message(&self, room_id: Uuid, message_id: Uuid, content: String) {
47067 let urls = extract_urls(&content);
47168 if urls.is_empty() {
@@ -527,51 +124,11 @@
527124
528125 /// 解析单条链接，返回可直接塞进 `MessageDto::embeds` 的一项。
529126 ///
530 /// 顺序很关键：注册表里能直接内嵌的平台（YouTube / Spotify / B 站正片）不联网；
531 /// 其余先跟随重定向，落地页命中可嵌入平台就升级成播放器，否则退回卡片。
127 /// `social` 的 resolver 链先跑静态平台注册表，未命中再走通用
128 /// HTML / oEmbed / 直链媒体解析；这里只负责把结果转成消息 DTO。
532129 async fn resolve_link(&self, url: &str) -> Option<serde_json::Value> {
533 if let Some(value) = social::embed_json(url)
534 && value["kind"] != "link"
535 {
536 return Some(value);
537 }
538 // 卡片（含短链：`b23.tv` 也注册成了卡片）先跟随重定向拿到落地页。
539 let row = self.unfurl_row(url).await;
540 let final_url = row
541 .as_ref()
542 .and_then(|row| row.final_url.clone())
543 .filter(|final_url| !final_url.is_empty() && final_url != url);
544 if let Some(final_url) = final_url.as_deref() {
545 if let Some(value) = social::embed_json(final_url) {
546 let mut value = value;
547 if value["kind"] == "link" {
548 // 落地页的元数据已经在上面那次抓取里拿到了，别再联网。
549 value = match row.as_ref().and_then(row_enrichment) {
550 Some(enrichment) => {
551 social::embed_json_enriched(final_url, &enrichment).unwrap_or(value)
552 }
553 None => value,
554 };
555 }
556 // 前端按消息里的原始文本匹配 embed，锚点必须留在原 URL 上。
557 value["url"] = serde_json::Value::String(url.to_string());
558 return Some(value);
559 }
560 // 落地页也不在注册表里：直接用抓到的元数据出普通卡片。
561 if let Some(row) = row.as_ref() {
562 return Some(link_embed_json(url, row));
563 }
564 }
565 // 注册表认识的卡片平台。
566 if let Some(base) = social::embed_json(url) {
567 let merged = match self.enrich_card(url).await {
568 Some(enrichment) => social::embed_json_enriched(url, &enrichment).unwrap_or(base),
569 None => base,
570 };
571 return Some(merged);
572 }
573 // 注册表不认识、也拿不到落地页：退化成 OpenGraph 卡片，别把链接吞掉。
574 Some(link_embed_json(url, row.as_ref()?))
130 let embed = social::unfurl(url).await.ok()?.into_iter().next()?;
131 serde_json::to_value(embed).ok()
575132 }
576133}
577134
@@ -587,7 +144,6 @@
587144
588145#[cfg(test)]
589146mod tests {
590 use super::*;
591147 use crate::services::messages::MessageEmbedDto;
592148
593149 /// 平台 embed 必须能原样填进 `MessageDto::embeds`，否则聊天里收到的一直是
@@ -606,7 +162,7 @@
606162 let value = social::embed_json(url).unwrap_or_else(|| panic!("{url} did not resolve"));
607163 let dto: MessageEmbedDto = serde_json::from_value(value)
608164 .unwrap_or_else(|error| panic!("{url} did not fit the DTO: {error}"));
609 assert_eq!(dto.url, url);
165 assert_eq!(dto.url.as_deref(), Some(url));
610166 assert!(
611167 dto.html
612168 .as_deref()
@@ -617,169 +173,17 @@
617173 }
618174
619175 #[test]
620 fn link_embed_fits_message_dto() {
621 let row = link_unfurls::Model {
622 url_hash: "hash".into(),
623 url: "https://example.com/".into(),
624 final_url: None,
625 oembed: None,
626 title: Some("Example".into()),
627 description: Some("desc".into()),
628 image_url: None,
629 site_name: Some("Example".into()),
630 status: UnfurlStatus::Ready,
631 fetched_at: time::OffsetDateTime::UNIX_EPOCH,
632 };
633 let dto: MessageEmbedDto =
634 serde_json::from_value(link_embed_json("https://example.com/", &row)).unwrap();
176 fn legacy_link_embed_still_deserializes() {
177 let value = serde_json::json!({
178 "type": "link",
179 "url": "https://example.com/",
180 "title": "Example",
181 "description": "desc",
182 "site_name": "Example",
183 });
184 let dto: MessageEmbedDto = serde_json::from_value(value).unwrap();
635185 assert_eq!(dto.embed_type, "link");
636186 assert_eq!(dto.title.as_deref(), Some("Example"));
637187 assert!(dto.html.is_none());
638 }
639
640 /// 卡片类平台联网补全后要塞进同一条 embed：标题 / 简介 / 封面进卡片，
641 /// 外部域名的 iframe 不会被采用。
642 #[test]
643 fn card_embeds_take_over_the_fetched_metadata() {
644 let url = "https://item.taobao.com/item.htm?id=123456";
645 let enrichment = social::Enrichment {
646 title: Some("商品标题".into()),
647 description: Some("商品简介".into()),
648 image_url: Some("https://img.alicdn.com/cover.jpg".into()),
649 frame_src: Some("https://evil.example/frame".into()),
650 };
651 let value = social::embed_json_enriched(url, &enrichment).expect("taobao resolves");
652 let dto: MessageEmbedDto = serde_json::from_value(value).expect("fits the dto");
653 assert_eq!(dto.embed_type, "platform");
654 assert_eq!(dto.kind.as_deref(), Some("link"));
655 assert_eq!(dto.title.as_deref(), Some("商品标题"));
656 assert_eq!(dto.description.as_deref(), Some("商品简介"));
657 assert_eq!(
658 dto.image_url.as_deref(),
659 Some("https://img.alicdn.com/cover.jpg")
660 );
661 let html = dto.html.expect("card html");
662 assert!(html.contains("gd-embed-thumb"), "{html}");
663 assert!(html.contains("商品简介"), "{html}");
664 assert!(!html.contains("evil.example"), "{html}");
665 }
666
667 /// 只发 `twitter:` 卡片、不发 `og:` 的站点以前会拿到空卡片。
668 #[test]
669 fn twitter_tags_back_up_missing_opengraph() {
670 let head = r#"<head>
671 <meta name="twitter:title" content="仅 Twitter 卡片">
672 <meta name="twitter:description" content="描述">
673 <meta name="twitter:image" content="https://cdn.example.com/cover.jpg">
674 </head>"#;
675 assert_eq!(
676 meta_any(head, &["og:title", "twitter:title"]).as_deref(),
677 Some("仅 Twitter 卡片")
678 );
679 assert_eq!(
680 meta_any(head, &["og:image", "og:image:url", "twitter:image"]).as_deref(),
681 Some("https://cdn.example.com/cover.jpg")
682 );
683 }
684
685 /// OpenGraph 缺失时的 JSON-LD 兜底，含 `@graph` 容器与对象形态的 `image`。
686 #[test]
687 fn json_ld_backs_up_missing_opengraph() {
688 let head = r#"<head><script type="application/ld+json">
689 {"@graph":[{"@type":"NewsArticle","headline":"结构化标题",
690 "description":"结构化摘要","image":{"url":"https://cdn.example.com/ld.jpg"}}]}
691 </script></head>"#;
692 let (title, description, image) = json_ld_meta(head);
693 assert_eq!(title.as_deref(), Some("结构化标题"));
694 assert_eq!(description.as_deref(), Some("结构化摘要"));
695 assert_eq!(image.as_deref(), Some("https://cdn.example.com/ld.jpg"));
696 }
697
698 /// 协议相对的封面地址必须补成绝对 URL，否则前端按聊天页域名解析、图片永远裂。
699 #[test]
700 fn protocol_relative_images_become_absolute() {
701 assert_eq!(
702 absolute_url("//n.sinaimg.cn/cover.jpg", "https://finance.sina.com.cn/a").as_deref(),
703 Some("https://n.sinaimg.cn/cover.jpg")
704 );
705 assert_eq!(
706 absolute_url("/img/cover.jpg", "https://example.com/post/1").as_deref(),
707 Some("https://example.com/img/cover.jpg")
708 );
709 assert!(absolute_url("javascript:alert(1)", "https://example.com/").is_none());
710 }
711
712 /// 页面自声明的 oEmbed 端点要能被认出来（带单双引号、大小写混排）。
713 #[test]
714 fn link_rel_oembed_is_discovered() {
715 let head = r#"<head><link rel="alternate" type="application/json+oembed"
716 href="https://example.com/oembed?url=x&amp;format=json"></head>"#;
717 assert_eq!(
718 link_oembed(head).as_deref(),
719 Some("https://example.com/oembed?url=x&format=json")
720 );
721 assert!(link_oembed("<head><link rel='stylesheet' href='/a.css'></head>").is_none());
722 assert!(
723 link_oembed("<head><link rel='alternate' type='text/xml+oembed' href='/o'></head>")
724 .is_none()
725 );
726 }
727
728 /// GBK 中文站按 `content-type` 解码，不能出乱码。
729 #[test]
730 fn gbk_pages_decode_from_the_declared_charset() {
731 let (encoded, _, _) = encoding_rs::GBK.encode("中文标题");
732 let text = outbound::decode_body(&encoded, "text/html; charset=gbk");
733 assert_eq!(text, "中文标题");
734 let text = outbound::decode_body(&encoded, "text/html; charset=GB2312");
735 assert_eq!(text, "中文标题");
736 }
737
738 /// 没声明 charset 时 UTF-8 直通；非法 UTF-8 回退 GB18030 而不是替换成 U+FFFD。
739 #[test]
740 fn undeclared_charsets_fall_back_to_utf8_then_gb18030() {
741 assert_eq!(
742 outbound::decode_body("标题".as_bytes(), "text/html"),
743 "标题"
744 );
745 let (encoded, _, _) = encoding_rs::GB18030.encode("中文标题");
746 assert_eq!(outbound::decode_body(&encoded, "text/html"), "中文标题");
747 }
748
749 /// Wikipedia 这类带括号的路径不能被当成句尾括号裁掉。
750 #[test]
751 fn urls_keep_balanced_parentheses() {
752 let urls =
753 extract_urls("看看 https://en.wikipedia.org/wiki/Rust_(programming_language) 这个");
754 assert_eq!(
755 urls,
756 vec!["https://en.wikipedia.org/wiki/Rust_(programming_language)".to_string()]
757 );
758 }
759
760 /// 被中文括号或引号包起来的链接要能剥壳；末尾多余标点要去掉。
761 #[test]
762 fn urls_are_unwrapped_from_surrounding_punctuation() {
763 assert_eq!(
764 extract_urls("（详见 https://example.com/a/b）"),
765 vec!["https://example.com/a/b".to_string()]
766 );
767 assert_eq!(
768 extract_urls("见 https://example.com/a/b."),
769 vec!["https://example.com/a/b".to_string()]
770 );
771 assert_eq!(
772 extract_urls("「https://example.com/x」"),
773 vec!["https://example.com/x".to_string()]
774 );
775 }
776
777 /// 包裹性括号要剥掉，但属于路径的括号要留住。
778 #[test]
779 fn wrapper_parenthesis_is_dropped_but_path_parenthesis_survives() {
780 assert_eq!(
781 extract_urls("(https://en.wikipedia.org/wiki/Rust_(programming_language))"),
782 vec!["https://en.wikipedia.org/wiki/Rust_(programming_language)".to_string()]
783 );
784188 }
785189}
Added lib/social/src/resolvers/mod.rs View lib/social/src/resolvers/mod.rs
@@ -0,0 +1,32 @@
1pub mod page;
2pub mod registry;
3
4use url::Url;
5
6use crate::UnfurlError;
7use crate::resolver::{BoxFuture, ResolveContext, Resolver};
8use crate::types::MessageEmbed;
9
10pub use registry::RegistryResolver;
11
12pub fn build_chain() -> Vec<Box<dyn Resolver>> {
13 vec![Box::new(RegistryResolver), Box::new(DefaultResolver)]
14}
15
16pub struct DefaultResolver;
17
18impl Resolver for DefaultResolver {
19 fn matches(&self, _url: &Url) -> bool {
20 true
21 }
22
23 fn resolve<'a>(
24 &'a self,
25 context: &'a ResolveContext<'_>,
26 ) -> BoxFuture<'a, Result<Vec<MessageEmbed>, UnfurlError>> {
27 Box::pin(async move {
28 let embed = page::resolve_page(context).await?;
29 Ok(embed.into_iter().collect())
30 })
31 }
32}
Added lib/social/src/resolvers/page.rs View lib/social/src/resolvers/page.rs
@@ -0,0 +1,223 @@
1use url::Url;
2
3use crate::media::{self, MediaKind};
4use crate::types::{EmbedAuthor, EmbedMedia, EmbedProvider, MessageEmbed};
5use crate::{ResolveContext, UnfurlError, html, http, normalize, oembed};
6
7const OEMBED_MAX_BYTES: usize = 262_144;
8
9pub async fn resolve_page(
10 context: &ResolveContext<'_>,
11) -> Result<Option<MessageEmbed>, UnfurlError> {
12 Ok(resolve_page_final(context).await?.1)
13}
14
15/// 同 [`resolve_page`]，但同时返回跟随重定向后的落地地址。
16///
17/// 短链（`b23.tv` / `t.co`）只有拿到落地页才能重新命中平台注册表并升级成播放器。
18pub async fn resolve_page_final(
19 context: &ResolveContext<'_>,
20) -> Result<(Url, Option<MessageEmbed>), UnfurlError> {
21 let fetched = http::fetch(
22 context.url.as_str(),
23 "text/html,application/xhtml+xml,application/json;q=0.9,*/*;q=0.8",
24 context.options.max_bytes,
25 )
26 .await?;
27 let final_url = Url::parse(&fetched.final_url).unwrap_or_else(|_| context.url.clone());
28 if !(200..300).contains(&fetched.status) {
29 return Ok((final_url, None));
30 }
31 let content_type = fetched.content_type.to_ascii_lowercase();
32
33 if let Some(kind) =
34 media::from_content_type(&content_type).or_else(|| media::from_url(&final_url))
35 {
36 return Ok((
37 final_url.clone(),
38 Some(media_embed(&final_url, kind, &content_type)),
39 ));
40 }
41 if !content_type.is_empty() && !content_type.contains("html") && !content_type.contains("xml") {
42 return Ok((final_url, None));
43 }
44
45 let page = html::parse(&fetched.body, &final_url);
46 let mut oembed = None;
47 if let Some(endpoint) = page.oembed.as_deref() {
48 oembed = oembed::fetch(endpoint, OEMBED_MAX_BYTES).await;
49 }
50 if oembed.is_none()
51 && let Some(endpoint) = crate::oembed_endpoint(context.url.as_str())
52 {
53 oembed = oembed::fetch(&endpoint.endpoint, OEMBED_MAX_BYTES).await;
54 }
55
56 Ok((
57 final_url.clone(),
58 build_embed(context, &final_url, page, oembed),
59 ))
60}
61
62fn build_embed(
63 context: &ResolveContext<'_>,
64 final_url: &Url,
65 page: html::PageMetadata,
66 oembed: Option<oembed::OEmbed>,
67) -> Option<MessageEmbed> {
68 let oembed = oembed.unwrap_or_default();
69 let title = oembed.title.clone().or(page.title);
70 let description = oembed.description.clone().or(page.description);
71 let image_url = oembed
72 .thumbnail_url
73 .as_deref()
74 .and_then(|value| http::absolute(final_url, value))
75 .or(page.image);
76 let image = image_url.as_ref().map(|url| EmbedMedia {
77 url: Some(url.clone()),
78 ..EmbedMedia::default()
79 });
80 let video = page
81 .video
82 .as_deref()
83 .and_then(|value| http::absolute(final_url, value))
84 .map(|url| EmbedMedia {
85 url: Some(url),
86 ..EmbedMedia::default()
87 });
88 let audio = page
89 .audio
90 .as_deref()
91 .and_then(|value| http::absolute(final_url, value))
92 .map(|url| EmbedMedia {
93 url: Some(url),
94 ..EmbedMedia::default()
95 });
96 let player_src = oembed
97 .html
98 .as_deref()
99 .and_then(oembed::iframe_src)
100 .and_then(|src| http::absolute(final_url, &src))
101 .and_then(|src| frame_allowed(&src, final_url));
102 let player = player_src.clone().or_else(|| {
103 page.player
104 .clone()
105 .and_then(|value| http::absolute(final_url, &value))
106 .and_then(|value| frame_allowed(&value, final_url))
107 });
108 let html = player_src.map(|src| safe_iframe(&src, oembed.width, oembed.height));
109 let embed_type = if html.is_some() {
110 "iframe"
111 } else if video.is_some() {
112 "video"
113 } else if audio.is_some() {
114 "audio"
115 } else if image.is_some() {
116 "image"
117 } else {
118 "link"
119 };
120 let author = oembed.author_name.clone().map(|name| EmbedAuthor {
121 name,
122 url: oembed.author_url.clone(),
123 ..EmbedAuthor::default()
124 });
125 let provider_name = oembed.provider_name.clone().or(page.site_name.clone());
126 let provider = provider_name.map(|name| EmbedProvider {
127 name,
128 url: oembed.provider_url.clone(),
129 });
130
131 let mut embed = MessageEmbed {
132 embed_type: embed_type.to_owned(),
133 url: Some(context.original_url.to_string()),
134 title,
135 description,
136 image: image.clone(),
137 video,
138 audio,
139 author,
140 provider,
141 html,
142 html_width: oembed.width,
143 html_height: oembed.height,
144 image_url: image_url.clone(),
145 site_name: page.site_name.clone(),
146 src: player,
147 height: oembed.height,
148 kind: Some(embed_type.to_owned()),
149 ..MessageEmbed::default()
150 };
151 normalize::normalize(&mut embed);
152 (!embed.is_empty()).then_some(embed)
153}
154
155const PLAYER_HOSTS: [&str; 12] = [
156 "youtube.com",
157 "youtube-nocookie.com",
158 "youtu.be",
159 "vimeo.com",
160 "soundcloud.com",
161 "spotify.com",
162 "twitch.tv",
163 "twitter.com",
164 "x.com",
165 "reddit.com",
166 "tiktok.com",
167 "codepen.io",
168];
169
170/// iframe 只接受页面自身域名或已知播放器域名：页面自声明的 oEmbed 端点
171/// 不应把任意第三方页面塞进消息流。
172fn frame_allowed(src: &str, base: &Url) -> Option<String> {
173 let url = Url::parse(src).ok()?;
174 if !matches!(url.scheme(), "http" | "https") {
175 return None;
176 }
177 let host = url.host_str()?.trim_end_matches('.').to_ascii_lowercase();
178 let base_host = base.host_str()?.trim_end_matches('.').to_ascii_lowercase();
179 let same_site = host == base_host
180 || host.ends_with(&format!(".{base_host}"))
181 || base_host.ends_with(&format!(".{host}"));
182 let known = PLAYER_HOSTS
183 .iter()
184 .any(|domain| host == *domain || host.ends_with(&format!(".{domain}")));
185 (same_site || known).then(|| url.to_string())
186}
187
188fn media_embed(url: &Url, kind: MediaKind, content_type: &str) -> MessageEmbed {
189 let media = EmbedMedia {
190 url: Some(url.to_string()),
191 content_type: (!content_type.is_empty()).then(|| content_type.to_owned()),
192 ..EmbedMedia::default()
193 };
194 let filename = url
195 .path_segments()
196 .and_then(|mut segments| segments.next_back())
197 .filter(|value| !value.is_empty())
198 .map(str::to_owned);
199 let mut embed = MessageEmbed {
200 embed_type: kind.embed_type().to_owned(),
201 url: Some(url.to_string()),
202 title: filename,
203 image: (kind == MediaKind::Image).then(|| media.clone()),
204 video: (kind == MediaKind::Video).then(|| media.clone()),
205 audio: (kind == MediaKind::Audio).then(|| media.clone()),
206 image_url: (kind == MediaKind::Image).then(|| url.to_string()),
207 kind: Some(kind.embed_type().to_owned()),
208 ..MessageEmbed::default()
209 };
210 normalize::normalize(&mut embed);
211 embed
212}
213
214fn safe_iframe(src: &str, width: Option<u32>, height: Option<u32>) -> String {
215 let width = width.unwrap_or(640);
216 let height = height.unwrap_or(360);
217 format!(
218 "<iframe class=\"gd-embed gd-embed-frame\" src=\"{}\" width=\"{width}\" height=\"{height}\" \
219 loading=\"lazy\" referrerpolicy=\"strict-origin-when-cross-origin\" allowfullscreen \
220 sandbox=\"allow-scripts allow-same-origin allow-popups allow-forms allow-presentation\"></iframe>",
221 crate::escape_html(src),
222 )
223}
Added lib/social/src/resolvers/registry.rs View lib/social/src/resolvers/registry.rs
@@ -0,0 +1,60 @@
1use url::Url;
2
3use crate::resolver::{BoxFuture, ResolveContext, Resolver};
4use crate::types::MessageEmbed;
5use crate::{EmbedKind, UnfurlError};
6
7pub struct RegistryResolver;
8
9impl Resolver for RegistryResolver {
10 fn matches(&self, url: &Url) -> bool {
11 crate::resolve::parse(url.as_str()).is_some()
12 }
13
14 fn resolve<'a>(
15 &'a self,
16 context: &'a ResolveContext<'_>,
17 ) -> BoxFuture<'a, Result<Vec<MessageEmbed>, UnfurlError>> {
18 Box::pin(async move {
19 let resolved = crate::resolve::parse(context.url.as_str())
20 .ok_or_else(|| UnfurlError::InvalidUrl("unknown platform".into()))?;
21 let mut embed = MessageEmbed::from_resolved(&resolved);
22 if resolved.kind == EmbedKind::Link
23 && let Ok((final_url, page)) = super::page::resolve_page_final(context).await
24 {
25 // 短链落地页命中可直接内嵌的平台时，升级成播放器但保留原始 URL。
26 if final_url != context.url
27 && let Some(landing) = crate::resolve::parse(final_url.as_str())
28 && landing.kind != EmbedKind::Link
29 {
30 let mut upgraded = MessageEmbed::from_resolved(&landing);
31 upgraded.url = Some(context.original_url.to_string());
32 crate::normalize::normalize(&mut upgraded);
33 return Ok(vec![upgraded]);
34 }
35 if let Some(page) = page {
36 embed.title = page.title.or(embed.title);
37 embed.description = page.description.or(embed.description);
38 if embed.image.is_none() {
39 embed.image = page.image;
40 embed.image_url = page.image_url;
41 }
42 embed.author = page.author.or(embed.author);
43 embed.provider = page.provider.or(embed.provider);
44 embed.site_name = page.site_name.or(embed.site_name);
45 if page.html.is_some() {
46 embed.html = page.html;
47 embed.embed_type = page.embed_type;
48 embed.html_width = page.html_width;
49 embed.html_height = page.html_height;
50 embed.src = page.src;
51 embed.height = page.height;
52 embed.kind = page.kind;
53 }
54 }
55 }
56 crate::normalize::normalize(&mut embed);
57 Ok(vec![embed])
58 })
59 }
60}
Added lib/social/src/error.rs View lib/social/src/error.rs
@@ -0,0 +1,11 @@
1#[derive(Debug, thiserror::Error)]
2pub enum UnfurlError {
3 #[error("invalid url: {0}")]
4 InvalidUrl(String),
5 #[error("blocked url: {0}")]
6 BlockedUrl(String),
7 #[error("fetch failed: {0}")]
8 Fetch(String),
9 #[error("response too large")]
10 TooLarge,
11}
Added lib/social/src/html.rs View lib/social/src/html.rs
@@ -0,0 +1,244 @@
1use url::Url;
2
3use crate::http::absolute;
4
5#[derive(Debug, Default, Clone)]
6pub struct PageMetadata {
7 pub title: Option<String>,
8 pub description: Option<String>,
9 pub image: Option<String>,
10 pub site_name: Option<String>,
11 pub video: Option<String>,
12 pub audio: Option<String>,
13 pub player: Option<String>,
14 pub theme_color: Option<String>,
15 pub oembed: Option<String>,
16 pub textual: bool,
17}
18
19pub fn parse(html: &str, base: &Url) -> PageMetadata {
20 let head = match html.find("</head>") {
21 Some(index) => &html[..index],
22 None => html,
23 };
24 let (ld_title, ld_description, ld_image) = json_ld_meta(head);
25 let title = meta_any(head, &["og:title", "twitter:title"])
26 .or(ld_title)
27 .or_else(|| page_title(head));
28 let description = meta_any(
29 head,
30 &["og:description", "twitter:description", "description"],
31 )
32 .or(ld_description);
33 let image = meta_any(head, &["og:image", "og:image:url", "twitter:image"])
34 .or(ld_image)
35 .and_then(|value| absolute(base, &value));
36 PageMetadata {
37 title: title.map(|value| value.chars().take(300).collect()),
38 description: description.map(|value| value.chars().take(1200).collect()),
39 image,
40 site_name: meta_any(head, &["og:site_name", "twitter:site", "application-name"]),
41 video: meta_any(
42 head,
43 &[
44 "og:video",
45 "og:video:url",
46 "og:video:secure_url",
47 "twitter:player:stream",
48 ],
49 )
50 .and_then(|value| absolute(base, &value)),
51 audio: meta_any(head, &["og:audio", "og:audio:url"])
52 .and_then(|value| absolute(base, &value)),
53 player: meta_any(head, &["twitter:player"]).and_then(|value| absolute(base, &value)),
54 theme_color: meta_any(head, &["theme-color"]),
55 oembed: link_oembed(head).and_then(|value| absolute(base, &value)),
56 textual: meta_any(
57 head,
58 &[
59 "og:title",
60 "og:description",
61 "og:site_name",
62 "twitter:title",
63 "twitter:description",
64 ],
65 )
66 .is_some(),
67 }
68}
69
70fn meta_content(head: &str, property: &str) -> Option<String> {
71 let lower = head.to_ascii_lowercase();
72 let mut search_from = 0;
73 loop {
74 let index = lower[search_from..].find("<meta")? + search_from;
75 let end = lower[index..].find('>')? + index;
76 let tag = &head[index..=end];
77 let tag_lower = &lower[index..=end];
78 search_from = end + 1;
79 let matches = tag_lower.contains(&format!("property=\"{property}\""))
80 || tag_lower.contains(&format!("property='{property}'"))
81 || tag_lower.contains(&format!("name=\"{property}\""))
82 || tag_lower.contains(&format!("name='{property}'"));
83 if !matches {
84 continue;
85 }
86 for attr in ["content=\"", "content='"] {
87 if let Some(start) = tag.find(attr) {
88 let value_start = start + attr.len();
89 let quote = attr.chars().last()?;
90 if let Some(value_end) = tag[value_start..].find(quote) {
91 let value = tag[value_start..value_start + value_end].trim();
92 if !value.is_empty() {
93 return Some(decode_entities(value));
94 }
95 }
96 }
97 }
98 }
99}
100
101fn meta_any(head: &str, properties: &[&str]) -> Option<String> {
102 properties
103 .iter()
104 .find_map(|property| meta_content(head, property))
105}
106
107fn page_title(head: &str) -> Option<String> {
108 let lower = head.to_ascii_lowercase();
109 let start = lower.find("<title")?;
110 let content_start = lower[start..].find('>')? + start + 1;
111 let end = lower[content_start..].find("</title>")? + content_start;
112 let title = head[content_start..end].trim();
113 (!title.is_empty()).then(|| decode_entities(title).chars().take(200).collect())
114}
115
116fn link_oembed(head: &str) -> Option<String> {
117 let lower = head.to_ascii_lowercase();
118 let mut search_from = 0;
119 loop {
120 let index = lower[search_from..].find("<link")? + search_from;
121 let end = lower[index..].find('>')? + index;
122 let tag = &head[index..=end];
123 let tag_lower = &lower[index..=end];
124 search_from = end + 1;
125 if !tag_lower.contains("oembed") || !tag_lower.contains("json") {
126 continue;
127 }
128 for attr in ["href=\"", "href='"] {
129 if let Some(start) = tag.find(attr) {
130 let value_start = start + attr.len();
131 let quote = attr.chars().last()?;
132 if let Some(value_end) = tag[value_start..].find(quote) {
133 let value = tag[value_start..value_start + value_end].trim();
134 if !value.is_empty() {
135 return Some(decode_entities(value));
136 }
137 }
138 }
139 }
140 }
141}
142
143fn json_ld_value(block: &serde_json::Value, key: &str) -> Option<String> {
144 let node = block.get(key)?;
145 let text = match node {
146 serde_json::Value::String(value) => value.clone(),
147 serde_json::Value::Object(map) => map.get("url")?.as_str()?.to_owned(),
148 serde_json::Value::Array(items) => items.first().and_then(|item| match item {
149 serde_json::Value::String(value) => Some(value.clone()),
150 serde_json::Value::Object(map) => map.get("url")?.as_str().map(str::to_owned),
151 _ => None,
152 })?,
153 _ => return None,
154 };
155 let trimmed = text.trim();
156 (!trimmed.is_empty()).then(|| decode_entities(trimmed))
157}
158
159fn json_ld_meta(head: &str) -> (Option<String>, Option<String>, Option<String>) {
160 let lower = head.to_ascii_lowercase();
161 let mut search_from = 0;
162 while let Some(index) = lower[search_from..]
163 .find("<script")
164 .map(|i| i + search_from)
165 {
166 let Some(end) = lower[index..].find('>').map(|i| i + index) else {
167 break;
168 };
169 let open_tag = &lower[index..=end];
170 search_from = end + 1;
171 if !open_tag.contains("application/ld+json") {
172 continue;
173 }
174 let Some(close) = lower[search_from..]
175 .find("</script>")
176 .map(|i| i + search_from)
177 else {
178 break;
179 };
180 let raw = &head[search_from..close];
181 search_from = close + "</script>".len();
182 let Ok(value) = serde_json::from_str::<serde_json::Value>(raw.trim()) else {
183 continue;
184 };
185 let blocks: Vec<&serde_json::Value> = match &value {
186 serde_json::Value::Array(items) => items.iter().collect(),
187 _ => vec![&value],
188 };
189 for block in blocks {
190 let title = json_ld_value(block, "headline").or_else(|| json_ld_value(block, "name"));
191 let description = json_ld_value(block, "description");
192 let image = json_ld_value(block, "image");
193 if title.is_some() || description.is_some() || image.is_some() {
194 return (title, description, image);
195 }
196 }
197 }
198 (None, None, None)
199}
200
201pub fn decode_entities(value: &str) -> String {
202 let mut output = String::with_capacity(value.len());
203 let mut rest = value;
204 while let Some(index) = rest.find('&') {
205 output.push_str(&rest[..index]);
206 rest = &rest[index..];
207 let Some(end) = rest.find(';').filter(|end| *end <= 10) else {
208 output.push('&');
209 rest = &rest[1..];
210 continue;
211 };
212 let entity = &rest[1..end];
213 let decoded = match entity {
214 "amp" => Some('&'),
215 "lt" => Some('<'),
216 "gt" => Some('>'),
217 "quot" => Some('"'),
218 "apos" | "#39" => Some('\''),
219 "nbsp" => Some(' '),
220 _ => entity
221 .strip_prefix("#x")
222 .or_else(|| entity.strip_prefix("#X"))
223 .and_then(|hex| u32::from_str_radix(hex, 16).ok())
224 .or_else(|| {
225 entity
226 .strip_prefix('#')
227 .and_then(|dec| dec.parse::<u32>().ok())
228 })
229 .and_then(char::from_u32),
230 };
231 match decoded {
232 Some(ch) => {
233 output.push(ch);
234 rest = &rest[end + 1..];
235 }
236 None => {
237 output.push('&');
238 rest = &rest[1..];
239 }
240 }
241 }
242 output.push_str(rest);
243 output
244}
Added lib/social/src/http.rs View lib/social/src/http.rs
@@ -0,0 +1,132 @@
1use std::time::Duration;
2
3use reqwest::redirect::Policy;
4use url::Url;
5
6use crate::{UnfurlError, net};
7
8const TIMEOUT_SECS: u64 = 10;
9const MAX_REDIRECTS: usize = 6;
10const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
11(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
12
13pub struct Fetched {
14 pub final_url: String,
15 pub status: u16,
16 pub content_type: String,
17 pub body: String,
18}
19
20pub async fn fetch(raw: &str, accept: &str, max_bytes: usize) -> Result<Fetched, UnfurlError> {
21 let mut current = net::parse(raw)?;
22 for _ in 0..=MAX_REDIRECTS {
23 net::validate_static(&current)?;
24 let host = current
25 .host_str()
26 .ok_or_else(|| UnfurlError::InvalidUrl("host is required".into()))?
27 .to_owned();
28 let port = current.port_or_known_default().unwrap_or(443);
29 let addrs = net::resolve_public(&host, port).await?;
30
31 let mut builder = reqwest::Client::builder()
32 .timeout(Duration::from_secs(TIMEOUT_SECS))
33 .redirect(Policy::none())
34 .user_agent(USER_AGENT);
35 for addr in &addrs {
36 builder = builder.resolve(&host, *addr);
37 }
38 let client = builder
39 .build()
40 .map_err(|error| UnfurlError::Fetch(error.to_string()))?;
41
42 let response = client
43 .get(current.clone())
44 .header("accept", accept)
45 .send()
46 .await
47 .map_err(|error| UnfurlError::Fetch(error.to_string()))?;
48
49 if response.status().is_redirection() {
50 let location = response
51 .headers()
52 .get("location")
53 .and_then(|value| value.to_str().ok())
54 .ok_or_else(|| UnfurlError::Fetch("redirect without location".into()))?;
55 let next = current
56 .join(location)
57 .map_err(|error| UnfurlError::InvalidUrl(error.to_string()))?;
58 net::validate_static(&next)?;
59 current = next;
60 continue;
61 }
62
63 let status = response.status().as_u16();
64 let content_type = response
65 .headers()
66 .get("content-type")
67 .and_then(|value| value.to_str().ok())
68 .unwrap_or_default()
69 .to_owned();
70 let mut body = Vec::new();
71 let mut stream = response;
72 while let Some(chunk) = stream
73 .chunk()
74 .await
75 .map_err(|error| UnfurlError::Fetch(error.to_string()))?
76 {
77 let remaining = max_bytes.saturating_sub(body.len());
78 if remaining == 0 {
79 break;
80 }
81 if chunk.len() > remaining {
82 body.extend_from_slice(&chunk[..remaining]);
83 break;
84 }
85 body.extend_from_slice(&chunk);
86 }
87 return Ok(Fetched {
88 final_url: current.to_string(),
89 status,
90 content_type: content_type.clone(),
91 body: decode_body(&body, &content_type),
92 });
93 }
94 Err(UnfurlError::Fetch("too many redirects".into()))
95}
96
97fn charset_of(content_type: &str) -> Option<String> {
98 let lower = content_type.to_ascii_lowercase();
99 let start = lower.find("charset=")? + "charset=".len();
100 let rest = &content_type[start..];
101 let value = rest
102 .split(|c: char| c == ';' || c == ',' || c.is_whitespace())
103 .next()
104 .unwrap_or("")
105 .trim_matches(|c: char| c == '"' || c == '\'');
106 (!value.is_empty()).then(|| value.to_string())
107}
108
109pub fn decode_body(body: &[u8], content_type: &str) -> String {
110 if let Some(label) = charset_of(content_type)
111 && let Some(encoding) = encoding_rs::Encoding::for_label(label.as_bytes())
112 {
113 let (text, _, _) = encoding.decode(body);
114 return text.into_owned();
115 }
116 match std::str::from_utf8(body) {
117 Ok(text) => text.to_owned(),
118 Err(_) => {
119 let (text, _, _) = encoding_rs::GB18030.decode(body);
120 text.into_owned()
121 }
122 }
123}
124
125pub fn absolute(base: &Url, value: &str) -> Option<String> {
126 let trimmed = value.trim();
127 if trimmed.is_empty() {
128 return None;
129 }
130 let url = base.join(trimmed).ok()?;
131 matches!(url.scheme(), "http" | "https").then(|| url.to_string())
132}
Modified lib/social/src/lib.rs View lib/social/src/lib.rs
@@ -8,15 +8,28 @@
88//! 注册表是纯静态数据（[`parse::PLATFORMS`]，每个平台一个文件），匹配与渲染都不做
99//! 网络请求，因此可以在消息写入路径上同步调用。
1010
11pub mod error;
12pub mod html;
13pub mod http;
14pub mod media;
15pub mod net;
16pub mod normalize;
17pub mod oembed;
1118pub mod parse;
1219pub mod render;
1320pub mod resolve;
21pub mod resolver;
22pub mod resolvers;
23pub mod types;
1424
1525use serde::{Deserialize, Serialize};
1626
27pub use error::UnfurlError;
1728pub use parse::{EmbedRule, Platform, provider_count, providers};
1829pub use render::{escape_html, render_html};
1930pub use resolve::{embed, embed_json, embed_json_enriched, is_supported, oembed_endpoint, parse};
31pub use resolver::{ResolveContext, Resolver, UnfurlEngine, UnfurlOptions, unfurl};
32pub use types::{EmbedAuthor, EmbedField, EmbedFooter, EmbedMedia, EmbedProvider, MessageEmbed};
2033
2134#[derive(Debug, Clone, Copy, PartialEq, Eq, Serialize, Deserialize, utoipa::ToSchema)]
2235#[serde(rename_all = "snake_case")]
Added lib/social/src/media.rs View lib/social/src/media.rs
@@ -0,0 +1,50 @@
1use url::Url;
2
3#[derive(Debug, Clone, Copy, PartialEq, Eq)]
4pub enum MediaKind {
5 Image,
6 Video,
7 Audio,
8}
9
10impl MediaKind {
11 pub const fn embed_type(self) -> &'static str {
12 match self {
13 Self::Image => "image",
14 Self::Video => "video",
15 Self::Audio => "audio",
16 }
17 }
18}
19
20pub fn from_url(url: &Url) -> Option<MediaKind> {
21 let path = url.path().to_ascii_lowercase();
22 let dot = path.rfind('.')?;
23 match &path[dot..] {
24 ".png" | ".jpg" | ".jpeg" | ".gif" | ".webp" | ".avif" | ".svg" | ".bmp" | ".apng"
25 | ".heic" | ".heif" | ".tif" | ".tiff" => Some(MediaKind::Image),
26 ".mp4" | ".webm" | ".mov" | ".avi" | ".mkv" | ".m4v" | ".ogv" | ".mpeg" | ".mpg"
27 | ".3gp" | ".3g2" | ".m3u8" | ".ts" => Some(MediaKind::Video),
28 ".mp3" | ".ogg" | ".wav" | ".flac" | ".m4a" | ".aac" | ".opus" | ".weba" | ".oga"
29 | ".aif" | ".aiff" | ".amr" | ".mid" | ".midi" => Some(MediaKind::Audio),
30 _ => None,
31 }
32}
33
34pub fn from_content_type(content_type: &str) -> Option<MediaKind> {
35 let value = content_type
36 .split(';')
37 .next()
38 .unwrap_or_default()
39 .trim()
40 .to_ascii_lowercase();
41 if value.starts_with("image/") {
42 Some(MediaKind::Image)
43 } else if value.starts_with("video/") {
44 Some(MediaKind::Video)
45 } else if value.starts_with("audio/") {
46 Some(MediaKind::Audio)
47 } else {
48 None
49 }
50}
Added lib/social/src/net.rs View lib/social/src/net.rs
@@ -0,0 +1,93 @@
1use std::net::IpAddr;
2
3use url::Url;
4
5use crate::UnfurlError;
6
7pub fn is_public_ip(ip: &IpAddr) -> bool {
8 match ip {
9 IpAddr::V4(v4) => {
10 !v4.is_loopback()
11 && !v4.is_private()
12 && !v4.is_link_local()
13 && !v4.is_broadcast()
14 && !v4.is_documentation()
15 && !v4.is_unspecified()
16 && !(v4.octets()[0] == 100 && (v4.octets()[1] & 0b1100_0000) == 0b0100_0000)
17 }
18 IpAddr::V6(v6) => {
19 !v6.is_loopback()
20 && !v6.is_unspecified()
21 && !v6.is_multicast()
22 && {
23 let segments = v6.segments();
24 !(segments[0] == 0x0064 && segments[1] == 0xff9b)
25 && (segments[0] & 0xfe00) != 0xfc00
26 && (segments[0] & 0xffc0) != 0xfe80
27 }
28 && match v6.to_ipv4_mapped() {
29 Some(v4) => is_public_ip(&IpAddr::V4(v4)),
30 None => true,
31 }
32 }
33 }
34}
35
36pub fn parse(raw: &str) -> Result<Url, UnfurlError> {
37 let trimmed = raw.trim();
38 if trimmed.is_empty() || trimmed.len() > 8192 {
39 return Err(UnfurlError::InvalidUrl("url is empty or too long".into()));
40 }
41 if trimmed.bytes().any(|byte| byte < 0x20 || byte == 0x7f) {
42 return Err(UnfurlError::InvalidUrl("url contains control bytes".into()));
43 }
44 let url = Url::parse(trimmed).map_err(|error| UnfurlError::InvalidUrl(error.to_string()))?;
45 validate_static(&url)?;
46 Ok(url)
47}
48
49pub fn validate_static(url: &Url) -> Result<(), UnfurlError> {
50 if !matches!(url.scheme(), "http" | "https") {
51 return Err(UnfurlError::BlockedUrl(format!(
52 "scheme {} is not allowed",
53 url.scheme()
54 )));
55 }
56 if !url.username().is_empty() || url.password().is_some() {
57 return Err(UnfurlError::BlockedUrl(
58 "credentials in url are not allowed".into(),
59 ));
60 }
61 let host = url
62 .host_str()
63 .ok_or_else(|| UnfurlError::InvalidUrl("host is required".into()))?;
64 if let Ok(ip) = host.parse::<IpAddr>()
65 && !is_public_ip(&ip)
66 {
67 return Err(UnfurlError::BlockedUrl(
68 "url points at a private or loopback address".into(),
69 ));
70 }
71 Ok(())
72}
73
74pub async fn resolve_public(
75 host: &str,
76 port: u16,
77) -> Result<Vec<std::net::SocketAddr>, UnfurlError> {
78 let addrs = tokio::net::lookup_host((host, port))
79 .await
80 .map_err(|error| UnfurlError::Fetch(format!("dns lookup failed: {error}")))?
81 .collect::<Vec<_>>();
82 if addrs.is_empty() {
83 return Err(UnfurlError::Fetch(
84 "dns resolution returned no address".into(),
85 ));
86 }
87 if addrs.iter().any(|addr| !is_public_ip(&addr.ip())) {
88 return Err(UnfurlError::BlockedUrl(
89 "url resolves to a private or loopback address".into(),
90 ));
91 }
92 Ok(addrs)
93}
Added lib/social/src/normalize.rs View lib/social/src/normalize.rs
@@ -0,0 +1,100 @@
1use crate::types::{EmbedAuthor, EmbedField, EmbedMedia, EmbedProvider, MessageEmbed};
2
3const TITLE_MAX: usize = 256;
4const DESCRIPTION_MAX: usize = 4096;
5const DESCRIPTION_LINES: usize = 12;
6const FIELD_NAME_MAX: usize = 256;
7const FIELD_VALUE_MAX: usize = 1024;
8const MAX_FIELDS: usize = 25;
9const HTML_MAX: usize = 64 * 1024;
10
11pub fn normalize(embed: &mut MessageEmbed) {
12 embed.title = embed
13 .title
14 .as_deref()
15 .map(|value| truncate(value, TITLE_MAX));
16 embed.description = embed
17 .description
18 .as_deref()
19 .map(|value| clamp_lines(&truncate(value, DESCRIPTION_MAX), DESCRIPTION_LINES));
20 embed.author = embed.author.take().and_then(normalize_author);
21 embed.provider = embed.provider.take().and_then(normalize_provider);
22 for media in [
23 embed.thumbnail.as_mut(),
24 embed.image.as_mut(),
25 embed.video.as_mut(),
26 embed.audio.as_mut(),
27 ]
28 .into_iter()
29 .flatten()
30 {
31 normalize_media(media);
32 }
33 embed.html = embed.html.as_deref().map(|value| truncate(value, HTML_MAX));
34 embed.fields = embed.fields.take().map(|fields| {
35 fields
36 .into_iter()
37 .take(MAX_FIELDS)
38 .filter_map(|mut field| {
39 let name = truncate(&field.name, FIELD_NAME_MAX);
40 if name.is_empty() {
41 return None;
42 }
43 field.name = name;
44 field.value = truncate(&field.value, FIELD_VALUE_MAX);
45 Some(field)
46 })
47 .collect::<Vec<EmbedField>>()
48 });
49 if let Some(children) = embed.children.take() {
50 embed.children = Some(
51 children
52 .into_iter()
53 .map(|mut child| {
54 normalize(&mut child);
55 child
56 })
57 .collect(),
58 );
59 }
60}
61
62fn normalize_author(mut author: EmbedAuthor) -> Option<EmbedAuthor> {
63 author.name = truncate(&author.name, TITLE_MAX);
64 (!author.name.is_empty()).then_some(author)
65}
66
67fn normalize_provider(mut provider: EmbedProvider) -> Option<EmbedProvider> {
68 provider.name = truncate(&provider.name, TITLE_MAX);
69 (!provider.name.is_empty()).then_some(provider)
70}
71
72fn normalize_media(media: &mut EmbedMedia) {
73 media.description = media
74 .description
75 .as_deref()
76 .map(|value| truncate(value, DESCRIPTION_MAX));
77}
78
79pub fn truncate(value: &str, max: usize) -> String {
80 if value.chars().count() <= max {
81 return value.to_owned();
82 }
83 let mut output: String = value.chars().take(max).collect();
84 output.push('…');
85 output
86}
87
88fn clamp_lines(value: &str, max_lines: usize) -> String {
89 let mut output = String::with_capacity(value.len());
90 for (index, line) in value.lines().enumerate() {
91 if index >= max_lines {
92 break;
93 }
94 if index > 0 {
95 output.push('\n');
96 }
97 output.push_str(line);
98 }
99 output
100}
Added lib/social/src/oembed.rs View lib/social/src/oembed.rs
@@ -0,0 +1,81 @@
1use crate::http;
2
3#[derive(Debug, Clone, Default)]
4pub struct OEmbed {
5 pub oembed_type: Option<String>,
6 pub title: Option<String>,
7 pub description: Option<String>,
8 pub provider_name: Option<String>,
9 pub provider_url: Option<String>,
10 pub author_name: Option<String>,
11 pub author_url: Option<String>,
12 pub thumbnail_url: Option<String>,
13 pub url: Option<String>,
14 pub html: Option<String>,
15 pub width: Option<u32>,
16 pub height: Option<u32>,
17}
18
19pub async fn fetch(endpoint: &str, max_bytes: usize) -> Option<OEmbed> {
20 let fetched = http::fetch(endpoint, "application/json", max_bytes)
21 .await
22 .ok()?;
23 if !(200..300).contains(&fetched.status) {
24 return None;
25 }
26 parse(&fetched.body)
27}
28
29pub fn parse(body: &str) -> Option<OEmbed> {
30 let value: serde_json::Value = serde_json::from_str(body).ok()?;
31 if !value.is_object() {
32 return None;
33 }
34 let oembed = OEmbed {
35 oembed_type: string_field(&value, "type"),
36 title: string_field(&value, "title"),
37 description: string_field(&value, "description"),
38 provider_name: string_field(&value, "provider_name"),
39 provider_url: string_field(&value, "provider_url"),
40 author_name: string_field(&value, "author_name"),
41 author_url: string_field(&value, "author_url"),
42 thumbnail_url: string_field(&value, "thumbnail_url"),
43 url: string_field(&value, "url"),
44 html: string_field(&value, "html"),
45 width: number_field(&value, "width"),
46 height: number_field(&value, "height"),
47 };
48 (oembed.title.is_some()
49 || oembed.description.is_some()
50 || oembed.thumbnail_url.is_some()
51 || oembed.html.is_some())
52 .then_some(oembed)
53}
54
55fn string_field(value: &serde_json::Value, key: &str) -> Option<String> {
56 let text = value.get(key)?.as_str()?.trim();
57 (!text.is_empty()).then(|| text.to_owned())
58}
59
60fn number_field(value: &serde_json::Value, key: &str) -> Option<u32> {
61 let node = value.get(key)?;
62 node.as_u64()
63 .and_then(|value| u32::try_from(value).ok())
64 .or_else(|| node.as_str().and_then(|value| value.trim().parse().ok()))
65}
66
67pub fn iframe_src(html: &str) -> Option<String> {
68 let lower = html.to_ascii_lowercase();
69 let start = lower.find("<iframe")?;
70 let tail = &html[start..];
71 let index = tail.to_ascii_lowercase().find("src=")? + 4;
72 let after = tail[index..].trim_start();
73 let quote = after.chars().next()?;
74 if quote != '"' && quote != '\'' {
75 return None;
76 }
77 let rest = &after[1..];
78 let end = rest.find(quote)?;
79 let src = rest[..end].trim();
80 (!src.is_empty()).then(|| src.to_owned())
81}
Added lib/social/src/resolver.rs View lib/social/src/resolver.rs
@@ -0,0 +1,82 @@
1use std::future::Future;
2use std::pin::Pin;
3use std::sync::OnceLock;
4use std::time::Duration;
5
6use url::Url;
7
8use crate::UnfurlError;
9use crate::types::MessageEmbed;
10
11pub type BoxFuture<'a, T> = Pin<Box<dyn Future<Output = T> + Send + 'a>>;
12
13#[derive(Debug, Clone)]
14pub struct UnfurlOptions {
15 pub max_bytes: usize,
16 pub timeout: Duration,
17}
18
19impl Default for UnfurlOptions {
20 fn default() -> Self {
21 Self {
22 max_bytes: 1_048_576,
23 timeout: Duration::from_secs(10),
24 }
25 }
26}
27
28pub struct ResolveContext<'a> {
29 pub url: Url,
30 pub original_url: Url,
31 pub options: &'a UnfurlOptions,
32}
33
34pub trait Resolver: Send + Sync {
35 fn matches(&self, url: &Url) -> bool;
36
37 fn resolve<'a>(
38 &'a self,
39 context: &'a ResolveContext<'_>,
40 ) -> BoxFuture<'a, Result<Vec<MessageEmbed>, UnfurlError>>;
41}
42
43pub struct UnfurlEngine {
44 resolvers: Vec<Box<dyn Resolver>>,
45}
46
47impl UnfurlEngine {
48 pub fn new() -> Self {
49 Self {
50 resolvers: crate::resolvers::build_chain(),
51 }
52 }
53
54 pub async fn unfurl(&self, raw: &str) -> Result<Vec<MessageEmbed>, UnfurlError> {
55 let url = crate::net::parse(raw)?;
56 crate::net::validate_static(&url)?;
57 let options = UnfurlOptions::default();
58 let context = ResolveContext {
59 url: url.clone(),
60 original_url: url,
61 options: &options,
62 };
63 for resolver in &self.resolvers {
64 if resolver.matches(&context.url) {
65 return resolver.resolve(&context).await;
66 }
67 }
68 Ok(Vec::new())
69 }
70}
71
72impl Default for UnfurlEngine {
73 fn default() -> Self {
74 Self::new()
75 }
76}
77
78static ENGINE: OnceLock<UnfurlEngine> = OnceLock::new();
79
80pub async fn unfurl(raw: &str) -> Result<Vec<MessageEmbed>, UnfurlError> {
81 ENGINE.get_or_init(UnfurlEngine::new).unfurl(raw).await
82}
Added lib/social/src/types.rs View lib/social/src/types.rs
@@ -0,0 +1,239 @@
1use serde::{Deserialize, Serialize};
2
3use crate::{EmbedKind, Resolved};
4
5fn default_embed_type() -> String {
6 "link".to_owned()
7}
8
9/// Discord 风格的消息 embed。
10///
11/// `platform` / `kind` / `src` / `image_url` 等字段是旧消息与旧缓存的兼容外壳：
12/// 历史 `chat_message.embeds` 里存的还是老结构，反序列化时不能丢字段；新写入
13/// 同时填富模型和这些兼容字段，前端可以逐步迁移。
14#[derive(Debug, Clone, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
15#[serde(default)]
16pub struct MessageEmbed {
17 #[serde(rename = "type", default = "default_embed_type")]
18 pub embed_type: String,
19 #[serde(skip_serializing_if = "Option::is_none")]
20 pub url: Option<String>,
21 #[serde(skip_serializing_if = "Option::is_none")]
22 pub title: Option<String>,
23 #[serde(skip_serializing_if = "Option::is_none")]
24 pub description: Option<String>,
25 #[serde(skip_serializing_if = "Option::is_none")]
26 pub color: Option<u32>,
27 #[serde(skip_serializing_if = "Option::is_none")]
28 pub timestamp: Option<String>,
29 #[serde(skip_serializing_if = "Option::is_none")]
30 pub thumbnail: Option<EmbedMedia>,
31 #[serde(skip_serializing_if = "Option::is_none")]
32 pub image: Option<EmbedMedia>,
33 #[serde(skip_serializing_if = "Option::is_none")]
34 pub video: Option<EmbedMedia>,
35 #[serde(skip_serializing_if = "Option::is_none")]
36 pub audio: Option<EmbedMedia>,
37 #[serde(skip_serializing_if = "Option::is_none")]
38 pub author: Option<EmbedAuthor>,
39 #[serde(skip_serializing_if = "Option::is_none")]
40 pub provider: Option<EmbedProvider>,
41 #[serde(skip_serializing_if = "Option::is_none")]
42 pub footer: Option<EmbedFooter>,
43 #[serde(skip_serializing_if = "Option::is_none")]
44 pub fields: Option<Vec<EmbedField>>,
45 #[serde(skip_serializing_if = "Option::is_none")]
46 pub html: Option<String>,
47 #[serde(skip_serializing_if = "Option::is_none")]
48 pub html_width: Option<u32>,
49 #[serde(skip_serializing_if = "Option::is_none")]
50 pub html_height: Option<u32>,
51 #[serde(skip_serializing_if = "Option::is_none")]
52 #[schema(no_recursion)]
53 pub children: Option<Vec<MessageEmbed>>,
54 #[serde(skip_serializing_if = "Option::is_none")]
55 pub nsfw: Option<bool>,
56
57 #[serde(skip_serializing_if = "Option::is_none")]
58 pub platform: Option<String>,
59 #[serde(skip_serializing_if = "Option::is_none")]
60 pub platform_name: Option<String>,
61 #[serde(skip_serializing_if = "Option::is_none")]
62 pub category: Option<String>,
63 #[serde(skip_serializing_if = "Option::is_none")]
64 pub kind: Option<String>,
65 #[serde(skip_serializing_if = "Option::is_none")]
66 pub aspect: Option<String>,
67 #[serde(skip_serializing_if = "Option::is_none")]
68 pub height: Option<u32>,
69 #[serde(skip_serializing_if = "Option::is_none")]
70 pub src: Option<String>,
71 #[serde(skip_serializing_if = "Option::is_none")]
72 pub image_url: Option<String>,
73 #[serde(skip_serializing_if = "Option::is_none")]
74 pub site_name: Option<String>,
75}
76
77impl Default for MessageEmbed {
78 fn default() -> Self {
79 Self {
80 embed_type: default_embed_type(),
81 url: None,
82 title: None,
83 description: None,
84 color: None,
85 timestamp: None,
86 thumbnail: None,
87 image: None,
88 video: None,
89 audio: None,
90 author: None,
91 provider: None,
92 footer: None,
93 fields: None,
94 html: None,
95 html_width: None,
96 html_height: None,
97 children: None,
98 nsfw: None,
99 platform: None,
100 platform_name: None,
101 category: None,
102 kind: None,
103 aspect: None,
104 height: None,
105 src: None,
106 image_url: None,
107 site_name: None,
108 }
109 }
110}
111
112impl MessageEmbed {
113 pub fn from_resolved(resolved: &Resolved) -> Self {
114 let media_url = resolved.src.clone().unwrap_or_else(|| resolved.url.clone());
115 let image_url = resolved
116 .image_url
117 .clone()
118 .or_else(|| (resolved.kind == EmbedKind::Image).then(|| media_url.clone()));
119 let media = |url: &str| EmbedMedia {
120 url: Some(url.to_owned()),
121 ..EmbedMedia::default()
122 };
123 let image = image_url.as_deref().map(media);
124 let video = (resolved.kind == EmbedKind::Video).then(|| media(&media_url));
125 let audio = (resolved.kind == EmbedKind::Audio).then(|| media(&media_url));
126 Self {
127 embed_type: resolved.kind.as_str().to_owned(),
128 url: Some(resolved.url.clone()),
129 title: Some(resolved.title.clone()),
130 description: resolved.description.clone(),
131 image: image.clone(),
132 video,
133 audio,
134 provider: Some(EmbedProvider {
135 name: resolved.platform_name.to_owned(),
136 url: None,
137 }),
138 html: Some(crate::render_html(resolved)),
139 html_height: resolved.height,
140 platform: Some(resolved.platform.to_owned()),
141 platform_name: Some(resolved.platform_name.to_owned()),
142 category: Some(resolved.category.as_str().to_owned()),
143 kind: Some(resolved.kind.as_str().to_owned()),
144 aspect: Some(resolved.aspect.as_str().to_owned()),
145 height: resolved.height,
146 src: resolved.src.clone(),
147 image_url,
148 site_name: Some(resolved.platform_name.to_owned()),
149 ..Self::default()
150 }
151 }
152
153 pub fn is_empty(&self) -> bool {
154 self.title.is_none()
155 && self.description.is_none()
156 && self.image.is_none()
157 && self.video.is_none()
158 && self.audio.is_none()
159 && self.html.is_none()
160 }
161
162 pub fn kind(&self) -> EmbedKind {
163 match self.embed_type.as_str() {
164 "iframe" => EmbedKind::Iframe,
165 "video" => EmbedKind::Video,
166 "audio" => EmbedKind::Audio,
167 "image" => EmbedKind::Image,
168 _ => EmbedKind::Link,
169 }
170 }
171
172 pub fn to_json(&self) -> serde_json::Value {
173 serde_json::to_value(self).unwrap_or(serde_json::Value::Null)
174 }
175}
176
177#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
178#[serde(default)]
179pub struct EmbedMedia {
180 #[serde(skip_serializing_if = "Option::is_none")]
181 pub url: Option<String>,
182 #[serde(skip_serializing_if = "Option::is_none")]
183 pub proxy_url: Option<String>,
184 #[serde(skip_serializing_if = "Option::is_none")]
185 pub content_type: Option<String>,
186 #[serde(skip_serializing_if = "Option::is_none")]
187 pub content_hash: Option<String>,
188 #[serde(skip_serializing_if = "Option::is_none")]
189 pub width: Option<u32>,
190 #[serde(skip_serializing_if = "Option::is_none")]
191 pub height: Option<u32>,
192 #[serde(skip_serializing_if = "Option::is_none")]
193 pub description: Option<String>,
194 #[serde(skip_serializing_if = "Option::is_none")]
195 pub placeholder: Option<String>,
196 #[serde(skip_serializing_if = "Option::is_none")]
197 pub duration: Option<u32>,
198 #[serde(default)]
199 pub flags: u32,
200}
201
202#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
203#[serde(default)]
204pub struct EmbedAuthor {
205 pub name: String,
206 #[serde(skip_serializing_if = "Option::is_none")]
207 pub url: Option<String>,
208 #[serde(skip_serializing_if = "Option::is_none")]
209 pub icon_url: Option<String>,
210 #[serde(skip_serializing_if = "Option::is_none")]
211 pub proxy_icon_url: Option<String>,
212}
213
214#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
215#[serde(default)]
216pub struct EmbedProvider {
217 pub name: String,
218 #[serde(skip_serializing_if = "Option::is_none")]
219 pub url: Option<String>,
220}
221
222#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
223#[serde(default)]
224pub struct EmbedFooter {
225 pub text: String,
226 #[serde(skip_serializing_if = "Option::is_none")]
227 pub icon_url: Option<String>,
228 #[serde(skip_serializing_if = "Option::is_none")]
229 pub proxy_icon_url: Option<String>,
230}
231
232#[derive(Debug, Clone, Default, PartialEq, Serialize, Deserialize, utoipa::ToSchema)]
233#[serde(default)]
234pub struct EmbedField {
235 pub name: String,
236 pub value: String,
237 #[serde(default)]
238 pub inline: bool,
239}
Modified lib/social/Cargo.toml View lib/social/Cargo.toml
@@ -4,9 +4,13 @@
44edition.workspace = true
55
66[dependencies]
7encoding_rs = { workspace = true }
78percent-encoding = { workspace = true }
89regex = { workspace = true }
10reqwest = { workspace = true }
911serde = { workspace = true, features = ["derive"] }
1012serde_json = { workspace = true }
13thiserror = { workspace = true }
14tokio = { workspace = true, features = ["net", "time"] }
1115url = { workspace = true }
1216utoipa = { workspace = true }
Modified Cargo.lock View Cargo.lock
@@ -7228,10 +7228,14 @@
72287228name = "social"
72297229version = "1.0.0"
72307230dependencies = [
7231 "encoding_rs",
72317232 "percent-encoding 2.3.2",
72327233 "regex",
7234 "reqwest 0.12.28",
72337235 "serde",
72347236 "serde_json",
7237 "thiserror 2.0.21",
7238 "tokio",
72357239 "url 2.5.8",
72367240 "utoipa 5.5.0",
72377241]