b6f7f73
ZhenYi
refactor: reorganize configuration files and update module paths for improved structure
b6f7f73
use url::Url;
use crate::http::absolute;
#[derive(Debug, Default, Clone)]
pub struct PageMetadata {
pub title: Option<String>,
pub description: Option<String>,
pub image: Option<String>,
pub site_name: Option<String>,
pub video: Option<String>,
pub audio: Option<String>,
pub player: Option<String>,
pub theme_color: Option<String>,
pub oembed: Option<String>,
}
pub fn parse(html: &str, base: &Url) -> PageMetadata {
let head = match html.find("</head>") {
Some(index) => &html[..index],
None => html,
};
let (ld_title, ld_description, ld_image) = json_ld_meta(head);
let title = meta_any(head, &["og:title", "twitter:title"])
.or(ld_title)
.or_else(|| page_title(head));
let description = meta_any(
head,
&["og:description", "twitter:description", "description"],
)
.or(ld_description);
let image = meta_any(head, &["og:image", "og:image:url", "twitter:image"])
.or(ld_image)
.and_then(|value| absolute(base, &value));
PageMetadata {
title: title.map(|value| value.chars().take(300).collect()),
description: description.map(|value| value.chars().take(1200).collect()),
image,
site_name: meta_any(head, &["og:site_name", "twitter:site", "application-name"]),
video: meta_any(
head,
&[
"og:video",
"og:video:url",
"og:video:secure_url",
"twitter:player:stream",
],
)
.and_then(|value| absolute(base, &value)),
audio: meta_any(head, &["og:audio", "og:audio:url"])
.and_then(|value| absolute(base, &value)),
player: meta_any(head, &["twitter:player"]).and_then(|value| absolute(base, &value)),
theme_color: meta_any(head, &["theme-color"]),
oembed: link_oembed(head).and_then(|value| absolute(base, &value)),
}
}
fn meta_content(head: &str, property: &str) -> Option<String> {
let lower = head.to_ascii_lowercase();
let mut search_from = 0;
loop {
let index = lower[search_from..].find("<meta")? + search_from;
let end = lower[index..].find('>')? + index;
let tag = &head[index..=end];
let tag_lower = &lower[index..=end];
search_from = end + 1;
let matches = tag_lower.contains(&format!("property=\"{property}\""))
|| tag_lower.contains(&format!("property='{property}'"))
|| tag_lower.contains(&format!("name=\"{property}\""))
|| tag_lower.contains(&format!("name='{property}'"));
if !matches {
continue;
}
for attr in ["content=\"", "content='"] {
if let Some(start) = tag.find(attr) {
let value_start = start + attr.len();
let quote = attr.chars().last()?;
if let Some(value_end) = tag[value_start..].find(quote) {
let value = tag[value_start..value_start + value_end].trim();
if !value.is_empty() {
return Some(decode_entities(value));
}
}
}
}
}
}
fn meta_any(head: &str, properties: &[&str]) -> Option<String> {
properties
.iter()
.find_map(|property| meta_content(head, property))
}
fn page_title(head: &str) -> Option<String> {
let lower = head.to_ascii_lowercase();
let start = lower.find("<title")?;
let content_start = lower[start..].find('>')? + start + 1;
let end = lower[content_start..].find("</title>")? + content_start;
let title = head[content_start..end].trim();
(!title.is_empty()).then(|| decode_entities(title).chars().take(200).collect())
}
fn link_oembed(head: &str) -> Option<String> {
let lower = head.to_ascii_lowercase();
let mut search_from = 0;
loop {
let index = lower[search_from..].find("<link")? + search_from;
let end = lower[index..].find('>')? + index;
let tag = &head[index..=end];
let tag_lower = &lower[index..=end];
search_from = end + 1;
if !tag_lower.contains("oembed") || !tag_lower.contains("json") {
continue;
}
for attr in ["href=\"", "href='"] {
if let Some(start) = tag.find(attr) {
let value_start = start + attr.len();
let quote = attr.chars().last()?;
if let Some(value_end) = tag[value_start..].find(quote) {
let value = tag[value_start..value_start + value_end].trim();
if !value.is_empty() {
return Some(decode_entities(value));
}
}
}
}
}
}
fn json_ld_value(block: &serde_json::Value, key: &str) -> Option<String> {
let node = block.get(key)?;
let text = match node {
serde_json::Value::String(value) => value.clone(),
serde_json::Value::Object(map) => map.get("url")?.as_str()?.to_owned(),
serde_json::Value::Array(items) => items.first().and_then(|item| match item {
serde_json::Value::String(value) => Some(value.clone()),
serde_json::Value::Object(map) => map.get("url")?.as_str().map(str::to_owned),
_ => None,
})?,
_ => return None,
};
let trimmed = text.trim();
(!trimmed.is_empty()).then(|| decode_entities(trimmed))
}
fn json_ld_meta(head: &str) -> (Option<String>, Option<String>, Option<String>) {
let lower = head.to_ascii_lowercase();
let mut search_from = 0;
while let Some(index) = lower[search_from..]
.find("<script")
.map(|i| i + search_from)
{
let Some(end) = lower[index..].find('>').map(|i| i + index) else {
break;
};
let open_tag = &lower[index..=end];
search_from = end + 1;
if !open_tag.contains("application/ld+json") {
continue;
}
let Some(close) = lower[search_from..]
.find("</script>")
.map(|i| i + search_from)
else {
break;
};
let raw = &head[search_from..close];
search_from = close + "</script>".len();
let Ok(value) = serde_json::from_str::<serde_json::Value>(raw.trim()) else {
continue;
};
let blocks: Vec<&serde_json::Value> = match &value {
serde_json::Value::Array(items) => items.iter().collect(),
_ => vec![&value],
};
for block in blocks {
let title = json_ld_value(block, "headline").or_else(|| json_ld_value(block, "name"));
let description = json_ld_value(block, "description");
let image = json_ld_value(block, "image");
if title.is_some() || description.is_some() || image.is_some() {
return (title, description, image);
}
}
}
(None, None, None)
}
pub fn decode_entities(value: &str) -> String {
let mut output = String::with_capacity(value.len());
let mut rest = value;
while let Some(index) = rest.find('&') {
output.push_str(&rest[..index]);
rest = &rest[index..];
let Some(end) = rest.find(';').filter(|end| *end <= 10) else {
output.push('&');
rest = &rest[1..];
continue;
};
let entity = &rest[1..end];
let decoded = match entity {
"amp" => Some('&'),
"lt" => Some('<'),
"gt" => Some('>'),
"quot" => Some('"'),
"apos" | "#39" => Some('\''),
"nbsp" => Some(' '),
_ => entity
.strip_prefix("#x")
.or_else(|| entity.strip_prefix("#X"))
.and_then(|hex| u32::from_str_radix(hex, 16).ok())
.or_else(|| {
entity
.strip_prefix('#')
.and_then(|dec| dec.parse::<u32>().ok())
})
.and_then(char::from_u32),
};
match decoded {
Some(ch) => {
output.push(ch);
rest = &rest[end + 1..];
}
None => {
output.push('&');
rest = &rest[1..];
}
}
}
output.push_str(rest);
output
}