use url::Url; use crate::http::absolute; #[derive(Debug, Default, Clone)] pub struct PageMetadata { pub title: Option, pub description: Option, pub image: Option, pub site_name: Option, pub video: Option, pub audio: Option, pub player: Option, pub theme_color: Option, pub oembed: Option, } pub fn parse(html: &str, base: &Url) -> PageMetadata { let head = match html.find("") { Some(index) => &html[..index], None => html, }; let (ld_title, ld_description, ld_image) = json_ld_meta(head); let title = meta_any(head, &["og:title", "twitter:title"]) .or(ld_title) .or_else(|| page_title(head)); let description = meta_any( head, &["og:description", "twitter:description", "description"], ) .or(ld_description); let image = meta_any(head, &["og:image", "og:image:url", "twitter:image"]) .or(ld_image) .and_then(|value| absolute(base, &value)); PageMetadata { title: title.map(|value| value.chars().take(300).collect()), description: description.map(|value| value.chars().take(1200).collect()), image, site_name: meta_any(head, &["og:site_name", "twitter:site", "application-name"]), video: meta_any( head, &[ "og:video", "og:video:url", "og:video:secure_url", "twitter:player:stream", ], ) .and_then(|value| absolute(base, &value)), audio: meta_any(head, &["og:audio", "og:audio:url"]) .and_then(|value| absolute(base, &value)), player: meta_any(head, &["twitter:player"]).and_then(|value| absolute(base, &value)), theme_color: meta_any(head, &["theme-color"]), oembed: link_oembed(head).and_then(|value| absolute(base, &value)), } } fn meta_content(head: &str, property: &str) -> Option { let lower = head.to_ascii_lowercase(); let mut search_from = 0; loop { let index = lower[search_from..].find("')? + index; let tag = &head[index..=end]; let tag_lower = &lower[index..=end]; search_from = end + 1; let matches = tag_lower.contains(&format!("property=\"{property}\"")) || tag_lower.contains(&format!("property='{property}'")) || tag_lower.contains(&format!("name=\"{property}\"")) || tag_lower.contains(&format!("name='{property}'")); if !matches { continue; } for attr in ["content=\"", "content='"] { if let Some(start) = tag.find(attr) { let value_start = start + attr.len(); let quote = attr.chars().last()?; if let Some(value_end) = tag[value_start..].find(quote) { let value = tag[value_start..value_start + value_end].trim(); if !value.is_empty() { return Some(decode_entities(value)); } } } } } } fn meta_any(head: &str, properties: &[&str]) -> Option { properties .iter() .find_map(|property| meta_content(head, property)) } fn page_title(head: &str) -> Option { let lower = head.to_ascii_lowercase(); let start = lower.find("')? + start + 1; let end = lower[content_start..].find("")? + content_start; let title = head[content_start..end].trim(); (!title.is_empty()).then(|| decode_entities(title).chars().take(200).collect()) } fn link_oembed(head: &str) -> Option { let lower = head.to_ascii_lowercase(); let mut search_from = 0; loop { let index = lower[search_from..].find("')? + index; let tag = &head[index..=end]; let tag_lower = &lower[index..=end]; search_from = end + 1; if !tag_lower.contains("oembed") || !tag_lower.contains("json") { continue; } for attr in ["href=\"", "href='"] { if let Some(start) = tag.find(attr) { let value_start = start + attr.len(); let quote = attr.chars().last()?; if let Some(value_end) = tag[value_start..].find(quote) { let value = tag[value_start..value_start + value_end].trim(); if !value.is_empty() { return Some(decode_entities(value)); } } } } } } fn json_ld_value(block: &serde_json::Value, key: &str) -> Option { let node = block.get(key)?; let text = match node { serde_json::Value::String(value) => value.clone(), serde_json::Value::Object(map) => map.get("url")?.as_str()?.to_owned(), serde_json::Value::Array(items) => items.first().and_then(|item| match item { serde_json::Value::String(value) => Some(value.clone()), serde_json::Value::Object(map) => map.get("url")?.as_str().map(str::to_owned), _ => None, })?, _ => return None, }; let trimmed = text.trim(); (!trimmed.is_empty()).then(|| decode_entities(trimmed)) } fn json_ld_meta(head: &str) -> (Option, Option, Option) { let lower = head.to_ascii_lowercase(); let mut search_from = 0; while let Some(index) = lower[search_from..] .find("').map(|i| i + index) else { break; }; let open_tag = &lower[index..=end]; search_from = end + 1; if !open_tag.contains("application/ld+json") { continue; } let Some(close) = lower[search_from..] .find("") .map(|i| i + search_from) else { break; }; let raw = &head[search_from..close]; search_from = close + "".len(); let Ok(value) = serde_json::from_str::(raw.trim()) else { continue; }; let blocks: Vec<&serde_json::Value> = match &value { serde_json::Value::Array(items) => items.iter().collect(), _ => vec![&value], }; for block in blocks { let title = json_ld_value(block, "headline").or_else(|| json_ld_value(block, "name")); let description = json_ld_value(block, "description"); let image = json_ld_value(block, "image"); if title.is_some() || description.is_some() || image.is_some() { return (title, description, image); } } } (None, None, None) } pub fn decode_entities(value: &str) -> String { let mut output = String::with_capacity(value.len()); let mut rest = value; while let Some(index) = rest.find('&') { output.push_str(&rest[..index]); rest = &rest[index..]; let Some(end) = rest.find(';').filter(|end| *end <= 10) else { output.push('&'); rest = &rest[1..]; continue; }; let entity = &rest[1..end]; let decoded = match entity { "amp" => Some('&'), "lt" => Some('<'), "gt" => Some('>'), "quot" => Some('"'), "apos" | "#39" => Some('\''), "nbsp" => Some(' '), _ => entity .strip_prefix("#x") .or_else(|| entity.strip_prefix("#X")) .and_then(|hex| u32::from_str_radix(hex, 16).ok()) .or_else(|| { entity .strip_prefix('#') .and_then(|dec| dec.parse::().ok()) }) .and_then(char::from_u32), }; match decoded { Some(ch) => { output.push(ch); rest = &rest[end + 1..]; } None => { output.push('&'); rest = &rest[1..]; } } } output.push_str(rest); output }