use std::time::Duration; use reqwest::redirect::Policy; use url::Url; use crate::{UnfurlError, net}; const TIMEOUT_SECS: u64 = 10; const MAX_REDIRECTS: usize = 6; const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \ (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36"; pub struct Fetched { pub final_url: String, pub status: u16, pub content_type: String, pub body: String, } pub async fn fetch(raw: &str, accept: &str, max_bytes: usize) -> Result { let mut current = net::parse(raw)?; for _ in 0..=MAX_REDIRECTS { net::validate_static(¤t)?; let host = current .host_str() .ok_or_else(|| UnfurlError::InvalidUrl("host is required".into()))? .to_owned(); let port = current.port_or_known_default().unwrap_or(443); let addrs = net::resolve_public(&host, port).await?; let mut builder = reqwest::Client::builder() .timeout(Duration::from_secs(TIMEOUT_SECS)) .redirect(Policy::none()) .user_agent(USER_AGENT); for addr in &addrs { builder = builder.resolve(&host, *addr); } let client = builder .build() .map_err(|error| UnfurlError::Fetch(error.to_string()))?; let response = client .get(current.clone()) .header("accept", accept) .send() .await .map_err(|error| UnfurlError::Fetch(error.to_string()))?; if response.status().is_redirection() { let location = response .headers() .get("location") .and_then(|value| value.to_str().ok()) .ok_or_else(|| UnfurlError::Fetch("redirect without location".into()))?; let next = current .join(location) .map_err(|error| UnfurlError::InvalidUrl(error.to_string()))?; net::validate_static(&next)?; current = next; continue; } let status = response.status().as_u16(); let content_type = response .headers() .get("content-type") .and_then(|value| value.to_str().ok()) .unwrap_or_default() .to_owned(); let mut body = Vec::new(); let mut stream = response; while let Some(chunk) = stream .chunk() .await .map_err(|error| UnfurlError::Fetch(error.to_string()))? { let remaining = max_bytes.saturating_sub(body.len()); if remaining == 0 { break; } if chunk.len() > remaining { body.extend_from_slice(&chunk[..remaining]); break; } body.extend_from_slice(&chunk); } return Ok(Fetched { final_url: current.to_string(), status, content_type: content_type.clone(), body: decode_body(&body, &content_type), }); } Err(UnfurlError::Fetch("too many redirects".into())) } fn charset_of(content_type: &str) -> Option { let lower = content_type.to_ascii_lowercase(); let start = lower.find("charset=")? + "charset=".len(); let rest = &content_type[start..]; let value = rest .split(|c: char| c == ';' || c == ',' || c.is_whitespace()) .next() .unwrap_or("") .trim_matches(|c: char| c == '"' || c == '\''); (!value.is_empty()).then(|| value.to_string()) } pub fn decode_body(body: &[u8], content_type: &str) -> String { if let Some(label) = charset_of(content_type) && let Some(encoding) = encoding_rs::Encoding::for_label(label.as_bytes()) { let (text, _, _) = encoding.decode(body); return text.into_owned(); } match std::str::from_utf8(body) { Ok(text) => text.to_owned(), Err(_) => { let (text, _, _) = encoding_rs::GB18030.decode(body); text.into_owned() } } } pub fn absolute(base: &Url, value: &str) -> Option { let trimmed = value.trim(); if trimmed.is_empty() { return None; } let url = base.join(trimmed).ok()?; matches!(url.scheme(), "http" | "https").then(|| url.to_string()) }