main
use std::time::Duration;
use reqwest::redirect::Policy;
use url::Url;
use crate::{UnfurlError, net};
const TIMEOUT_SECS: u64 = 10;
const MAX_REDIRECTS: usize = 6;
const USER_AGENT: &str = "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 \
(KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36";
pub struct Fetched {
pub final_url: String,
pub status: u16,
pub content_type: String,
pub body: String,
}
pub async fn fetch(raw: &str, accept: &str, max_bytes: usize) -> Result<Fetched, UnfurlError> {
let mut current = net::parse(raw)?;
for _ in 0..=MAX_REDIRECTS {
net::validate_static(¤t)?;
let host = current
.host_str()
.ok_or_else(|| UnfurlError::InvalidUrl("host is required".into()))?
.to_owned();
let port = current.port_or_known_default().unwrap_or(443);
let addrs = net::resolve_public(&host, port).await?;
let mut builder = reqwest::Client::builder()
.timeout(Duration::from_secs(TIMEOUT_SECS))
.redirect(Policy::none())
.user_agent(USER_AGENT);
for addr in &addrs {
builder = builder.resolve(&host, *addr);
}
let client = builder
.build()
.map_err(|error| UnfurlError::Fetch(error.to_string()))?;
let response = client
.get(current.clone())
.header("accept", accept)
.send()
.await
.map_err(|error| UnfurlError::Fetch(error.to_string()))?;
if response.status().is_redirection() {
let location = response
.headers()
.get("location")
.and_then(|value| value.to_str().ok())
.ok_or_else(|| UnfurlError::Fetch("redirect without location".into()))?;
let next = current
.join(location)
.map_err(|error| UnfurlError::InvalidUrl(error.to_string()))?;
net::validate_static(&next)?;
current = next;
continue;
}
let status = response.status().as_u16();
let content_type = response
.headers()
.get("content-type")
.and_then(|value| value.to_str().ok())
.unwrap_or_default()
.to_owned();
let mut body = Vec::new();
let mut stream = response;
while let Some(chunk) = stream
.chunk()
.await
.map_err(|error| UnfurlError::Fetch(error.to_string()))?
{
let remaining = max_bytes.saturating_sub(body.len());
if remaining == 0 {
break;
}
if chunk.len() > remaining {
body.extend_from_slice(&chunk[..remaining]);
break;
}
body.extend_from_slice(&chunk);
}
return Ok(Fetched {
final_url: current.to_string(),
status,
content_type: content_type.clone(),
body: decode_body(&body, &content_type),
});
}
Err(UnfurlError::Fetch("too many redirects".into()))
}
fn charset_of(content_type: &str) -> Option<String> {
let lower = content_type.to_ascii_lowercase();
let start = lower.find("charset=")? + "charset=".len();
let rest = &content_type[start..];
let value = rest
.split(|c: char| c == ';' || c == ',' || c.is_whitespace())
.next()
.unwrap_or("")
.trim_matches(|c: char| c == '"' || c == '\'');
(!value.is_empty()).then(|| value.to_string())
}
pub fn decode_body(body: &[u8], content_type: &str) -> String {
if let Some(label) = charset_of(content_type)
&& let Some(encoding) = encoding_rs::Encoding::for_label(label.as_bytes())
{
let (text, _, _) = encoding.decode(body);
return text.into_owned();
}
match std::str::from_utf8(body) {
Ok(text) => text.to_owned(),
Err(_) => {
let (text, _, _) = encoding_rs::GB18030.decode(body);
text.into_owned()
}
}
}
pub fn absolute(base: &Url, value: &str) -> Option<String> {
let trimmed = value.trim();
if trimmed.is_empty() {
return None;
}
let url = base.join(trimmed).ok()?;
matches!(url.scheme(), "http" | "https").then(|| url.to_string())
}