feat(otel): export traces and metrics to OpenObserve

telemetry::init now installs a real SDK tracer provider (kept in a OnceLock
so shutdown can flush the batch processor) and puts the env filter on the
registry instead of the fmt layer, so the level applies to every layer.
apihost treats a failing meter provider as "metrics disabled" rather than
refusing to boot, and flushes spans on the way out.

chart: every workload gets OTEL_SERVICE_NAME from its own component label
via the downward API - without it all eleven processes collapse into one
unknown_service:<bin> on the receiving side.

prod values: OTLP endpoint (org id in the path, not a credential),
parentbased_traceidratio sampling at 0.2 because TraceLayer spans every
HTTP request including the static assets apihost serves, and the two
Authorization header keys documented as living in PROD_SECRET_VALUES.
Metrics deliberately carry no stream-name: pinning one would push every
metric into a single OpenObserve stream and break the per-metric browser.
Signal-specific variables override rather than merge with the generic one,
which is why the traces header repeats the Authorization value.

ZhenYi committed 1 month ago

parent ddce65d commit 21bacd8 Browse files

app/apihost/src/main.rs +11 −3 View

@@ -25,8 +25,13 @@

2525 .context("building AppState")?;

2626 info!("dependencies ready");

2727

28 let meter_provider = webapi::metrics::init_meter_provider()

29 .map_err(|err| anyhow::anyhow!("failed to init meter provider: {err}"))?;

28 let meter_provider = match webapi::metrics::init_meter_provider() {

29 Ok(provider) => Some(provider),

30 Err(err) => {

31 warn!(error = %err, "failed to init meter provider; metrics disabled");

32 None

33 }

34 };

3035

3136 tokio::spawn(chat::worker::run_background_tasks(state.clone()));

3237 tokio::spawn(auth::worker::run_deletion_executor(state.clone()));

@@ -69,9 +74,12 @@

6974 .await

7075 .context("axum server error")?;

7176

72 if let Err(err) = meter_provider.shutdown() {

77 if let Some(meter_provider) = meter_provider

78 && let Err(err) = meter_provider.shutdown()

79 {

7380 warn!(error = %err, "failed to shut down meter provider");

7481 }

82 export::telemetry::flush_traces();

7583

7684 info!("apihost shut down");

7785 Ok(())

chart/templates/_helpers.tpl +6 −0 View

@@ -151,6 +151,12 @@

151151{{- if .Values.metrics.otlpEndpoint }}

152152- name: OTEL_EXPORTER_OTLP_ENDPOINT

153153 value: {{ .Values.metrics.otlpEndpoint | quote }}

154# 服务名取各自的 component label（downward API），一处覆盖全部 workload；

155# 不设的话所有进程在观测端会塌成一个 unknown_service:<bin>。

156- name: OTEL_SERVICE_NAME

157 valueFrom:

158 fieldRef:

159 fieldPath: "metadata.labels['app.kubernetes.io/component']"

154160{{- end }}

155161{{- range $k, $v := .Values.config.env }}

156162{{- if not (hasPrefix "APP_WEB__WEB_BASE_URL" $k) }}

deploy/values/prod.yaml +17 −0 View

@@ -70,6 +70,23 @@

7070

7171 RUST_LOG: "info,axum_session::session_store=error"

7272

73 # ---- OpenTelemetry ----

74 # endpoint 指向 OpenObserve 的 OTLP HTTP 入口（org id 在路径里，非凭据）；

75 # SDK 自己往 endpoint 后拼 /v1/traces 与 /v1/metrics。

76 # 凭据在 PROD_SECRET_VALUES 里分两个键：

77 # OTEL_EXPORTER_OTLP_HEADERS = Authorization=... （所有信号共用）

78 # OTEL_EXPORTER_OTLP_TRACES_HEADERS = Authorization=...,stream-name=default

79 # traces 按用户给的配置落到 default 流；metrics 故意不带 stream-name，

80 # 否则 OO 会把所有指标塞进一条流，Metrics 页面按指标名浏览就废了。

81 # 注意 signal-specific 变量是整体覆盖通用变量、不是合并，所以 traces 那条要带全。

82 # 采样：TraceLayer 给每个 HTTP 请求建 span，静态资源也在 apihost 上，全采会把

83 # 存储喂满；parentbased 保证被采样的请求链路完整（子 span 跟随父决策）。

84 OTEL_TRACES_SAMPLER: "parentbased_traceidratio"

85 OTEL_TRACES_SAMPLER_ARG: "0.2"

87metrics:

88 otlpEndpoint: "https://o2.gitdata.me/api/3JmB4YFMnPffh11t8283QXAwW36"

7390ingress:

7491 enabled: true

7592 className: nginx

lib/export/src/telemetry.rs +69 −1 View

@@ -1,9 +1,64 @@

1use std::sync::OnceLock;

3use opentelemetry::trace::TracerProvider as _;

4use opentelemetry_sdk::trace::SdkTracerProvider;

15use tracing::{info, warn};

26use tracing_subscriber::EnvFilter;

7use tracing_subscriber::layer::SubscriberExt;

8use tracing_subscriber::util::SubscriberInitExt;

10/// Installed tracer provider, kept so shutdown can flush the batch processor.

11static TRACER_PROVIDER: OnceLock<SdkTracerProvider> = OnceLock::new();

312

413pub fn init_tracing() {

14 // The filter is installed on the registry rather than on the fmt layer so it

15 // governs span *creation*: without it every debug-level span in the process

16 // (hyper/tower internals) becomes an exported span.

517 let filter = EnvFilter::try_from_default_env().unwrap_or_else(|_| EnvFilter::new("info"));

6 tracing_subscriber::fmt().with_env_filter(filter).init();

18 let subscriber = tracing_subscriber::registry()

19 .with(tracing_subscriber::fmt::layer())

20 .with(filter);

22 match tracer_provider() {

23 Some(provider) => {

24 let _ = TRACER_PROVIDER.set(provider.clone());

25 subscriber

26 .with(tracing_opentelemetry::layer().with_tracer(provider.tracer("gitdata")))

27 .init();

28 info!("opentelemetry trace export enabled");

29 }

30 None => subscriber.init(),

31 }

32}

34/// Traces are opt-in through the standard OTLP environment variables: setting

35/// `OTEL_EXPORTER_OTLP_ENDPOINT` turns export on, anything else (including a bad

36/// endpoint) only downgrades to log-only. Telemetry must never stop a host from

37/// booting, so every failure path here logs and returns `None`.

38fn tracer_provider() -> Option<SdkTracerProvider> {

39 if std::env::var("OTEL_EXPORTER_OTLP_ENDPOINT")

40 .unwrap_or_default()

41 .is_empty()

42 {

43 return None;

44 }

46 let exporter = match opentelemetry_otlp::SpanExporter::builder()

47 .with_http()

48 .build()

49 {

50 Ok(exporter) => exporter,

51 Err(err) => {

52 eprintln!("failed to build otlp span exporter, traces disabled: {err}");

53 return None;

54 }

55 };

57 Some(

58 SdkTracerProvider::builder()

59 .with_batch_exporter(exporter)

60 .build(),

61 )

762}

863

964pub async fn shutdown_signal() {

@@ -33,4 +88,17 @@

3388 _ = ctrl_c => info!("received SIGINT"),

3489 _ = terminate => info!("received SIGTERM"),

3590 }

92 flush_traces();

93}

95/// Drains spans queued in the batch processor. Without this the tail of every

96/// deployment's trace output is lost on rollout, which is exactly when you look

97/// at traces.

98pub fn flush_traces() {

99 if let Some(provider) = TRACER_PROVIDER.get()

100 && let Err(err) = provider.shutdown()

101 {

102 warn!(error = %err, "failed to flush tracer provider");

103 }

36104}