| 1 | //! `GET /metrics` — Prometheus exposition (spec §7, §10). |
| 2 | //! |
| 3 | //! > `GET /metrics` — **bind to loopback only**. |
| 4 | //! |
| 5 | //! Dogfood serves one port, so "bind to loopback only" is enforced per request |
| 6 | //! rather than per socket: the peer address must be a loopback address, or the |
| 7 | //! response is the same 404 any unknown path gets. A 403 would confirm the |
| 8 | //! endpoint exists, and the metrics say how many private repositories there are. |
| 9 | //! |
| 10 | //! The peer address comes from `ConnectInfo` — the real TCP peer — and never |
| 11 | //! from `X-Forwarded-For`, which the client controls. Behind the edge proxy that |
| 12 | //! means metrics are unreachable from outside the host, which is the point; |
| 13 | //! scrape it with `docker exec` or over the compose network. |
| 14 | //! |
| 15 | //! The metric set is the one §10 says to alert on: job queue depth, indexer |
| 16 | //! failures, and push latency. Everything else here is cheap context. |
| 17 | |
| 18 | use axum::extract::State; |
| 19 | use axum::response::{IntoResponse, Response}; |
| 20 | |
| 21 | use crate::error::{AppError, AppResult}; |
| 22 | use crate::state::{AppState, PeerAddr}; |
| 23 | |
| 24 | pub async fn metrics(State(state): State<AppState>, peer: PeerAddr) -> AppResult<Response> { |
| 25 | // A peer that cannot be proven local is treated exactly like a remote one. |
| 26 | // Defaulting the other way would serve metrics to anybody the moment the |
| 27 | // server was reconfigured without connect info. |
| 28 | if !peer.is_loopback() { |
| 29 | tracing::debug!(peer = ?peer.ip(), "refusing /metrics: not a loopback peer"); |
| 30 | return Err(AppError::NotFound); |
| 31 | } |
| 32 | |
| 33 | let mut out = String::with_capacity(2048); |
| 34 | |
| 35 | // ── the three §10 alerting signals ─────────────────────────────────────── |
| 36 | |
| 37 | let (ready, failed, oldest): (i64, i64, Option<f64>) = sqlx::query_as( |
| 38 | "SELECT |
| 39 | count(*) FILTER (WHERE locked_at IS NULL AND attempts < max_attempts), |
| 40 | count(*) FILTER (WHERE attempts >= max_attempts), |
| 41 | EXTRACT(EPOCH FROM (now() - min(run_at)))::float8 |
| 42 | FILTER (WHERE locked_at IS NULL) |
| 43 | FROM jobs", |
| 44 | ) |
| 45 | .fetch_one(&state.db) |
| 46 | .await |
| 47 | .unwrap_or((0, 0, None)); |
| 48 | |
| 49 | gauge(&mut out, "dogfood_jobs_queued", "Jobs waiting to run.", ready as f64); |
| 50 | gauge( |
| 51 | &mut out, |
| 52 | "dogfood_jobs_failed", |
| 53 | "Jobs that exhausted their retries. Alert on any increase.", |
| 54 | failed as f64, |
| 55 | ); |
| 56 | gauge( |
| 57 | &mut out, |
| 58 | "dogfood_jobs_oldest_seconds", |
| 59 | "Age of the oldest queued job. The queue-depth alert that actually matters.", |
| 60 | oldest.unwrap_or(0.0), |
| 61 | ); |
| 62 | |
| 63 | // Push latency, as the lag between a push landing and its index job being |
| 64 | // picked up. Measured from the rows rather than instrumented in-process so |
| 65 | // it survives a restart of either service. |
| 66 | let push_lag: Option<f64> = sqlx::query_scalar( |
| 67 | "SELECT EXTRACT(EPOCH FROM (now() - max(pushed_at)))::float8 FROM revisions", |
| 68 | ) |
| 69 | .fetch_one(&state.db) |
| 70 | .await |
| 71 | .unwrap_or(None); |
| 72 | gauge( |
| 73 | &mut out, |
| 74 | "dogfood_seconds_since_last_push", |
| 75 | "Seconds since the most recent indexed revision.", |
| 76 | push_lag.unwrap_or(-1.0), |
| 77 | ); |
| 78 | |
| 79 | // ── inventory ──────────────────────────────────────────────────────────── |
| 80 | |
| 81 | for (metric, help, sql) in [ |
| 82 | ("dogfood_users", "Provisioned users.", "SELECT count(*) FROM users"), |
| 83 | ("dogfood_repos", "Repositories.", "SELECT count(*) FROM repos"), |
| 84 | ( |
| 85 | "dogfood_repos_private", |
| 86 | "Private repositories.", |
| 87 | "SELECT count(*) FROM repos WHERE visibility = 'private'", |
| 88 | ), |
| 89 | ("dogfood_changes", "Changes indexed.", "SELECT count(*) FROM changes"), |
| 90 | ( |
| 91 | "dogfood_changes_conflicted", |
| 92 | "Changes whose head revision is conflicted.", |
| 93 | "SELECT count(*) FROM changes WHERE conflicted", |
| 94 | ), |
| 95 | ("dogfood_revisions", "Revisions indexed.", "SELECT count(*) FROM revisions"), |
| 96 | ("dogfood_issues_open", "Open issues.", "SELECT count(*) FROM issues WHERE state = 'open'"), |
| 97 | ( |
| 98 | "dogfood_comments_orphaned", |
| 99 | "Inline comments whose anchor no longer exists.", |
| 100 | "SELECT count(*) FROM comments WHERE anchor_state = 'orphaned'", |
| 101 | ), |
| 102 | ( |
| 103 | "dogfood_highlight_cache_entries", |
| 104 | "Cached syntax-highlighted blobs.", |
| 105 | "SELECT count(*) FROM highlight_cache", |
| 106 | ), |
| 107 | ] { |
| 108 | let n: i64 = sqlx::query_scalar(sql).fetch_one(&state.db).await.unwrap_or(-1); |
| 109 | gauge(&mut out, metric, help, n as f64); |
| 110 | } |
| 111 | |
| 112 | // Reachability of the database is already the substance of every query |
| 113 | // above; a separate probe would only be able to disagree with them. |
| 114 | gauge(&mut out, "dogfood_up", "Always 1 when the process is serving.", 1.0); |
| 115 | |
| 116 | Ok(( |
| 117 | [( |
| 118 | axum::http::header::CONTENT_TYPE, |
| 119 | "text/plain; version=0.0.4; charset=utf-8", |
| 120 | )], |
| 121 | out, |
| 122 | ) |
| 123 | .into_response()) |
| 124 | } |
| 125 | |
| 126 | fn gauge(out: &mut String, name: &str, help: &str, value: f64) { |
| 127 | use std::fmt::Write as _; |
| 128 | let _ = writeln!(out, "# HELP {name} {help}"); |
| 129 | let _ = writeln!(out, "# TYPE {name} gauge"); |
| 130 | let _ = writeln!(out, "{name} {value}"); |
| 131 | } |
| 132 | |
| 133 | #[cfg(test)] |
| 134 | mod tests { |
| 135 | use super::gauge; |
| 136 | |
| 137 | #[test] |
| 138 | fn exposition_format_is_well_formed() { |
| 139 | let mut s = String::new(); |
| 140 | gauge(&mut s, "dogfood_x", "A thing.", 3.0); |
| 141 | assert_eq!( |
| 142 | s, |
| 143 | "# HELP dogfood_x A thing.\n# TYPE dogfood_x gauge\ndogfood_x 3\n" |
| 144 | ); |
| 145 | } |
| 146 | |
| 147 | #[test] |
| 148 | fn loopback_detection_covers_both_families() { |
| 149 | use std::net::IpAddr; |
| 150 | assert!("127.0.0.1".parse::<IpAddr>().unwrap().is_loopback()); |
| 151 | assert!("::1".parse::<IpAddr>().unwrap().is_loopback()); |
| 152 | // The container's own bridge address is *not* loopback, which is what |
| 153 | // keeps /metrics off the compose network by default. |
| 154 | assert!(!"172.17.0.2".parse::<IpAddr>().unwrap().is_loopback()); |
| 155 | assert!(!"10.0.0.1".parse::<IpAddr>().unwrap().is_loopback()); |
| 156 | } |
| 157 | } |
157 lines · Rust