| 1 | //! `GET /metrics` — Prometheus exposition (spec §7, §10). | |
| 2 | //! | |
| 3 | //! > `GET /metrics` — **bind to loopback only**. | |
| 4 | //! | |
| 5 | //! Dogfood serves one port, so "bind to loopback only" is enforced per request | |
| 6 | //! rather than per socket: the peer address must be a loopback address, or the | |
| 7 | //! response is the same 404 any unknown path gets. A 403 would confirm the | |
| 8 | //! endpoint exists, and the metrics say how many private repositories there are. | |
| 9 | //! | |
| 10 | //! The peer address comes from `ConnectInfo` — the real TCP peer — and never | |
| 11 | //! from `X-Forwarded-For`, which the client controls. Behind the edge proxy that | |
| 12 | //! means metrics are unreachable from outside the host, which is the point; | |
| 13 | //! scrape it with `docker exec` or over the compose network. | |
| 14 | //! | |
| 15 | //! The metric set is the one §10 says to alert on: job queue depth, indexer | |
| 16 | //! failures, and push latency. Everything else here is cheap context. | |
| 17 | ||
| 18 | use axum::extract::State; | |
| 19 | use axum::response::{IntoResponse, Response}; | |
| 20 | ||
| 21 | use crate::error::{AppError, AppResult}; | |
| 22 | use crate::state::{AppState, PeerAddr}; | |
| 23 | ||
| 24 | pub async fn metrics(State(state): State<AppState>, peer: PeerAddr) -> AppResult<Response> { | |
| 25 | // A peer that cannot be proven local is treated exactly like a remote one. | |
| 26 | // Defaulting the other way would serve metrics to anybody the moment the | |
| 27 | // server was reconfigured without connect info. | |
| 28 | if !peer.is_loopback() { | |
| 29 | tracing::debug!(peer = ?peer.ip(), "refusing /metrics: not a loopback peer"); | |
| 30 | return Err(AppError::NotFound); | |
| 31 | } | |
| 32 | ||
| 33 | let mut out = String::with_capacity(2048); | |
| 34 | ||
| 35 | // ── the three §10 alerting signals ─────────────────────────────────────── | |
| 36 | ||
| 37 | let (ready, failed, oldest): (i64, i64, Option<f64>) = sqlx::query_as( | |
| 38 | "SELECT | |
| 39 | count(*) FILTER (WHERE locked_at IS NULL AND attempts < max_attempts), | |
| 40 | count(*) FILTER (WHERE attempts >= max_attempts), | |
| 41 | EXTRACT(EPOCH FROM (now() - min(run_at)))::float8 | |
| 42 | FILTER (WHERE locked_at IS NULL) | |
| 43 | FROM jobs", | |
| 44 | ) | |
| 45 | .fetch_one(&state.db) | |
| 46 | .await | |
| 47 | .unwrap_or((0, 0, None)); | |
| 48 | ||
| 49 | gauge(&mut out, "dogfood_jobs_queued", "Jobs waiting to run.", ready as f64); | |
| 50 | gauge( | |
| 51 | &mut out, | |
| 52 | "dogfood_jobs_failed", | |
| 53 | "Jobs that exhausted their retries. Alert on any increase.", | |
| 54 | failed as f64, | |
| 55 | ); | |
| 56 | gauge( | |
| 57 | &mut out, | |
| 58 | "dogfood_jobs_oldest_seconds", | |
| 59 | "Age of the oldest queued job. The queue-depth alert that actually matters.", | |
| 60 | oldest.unwrap_or(0.0), | |
| 61 | ); | |
| 62 | ||
| 63 | // Push latency, as the lag between a push landing and its index job being | |
| 64 | // picked up. Measured from the rows rather than instrumented in-process so | |
| 65 | // it survives a restart of either service. | |
| 66 | let push_lag: Option<f64> = sqlx::query_scalar( | |
| 67 | "SELECT EXTRACT(EPOCH FROM (now() - max(pushed_at)))::float8 FROM revisions", | |
| 68 | ) | |
| 69 | .fetch_one(&state.db) | |
| 70 | .await | |
| 71 | .unwrap_or(None); | |
| 72 | gauge( | |
| 73 | &mut out, | |
| 74 | "dogfood_seconds_since_last_push", | |
| 75 | "Seconds since the most recent indexed revision.", | |
| 76 | push_lag.unwrap_or(-1.0), | |
| 77 | ); | |
| 78 | ||
| 79 | // ── inventory ──────────────────────────────────────────────────────────── | |
| 80 | ||
| 81 | for (metric, help, sql) in [ | |
| 82 | ("dogfood_users", "Provisioned users.", "SELECT count(*) FROM users"), | |
| 83 | ("dogfood_repos", "Repositories.", "SELECT count(*) FROM repos"), | |
| 84 | ( | |
| 85 | "dogfood_repos_private", | |
| 86 | "Private repositories.", | |
| 87 | "SELECT count(*) FROM repos WHERE visibility = 'private'", | |
| 88 | ), | |
| 89 | ("dogfood_changes", "Changes indexed.", "SELECT count(*) FROM changes"), | |
| 90 | ( | |
| 91 | "dogfood_changes_conflicted", | |
| 92 | "Changes whose head revision is conflicted.", | |
| 93 | "SELECT count(*) FROM changes WHERE conflicted", | |
| 94 | ), | |
| 95 | ("dogfood_revisions", "Revisions indexed.", "SELECT count(*) FROM revisions"), | |
| 96 | ("dogfood_issues_open", "Open issues.", "SELECT count(*) FROM issues WHERE state = 'open'"), | |
| 97 | ( | |
| 98 | "dogfood_comments_orphaned", | |
| 99 | "Inline comments whose anchor no longer exists.", | |
| 100 | "SELECT count(*) FROM comments WHERE anchor_state = 'orphaned'", | |
| 101 | ), | |
| 102 | ( | |
| 103 | "dogfood_highlight_cache_entries", | |
| 104 | "Cached syntax-highlighted blobs.", | |
| 105 | "SELECT count(*) FROM highlight_cache", | |
| 106 | ), | |
| 107 | ] { | |
| 108 | let n: i64 = sqlx::query_scalar(sql).fetch_one(&state.db).await.unwrap_or(-1); | |
| 109 | gauge(&mut out, metric, help, n as f64); | |
| 110 | } | |
| 111 | ||
| 112 | // Reachability of the database is already the substance of every query | |
| 113 | // above; a separate probe would only be able to disagree with them. | |
| 114 | gauge(&mut out, "dogfood_up", "Always 1 when the process is serving.", 1.0); | |
| 115 | ||
| 116 | Ok(( | |
| 117 | [( | |
| 118 | axum::http::header::CONTENT_TYPE, | |
| 119 | "text/plain; version=0.0.4; charset=utf-8", | |
| 120 | )], | |
| 121 | out, | |
| 122 | ) | |
| 123 | .into_response()) | |
| 124 | } | |
| 125 | ||
| 126 | fn gauge(out: &mut String, name: &str, help: &str, value: f64) { | |
| 127 | use std::fmt::Write as _; | |
| 128 | let _ = writeln!(out, "# HELP {name} {help}"); | |
| 129 | let _ = writeln!(out, "# TYPE {name} gauge"); | |
| 130 | let _ = writeln!(out, "{name} {value}"); | |
| 131 | } | |
| 132 | ||
| 133 | #[cfg(test)] | |
| 134 | mod tests { | |
| 135 | use super::gauge; | |
| 136 | ||
| 137 | #[test] | |
| 138 | fn exposition_format_is_well_formed() { | |
| 139 | let mut s = String::new(); | |
| 140 | gauge(&mut s, "dogfood_x", "A thing.", 3.0); | |
| 141 | assert_eq!( | |
| 142 | s, | |
| 143 | "# HELP dogfood_x A thing.\n# TYPE dogfood_x gauge\ndogfood_x 3\n" | |
| 144 | ); | |
| 145 | } | |
| 146 | ||
| 147 | #[test] | |
| 148 | fn loopback_detection_covers_both_families() { | |
| 149 | use std::net::IpAddr; | |
| 150 | assert!("127.0.0.1".parse::<IpAddr>().unwrap().is_loopback()); | |
| 151 | assert!("::1".parse::<IpAddr>().unwrap().is_loopback()); | |
| 152 | // The container's own bridge address is *not* loopback, which is what | |
| 153 | // keeps /metrics off the compose network by default. | |
| 154 | assert!(!"172.17.0.2".parse::<IpAddr>().unwrap().is_loopback()); | |
| 155 | assert!(!"10.0.0.1".parse::<IpAddr>().unwrap().is_loopback()); | |
| 156 | } | |
| 157 | } |
157 lines · Rust