dogfood
Jump to…
snowcorepublic2 bookmarks
snowfix: 500 error because of database is patchedyuzpxzopsouq1mo
mainroot / crates / df-web / src / git_http / mod.rsBlameRaw
1//! Git smart HTTP (spec §7).
2//!
3//! `jj git push` and plain `git push` are both ordinary Git pushes over this
4//! transport — the server never runs `jj`. We shell out to `git upload-pack`
5//! and `git receive-pack`, which is what every forge does: reimplementing the
6//! pack protocol buys nothing and gets the edge cases wrong.
7//!
8//! Authentication is HTTP Basic with a personal access token as the password
9//! (spec §6), because Git clients cannot perform an OIDC browser flow.
10
11use std::process::Stdio;
12
13use axum::extract::{Path as UrlPath, Query, State};
14use axum::http::{header, HeaderMap, StatusCode};
15use axum::response::{IntoResponse, Response};
16use df_db::ids::new_id;
17use df_store::RepoId;
18use serde::Deserialize;
19use tokio::io::AsyncWriteExt;
20use uuid::Uuid;
21
22use crate::error::{AppError, AppResult};
23use crate::repo_ctx::RepoContext;
24use crate::state::AppState;
25
26mod auth;
27
28/// Wall-clock cap on any Git subprocess.
29///
30/// Spec §9: "A `git-upload-pack` on a maliciously constructed repo can consume
31/// unbounded CPU — run it with a timeout."
32const GIT_TIMEOUT: std::time::Duration = std::time::Duration::from_secs(300);
33
34/// Cap on a fetch request body.
35///
36/// `git-upload-pack` receives want/have negotiation lines, not objects. Even a
37/// pathological client with an enormous have-list stays far below this, so a
38/// body larger than it is not a fetch.
39const UPLOAD_PACK_REQUEST_LIMIT: u64 = 16 * 1024 * 1024;
40
41#[derive(Deserialize)]
42pub struct ServiceQuery {
43 pub service: Option<String>,
44}
45
46/// `GET /{owner}/{repo}.git/info/refs?service=…`
47///
48/// Reference advertisement. This is the first request any clone or push makes,
49/// so it is also where authentication is challenged.
50pub async fn info_refs(
51 State(state): State<AppState>,
52 UrlPath((owner, repo)): UrlPath<(String, String)>,
53 Query(q): Query<ServiceQuery>,
54 headers: HeaderMap,
55) -> AppResult<Response> {
56 let service = q.service.unwrap_or_default();
57
58 // Only the two real services. Anything else is either a probe or an
59 // attempt to reach a different git command.
60 let write = match service.as_str() {
61 "git-upload-pack" => false,
62 "git-receive-pack" => true,
63 _ => return Err(AppError::NotFound),
64 };
65
66 let repo = strip_git_suffix(&repo);
67 let ctx = match resolve(&state, &owner, &repo, &headers, write).await? {
68 Resolved::Ok(ctx) => ctx,
69 Resolved::Challenge => return Ok(unauthorized()),
70 };
71
72 let dir = repo_dir(&state, &ctx);
73 let out = run_git(
74 &[
75 service.trim_start_matches("git-"),
76 "--stateless-rpc",
77 "--advertise-refs",
78 ".",
79 ],
80 &dir,
81 GitInput::Empty,
82 None,
83 )
84 .await?;
85
86 // The advertisement is prefixed with a pkt-line naming the service.
87 let mut body = pkt_line(&format!("# service={service}\n"));
88 body.extend_from_slice(b"0000");
89 body.extend_from_slice(&out);
90
91 Ok((
92 StatusCode::OK,
93 [
94 (
95 header::CONTENT_TYPE,
96 format!("application/x-{service}-advertisement"),
97 ),
98 (header::CACHE_CONTROL, "no-cache".into()),
99 ],
100 body,
101 )
102 .into_response())
103}
104
105/// `POST /{owner}/{repo}.git/git-upload-pack` — a clone or fetch.
106pub async fn upload_pack(
107 State(state): State<AppState>,
108 UrlPath((owner, repo)): UrlPath<(String, String)>,
109 headers: HeaderMap,
110 body: axum::body::Body,
111) -> AppResult<Response> {
112 let repo = strip_git_suffix(&repo);
113 let ctx = match resolve(&state, &owner, &repo, &headers, false).await? {
114 Resolved::Ok(ctx) => ctx,
115 Resolved::Challenge => return Ok(unauthorized()),
116 };
117
118 let dir = repo_dir(&state, &ctx);
119 let out = run_git(
120 &["upload-pack", "--stateless-rpc", "."],
121 &dir,
122 // A fetch request is want/have lines, not a pack. The cap is generous
123 // for that and still bounded.
124 GitInput::Stream { body, max_bytes: UPLOAD_PACK_REQUEST_LIMIT },
125 // A fetch does not run the pre-receive hook.
126 None,
127 )
128 .await?;
129
130 Ok(git_response("git-upload-pack-result", out))
131}
132
133/// `POST /{owner}/{repo}.git/git-receive-pack` — a push.
134///
135/// After the pack lands, an `IndexPush` job is enqueued and we return
136/// immediately; indexing happens in the worker (spec §4).
137pub async fn receive_pack(
138 State(state): State<AppState>,
139 UrlPath((owner, repo)): UrlPath<(String, String)>,
140 headers: HeaderMap,
141 body: axum::body::Body,
142) -> AppResult<Response> {
143 let repo = strip_git_suffix(&repo);
144
145 // Push requires write, so an anonymous request is challenged rather than
146 // 404'd — the client needs to know to send credentials.
147 let ctx = match resolve(&state, &owner, &repo, &headers, true).await? {
148 Resolved::Ok(ctx) => ctx,
149 Resolved::Challenge => return Ok(unauthorized()),
150 };
151
152 // An archived repository is read-only. The settings page promises that, so
153 // it has to be true on the wire and not only in the UI.
154 if ctx.repo.archived {
155 return Err(AppError::BadRequest(
156 "this repository is archived and does not accept pushes".into(),
157 ));
158 }
159
160 // The pack cap (spec §9) is enforced as the body streams, in `run_git` —
161 // `Content-Length` is a claim by the client and checking it here would be
162 // checking the claim rather than the bytes.
163 let limit = state.config.max_pack_bytes;
164
165 let pusher = auth::authenticated_user_id(&state, &headers).await;
166
167 let dir = repo_dir(&state, &ctx);
168 let out = run_git(
169 &["receive-pack", "--stateless-rpc", "."],
170 &dir,
171 GitInput::Stream { body, max_bytes: limit },
172 Some(HookEnv {
173 database_url: state.config.database_url.clone(),
174 pusher_id: pusher,
175 }),
176 )
177 .await?;
178
179 // Enqueue indexing. A failure here must not fail the push — the objects are
180 // already durably written, and `dogfood-admin reindex` can recover the
181 // index. Failing the push would make the client retry a push that already
182 // succeeded.
183 if let Err(e) = enqueue_index(&state, ctx.repo.id, pusher).await {
184 tracing::error!(repo = %ctx.repo.id, "enqueuing IndexPush failed: {e:#}");
185 }
186
187 if let Err(e) = sqlx::query("UPDATE repos SET pushed_at = now() WHERE id = $1")
188 .bind(ctx.repo.id)
189 .execute(&state.db)
190 .await
191 {
192 tracing::warn!("recording push time failed: {e}");
193 }
194
195 Ok(git_response("git-receive-pack-result", out))
196}
197
198/// Queue an indexing job for a repository.
199pub async fn enqueue_index(
200 state: &AppState,
201 repo_id: Uuid,
202 pushed_by: Option<Uuid>,
203) -> anyhow::Result<()> {
204 sqlx::query("INSERT INTO jobs (id, kind, payload) VALUES ($1, 'index_push', $2)")
205 .bind(new_id())
206 .bind(serde_json::json!({
207 "repo_id": repo_id,
208 "pushed_by": pushed_by,
209 }))
210 .execute(&state.db)
211 .await?;
212 Ok(())
213}
214
215// ─── plumbing ────────────────────────────────────────────────────────────────
216
217enum Resolved {
218 Ok(RepoContext),
219 /// Credentials are needed, or the ones supplied were wrong.
220 Challenge,
221}
222
223/// Resolve the repository and authorize the operation.
224///
225/// Read access to a public repo is anonymous. Anything else needs a token.
226/// Returning `Challenge` rather than 404 for an unauthenticated *write* is
227/// deliberate: without it, `git push` to a private repo fails with a confusing
228/// "not found" instead of prompting for credentials. It leaks only that a push
229/// endpoint exists, which is true of every path on the host.
230async fn resolve(
231 state: &AppState,
232 owner: &str,
233 name: &str,
234 headers: &HeaderMap,
235 write: bool,
236) -> AppResult<Resolved> {
237 let user = auth::authenticate(state, headers).await?;
238
239 let ctx = match RepoContext::load(state, owner, name, user.as_ref()).await {
240 Ok(ctx) => ctx,
241 Err(AppError::NotFound) => {
242 // Unauthenticated: could be a private repo. Challenge so a client
243 // with credentials can retry.
244 return Ok(if user.is_none() {
245 Resolved::Challenge
246 } else {
247 return Err(AppError::NotFound);
248 });
249 }
250 Err(e) => return Err(e),
251 };
252
253 if write && !ctx.access.can_push() {
254 return Ok(if user.is_none() {
255 Resolved::Challenge
256 } else {
257 // Authenticated but not permitted: say so plainly, since they can
258 // already see the repository.
259 return Err(AppError::Forbidden);
260 });
261 }
262
263 Ok(Resolved::Ok(ctx))
264}
265
266fn unauthorized() -> Response {
267 (
268 StatusCode::UNAUTHORIZED,
269 [(
270 header::WWW_AUTHENTICATE,
271 "Basic realm=\"Dogfood\", charset=\"UTF-8\"",
272 )],
273 "Authentication required. Use your handle and a personal access token.\n",
274 )
275 .into_response()
276}
277
278fn repo_dir(state: &AppState, ctx: &RepoContext) -> std::path::PathBuf {
279 // Sharded exactly as GitStore lays it out. Derived from the UUID, never
280 // from a name (spec §3).
281 let hex = RepoId(ctx.repo.id).0.simple().to_string();
282 std::path::Path::new(&state.config.repo_root)
283 .join(&hex[..2])
284 .join(format!("{hex}.git"))
285}
286
287/// Strip a trailing `.git` from the URL segment.
288fn strip_git_suffix(s: &str) -> String {
289 s.strip_suffix(".git").unwrap_or(s).to_string()
290}
291
292fn git_response(content_type: &str, body: Vec<u8>) -> Response {
293 (
294 StatusCode::OK,
295 [
296 (
297 header::CONTENT_TYPE,
298 format!("application/x-{content_type}"),
299 ),
300 (header::CACHE_CONTROL, "no-cache".into()),
301 ],
302 body,
303 )
304 .into_response()
305}
306
307/// Encode a pkt-line.
308fn pkt_line(s: &str) -> Vec<u8> {
309 let mut out = format!("{:04x}", s.len() + 4).into_bytes();
310 out.extend_from_slice(s.as_bytes());
311 out
312}
313
314/// What to feed a git subprocess on stdin.
315enum GitInput {
316 /// Nothing — the ref advertisement takes no input.
317 Empty,
318 /// The request body, streamed. Never collected into memory: a pack is
319 /// hundreds of megabytes by design, and buffering one per concurrent push
320 /// is how a forge runs a host out of RAM.
321 Stream {
322 body: axum::body::Body,
323 /// Hard cap. Exceeding it kills the transfer rather than letting an
324 /// unbounded pack through (spec §9).
325 max_bytes: u64,
326 },
327}
328
329/// What the pre-receive hook needs from its environment.
330///
331/// `git` is spawned with a cleared environment, so anything the hook needs has
332/// to be handed to it explicitly. Without `DATABASE_URL` the hook cannot check
333/// protected bookmarks — and it used to skip that check silently, which meant
334/// bookmark protection did not exist over HTTPS at all.
335struct HookEnv {
336 database_url: String,
337 pusher_id: Option<Uuid>,
338}
339
340/// Run a git subprocess with a timeout, feeding it `input`.
341async fn run_git(
342 args: &[&str],
343 dir: &std::path::Path,
344 input: GitInput,
345 hook_env: Option<HookEnv>,
346) -> AppResult<Vec<u8>> {
347 if !dir.is_dir() {
348 // The database knows about a repository the filesystem does not. Real
349 // after a partial restore (spec §10).
350 tracing::error!(?dir, "repository directory is missing");
351 return Err(AppError::NotFound);
352 }
353
354 let mut cmd = tokio::process::Command::new("git");
355 cmd.args(args)
356 .current_dir(dir)
357 .stdin(Stdio::piped())
358 .stdout(Stdio::piped())
359 .stderr(Stdio::piped())
360 // Without this the `GIT_TIMEOUT` below bounds how long we *wait*, not
361 // how long git runs: tokio does not kill a child when its handle is
362 // dropped unless asked, so a timed-out `upload-pack` on a hostile
363 // repository would keep burning CPU with nothing left watching it.
364 .kill_on_drop(true)
365 // Never let a repository's own config influence what we run, and never
366 // let git prompt for anything.
367 .env_clear()
368 .env("PATH", "/usr/bin:/bin")
369 .env("GIT_TERMINAL_PROMPT", "0")
370 .env("GIT_CONFIG_NOSYSTEM", "1")
371 .env("HOME", "/nonexistent")
372 // Advertise object-format and allow the protocol v2 the client asks for.
373 .env("GIT_PROTOCOL", "version=2");
374
375 // Only the push path installs these, and only the push path runs the hook.
376 if let Some(hook) = hook_env {
377 cmd.env("DATABASE_URL", hook.database_url);
378 // Marks the check as mandatory: the hook refuses the push if it is set
379 // and the check cannot be performed, rather than failing open.
380 cmd.env("DOGFOOD_ENFORCE", "1");
381 if let Some(id) = hook.pusher_id {
382 cmd.env("DOGFOOD_PUSHER_ID", id.to_string());
383 }
384 }
385
386 let mut child = cmd
387 .spawn()
388 .map_err(|e| AppError::Internal(anyhow::anyhow!("spawning git: {e}")))?;
389
390 if let Some(mut stdin) = child.stdin.take() {
391 // Write on a task: a large pack exceeds the pipe buffer many times over,
392 // and writing inline would deadlock against a child that is blocked
393 // writing stdout.
394 match input {
395 GitInput::Empty => {
396 let _ = stdin.shutdown().await;
397 }
398 GitInput::Stream { body, max_bytes } => {
399 tokio::spawn(async move {
400 use futures::StreamExt;
401
402 let mut stream = body.into_data_stream();
403 let mut written: u64 = 0;
404
405 while let Some(chunk) = stream.next().await {
406 let chunk = match chunk {
407 Ok(c) => c,
408 Err(e) => {
409 tracing::debug!("reading the request body failed: {e}");
410 break;
411 }
412 };
413
414 written += chunk.len() as u64;
415 if written > max_bytes {
416 // Dropping stdin makes git see a truncated pack and
417 // fail, which is what we want: the objects are not
418 // written and the client is told the push failed.
419 tracing::warn!(
420 max_bytes,
421 "request body exceeded the pack limit; aborting the transfer"
422 );
423 break;
424 }
425
426 if let Err(e) = stdin.write_all(&chunk).await {
427 tracing::debug!("writing to git stdin failed: {e}");
428 break;
429 }
430 }
431
432 let _ = stdin.shutdown().await;
433 });
434 }
435 }
436 }
437
438 let out = match tokio::time::timeout(GIT_TIMEOUT, child.wait_with_output()).await {
439 Ok(Ok(o)) => o,
440 Ok(Err(e)) => {
441 return Err(AppError::Internal(anyhow::anyhow!("running git: {e}")));
442 }
443 Err(_) => {
444 // Dropping the timed-out future drops the `Child`, and
445 // `kill_on_drop` above turns that into a SIGKILL. The process is
446 // gone by the time this returns rather than orphaned.
447 tracing::error!(?args, "git operation timed out; killing it");
448 return Err(AppError::Internal(anyhow::anyhow!("git timed out")));
449 }
450 };
451
452 if !out.status.success() {
453 let stderr = String::from_utf8_lossy(&out.stderr);
454 tracing::error!(?args, status = ?out.status, "git failed: {stderr}");
455 return Err(AppError::Internal(anyhow::anyhow!("git failed: {stderr}")));
456 }
457
458 Ok(out.stdout)
459}
460
461#[cfg(test)]
462mod tests {
463 use super::*;
464
465 #[test]
466 fn pkt_lines_carry_a_four_byte_hex_length() {
467 // The length is inclusive of the 4-byte header itself.
468 // "# service=git-upload-pack\n" is 26 bytes, +4 = 30 = 0x1e.
469 let line = "# service=git-upload-pack\n";
470 assert_eq!(line.len(), 26);
471 let p = pkt_line(line);
472 assert_eq!(&p[..4], b"001e");
473 assert_eq!(p.len(), 0x1e);
474 }
475
476 #[test]
477 fn pkt_line_of_empty_input_is_just_the_header() {
478 assert_eq!(pkt_line(""), b"0004");
479 }
480
481 #[test]
482 fn strips_only_a_trailing_git_suffix() {
483 assert_eq!(strip_git_suffix("repo.git"), "repo");
484 assert_eq!(strip_git_suffix("repo"), "repo");
485 // A repository legitimately named `x.github` must not be truncated.
486 assert_eq!(strip_git_suffix("x.github"), "x.github");
487 assert_eq!(strip_git_suffix("my.git.repo"), "my.git.repo");
488 }
489}

489 lines · Rust