| 1 | //! Cached syntax highlighting (spec §8). |
| 2 | //! |
| 3 | //! > Highlight in the worker and cache the rendered HTML in Postgres keyed by |
| 4 | //! > blob OID — highlighting the same file on every request is the easiest |
| 5 | //! > performance mistake to make here. |
| 6 | //! |
| 7 | //! **Deviation, deliberate.** Highlighting happens here on a cache miss rather |
| 8 | //! than only in the worker. Precomputing in the worker means the *first* view of |
| 9 | //! any file renders unhighlighted, which is the view that matters most after a |
| 10 | //! push. The property the spec is actually protecting — never highlight the same |
| 11 | //! bytes twice — is provided by the cache, which is what this module is. The |
| 12 | //! parse itself runs on a blocking thread so a large file cannot stall the |
| 13 | //! async runtime. |
| 14 | //! |
| 15 | //! The cache key is the blob's content address, so it is shared across |
| 16 | //! repositories and across revisions: a file that did not change between two |
| 17 | //! revisions is highlighted once, ever. Because the key is a pure function of |
| 18 | //! the bytes, there is no invalidation problem — different content is a |
| 19 | //! different key. |
| 20 | |
| 21 | use df_render::highlight::{self, Skipped}; |
| 22 | use df_store::Blob; |
| 23 | use sqlx::PgPool; |
| 24 | |
| 25 | /// Highlighted lines, or the reason there are none. |
| 26 | pub struct Rendered { |
| 27 | /// One HTML fragment per line. Empty when `skipped` is set. |
| 28 | pub lines: Vec<String>, |
| 29 | pub language: Option<String>, |
| 30 | /// Set when the file is to be rendered as plain text, with the reason. |
| 31 | pub skipped: Option<Skipped>, |
| 32 | } |
| 33 | |
| 34 | impl Rendered { |
| 35 | fn plain(reason: Skipped) -> Self { |
| 36 | Rendered { lines: Vec::new(), language: None, skipped: Some(reason) } |
| 37 | } |
| 38 | } |
| 39 | |
| 40 | /// Highlight a blob, consulting and populating the Postgres cache. |
| 41 | /// |
| 42 | /// Never fails: a database error degrades to highlighting without the cache, |
| 43 | /// and a highlighting failure degrades to plain text. A syntax colour is not |
| 44 | /// worth a 500. |
| 45 | pub async fn render(db: &PgPool, blob: &Blob) -> Rendered { |
| 46 | let Some(text) = blob.text() else { |
| 47 | return Rendered::plain(Skipped::NoGrammar); |
| 48 | }; |
| 49 | |
| 50 | // No grammar means nothing to cache — decided from the path alone, so this |
| 51 | // costs one table lookup and avoids a pointless database round trip for |
| 52 | // every LICENSE and .gitignore in the tree. |
| 53 | let Some(lang) = highlight::language_for_path(&blob.path) else { |
| 54 | return Rendered::plain(Skipped::NoGrammar); |
| 55 | }; |
| 56 | |
| 57 | match load(db, &blob.content_id).await { |
| 58 | Ok(Some(hit)) => return hit, |
| 59 | Ok(None) => {} |
| 60 | Err(e) => tracing::warn!("highlight cache read failed, rendering uncached: {e}"), |
| 61 | } |
| 62 | |
| 63 | // tree-sitter is CPU-bound and this is an async handler; parsing a megabyte |
| 64 | // on the runtime thread would stall every other request on that worker. |
| 65 | let path = blob.path.clone(); |
| 66 | let source = text.to_owned(); |
| 67 | let highlighted = |
| 68 | match tokio::task::spawn_blocking(move || highlight::highlight(&path, &source)).await { |
| 69 | Ok(r) => r, |
| 70 | Err(e) => { |
| 71 | tracing::error!("highlighting panicked: {e}"); |
| 72 | return Rendered::plain(Skipped::Failed); |
| 73 | } |
| 74 | }; |
| 75 | |
| 76 | let out = match highlighted { |
| 77 | Ok(h) => Rendered { |
| 78 | lines: h.lines, |
| 79 | language: Some(h.language.to_string()), |
| 80 | skipped: None, |
| 81 | }, |
| 82 | Err(reason) => Rendered::plain(reason), |
| 83 | }; |
| 84 | |
| 85 | // Only successful renders are cached. Caching a rejection would save |
| 86 | // nothing: every rejection is decided from the path or the size, both of |
| 87 | // which are already known before the lookup. |
| 88 | if out.skipped.is_none() { |
| 89 | if let Err(e) = store(db, &blob.content_id, lang.display, &out.lines).await { |
| 90 | tracing::warn!("highlight cache write failed: {e}"); |
| 91 | } |
| 92 | } |
| 93 | |
| 94 | out |
| 95 | } |
| 96 | |
| 97 | /// The separator between cached line fragments. |
| 98 | /// |
| 99 | /// A form feed rather than a newline: the HTML fragments themselves are free of |
| 100 | /// newlines, but a fragment could in principle contain one from a multi-line |
| 101 | /// token, and splitting on the wrong character would silently corrupt the line |
| 102 | /// numbering. U+000C cannot appear in the renderer's output. |
| 103 | const LINE_SEP: char = '\u{c}'; |
| 104 | |
| 105 | async fn load(db: &PgPool, content_id: &str) -> sqlx::Result<Option<Rendered>> { |
| 106 | let row: Option<(Option<String>, String)> = |
| 107 | sqlx::query_as("SELECT language, html FROM highlight_cache WHERE blob_oid = $1") |
| 108 | .bind(content_id) |
| 109 | .fetch_optional(db) |
| 110 | .await?; |
| 111 | |
| 112 | Ok(row.map(|(language, html)| Rendered { |
| 113 | lines: if html.is_empty() { |
| 114 | Vec::new() |
| 115 | } else { |
| 116 | html.split(LINE_SEP).map(str::to_owned).collect() |
| 117 | }, |
| 118 | language, |
| 119 | skipped: None, |
| 120 | })) |
| 121 | } |
| 122 | |
| 123 | async fn store(db: &PgPool, content_id: &str, language: &str, lines: &[String]) -> sqlx::Result<()> { |
| 124 | let html = lines.join(&LINE_SEP.to_string()); |
| 125 | |
| 126 | // ON CONFLICT DO NOTHING: two requests racing on the same blob both compute |
| 127 | // the same bytes, so whichever lands first is correct and the other is a |
| 128 | // no-op rather than a lock wait. |
| 129 | sqlx::query( |
| 130 | "INSERT INTO highlight_cache (blob_oid, language, html, bytes) |
| 131 | VALUES ($1, $2, $3, $4) |
| 132 | ON CONFLICT (blob_oid) DO NOTHING", |
| 133 | ) |
| 134 | .bind(content_id) |
| 135 | .bind(language) |
| 136 | .bind(&html) |
| 137 | .bind(html.len() as i32) |
| 138 | .execute(db) |
| 139 | .await?; |
| 140 | |
| 141 | Ok(()) |
| 142 | } |
142 lines · Rust