snow/core · Dogfood

snowinitial commitqoxwzsukwmkx1mo

mainroot / crates / df-web / src / highlight_cache.rsHide Blame Raw

Matt W	1	//! Cached syntax highlighting (spec §8).
Matt W	2	//!
Matt W	3	//! > Highlight in the worker and cache the rendered HTML in Postgres keyed by
Matt W	4	//! > blob OID — highlighting the same file on every request is the easiest
Matt W	5	//! > performance mistake to make here.
Matt W	6	//!
Matt W	7	//! Deviation, deliberate. Highlighting happens here on a cache miss rather
Matt W	8	//! than only in the worker. Precomputing in the worker means the first view of
Matt W	9	//! any file renders unhighlighted, which is the view that matters most after a
Matt W	10	//! push. The property the spec is actually protecting — never highlight the same
Matt W	11	//! bytes twice — is provided by the cache, which is what this module is. The
Matt W	12	//! parse itself runs on a blocking thread so a large file cannot stall the
Matt W	13	//! async runtime.
Matt W	14	//!
Matt W	15	//! The cache key is the blob's content address, so it is shared across
Matt W	16	//! repositories and across revisions: a file that did not change between two
Matt W	17	//! revisions is highlighted once, ever. Because the key is a pure function of
Matt W	18	//! the bytes, there is no invalidation problem — different content is a
Matt W	19	//! different key.
Matt W	20
Matt W	21	use df_render::highlight::{self, Skipped};
Matt W	22	use df_store::Blob;
Matt W	23	use sqlx::PgPool;
Matt W	24
Matt W	25	/// Highlighted lines, or the reason there are none.
Matt W	26	pub struct Rendered {
Matt W	27	/// One HTML fragment per line. Empty when `skipped` is set.
Matt W	28	pub lines: Vec<String>,
Matt W	29	pub language: Option<String>,
Matt W	30	/// Set when the file is to be rendered as plain text, with the reason.
Matt W	31	pub skipped: Option<Skipped>,
Matt W	32	}
Matt W	33
Matt W	34	impl Rendered {
Matt W	35	fn plain(reason: Skipped) -> Self {
Matt W	36	Rendered { lines: Vec::new(), language: None, skipped: Some(reason) }
Matt W	37	}
Matt W	38	}
Matt W	39
Matt W	40	/// Highlight a blob, consulting and populating the Postgres cache.
Matt W	41	///
Matt W	42	/// Never fails: a database error degrades to highlighting without the cache,
Matt W	43	/// and a highlighting failure degrades to plain text. A syntax colour is not
Matt W	44	/// worth a 500.
Matt W	45	pub async fn render(db: &PgPool, blob: &Blob) -> Rendered {
Matt W	46	let Some(text) = blob.text() else {
Matt W	47	return Rendered::plain(Skipped::NoGrammar);
Matt W	48	};
Matt W	49
Matt W	50	// No grammar means nothing to cache — decided from the path alone, so this
Matt W	51	// costs one table lookup and avoids a pointless database round trip for
Matt W	52	// every LICENSE and .gitignore in the tree.
Matt W	53	let Some(lang) = highlight::language_for_path(&blob.path) else {
Matt W	54	return Rendered::plain(Skipped::NoGrammar);
Matt W	55	};
Matt W	56
Matt W	57	match load(db, &blob.content_id).await {
Matt W	58	Ok(Some(hit)) => return hit,
Matt W	59	Ok(None) => {}
Matt W	60	Err(e) => tracing::warn!("highlight cache read failed, rendering uncached: {e}"),
Matt W	61	}
Matt W	62
Matt W	63	// tree-sitter is CPU-bound and this is an async handler; parsing a megabyte
Matt W	64	// on the runtime thread would stall every other request on that worker.
Matt W	65	let path = blob.path.clone();
Matt W	66	let source = text.to_owned();
Matt W	67	let highlighted =
Matt W	68	match tokio::task::spawn_blocking(move \|\| highlight::highlight(&path, &source)).await {
Matt W	69	Ok(r) => r,
Matt W	70	Err(e) => {
Matt W	71	tracing::error!("highlighting panicked: {e}");
Matt W	72	return Rendered::plain(Skipped::Failed);
Matt W	73	}
Matt W	74	};
Matt W	75
Matt W	76	let out = match highlighted {
Matt W	77	Ok(h) => Rendered {
Matt W	78	lines: h.lines,
Matt W	79	language: Some(h.language.to_string()),
Matt W	80	skipped: None,
Matt W	81	},
Matt W	82	Err(reason) => Rendered::plain(reason),
Matt W	83	};
Matt W	84
Matt W	85	// Only successful renders are cached. Caching a rejection would save
Matt W	86	// nothing: every rejection is decided from the path or the size, both of
Matt W	87	// which are already known before the lookup.
Matt W	88	if out.skipped.is_none() {
Matt W	89	if let Err(e) = store(db, &blob.content_id, lang.display, &out.lines).await {
Matt W	90	tracing::warn!("highlight cache write failed: {e}");
Matt W	91	}
Matt W	92	}
Matt W	93
Matt W	94	out
Matt W	95	}
Matt W	96
Matt W	97	/// The separator between cached line fragments.
Matt W	98	///
Matt W	99	/// A form feed rather than a newline: the HTML fragments themselves are free of
Matt W	100	/// newlines, but a fragment could in principle contain one from a multi-line
Matt W	101	/// token, and splitting on the wrong character would silently corrupt the line
Matt W	102	/// numbering. U+000C cannot appear in the renderer's output.
Matt W	103	const LINE_SEP: char = '\u{c}';
Matt W	104
Matt W	105	async fn load(db: &PgPool, content_id: &str) -> sqlx::Result<Option<Rendered>> {
Matt W	106	let row: Option<(Option<String>, String)> =
Matt W	107	sqlx::query_as("SELECT language, html FROM highlight_cache WHERE blob_oid = $1")
Matt W	108	.bind(content_id)
Matt W	109	.fetch_optional(db)
Matt W	110	.await?;
Matt W	111
Matt W	112	Ok(row.map(\|(language, html)\| Rendered {
Matt W	113	lines: if html.is_empty() {
Matt W	114	Vec::new()
Matt W	115	} else {
Matt W	116	html.split(LINE_SEP).map(str::to_owned).collect()
Matt W	117	},
Matt W	118	language,
Matt W	119	skipped: None,
Matt W	120	}))
Matt W	121	}
Matt W	122
Matt W	123	async fn store(db: &PgPool, content_id: &str, language: &str, lines: &[String]) -> sqlx::Result<()> {
Matt W	124	let html = lines.join(&LINE_SEP.to_string());
Matt W	125
Matt W	126	// ON CONFLICT DO NOTHING: two requests racing on the same blob both compute
Matt W	127	// the same bytes, so whichever lands first is correct and the other is a
Matt W	128	// no-op rather than a lock wait.
Matt W	129	sqlx::query(
Matt W	130	"INSERT INTO highlight_cache (blob_oid, language, html, bytes)
Matt W	131	VALUES ($1, $2, $3, $4)
Matt W	132	ON CONFLICT (blob_oid) DO NOTHING",
Matt W	133	)
Matt W	134	.bind(content_id)
Matt W	135	.bind(language)
Matt W	136	.bind(&html)
Matt W	137	.bind(html.len() as i32)
Matt W	138	.execute(db)
Matt W	139	.await?;
Matt W	140
Matt W	141	Ok(())
Matt W	142	}

142 lines · Rust