dogfood
Jump to…
snowcorepublic2 bookmarks
snowinitial commitqoxwzsukwmkx1mo
mainroot / crates / df-render / src / highlight.rsBlameRaw
1//! Syntax highlighting (spec §8).
2//!
3//! > **Syntax highlighting** with `tree-sitter` and a curated grammar set
4//! > (roughly the top 25 languages), falling back to plain text. […] Skip files
5//! > over 1 MB or with lines over 5000 characters, and render those as plain
6//! > text with a notice.
7//!
8//! The output is a `Vec<String>` of **per-line HTML fragments**, not one blob of
9//! markup. The blob view needs a line-number gutter and anchorable line ids, and
10//! producing lines here is what lets the diff view highlight a single side of a
11//! hunk without re-parsing.
12//!
13//! Two safety properties this module is responsible for:
14//!
15//! * **Escaping.** The source is repository content, which is attacker
16//! controlled. `tree-sitter-highlight`'s `HtmlRenderer` escapes `<`, `&`, `'`
17//! and `"` in the text it emits, and every attribute this module adds is a
18//! fixed string chosen from the highlight-name table — no user data ever
19//! reaches an attribute value. `escaping_is_not_optional` pins that.
20//! * **Termination.** A pathological file must degrade, not hang the process.
21//! The size and line-length caps are checked *before* the parser runs, and a
22//! highlight error returns `None` rather than propagating.
23
24use std::collections::HashMap;
25use std::sync::LazyLock;
26
27use tree_sitter_highlight::{HighlightConfiguration, Highlighter, HtmlRenderer};
28
29/// Files above this are served as plain text (spec §8).
30pub const MAX_BYTES: usize = 1024 * 1024;
31
32/// A single line this long means the file is generated or minified. Parsing it
33/// is slow and the result is unreadable either way (spec §8).
34pub const MAX_LINE_CHARS: usize = 5_000;
35
36/// Highlight capture names, in the order the CSS classes are derived from.
37///
38/// The index into this array is what `tree-sitter-highlight` hands back, so the
39/// order is load-bearing: it must match [`CLASSES`] exactly.
40const HIGHLIGHT_NAMES: &[&str] = &[
41 "attribute",
42 "boolean",
43 "character",
44 "comment",
45 "comment.documentation",
46 "constant",
47 "constant.builtin",
48 "constructor",
49 "embedded",
50 "escape",
51 "function",
52 "function.builtin",
53 "function.method",
54 "keyword",
55 "label",
56 "module",
57 "number",
58 "operator",
59 "property",
60 "punctuation",
61 "punctuation.bracket",
62 "punctuation.delimiter",
63 "punctuation.special",
64 "string",
65 "string.escape",
66 "string.special",
67 "tag",
68 "type",
69 "type.builtin",
70 "variable",
71 "variable.builtin",
72 "variable.parameter",
73];
74
75/// The `class="…"` attribute emitted for each highlight index.
76///
77/// Precomputed as complete attribute bytes so the render callback is a slice
78/// copy rather than a format call per token — highlighting a large file emits a
79/// great many of these.
80static CLASSES: LazyLock<Vec<Vec<u8>>> = LazyLock::new(|| {
81 HIGHLIGHT_NAMES
82 .iter()
83 .map(|name| {
84 // `hl-function-builtin` also carries `hl-function`, so the stylesheet
85 // can theme a whole family and refine one member of it.
86 let mut classes = String::new();
87 let mut acc = String::new();
88 for part in name.split('.') {
89 if !acc.is_empty() {
90 acc.push('-');
91 }
92 acc.push_str(part);
93 if !classes.is_empty() {
94 classes.push(' ');
95 }
96 classes.push_str("hl-");
97 classes.push_str(&acc);
98 }
99 format!(r#"class="{classes}""#).into_bytes()
100 })
101 .collect()
102});
103
104/// A highlighted file.
105#[derive(Debug)]
106pub struct Highlighted {
107 /// The grammar that was used, for the "detected as Rust" affordance.
108 pub language: &'static str,
109 /// One HTML fragment per line, already escaped. No trailing newline.
110 pub lines: Vec<String>,
111}
112
113/// Why a file was not highlighted.
114///
115/// Distinguished from "no grammar" so the view can say *why* — a 4 MB file
116/// rendering as plain text is a decision the reader should be told about, and a
117/// `.md` file rendering plain because it is minified is worth explaining.
118#[derive(Debug, Clone, Copy, PartialEq, Eq)]
119pub enum Skipped {
120 /// Over [`MAX_BYTES`].
121 TooLarge,
122 /// Contains a line over [`MAX_LINE_CHARS`].
123 LineTooLong,
124 /// No grammar for this file type.
125 NoGrammar,
126 /// The grammar failed on this input. Rare, and never fatal.
127 Failed,
128}
129
130/// Highlight `source`, choosing a grammar from `path`.
131///
132/// Returns `Err(Skipped)` whenever the file should be shown as plain text —
133/// there is deliberately no partial result, because half a highlighted file is
134/// worse than none.
135pub fn highlight(path: &str, source: &str) -> Result<Highlighted, Skipped> {
136 if source.len() > MAX_BYTES {
137 return Err(Skipped::TooLarge);
138 }
139 if source.lines().any(|l| l.chars().count() > MAX_LINE_CHARS) {
140 return Err(Skipped::LineTooLong);
141 }
142
143 let lang = language_for_path(path).ok_or(Skipped::NoGrammar)?;
144 let config = GRAMMARS.get(lang.grammar).ok_or(Skipped::NoGrammar)?;
145
146 let mut highlighter = Highlighter::new();
147 // The injection callback resolves embedded languages (JS inside HTML, and so
148 // on). Returning `None` means "render the embedded region with the outer
149 // grammar", which is correct-but-plain rather than wrong.
150 let events = highlighter
151 .highlight(config, source.as_bytes(), None, |name| GRAMMARS.get(name))
152 .map_err(|_| Skipped::Failed)?;
153
154 let mut renderer = HtmlRenderer::new();
155 renderer
156 .render(events, source.as_bytes(), &|h, out: &mut Vec<u8>| {
157 if let Some(class) = CLASSES.get(h.0) {
158 out.extend_from_slice(class);
159 }
160 })
161 .map_err(|_| Skipped::Failed)?;
162
163 // `HtmlRenderer::lines` yields each line with a trailing newline and with
164 // every span opened on that line also closed on it, so a line is a valid
165 // standalone fragment.
166 let lines = renderer
167 .lines()
168 .map(|l| l.trim_end_matches('\n').to_owned())
169 .collect();
170
171 Ok(Highlighted { language: lang.display, lines })
172}
173
174/// A grammar we can highlight with.
175#[derive(Debug, Clone, Copy)]
176pub struct Language {
177 /// Key into [`GRAMMARS`]. Also the name injections are looked up by, so it
178 /// matches tree-sitter's own language names.
179 pub grammar: &'static str,
180 /// Human-readable, for the UI.
181 pub display: &'static str,
182}
183
184/// Pick a grammar for a path.
185///
186/// Whole filenames are checked before extensions, so `Dockerfile` and
187/// `.bashrc` resolve even though they have no useful extension.
188pub fn language_for_path(path: &str) -> Option<Language> {
189 let name = path.rsplit('/').next().unwrap_or(path);
190 let lower = name.to_ascii_lowercase();
191
192 let by_name = match lower.as_str() {
193 "makefile" | "gnumakefile" | "dockerfile" | "containerfile" | "justfile" => Some("bash"),
194 ".bashrc" | ".bash_profile" | ".profile" | ".zshrc" | ".zprofile" => Some("bash"),
195 "cargo.lock" | "gemfile.lock" => Some("toml"),
196 "gemfile" | "rakefile" | "podfile" => Some("ruby"),
197 "cmakelists.txt" => None,
198 _ => None,
199 };
200 if let Some(g) = by_name {
201 return GRAMMAR_DISPLAY
202 .iter()
203 .find(|(k, _)| *k == g)
204 .map(|(grammar, display)| Language { grammar, display });
205 }
206
207 // `.tar.gz` must not resolve as `gz`; only the final component matters.
208 let ext = lower.rsplit_once('.').map(|(_, e)| e)?;
209
210 let grammar = match ext {
211 "rs" => "rust",
212 "py" | "pyi" | "pyw" => "python",
213 "js" | "mjs" | "cjs" | "jsx" => "javascript",
214 "ts" | "mts" | "cts" => "typescript",
215 "tsx" => "tsx",
216 "go" => "go",
217 "c" | "h" => "c",
218 "cc" | "cpp" | "cxx" | "hpp" | "hh" | "hxx" => "cpp",
219 "java" => "java",
220 "rb" | "rake" | "gemspec" => "ruby",
221 "json" | "jsonc" | "webmanifest" => "json",
222 "toml" => "toml",
223 "yaml" | "yml" => "yaml",
224 "html" | "htm" | "xhtml" => "html",
225 "css" | "scss" => "css",
226 "sh" | "bash" | "zsh" | "ksh" => "bash",
227 "md" | "markdown" | "mdown" => "markdown",
228 "sql" => "sql",
229 "php" => "php",
230 "cs" => "c_sharp",
231 "scala" | "sbt" | "sc" => "scala",
232 "hs" => "haskell",
233 "swift" => "swift",
234 "ex" | "exs" => "elixir",
235 "lua" => "lua",
236 "zig" => "zig",
237 "nix" => "nix",
238 _ => return None,
239 };
240
241 GRAMMAR_DISPLAY
242 .iter()
243 .find(|(k, _)| *k == grammar)
244 .map(|(grammar, display)| Language { grammar, display })
245}
246
247/// Grammar key → display name. Also the authoritative list of what is loaded.
248const GRAMMAR_DISPLAY: &[(&str, &str)] = &[
249 ("bash", "Shell"),
250 ("c", "C"),
251 ("c_sharp", "C#"),
252 ("cpp", "C++"),
253 ("css", "CSS"),
254 ("elixir", "Elixir"),
255 ("go", "Go"),
256 ("haskell", "Haskell"),
257 ("html", "HTML"),
258 ("java", "Java"),
259 ("javascript", "JavaScript"),
260 ("json", "JSON"),
261 ("lua", "Lua"),
262 ("markdown", "Markdown"),
263 ("nix", "Nix"),
264 ("php", "PHP"),
265 ("python", "Python"),
266 ("ruby", "Ruby"),
267 ("rust", "Rust"),
268 ("scala", "Scala"),
269 ("sql", "SQL"),
270 ("swift", "Swift"),
271 ("toml", "TOML"),
272 ("tsx", "TSX"),
273 ("typescript", "TypeScript"),
274 ("yaml", "YAML"),
275 ("zig", "Zig"),
276];
277
278/// Every grammar, parsed and configured once.
279///
280/// `HighlightConfiguration::configure` is what binds capture names to the
281/// indices used by [`CLASSES`], and it mutates the configuration — so it happens
282/// here, once, rather than per request. Building these costs a few milliseconds
283/// each and the map is shared across every request thereafter.
284static GRAMMARS: LazyLock<HashMap<&'static str, HighlightConfiguration>> = LazyLock::new(|| {
285 let mut m = HashMap::new();
286
287 /// Build one configuration, skipping the grammar entirely if its queries do
288 /// not compile. A broken grammar must degrade to plain text, never panic at
289 /// the first request that touches it.
290 fn add(
291 m: &mut HashMap<&'static str, HighlightConfiguration>,
292 name: &'static str,
293 language: tree_sitter::Language,
294 highlights: &str,
295 injections: &str,
296 locals: &str,
297 ) {
298 match HighlightConfiguration::new(language, name, highlights, injections, locals) {
299 Ok(mut c) => {
300 c.configure(HIGHLIGHT_NAMES);
301 m.insert(name, c);
302 }
303 Err(e) => tracing_warn(name, &e.to_string()),
304 }
305 }
306
307 add(&mut m, "rust", tree_sitter_rust::LANGUAGE.into(),
308 tree_sitter_rust::HIGHLIGHTS_QUERY, tree_sitter_rust::INJECTIONS_QUERY, "");
309 add(&mut m, "python", tree_sitter_python::LANGUAGE.into(),
310 tree_sitter_python::HIGHLIGHTS_QUERY, "", "");
311 add(&mut m, "javascript", tree_sitter_javascript::LANGUAGE.into(),
312 tree_sitter_javascript::HIGHLIGHT_QUERY,
313 tree_sitter_javascript::INJECTIONS_QUERY,
314 tree_sitter_javascript::LOCALS_QUERY);
315 add(&mut m, "typescript", tree_sitter_typescript::LANGUAGE_TYPESCRIPT.into(),
316 // The TypeScript grammar's queries extend JavaScript's rather than
317 // repeating them, so both must be supplied or half the file is plain.
318 &format!("{}\n{}", tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY),
319 tree_sitter_javascript::INJECTIONS_QUERY,
320 tree_sitter_typescript::LOCALS_QUERY);
321 add(&mut m, "tsx", tree_sitter_typescript::LANGUAGE_TSX.into(),
322 &format!("{}\n{}", tree_sitter_javascript::HIGHLIGHT_QUERY, tree_sitter_typescript::HIGHLIGHTS_QUERY),
323 tree_sitter_javascript::INJECTIONS_QUERY,
324 tree_sitter_typescript::LOCALS_QUERY);
325 add(&mut m, "go", tree_sitter_go::LANGUAGE.into(),
326 tree_sitter_go::HIGHLIGHTS_QUERY, "", "");
327 add(&mut m, "c", tree_sitter_c::LANGUAGE.into(),
328 tree_sitter_c::HIGHLIGHT_QUERY, "", "");
329 add(&mut m, "cpp", tree_sitter_cpp::LANGUAGE.into(),
330 &format!("{}\n{}", tree_sitter_c::HIGHLIGHT_QUERY, tree_sitter_cpp::HIGHLIGHT_QUERY), "", "");
331 add(&mut m, "java", tree_sitter_java::LANGUAGE.into(),
332 tree_sitter_java::HIGHLIGHTS_QUERY, "", "");
333 add(&mut m, "ruby", tree_sitter_ruby::LANGUAGE.into(),
334 tree_sitter_ruby::HIGHLIGHTS_QUERY, "", tree_sitter_ruby::LOCALS_QUERY);
335 add(&mut m, "json", tree_sitter_json::LANGUAGE.into(),
336 tree_sitter_json::HIGHLIGHTS_QUERY, "", "");
337 add(&mut m, "toml", tree_sitter_toml_ng::LANGUAGE.into(),
338 tree_sitter_toml_ng::HIGHLIGHTS_QUERY, "", "");
339 add(&mut m, "yaml", tree_sitter_yaml::LANGUAGE.into(),
340 tree_sitter_yaml::HIGHLIGHTS_QUERY, "", "");
341 add(&mut m, "html", tree_sitter_html::LANGUAGE.into(),
342 tree_sitter_html::HIGHLIGHTS_QUERY, tree_sitter_html::INJECTIONS_QUERY, "");
343 add(&mut m, "css", tree_sitter_css::LANGUAGE.into(),
344 tree_sitter_css::HIGHLIGHTS_QUERY, "", "");
345 add(&mut m, "bash", tree_sitter_bash::LANGUAGE.into(),
346 tree_sitter_bash::HIGHLIGHT_QUERY, "", "");
347 add(&mut m, "markdown", tree_sitter_md::LANGUAGE.into(),
348 tree_sitter_md::HIGHLIGHT_QUERY_BLOCK, tree_sitter_md::INJECTION_QUERY_BLOCK, "");
349 add(&mut m, "sql", tree_sitter_sequel::LANGUAGE.into(),
350 tree_sitter_sequel::HIGHLIGHTS_QUERY, "", "");
351 add(&mut m, "php", tree_sitter_php::LANGUAGE_PHP.into(),
352 tree_sitter_php::HIGHLIGHTS_QUERY, tree_sitter_php::INJECTIONS_QUERY, "");
353 add(&mut m, "c_sharp", tree_sitter_c_sharp::LANGUAGE.into(),
354 tree_sitter_c_sharp::HIGHLIGHTS_QUERY, "", "");
355 add(&mut m, "scala", tree_sitter_scala::LANGUAGE.into(),
356 tree_sitter_scala::HIGHLIGHTS_QUERY, "", tree_sitter_scala::LOCALS_QUERY);
357 add(&mut m, "haskell", tree_sitter_haskell::LANGUAGE.into(),
358 tree_sitter_haskell::HIGHLIGHTS_QUERY, tree_sitter_haskell::INJECTIONS_QUERY,
359 tree_sitter_haskell::LOCALS_QUERY);
360 add(&mut m, "swift", tree_sitter_swift::LANGUAGE.into(),
361 tree_sitter_swift::HIGHLIGHTS_QUERY, tree_sitter_swift::INJECTIONS_QUERY,
362 tree_sitter_swift::LOCALS_QUERY);
363 add(&mut m, "elixir", tree_sitter_elixir::LANGUAGE.into(),
364 tree_sitter_elixir::HIGHLIGHTS_QUERY, tree_sitter_elixir::INJECTIONS_QUERY, "");
365 add(&mut m, "lua", tree_sitter_lua::LANGUAGE.into(),
366 tree_sitter_lua::HIGHLIGHTS_QUERY, tree_sitter_lua::INJECTIONS_QUERY,
367 tree_sitter_lua::LOCALS_QUERY);
368 add(&mut m, "zig", tree_sitter_zig::LANGUAGE.into(),
369 tree_sitter_zig::HIGHLIGHTS_QUERY, tree_sitter_zig::INJECTIONS_QUERY, "");
370 add(&mut m, "nix", tree_sitter_nix::LANGUAGE.into(),
371 tree_sitter_nix::HIGHLIGHTS_QUERY, tree_sitter_nix::INJECTIONS_QUERY, "");
372
373 m
374});
375
376/// `df-render` deliberately has no `tracing` dependency — it is a pure
377/// rendering crate — so a grammar that fails to load reports on stderr.
378/// This only ever runs once per process, at first use.
379fn tracing_warn(name: &str, message: &str) {
380 eprintln!("df-render: grammar {name} failed to load, files will render as plain text: {message}");
381}
382
383#[cfg(test)]
384mod tests {
385 use super::*;
386
387 #[test]
388 fn highlights_rust_and_splits_by_line() {
389 let out = highlight("src/main.rs", "fn main() {\n let x = 1;\n}\n").unwrap();
390 assert_eq!(out.language, "Rust");
391 assert_eq!(out.lines.len(), 3, "one fragment per source line");
392 assert!(out.lines[0].contains("hl-keyword"), "`fn` should be a keyword: {:?}", out.lines[0]);
393 // The Rust grammar captures integer literals as `constant.builtin`, not
394 // `number`. Asserting the family class rather than the leaf keeps this
395 // test about "the literal got classified" instead of about upstream's
396 // choice of capture name.
397 assert!(
398 out.lines[1].contains("hl-constant"),
399 "`1` should be classified: {:?}",
400 out.lines[1]
401 );
402 }
403
404 /// The security-critical property: repository content is attacker
405 /// controlled, and it reaches a page that renders on our origin.
406 #[test]
407 fn escaping_is_not_optional() {
408 let evil = "let s = \"</span><script>alert(1)</script>\";\n";
409 let out = highlight("x.rs", evil).unwrap();
410 let joined = out.lines.join("\n");
411 assert!(
412 !joined.contains("<script>"),
413 "raw markup escaped from the highlighter: {joined}"
414 );
415 assert!(joined.contains("&lt;script&gt;"), "expected escaped form: {joined}");
416 }
417
418 /// Even with no grammar the caller must be able to tell "plain" from
419 /// "broken", so every rejection carries a reason.
420 #[test]
421 fn unknown_extensions_fall_back_rather_than_erroring() {
422 assert_eq!(highlight("a.unknownext", "hello").unwrap_err(), Skipped::NoGrammar);
423 assert_eq!(highlight("noextension", "hello").unwrap_err(), Skipped::NoGrammar);
424 }
425
426 #[test]
427 fn the_spec_size_limits_are_enforced_before_parsing() {
428 let big = "a\n".repeat(MAX_BYTES);
429 assert_eq!(highlight("a.rs", &big).unwrap_err(), Skipped::TooLarge);
430
431 let minified = format!("let x = \"{}\";\n", "y".repeat(MAX_LINE_CHARS + 1));
432 assert_eq!(highlight("a.js", &minified).unwrap_err(), Skipped::LineTooLong);
433 }
434
435 #[test]
436 fn class_names_carry_the_whole_family() {
437 let idx = HIGHLIGHT_NAMES.iter().position(|n| *n == "function.builtin").unwrap();
438 let class = String::from_utf8(CLASSES[idx].clone()).unwrap();
439 assert_eq!(class, r#"class="hl-function hl-function-builtin""#);
440 }
441
442 #[test]
443 fn filenames_without_a_useful_extension_still_resolve() {
444 assert_eq!(language_for_path("Dockerfile").unwrap().display, "Shell");
445 assert_eq!(language_for_path("a/b/Makefile").unwrap().display, "Shell");
446 assert_eq!(language_for_path("Cargo.lock").unwrap().display, "TOML");
447 // Only the final extension counts.
448 assert!(language_for_path("archive.tar.gz").is_none());
449 assert_eq!(language_for_path("a.tar.rs").unwrap().display, "Rust");
450 }
451
452 /// Every grammar in the display table must actually have loaded. A typo in
453 /// a query, or an upstream grammar that stops compiling its queries against
454 /// the pinned `tree-sitter`, silently turns a language plain — this is the
455 /// test that notices.
456 #[test]
457 fn every_advertised_grammar_loads() {
458 for (key, display) in GRAMMAR_DISPLAY {
459 assert!(
460 GRAMMARS.contains_key(key),
461 "grammar {key} ({display}) is advertised but did not load"
462 );
463 }
464 assert!(GRAMMARS.len() >= 25, "spec §8 asks for roughly the top 25 languages");
465 }
466
467 /// A file that ends without a trailing newline must not lose its last line.
468 #[test]
469 fn a_missing_trailing_newline_does_not_drop_a_line() {
470 let out = highlight("a.rs", "fn a() {}\nfn b() {}").unwrap();
471 assert_eq!(out.lines.len(), 2);
472 assert!(out.lines[1].contains("b"));
473 }
474
475 #[test]
476 fn an_empty_file_highlights_to_nothing() {
477 let out = highlight("a.rs", "").unwrap();
478 assert!(out.lines.is_empty() || out.lines == vec![""]);
479 }
480}

480 lines · Rust