typescript 68 lines · 10 steps

How a regex tokenizer highlights code

A small lexer walks source text with ordered regex rules, then wraps each token in colored HTML spans.

Explained by highlit

Intermediate tokenizer regex lexing syntax-highlighting state-machine

1type TokenType = "keyword" | "string" | "comment" | "number" | "text";
2 
3interface Token {
4  type: TokenType;
5  value: string;
6}
7 
8const KEYWORDS = new Set([
9  "const", "let", "var", "function", "return", "if", "else",
10  "for", "while", "class", "import", "export", "new", "typeof",
11]);
12 
13const RULES: Array<{ type: TokenType; pattern: RegExp }> = [
14  { type: "comment", pattern: /^\/\/[^\n]*|^\/\*[\s\S]*?\*\// },
15  { type: "string", pattern: /^"(?:\\.|[^"\\])*"|^'(?:\\.|[^'\\])*'|^`(?:\\.|[^`\\])*`/ },
16  { type: "number", pattern: /^\d+(?:\.\d+)?/ },
17  { type: "text", pattern: /^[A-Za-z_$][\w$]*/ },
18  { type: "text", pattern: /^\s+/ },
19  { type: "text", pattern: /^[^\w\s]/ },
20];
21 
22export function tokenize(source: string): Token[] {
23  const tokens: Token[] = [];
24  let rest = source;
25 
26  while (rest.length > 0) {
27    let matched = false;
28 
29    for (const { type, pattern } of RULES) {
30      const match = pattern.exec(rest);
31      if (!match) continue;
32 
33      const value = match[0];
34      const resolved: TokenType =
35        type === "text" && KEYWORDS.has(value) ? "keyword" : type;
36 
37      tokens.push({ type: resolved, value });
38      rest = rest.slice(value.length);
39      matched = true;
40      break;
41    }
42 
43    if (!matched) {
44      tokens.push({ type: "text", value: rest[0] });
45      rest = rest.slice(1);
46    }
47  }
48 
49  return tokens;
50}
51 
52const COLORS: Record<TokenType, string> = {
53  keyword: "#c678dd",
54  string: "#98c379",
55  comment: "#5c6370",
56  number: "#d19a66",
57  text: "#abb2bf",
58};
59 
60export function highlight(source: string): string {
61  return tokenize(source)
62    .map(({ type, value }) => {
63      const escaped = value.replace(/&/g, "&amp;").replace(/</g, "&lt;");
64      if (type === "text") return escaped;
65      return `<span style="color:${COLORS[type]}">${escaped}</span>`;
66    })
67    .join("");
68}

01 / 01

STEP 01

Walkthrough

Space play ←→ step click any line

Three takeaways

1Ordering rules so the most specific patterns match first is what keeps a greedy lexer correct.
2Anchoring every pattern to the start of the remaining input lets you consume the string one token at a time.
3Separating tokenizing from rendering keeps the lexer reusable for outputs other than HTML.

Related explainers

typescript

import { Component, computed, DestroyRef, inject, input, output, signal } from '@angular/core';
import { takeUntilDestroyed } from '@angular/core/rxjs-interop';
import { interval, map, takeWhile } from 'rxjs';

How a signal-driven countdown works in Angular

signals reactivity rxjs

Intermediate 8 steps

rust

use std::time::Duration;
 
#[derive(Debug, PartialEq)]
pub enum ParseDurationError {

Parsing duration strings safely in Rust

parsing error-handling checked-arithmetic

Intermediate 8 steps

package logparse
 
import (
	"bufio"

Splitting multi-line logs with a Scanner

parsing streaming bufio

Intermediate 9 steps

java

public final class LogRedactor {
 
    private static final Pattern SECRET = Pattern.compile(
        "(?i)(password|token|api[_-]?key|secret|authorization)\\s*[=:]\\s*\\S+");

Streaming log redaction in Java

regex streaming-io try-with-resources

Intermediate 9 steps

package handlers
 
import (
	"net/http"

Custom validators and binding in Gin

validation struct-tags error-handling

Intermediate 8 steps

typescript

import { Component, inject, signal } from '@angular/core';
import { CdkDragDrop, DragDropModule, moveItemInArray } from '@angular/cdk/drag-drop';
import { HttpClient } from '@angular/common/http';
import { finalize } from 'rxjs';