javascript 50 lines · 8 steps

Building a syntax highlighter tokenizer

A prioritized list of anchored regexes turns source code into typed tokens, then wraps each in a styled span.

Explained by highlit
Intermediate tokenizer regular-expressions lexing syntax-highlighting html-escaping
1const TOKEN_SPECS = [
2 ["comment", /^\/\/[^\n]*|^\/\*[\s\S]*?\*\//],
3 ["string", /^"(?:\\.|[^"\\])*"|^'(?:\\.|[^'\\])*'|^`(?:\\.|[^`\\])*`/],
4 ["number", /^0[xX][\da-fA-F]+|^\d+(?:\.\d+)?(?:[eE][+-]?\d+)?/],
5 ["keyword", /^\b(?:const|let|var|function|return|if|else|for|while|class|new|import|export|from|await|async)\b/],
6 ["boolean", /^\b(?:true|false|null|undefined)\b/],
7 ["identifier", /^[A-Za-z_$][\w$]*/],
8 ["operator", /^[+\-*/%=<>!&|^~?:]+/],
9 ["punctuation", /^[{}()\[\];,.]/],
10 ["whitespace", /^\s+/],
11];
12 
13const ESCAPES = { "&": "&amp;", "<": "&lt;", ">": "&gt;" };
14const escapeHtml = (str) => str.replace(/[&<>]/g, (c) => ESCAPES[c]);
15 
16function tokenize(source) {
17 const tokens = [];
18 let rest = source;
19 
20 while (rest.length > 0) {
21 let matched = false;
22 
23 for (const [type, pattern] of TOKEN_SPECS) {
24 const match = pattern.exec(rest);
25 if (!match) continue;
26 
27 tokens.push({ type, value: match[0] });
28 rest = rest.slice(match[0].length);
29 matched = true;
30 break;
31 }
32 
33 if (!matched) {
34 tokens.push({ type: "unknown", value: rest[0] });
35 rest = rest.slice(1);
36 }
37 }
38 
39 return tokens;
40}
41 
42function highlight(source) {
43 return tokenize(source)
44 .map(({ type, value }) =>
45 type === "whitespace"
46 ? value
47 : `<span class="tok-${type}">${escapeHtml(value)}</span>`
48 )
49 .join("");
50}
01 / 01
STEP 01

Walkthrough

Space play ←→ step click any line
Three takeaways
  1. 1Ordering token rules by specificity lets earlier, greedier patterns win before broader ones can misclassify input.
  2. 2Anchoring every regex with ^ and slicing the matched length turns pattern matching into a forward-consuming scanner.
  3. 3Always escape user text before injecting it into HTML, even inside a display-only tool.

Related explainers

javascript 
import { NextResponse } from 'next/server';
 
const locales = ['en', 'fr', 'de', 'es'];
const defaultLocale = 'en';

Locale routing with Next.js middleware

middleware i18n content-negotiation
Intermediate 10 steps
javascript 
const express = require('express');
const cookieParser = require('cookie-parser');
 
const router = express.Router();

Remember-me login with signed cookies in Express

authentication signed-cookies sessions
Intermediate 9 steps
javascript 
export async function compressImage(file, { maxWidth = 1600, maxHeight = 1600, quality = 0.8, mimeType = 'image/jpeg' } = {}) {
  const bitmap = await createImageBitmap(file);
 
  let { width, height } = bitmap;

Compressing images in the browser with canvas

canvas image-processing promises
Intermediate 7 steps
javascript 
const express = require('express');
const multer = require('multer');
const path = require('path');
const crypto = require('crypto');

Safe image uploads with Multer in Express

file-upload multer validation
Intermediate 7 steps
javascript 
const express = require('express');
const jwt = require('jsonwebtoken');
const crypto = require('crypto');
 

Refresh token rotation in Express

jwt token-rotation authentication
Advanced 9 steps
javascript 
class StarRating {
  constructor(container, { max = 5, value = 0, onChange } = {}) {
    this.container = container;
    this.max = max;

Building an accessible star-rating widget

dom event-handling accessibility
Intermediate 7 steps

Share this explainer

Here's the card — post it anywhere.

Building a syntax highlighter tokenizer — share card
Made with highlit — turn any snippet into a walkthrough like this in about a minute.
Explain your code