Revamp regex scanning [publish]

.gitignore

@@ -1,3 +1,4 @@
 node_modules/
 dist/
 .eslintcache
+local-bench/

.prettierignore

@@ -3,3 +3,4 @@ dist/
 tests/snapshots/
 bench/source
 playground/output.css
+local-bench/out.css

CHANGELOG.md

@@ -2,6 +2,8 @@
 
 ## Unreleased
 
+## 0.7.0
+
 ### Remove dependency on LightningCSS
 
 This was here mostly because I wanted to get CSS modules and Tailwind working with esbuild. Now that esbuild support CSS modules, I can remove this coupling and makes this repo easier to re-use in other bundlers. This also mean I'm dropping from this repo features related to build tools, like `downwind.transform`, `cssModuleToJS` & `convertTargets`. The rest of the core downwind object API has also been updated to give more flexibility outside of built-in plugins.
@@ -34,9 +36,17 @@ This is not perfect, but I think that the cost of waiting few hundred millisecon
 
 There was a involuntary mismatch with Tailwind when applying the important modifier (`!`) with a variant, the implementation required to use `!hover:font-medium` instead of `hover:!font-medium`. This has been changed to match Tailwind syntax.
 
-### Support `/` in arbitrary values
+### Scanning update
+
+The read of https://marvinh.dev/blog/speeding-up-javascript-ecosystem-part-8/ make me realize the current regex approach was really inefficient: instead of search for "strings" and then for classes inside strings, we should directly grep all patterns that looks like a Tailwind classes, and by having a strict subset on the left border (``'"`\s}``), the first character (`a-z0-9![-`), the last char (`a-z0-9%]`) and the right border (``'"`\s:$``), we can get a good ratio of matches. This means that custom utils & shortcuts should be at least 2 chars.
+
+This change uses a lookbehind assertion for the left border, which means that if a playground was made, it would not work with Safari before 16.4.
+
+This new approach also allow for quotes inside custom values, which makes `before:content-['hello_world']` & `[&[data-selected="true"]]:bg-blue-100` now possible in downwind.
+
+The parser has been modified to parse arbitrary values before modifiers (opacity, line height) so that `/` can be used inside arbitrary values. `text-[calc(3rem/5)]/[calc(4rem/5)]` is now supported.
 
-The parser has been modifier to parse arbitrary values before modifiers (opacity, line height) so that `/` can be used inside arbitrary values. `text-[calc(3rem/5)]/[calc(4rem/5)]` is now supported.
+And the nice part is that it's also quite fast. When running (mac M1) on 281 tsx files of my production codebase, time running regex went from `90ms` to `13ms` (and 125543 to 33992 candidates). For the total time (init, scan & CSS generation), the time went from `117ms` to `36ms`. The perf update is completely crushed by the 'Interval check' change, but this is nice to see that a new approach with less limitations is also faster!
 
 ## 0.6.2
 

README.md

@@ -117,7 +117,6 @@ The implementation would work most of the time, but some shortcuts have been mad
 - `backgroundImage`, `backgroundPosition` and `fontFamily` are not supported
 - For prefix with collision (divide, border, bg, gradient steps, stroke, text, decoration, outline, ring, ring-offset), if the value doesn't match a CSS color (hex, rgb\[a], hsl\[a]) or a CSS variable it's interpreted as the "size" version. Using data types is not supported
 - Underscore are always mapped to space
-- Values with quotes are not possible (by design for fast scanning)
 - The theme function is not supported
 
 [Arbitrary properties](https://tailwindcss.com/docs/adding-custom-styles#arbitrary-properties) can be used to bypass some edge cases.
@@ -268,8 +267,13 @@ Then an object with few methods is returned:
     - If the token start `[`, looks for next `]:` and add the content as arbitrary variant. If no `]:`, test if it's an arbitrary value (`[color:red]`).
     - else search `:` and use the prefix to search in the variant map
   - Test if the remaining token is part of the static rules
-  - Search for `-[.+]` and then for the prefix possible arbitrary values maps
-  - If the token ends roughly `/\d+` or `/[.+]`, parse the end as a modifier
+  - Search for `-[`
+    - if matchs:
+      - search for the prefix in the arbitrary values maps, if not bail out
+      - search for `]/`
+        - if matchs, parse the left as arbitrary value and thr right as modifier
+        - else if ends with `]`, parse the left as arbitrary value
+    - else search for `/`, parse search for the left in the static rules map and parse the end as a modifier
 
 If the token matches a rule and is new it's added to an internal map structure by media queries. `true` is returned and this can be used to invalidate utils in developments.
 

package.json

@@ -3,7 +3,7 @@
   "description": "A bundler-first & PostCSS-independent implementation of Tailwind",
   "private": true,
   "type": "module",
-  "version": "0.6.2",
+  "version": "0.7.0",
   "author": "Arnaud Barré (https://github.com/ArnaudBarre)",
   "license": "MIT",
   "scripts": {
@@ -28,7 +28,7 @@
     "@arnaud-barre/config-loader": "^0.7.1"
   },
   "devDependencies": {
-    "@arnaud-barre/eslint-config": "^3.1.3",
+    "@arnaud-barre/eslint-config": "^3.1.4",
     "@arnaud-barre/prettier-plugin-sort-imports": "^0.1.2",
     "@arnaud-barre/tnode": "^0.19.2",
     "@types/node": "^20.8.10",

src/getEntries.ts

@@ -1,5 +1,4 @@
-import { getRules, type Rule, type Shortcut } from "./getRules.ts";
-import type { ResolvedConfig } from "./resolveConfig.ts";
+import type { Rule, Shortcut } from "./getRules.ts";
 import type {
   DirectionThemeRule,
   RuleMeta,
@@ -32,8 +31,7 @@ type ArbitraryEntry = {
 
 const allowNegativeRE = /^[1-9]|^0\./;
 
-export const getEntries = (config: ResolvedConfig) => {
-  const rules = getRules(config);
+export const getEntries = (rules: Rule[]) => {
   const rulesEntries = new Map<string, RuleEntry & { isArbitrary: false }>();
   const arbitraryEntries = new Map<string, ArbitraryEntry[]>();
   let order = 0;

src/index.ts

@@ -9,7 +9,7 @@ import {
   isThemeRule,
   type RuleEntry,
 } from "./getEntries.ts";
-import type { Rule } from "./getRules.ts";
+import { getRules, type Rule } from "./getRules.ts";
 import { resolveConfig } from "./resolveConfig.ts";
 import type {
   CSSEntries,
@@ -24,11 +24,11 @@ import {
   applyVariants,
   arbitraryPropertyMatchToLine,
   cssEntriesToLines,
-  escapeSelector,
   printBlock,
   printContainerClass,
   printScreenContainer,
 } from "./utils/print.ts";
+import { escapeSelector, selectorRE } from "./utils/regex.ts";
 import { themeGet } from "./utils/themeGet.ts";
 import { getVariants, type Variant } from "./variants.ts";
 
@@ -37,8 +37,6 @@ export const VERSION = __VERSION__;
 const applyRE = /[{\s]@apply ([^;}\n]+)([;}\n])/g;
 const screenRE = /screen\(([a-z-]+)\)/g;
 const themeRE = /theme\("([^)]+)"\)/g;
-const validSelectorRE = /^[a-z0-9.:/_[\]!#,%&>+~*()@-]+$/;
-const arbitraryPropertyRE = /^\[[^[\]:]+:[^[\]:]+]$/;
 
 type Match = {
   token: string;
@@ -71,7 +69,8 @@ export const initDownwindWithConfig = ({
   const config = resolveConfig(userConfig);
   const defaults = getDefaults(config);
   const variantsMap = getVariants(config);
-  const { rulesEntries, arbitraryEntries } = getEntries(config);
+  const rules = getRules(config);
+  const { rulesEntries, arbitraryEntries } = getEntries(rules);
 
   const usedKeyframes = new Set<string>();
   const usedDefaults = new Set<Default>();
@@ -158,8 +157,8 @@ export const initDownwindWithConfig = ({
   };
 
   const parseCache = new Map<string, Match>();
-  const parse = (token: string, skipBlockList?: boolean): Match | undefined => {
-    if (!skipBlockList && blockList.has(token)) return;
+  const parse = (token: string): Match | undefined => {
+    if (blockList.has(token)) return;
     const cachedValue = parseCache.get(token);
     if (cachedValue) return cachedValue;
 
@@ -172,14 +171,15 @@ export const initDownwindWithConfig = ({
       important = tokenWithoutVariants.startsWith("!");
       if (important) tokenWithoutVariants = tokenWithoutVariants.slice(1);
       if (tokenWithoutVariants.startsWith("[")) {
-        if (arbitraryPropertyRE.test(tokenWithoutVariants)) {
+        const index = tokenWithoutVariants.indexOf("]:");
+        if (index === -1) {
+          // This is not a custom variant. Test
           isArbitraryProperty = true;
           return "NO_VARIANT";
         } else if (important) {
-          return; // Using ! prefix is not valid for variant
+          // Using ! prefix is not valid for variant
+          return;
         }
-        const index = tokenWithoutVariants.indexOf("]:");
-        if (index === -1) return;
         const content = tokenWithoutVariants.slice(1, index);
         tokenWithoutVariants = tokenWithoutVariants.slice(index + 2);
         if (content.includes("&")) {
@@ -368,7 +368,7 @@ export const initDownwindWithConfig = ({
     let invalidateUtils = false;
     for (const token of tokens.split(" ")) {
       if (!token) continue;
-      const match = parse(token, true);
+      const match = parse(token);
       if (!match) {
         throw new DownwindError(`No rule matching "${token}"`, context);
       }
@@ -418,9 +418,12 @@ export const initDownwindWithConfig = ({
   };
 
   for (const token of config.safelist) {
-    const match = parse(token, true);
+    const match = parse(token);
     if (!match) {
-      throw new Error(`downwind: No rule matching "${token}" in safelist`);
+      throw new DownwindError(
+        `No rule matching "${token}" in safelist`,
+        JSON.stringify({ safelist: config.safelist }),
+      );
     }
     addMatch(match);
   }
@@ -487,11 +490,8 @@ export const initDownwindWithConfig = ({
       return { code: content, invalidateUtils };
     },
     scan: (code: string): boolean /* hasNewUtils */ => {
-      const tokens = code
-        .split(/[\s'"`;=]+/g)
-        .filter((t) => validSelectorRE.test(t) && !blockList.has(t));
       let hasNewUtils = false;
-      for (const token of tokens) {
+      for (const [token] of code.matchAll(selectorRE)) {
         const match = parse(token);
         if (match && addMatch(match)) hasNewUtils = true;
       }

src/utils/print.ts

@@ -2,9 +2,6 @@ import type { ResolvedConfig } from "../resolveConfig.ts";
 import type { Container, CSSEntries, RuleMeta } from "../types.d.ts";
 import type { Variant } from "../variants.ts";
 
-export const escapeSelector = (selector: string) =>
-  selector.replaceAll(/[.:/[\]!#,%&>+~*@()]/g, (c) => `\\${c}`);
-
 export const printBlock = (
   selector: string,
   lines: string[],

src/utils/regex.ts

@@ -0,0 +1,65 @@
+export const escapeSelector = (selector: string) => {
+  const escaped = selector.replaceAll(
+    // Keep in sync allowed chars below
+    /[.:/[\]!#='",%&>+~*@()]/g,
+    (c) => `\\${c}`,
+  );
+  return /^\d/.test(escaped) ? `\\${escaped}` : escaped;
+};
+
+const regularVariant = /[a-z0-9][a-z0-9-]+/;
+const dynamicVariant = /[a-z]+-\[[a-z:-]+]/;
+// & to position the selector
+// []=" for attributes: [&[type="email"]]
+// :>+*~.()_ for css selectors: [&:nth-child(3)] [&_p] [&>*] [.sidebar+&]
+// @ for media: [@media(min-width:900px)]
+const arbitraryVariant = /\[[a-z0-9&[\]=":>+*~.()_@-]+]/;
+const variant = new RegExp(
+  `(?:${regularVariant.source}|${dynamicVariant.source}|${arbitraryVariant.source}):`,
+);
+const variants = new RegExp(`(?:${variant.source})*`);
+
+// Opacity/line-height modifiers
+const regularModifier = /[a-z0-9]+/;
+// .% for opacity
+// ()+*/- for calc (line height)
+const arbitraryModifier = /\[[a-z0-9.%()+*/-]+]/;
+const modifier = new RegExp(
+  `/(?:${regularModifier.source}|${arbitraryModifier.source})`,
+);
+
+// % linear-background: via-40%
+// Requires at least 3 chars, only a constraint for custom utils
+const regularUtilities = /[a-z][a-z0-9-]*[a-z0-9%]/;
+// # for color
+// . for opacity
+// _, for linear-background
+// ' for content (before/after)
+// % for size
+// ()+*/- for calc
+const arbitraryValueSet = /[a-z0-9#._,'%()+*/-]+/;
+const arbitraryValues = new RegExp(
+  `[a-z][a-z-]*-\\[${arbitraryValueSet.source}]`,
+);
+
+const ruleBasedContent = new RegExp(
+  `-?(?:${regularUtilities.source}|${arbitraryValues.source})(?:${modifier.source})?`,
+);
+
+const arbitraryProperties = new RegExp(
+  `\\[[a-z][a-z-]+:${arbitraryValueSet.source}]`,
+);
+const selectorREWithoutBorders = new RegExp(
+  `${variants.source}!?(?:${ruleBasedContent.source}|${arbitraryProperties.source})`,
+);
+
+// } for template string: `${base}text-lg` (questionnable)
+const leftBorder = /(?<=['"`\s}])/;
+// : for object keys
+// $ for template string: `text-lg${base}` (questionnable)
+const rightBorder = /(?=['"`\s:$])/;
+
+export const selectorRE = new RegExp(
+  `${leftBorder.source}${selectorREWithoutBorders.source}${rightBorder.source}`,
+  "g",
+);

tests/generate.test.ts

@@ -27,7 +27,7 @@ const cases: [name: string, content: string, config?: UserConfig][] = [
   ["with-default", "rotate-12"],
   ["with-keyframes", "animate-spin"],
   ["gradients", "from-orange-200 via-purple-400 via-40% to-red-600"],
-  ["line-height modifier", "text-lg/8"],
+  ["line-height modifier", "text-lg/8 text-lg/tight"],
   [
     "box-shadow colors",
     "shadow shadow-lg shadow-none shadow-teal-800 shadow-[#dd2] shadow-[5px_10px_teal]",
@@ -91,14 +91,17 @@ const cases: [name: string, content: string, config?: UserConfig][] = [
     "line height modifier",
     "text-lg/[18px] text-[calc(3rem/5)]/7 text-[calc(3rem/5)]/[calc(4rem/5)]",
   ],
-  ["arbitrary-values-with-spaces", "grid grid-cols-[1fr_500px_2fr]"],
+  [
+    "arbitrary-values-with-spaces",
+    "grid grid-cols-[1fr_500px_2fr] before:content-['hello_world']",
+  ],
   [
     "arbitrary-properties",
     "[mask-type:luminance] hover:[mask-type:alpha] [background:repeating-linear-gradient(45deg,#606dbc,#606dbc_10px,#465298_10px,#465298_20px)]",
   ],
   [
     "arbitrary-variants",
-    "[html:has(&)]:bg-blue-500 [&:nth-child(3)]:underline [&>*]:p-4 [.sidebar:hover_&]:opacity-70",
+    '[html:has(&)]:bg-blue-500 [&:nth-child(3)]:underline [&>*]:p-4 [&[data-selected="true"]]:bg-blue-100 [.sidebar:hover_&]:opacity-70',
   ],
   ["arbitrary-media", "[@media(min-width:900px)]:block"],
   ["max-screen", "sm:max-md:p-2"],
@@ -115,8 +118,15 @@ const cases: [name: string, content: string, config?: UserConfig][] = [
   ["custom-config", "p-4 p-6 m-4", { theme: { padding: { 4: "4px" } } }],
   [
     "extend-config",
-    "p-4 p-6 m-4",
-    { theme: { extend: { padding: { 4: "4px" } } } },
+    "p-4 p-6 m-4 4xl:p-2",
+    {
+      theme: {
+        extend: {
+          screens: { "4xl": "2000px" },
+          padding: { 4: "4px" },
+        },
+      },
+    },
   ],
   [
     "static-plugin",
@@ -193,7 +203,7 @@ const snapshots = Object.fromEntries(
 let newSnapshot = "";
 for (const [name, content, config] of cases) {
   const downwind = initDownwindWithConfig({ config });
-  downwind.scan(content);
+  downwind.scan(`<div className="${content}" />`);
   const actual = `/* ${name}: ${content} */\n${downwind.generate()}\n`;
   if (shouldUpdateSnapshots) newSnapshot += actual;
   test(name, () => {

tests/regex.test.ts

@@ -0,0 +1,10 @@
+import assert from "node:assert";
+import { test } from "node:test";
+import { selectorRE } from "../src/utils/regex.ts";
+
+test("regex", () => {
+  assert.equal(
+    selectorRE.toString(),
+    /(?<=['"`\s}])(?:(?:[a-z0-9][a-z0-9-]+|[a-z]+-\[[a-z:-]+]|\[[a-z0-9&[\]=":>+*~.()_@-]+]):)*!?(?:-?(?:[a-z][a-z0-9-]*[a-z0-9%]|[a-z][a-z-]*-\[[a-z0-9#._,'%()+*/-]+])(?:\/(?:[a-z0-9]+|\[[a-z0-9.%()+*/-]+]))?|\[[a-z][a-z-]+:[a-z0-9#._,'%()+*/-]+])(?=['"`\s:$])/g.toString(),
+  );
+});

tests/run-tests.ts

@@ -8,6 +8,7 @@ declare global {
 }
 
 import("./generate.test.ts");
+import("./regex.test.ts");
 import("./preTransformCSS.test.ts");
 import("./codegen.test.ts");
 import("./esbuildPlugin.test.ts");