feat(rules): 建立共享排版契约并收束 WebKit 与 Unicode 基线
This commit is contained in:
@@ -0,0 +1,498 @@
|
||||
import assert from "node:assert/strict";
|
||||
import { readFile } from "node:fs/promises";
|
||||
import { dirname, join } from "node:path";
|
||||
import { fileURLToPath } from "node:url";
|
||||
import test from "node:test";
|
||||
|
||||
import Ajv2020 from "ajv/dist/2020.js";
|
||||
import { build } from "esbuild";
|
||||
import { JSDOM } from "jsdom";
|
||||
|
||||
const TEST_DIR = dirname(fileURLToPath(import.meta.url));
|
||||
const ROOT = dirname(TEST_DIR);
|
||||
const RULES_DIR = join(ROOT, "rules");
|
||||
|
||||
async function readJson(path)
|
||||
{
|
||||
return JSON.parse(await readFile(path, "utf8"));
|
||||
}
|
||||
|
||||
const [ rules, rulesSchema, vectorsDoc, vectorsSchema, packageJson, spacingCss ] = await Promise.all([
|
||||
readJson(join(RULES_DIR, "juzhen-rules.json")),
|
||||
readJson(join(RULES_DIR, "juzhen-rules.schema.json")),
|
||||
readJson(join(RULES_DIR, "golden-vectors.json")),
|
||||
readJson(join(RULES_DIR, "golden-vectors.schema.json")),
|
||||
readJson(join(ROOT, "package.json")),
|
||||
readFile(join(ROOT, "src/css/_spacing.css"), "utf8"),
|
||||
]);
|
||||
|
||||
const ajv = new Ajv2020({ allErrors: true, strict: true });
|
||||
const validateRules = ajv.compile(rulesSchema);
|
||||
const validateVectors = ajv.compile(vectorsSchema);
|
||||
|
||||
async function bundleSourceAdapter()
|
||||
{
|
||||
const source = [
|
||||
'export * from "./src/core/unicode.ts";',
|
||||
'export * from "./src/core/locale.ts";',
|
||||
'export { createJuzhen, normalizeOptions } from "./src/index.ts";',
|
||||
'export { jiyaPass, jiyaAdjacencyPass } from "./src/typeset/jiya.ts";',
|
||||
'export { longWordPass } from "./src/typeset/longword.ts";',
|
||||
'export { levelMarkPass } from "./src/typeset/levelmark.ts";',
|
||||
'export { jinzePass } from "./src/typeset/jinze.ts";',
|
||||
'export { spacingPass } from "./src/typeset/spacing.ts";',
|
||||
'export { orphanPass } from "./src/typeset/orphan.ts";',
|
||||
'export { lineEdgePass, gapTrimPass } from "./src/typeset/lineedge.ts";',
|
||||
].join("\n");
|
||||
const result = await build({
|
||||
stdin: {
|
||||
contents: source,
|
||||
loader: "ts",
|
||||
resolveDir: ROOT,
|
||||
sourcefile: "shared-rules-adapter.ts",
|
||||
},
|
||||
bundle: true,
|
||||
format: "esm",
|
||||
platform: "node",
|
||||
target: "es2022",
|
||||
write: false,
|
||||
logLevel: "silent",
|
||||
});
|
||||
const bundled = result.outputFiles[0].text + "\n//# sourceURL=shared-rules-adapter.mjs\n";
|
||||
const encoded = Buffer.from(bundled).toString("base64");
|
||||
return import("data:text/javascript;base64," + encoded);
|
||||
}
|
||||
|
||||
const adapter = await bundleSourceAdapter();
|
||||
const vectors = vectorsDoc.vectors;
|
||||
|
||||
function vectorsOf(category)
|
||||
{
|
||||
return vectors.filter((vector) => vector.category === category);
|
||||
}
|
||||
|
||||
function renderText(text, options = {})
|
||||
{
|
||||
const dom = new JSDOM("<!doctype html><body><p></p></body>");
|
||||
const paragraph = dom.window.document.querySelector("p");
|
||||
paragraph.textContent = text;
|
||||
const instance = adapter.createJuzhen({
|
||||
spacing: false,
|
||||
jiya: false,
|
||||
jinze: false,
|
||||
orphan: false,
|
||||
longWord: false,
|
||||
...options,
|
||||
});
|
||||
const globals = {
|
||||
document: globalThis.document,
|
||||
NodeFilter: globalThis.NodeFilter,
|
||||
};
|
||||
globalThis.document = dom.window.document;
|
||||
globalThis.NodeFilter = dom.window.NodeFilter;
|
||||
try
|
||||
{
|
||||
instance.render(paragraph);
|
||||
}
|
||||
finally
|
||||
{
|
||||
if (globals.document === undefined) { delete globalThis.document; }
|
||||
else { globalThis.document = globals.document; }
|
||||
if (globals.NodeFilter === undefined) { delete globalThis.NodeFilter; }
|
||||
else { globalThis.NodeFilter = globals.NodeFilter; }
|
||||
}
|
||||
return { dom, paragraph, instance };
|
||||
}
|
||||
|
||||
function punctuationCharacter(punctuationClass)
|
||||
{
|
||||
return {
|
||||
stop: "。",
|
||||
pause: "，",
|
||||
open: "「",
|
||||
close: "」",
|
||||
middle: "·",
|
||||
liga: "…",
|
||||
}[punctuationClass];
|
||||
}
|
||||
|
||||
function entryBounds(entry)
|
||||
{
|
||||
const start = Number.parseInt(entry.codePoint || entry.start, 16);
|
||||
const end = Number.parseInt(entry.codePoint || entry.end, 16);
|
||||
return { start, end };
|
||||
}
|
||||
|
||||
function assertOrderedCodePointSet(name, entries)
|
||||
{
|
||||
let previousEnd = -1;
|
||||
for (const entry of entries)
|
||||
{
|
||||
const { start, end } = entryBounds(entry);
|
||||
assert.ok(start >= 0 && end <= 0x10FFFF, name + " Unicode bounds");
|
||||
assert.ok(start <= end, name + " closed range order");
|
||||
assert.ok(start > previousEnd, name + " ranges must be sorted and non-overlapping");
|
||||
previousEnd = end;
|
||||
}
|
||||
}
|
||||
|
||||
function webEscape(codePoint)
|
||||
{
|
||||
const hex = codePoint.toString(16).toUpperCase();
|
||||
return codePoint <= 0xFFFF
|
||||
? "\\u" + hex.padStart(4, "0")
|
||||
: "\\u{" + hex + "}";
|
||||
}
|
||||
|
||||
function rangesToWebFragment(entries)
|
||||
{
|
||||
return entries.map((entry) =>
|
||||
{
|
||||
const { start, end } = entryBounds(entry);
|
||||
return start === end
|
||||
? webEscape(start)
|
||||
: webEscape(start) + "-" + webEscape(end);
|
||||
}).join("");
|
||||
}
|
||||
|
||||
function assertEquivalentMembership(name, entries, sourceFragment)
|
||||
{
|
||||
assertOrderedCodePointSet(name, entries);
|
||||
const canonical = new RegExp("^[" + rangesToWebFragment(entries) + "]$", "u");
|
||||
const source = new RegExp("^[" + sourceFragment + "]$", "u");
|
||||
for (let codePoint = 0; codePoint <= 0x10FFFF; codePoint += 1)
|
||||
{
|
||||
const character = String.fromCodePoint(codePoint);
|
||||
assert.equal(
|
||||
canonical.test(character),
|
||||
source.test(character),
|
||||
name + " differs at U+" + codePoint.toString(16).toUpperCase(),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function canonicalCodePoints(entries)
|
||||
{
|
||||
const out = [];
|
||||
for (const entry of entries)
|
||||
{
|
||||
const { start, end } = entryBounds(entry);
|
||||
for (let codePoint = start; codePoint <= end; codePoint += 1) { out.push(codePoint); }
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
test("shared rules: Ajv 2020 strict validation, rejection cases, exports, and coverage", () =>
|
||||
{
|
||||
assert.equal(rulesSchema.$schema, "https://json-schema.org/draft/2020-12/schema");
|
||||
assert.equal(vectorsSchema.$schema, "https://json-schema.org/draft/2020-12/schema");
|
||||
assert.equal(rules.schemaVersion, 1);
|
||||
assert.equal(rules.rulesetVersion, "1.0.0");
|
||||
assert.equal(rules.unicodeProfile.version, "17.0.0");
|
||||
assert.equal(vectorsDoc.schemaVersion, 1);
|
||||
assert.equal(vectorsDoc.rulesetVersion, rules.rulesetVersion);
|
||||
assert.equal(validateRules(rules), true, ajv.errorsText(validateRules.errors));
|
||||
assert.equal(validateVectors(vectorsDoc), true, ajv.errorsText(validateVectors.errors));
|
||||
|
||||
const extraRules = structuredClone(rules);
|
||||
extraRules.unexpected = true;
|
||||
assert.equal(validateRules(extraRules), false, "rules additionalProperties must be rejected");
|
||||
|
||||
const bogusJinze = structuredClone(vectorsDoc);
|
||||
const jinzeIndex = bogusJinze.vectors.findIndex((vector) => vector.category === "jinze");
|
||||
assert.notEqual(jinzeIndex, -1, "valid fixture must contain a jinze vector");
|
||||
bogusJinze.vectors[jinzeIndex] = {
|
||||
id: "bogus-jinze",
|
||||
category: "jinze",
|
||||
input: {},
|
||||
expected: { reason: "bogus" },
|
||||
};
|
||||
assert.equal(validateVectors(bogusJinze), false, "malformed jinze vector must be rejected");
|
||||
assert.equal(
|
||||
validateVectors.errors?.some((error) => error.keyword === "minItems"),
|
||||
false,
|
||||
"bogus jinze rejection must not rely on document-wide vector count",
|
||||
);
|
||||
|
||||
const extraVectorField = structuredClone(vectorsDoc);
|
||||
extraVectorField.vectors[0].input.unexpected = true;
|
||||
assert.equal(
|
||||
validateVectors(extraVectorField),
|
||||
false,
|
||||
"category input additionalProperties must be rejected",
|
||||
);
|
||||
|
||||
assert.ok(vectors.length >= 20, `expected at least 20 vectors, got ${vectors.length}`);
|
||||
const ids = vectors.map((vector) => vector.id);
|
||||
assert.equal(new Set(ids).size, ids.length, "golden vector ids must be unique");
|
||||
for (const category of [ "jinze", "orphan" ])
|
||||
{
|
||||
assert.ok(
|
||||
vectors.some((vector) => vector.category === category && vector.expected.atoms),
|
||||
`${category} must retain at least one cross-feature atom vector`,
|
||||
);
|
||||
}
|
||||
const requiredCategories = [
|
||||
"language-classification",
|
||||
"punctuation-classification",
|
||||
"punctuation-style",
|
||||
"punctuation-adjacency",
|
||||
"spacing",
|
||||
"ivs-atom",
|
||||
"jinze",
|
||||
"orphan",
|
||||
];
|
||||
for (const category of requiredCategories)
|
||||
{
|
||||
assert.ok(vectorsOf(category).length > 0, `missing vector category ${category}`);
|
||||
}
|
||||
|
||||
assert.deepEqual(packageJson.exports["./rules"], "./rules/juzhen-rules.json");
|
||||
assert.deepEqual(packageJson.exports["./rules/schema"], "./rules/juzhen-rules.schema.json");
|
||||
assert.deepEqual(packageJson.exports["./rules/vectors"], "./rules/golden-vectors.json");
|
||||
assert.deepEqual(
|
||||
packageJson.exports["./rules/vectors-schema"],
|
||||
"./rules/golden-vectors.schema.json",
|
||||
);
|
||||
assert.ok(packageJson.files.includes("rules"));
|
||||
});
|
||||
|
||||
test("shared rules: canonical character classes equal the Web source constants", () =>
|
||||
{
|
||||
const canonical = rules.characterClasses;
|
||||
assert.equal(canonical.representation, "unicode-code-point-ranges");
|
||||
assertEquivalentMembership("cjk", canonical.cjk, adapter.CJK);
|
||||
assertEquivalentMembership("ansAfterCjk", canonical.ansAfterCjk, adapter.ANS_AFTER_CJK);
|
||||
assertEquivalentMembership("ansBeforeCjk", canonical.ansBeforeCjk, adapter.ANS_BEFORE_CJK);
|
||||
assertEquivalentMembership(
|
||||
"variationSelectors",
|
||||
canonical.variationSelectors,
|
||||
adapter.VARIATION_SELECTORS,
|
||||
);
|
||||
assert.deepEqual(canonical.biaodian.classificationOrder, [
|
||||
"stop", "pause", "open", "close", "middle", "liga",
|
||||
]);
|
||||
for (const punctuationClass of canonical.biaodian.classificationOrder)
|
||||
{
|
||||
const entries = canonical.biaodian.classes[punctuationClass];
|
||||
assertOrderedCodePointSet("biaodian." + punctuationClass, entries);
|
||||
assert.deepEqual(
|
||||
canonicalCodePoints(entries),
|
||||
Array.from(adapter.DEFAULT_RULESET.biaodian[punctuationClass], (character) =>
|
||||
character.codePointAt(0)).sort((a, b) => a - b),
|
||||
"biaodian." + punctuationClass,
|
||||
);
|
||||
}
|
||||
|
||||
const extended = adapter.resolveRuleset({
|
||||
cjk: "〇",
|
||||
western: "★",
|
||||
biaodian: { stop: "‼" },
|
||||
});
|
||||
assert.equal(extended.cjk, adapter.CJK + "〇");
|
||||
assert.equal(extended.ansAfter, adapter.ANS_AFTER_CJK + "★");
|
||||
assert.equal(extended.ansBefore, adapter.ANS_BEFORE_CJK + "★");
|
||||
assert.equal(extended.biaodian.stop, adapter.DEFAULT_RULESET.biaodian.stop + "‼");
|
||||
assert.equal(canonical.overrideSemantics.mode, "append");
|
||||
assert.equal(canonical.overrideSemantics.removalAllowed, false);
|
||||
assert.equal(canonical.overrideSemantics.reclassificationAllowed, false);
|
||||
});
|
||||
|
||||
test("shared rules: locale defaults and all language vectors match the Web adapter", () =>
|
||||
{
|
||||
const config = { default: "zh-Hant", policy: {} };
|
||||
for (const [ langClass, expectedStyle ] of Object.entries(rules.locales.defaultPolicies))
|
||||
{
|
||||
assert.equal(adapter.styleFor(langClass, config), expectedStyle, langClass);
|
||||
}
|
||||
for (const vector of vectorsOf("language-classification"))
|
||||
{
|
||||
assert.equal(
|
||||
adapter.classify(vector.input.lang),
|
||||
vector.expected.langClass,
|
||||
vector.id,
|
||||
);
|
||||
}
|
||||
assert.equal(adapter.classify("zhx"), "other", "BCP-47 boundary must reject zhx");
|
||||
});
|
||||
|
||||
test("shared rules: punctuation classification and style vectors match Web behavior", () =>
|
||||
{
|
||||
for (const vector of vectorsOf("punctuation-classification"))
|
||||
{
|
||||
const actual = adapter.classifyBiaodian(vector.input.character);
|
||||
const expected = vector.expected.punctuationClass;
|
||||
assert.equal(actual, expected === null ? null : "bd-" + expected, vector.id);
|
||||
}
|
||||
|
||||
for (const vector of vectorsOf("punctuation-style"))
|
||||
{
|
||||
const halfClasses = rules.punctuationStyles.baselineHalfWidthClasses[vector.input.style];
|
||||
assert.equal(
|
||||
halfClasses.includes(vector.input.punctuationClass),
|
||||
vector.expected.halfWidth,
|
||||
vector.id + " canonical",
|
||||
);
|
||||
const character = punctuationCharacter(vector.input.punctuationClass);
|
||||
const { paragraph } = renderText(character, {
|
||||
jiya: true,
|
||||
lang: { default: "zh-Hant", style: vector.input.style },
|
||||
});
|
||||
const char = paragraph.querySelector("jz-char");
|
||||
assert.ok(char, vector.id + " must be classified by Web adapter");
|
||||
assert.equal(char.classList.contains("jz-half"), vector.expected.halfWidth, vector.id);
|
||||
}
|
||||
});
|
||||
|
||||
test("shared rules: punctuation adjacency vectors match Web behavior", () =>
|
||||
{
|
||||
for (const vector of vectorsOf("punctuation-adjacency"))
|
||||
{
|
||||
const leftCharacter = punctuationCharacter(vector.input.left);
|
||||
const rightCharacter = punctuationCharacter(vector.input.right);
|
||||
const { paragraph } = renderText(leftCharacter + rightCharacter, {
|
||||
jiya: true,
|
||||
lang: { default: "zh-Hant", style: "quanjiao" },
|
||||
});
|
||||
const chars = Array.from(paragraph.querySelectorAll("jz-char"));
|
||||
assert.equal(chars.length, 2, vector.id);
|
||||
assert.equal(
|
||||
chars[0].classList.contains("jz-half"),
|
||||
vector.expected.squeezeLeft,
|
||||
vector.id + " left",
|
||||
);
|
||||
assert.equal(
|
||||
chars[1].classList.contains("jz-half"),
|
||||
vector.expected.squeezeRight,
|
||||
vector.id + " right",
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("shared rules: spacing vectors match Web decisions and author-space normalization", () =>
|
||||
{
|
||||
assert.equal(rules.spacing.gapEm, 0.25);
|
||||
const cssGap = /jz-hws\s*\{[^}]*margin-right:\s*([0-9.]+)em\s*;/s.exec(spacingCss);
|
||||
assert.ok(cssGap, "Web adapter spacing CSS must declare jz-hws margin-right in em");
|
||||
assert.equal(Number.parseFloat(cssGap[1]), rules.spacing.gapEm, "Web CSS gap drift");
|
||||
for (const vector of vectorsOf("spacing"))
|
||||
{
|
||||
const { paragraph } = renderText(vector.input.text, { spacing: true });
|
||||
assert.equal(
|
||||
paragraph.querySelectorAll("jz-hws").length,
|
||||
vector.expected.managedGaps,
|
||||
vector.id + " gaps",
|
||||
);
|
||||
assert.equal(paragraph.textContent, vector.expected.normalizedText, vector.id + " text");
|
||||
if (Object.hasOwn(vector.expected, "gapEm"))
|
||||
{
|
||||
assert.equal(vector.expected.gapEm, rules.spacing.gapEm, vector.id + " gap width");
|
||||
}
|
||||
}
|
||||
});
|
||||
|
||||
test("shared rules: IVS vectors enforce base plus at most one selector", () =>
|
||||
{
|
||||
assert.equal(rules.atoms.maxVariationSelectors, 1);
|
||||
for (const vector of vectorsOf("ivs-atom"))
|
||||
{
|
||||
assert.deepEqual(
|
||||
adapter.splitVariationSequences(vector.input.text),
|
||||
vector.expected.atoms,
|
||||
vector.id,
|
||||
);
|
||||
}
|
||||
});
|
||||
|
||||
test("Web adapter only: pass groups, orders, levels, and compatibility no-ops do not drift", () =>
|
||||
{
|
||||
const actual = {
|
||||
charify: [ adapter.jiyaPass, adapter.longWordPass ],
|
||||
standalone: [
|
||||
adapter.levelMarkPass,
|
||||
adapter.jiyaAdjacencyPass,
|
||||
adapter.jinzePass,
|
||||
adapter.spacingPass,
|
||||
adapter.orphanPass,
|
||||
adapter.lineEdgePass,
|
||||
adapter.gapTrimPass,
|
||||
],
|
||||
};
|
||||
assert.deepEqual(
|
||||
actual.charify.map(({ name, order, level }) => ({ name, order, level })),
|
||||
[
|
||||
{ name: "jiya", order: 20, level: "text" },
|
||||
{ name: "longWord", order: 60, level: "paragraph" },
|
||||
],
|
||||
"Web charify group",
|
||||
);
|
||||
assert.deepEqual(
|
||||
actual.standalone.map(({ name, order, level }) => ({ name, order, level })),
|
||||
[
|
||||
{ name: "levelMark", order: 10, level: "text" },
|
||||
{ name: "jiyaAdjacency", order: 25, level: "text" },
|
||||
{ name: "jinze", order: 40, level: "paragraph" },
|
||||
{ name: "spacing", order: 50, level: "text" },
|
||||
{ name: "orphan", order: 70, level: "paragraph" },
|
||||
{ name: "jiyaLineEdge", order: 90, level: "paragraph" },
|
||||
{ name: "spacingGapTrim", order: 92, level: "paragraph" },
|
||||
],
|
||||
"Web standalone group",
|
||||
);
|
||||
const passNames = new Set([ ...actual.charify, ...actual.standalone ].map((pass) => pass.name));
|
||||
for (const option of [ "hanging", "biaodian", "emphasis", "ruby" ])
|
||||
{
|
||||
assert.equal(passNames.has(option), false, option + " must remain unregistered");
|
||||
}
|
||||
});
|
||||
|
||||
test("shared rules: jinze and orphan semantic vectors match Web behavior", () =>
|
||||
{
|
||||
for (const vector of vectorsOf("jinze"))
|
||||
{
|
||||
if (vector.expected.atoms)
|
||||
{
|
||||
assert.deepEqual(
|
||||
adapter.splitVariationSequences(vector.input.left + vector.input.right),
|
||||
vector.expected.atoms,
|
||||
vector.id + " atoms",
|
||||
);
|
||||
}
|
||||
const { paragraph } = renderText(vector.input.left + vector.input.right, {
|
||||
jiya: true,
|
||||
jinze: true,
|
||||
lang: { default: "zh-Hant", style: "quanjiao" },
|
||||
});
|
||||
assert.equal(
|
||||
paragraph.querySelector("jz-jinze") !== null,
|
||||
vector.expected.prohibitedBreak,
|
||||
vector.id,
|
||||
);
|
||||
}
|
||||
|
||||
const anyCjk = new RegExp(
|
||||
"[" + rangesToWebFragment(rules.characterClasses.cjk) + "]",
|
||||
"u",
|
||||
);
|
||||
const punctuation = new Set(
|
||||
Object.values(rules.characterClasses.biaodian.classes)
|
||||
.flatMap((entries) => canonicalCodePoints(entries)),
|
||||
);
|
||||
for (const vector of vectorsOf("orphan"))
|
||||
{
|
||||
const atoms = adapter.splitVariationSequences(vector.input.text);
|
||||
if (vector.expected.atoms)
|
||||
{
|
||||
assert.deepEqual(atoms, vector.expected.atoms, vector.id + " atoms");
|
||||
}
|
||||
const meaningful = atoms.filter((atom) =>
|
||||
anyCjk.test(atom) && !punctuation.has(atom.codePointAt(0)));
|
||||
assert.equal(meaningful.length, vector.expected.meaningfulCjkCount, vector.id + " count");
|
||||
|
||||
const { paragraph } = renderText(vector.input.text, {
|
||||
orphan: { chars: vector.input.minimum },
|
||||
});
|
||||
const orphan = paragraph.querySelector("jz-orphan");
|
||||
assert.equal(orphan ? orphan.textContent : null, vector.expected.protectedSuffix, vector.id);
|
||||
}
|
||||
});
|
||||
Reference in New Issue
Block a user