513 lines
18 KiB
JavaScript
513 lines
18 KiB
JavaScript
import assert from "node:assert/strict";
|
||
import { readFile } from "node:fs/promises";
|
||
import { dirname, join } from "node:path";
|
||
import { fileURLToPath } from "node:url";
|
||
import test from "node:test";
|
||
|
||
import Ajv2020 from "ajv/dist/2020.js";
|
||
import { build } from "esbuild";
|
||
import { JSDOM } from "jsdom";
|
||
|
||
const TEST_DIR = dirname(fileURLToPath(import.meta.url));
|
||
const ROOT = dirname(TEST_DIR);
|
||
const RULES_DIR = join(ROOT, "rules");
|
||
|
||
async function readJson(path)
|
||
{
|
||
return JSON.parse(await readFile(path, "utf8"));
|
||
}
|
||
|
||
const [ rules, rulesSchema, vectorsDoc, vectorsSchema, packageJson, spacingCss ] = await Promise.all([
|
||
readJson(join(RULES_DIR, "juzhen-rules.json")),
|
||
readJson(join(RULES_DIR, "juzhen-rules.schema.json")),
|
||
readJson(join(RULES_DIR, "golden-vectors.json")),
|
||
readJson(join(RULES_DIR, "golden-vectors.schema.json")),
|
||
readJson(join(ROOT, "package.json")),
|
||
readFile(join(ROOT, "src/css/_spacing.css"), "utf8"),
|
||
]);
|
||
|
||
const ajv = new Ajv2020({ allErrors: true, strict: true });
|
||
const validateRules = ajv.compile(rulesSchema);
|
||
const validateVectors = ajv.compile(vectorsSchema);
|
||
|
||
async function bundleSourceAdapter()
|
||
{
|
||
const source = [
|
||
'export * from "./src/core/unicode.ts";',
|
||
'export * from "./src/core/locale.ts";',
|
||
'export { createJuzhen, normalizeOptions } from "./src/index.ts";',
|
||
'export { jiyaPass, jiyaAdjacencyPass } from "./src/typeset/jiya.ts";',
|
||
'export { longWordPass } from "./src/typeset/longword.ts";',
|
||
'export { levelMarkPass } from "./src/typeset/levelmark.ts";',
|
||
'export { jinzePass } from "./src/typeset/jinze.ts";',
|
||
'export { spacingPass } from "./src/typeset/spacing.ts";',
|
||
'export { orphanPass } from "./src/typeset/orphan.ts";',
|
||
'export { lineEdgePass, gapTrimPass } from "./src/typeset/lineedge.ts";',
|
||
].join("\n");
|
||
const result = await build({
|
||
stdin: {
|
||
contents: source,
|
||
loader: "ts",
|
||
resolveDir: ROOT,
|
||
sourcefile: "shared-rules-adapter.ts",
|
||
},
|
||
bundle: true,
|
||
format: "esm",
|
||
platform: "node",
|
||
target: "es2022",
|
||
write: false,
|
||
logLevel: "silent",
|
||
});
|
||
const bundled = result.outputFiles[0].text + "\n//# sourceURL=shared-rules-adapter.mjs\n";
|
||
const encoded = Buffer.from(bundled).toString("base64");
|
||
return import("data:text/javascript;base64," + encoded);
|
||
}
|
||
|
||
const adapter = await bundleSourceAdapter();
|
||
const vectors = vectorsDoc.vectors;
|
||
|
||
function vectorsOf(category)
|
||
{
|
||
return vectors.filter((vector) => vector.category === category);
|
||
}
|
||
|
||
function renderText(text, options = {})
|
||
{
|
||
const dom = new JSDOM("<!doctype html><body><p></p></body>");
|
||
const paragraph = dom.window.document.querySelector("p");
|
||
paragraph.textContent = text;
|
||
const instance = adapter.createJuzhen({
|
||
spacing: false,
|
||
jiya: false,
|
||
jinze: false,
|
||
orphan: false,
|
||
longWord: false,
|
||
...options,
|
||
});
|
||
const globals = {
|
||
document: globalThis.document,
|
||
NodeFilter: globalThis.NodeFilter,
|
||
};
|
||
globalThis.document = dom.window.document;
|
||
globalThis.NodeFilter = dom.window.NodeFilter;
|
||
try
|
||
{
|
||
instance.render(paragraph);
|
||
}
|
||
finally
|
||
{
|
||
if (globals.document === undefined) { delete globalThis.document; }
|
||
else { globalThis.document = globals.document; }
|
||
if (globals.NodeFilter === undefined) { delete globalThis.NodeFilter; }
|
||
else { globalThis.NodeFilter = globals.NodeFilter; }
|
||
}
|
||
return { dom, paragraph, instance };
|
||
}
|
||
|
||
function punctuationCharacter(punctuationClass)
|
||
{
|
||
return {
|
||
stop: "。",
|
||
pause: "，",
|
||
open: "「",
|
||
close: "」",
|
||
middle: "·",
|
||
liga: "…",
|
||
}[punctuationClass];
|
||
}
|
||
|
||
function entryBounds(entry)
|
||
{
|
||
const start = Number.parseInt(entry.codePoint || entry.start, 16);
|
||
const end = Number.parseInt(entry.codePoint || entry.end, 16);
|
||
return { start, end };
|
||
}
|
||
|
||
function assertOrderedCodePointSet(name, entries)
|
||
{
|
||
let previousEnd = -1;
|
||
for (const entry of entries)
|
||
{
|
||
const { start, end } = entryBounds(entry);
|
||
assert.ok(start >= 0 && end <= 0x10FFFF, name + " Unicode bounds");
|
||
assert.ok(start <= end, name + " closed range order");
|
||
assert.ok(start > previousEnd, name + " ranges must be sorted and non-overlapping");
|
||
previousEnd = end;
|
||
}
|
||
}
|
||
|
||
function webEscape(codePoint)
|
||
{
|
||
const hex = codePoint.toString(16).toUpperCase();
|
||
return codePoint <= 0xFFFF
|
||
? "\\u" + hex.padStart(4, "0")
|
||
: "\\u{" + hex + "}";
|
||
}
|
||
|
||
function rangesToWebFragment(entries)
|
||
{
|
||
return entries.map((entry) =>
|
||
{
|
||
const { start, end } = entryBounds(entry);
|
||
return start === end
|
||
? webEscape(start)
|
||
: webEscape(start) + "-" + webEscape(end);
|
||
}).join("");
|
||
}
|
||
|
||
function assertEquivalentMembership(name, entries, sourceFragment)
|
||
{
|
||
assertOrderedCodePointSet(name, entries);
|
||
const canonical = new RegExp("^[" + rangesToWebFragment(entries) + "]$", "u");
|
||
const source = new RegExp("^[" + sourceFragment + "]$", "u");
|
||
for (let codePoint = 0; codePoint <= 0x10FFFF; codePoint += 1)
|
||
{
|
||
const character = String.fromCodePoint(codePoint);
|
||
assert.equal(
|
||
canonical.test(character),
|
||
source.test(character),
|
||
name + " differs at U+" + codePoint.toString(16).toUpperCase(),
|
||
);
|
||
}
|
||
}
|
||
|
||
function canonicalCodePoints(entries)
|
||
{
|
||
const out = [];
|
||
for (const entry of entries)
|
||
{
|
||
const { start, end } = entryBounds(entry);
|
||
for (let codePoint = start; codePoint <= end; codePoint += 1) { out.push(codePoint); }
|
||
}
|
||
return out;
|
||
}
|
||
|
||
test("shared rules: Ajv 2020 strict validation, rejection cases, exports, and coverage", () =>
|
||
{
|
||
assert.equal(rulesSchema.$schema, "https://json-schema.org/draft/2020-12/schema");
|
||
assert.equal(vectorsSchema.$schema, "https://json-schema.org/draft/2020-12/schema");
|
||
assert.equal(rules.schemaVersion, 1);
|
||
assert.equal(rules.rulesetVersion, "1.0.0");
|
||
assert.equal(rules.unicodeProfile.version, "17.0.0");
|
||
assert.equal(vectorsDoc.schemaVersion, 1);
|
||
assert.equal(vectorsDoc.rulesetVersion, rules.rulesetVersion);
|
||
assert.equal(validateRules(rules), true, ajv.errorsText(validateRules.errors));
|
||
assert.equal(validateVectors(vectorsDoc), true, ajv.errorsText(validateVectors.errors));
|
||
|
||
const extraRules = structuredClone(rules);
|
||
extraRules.unexpected = true;
|
||
assert.equal(validateRules(extraRules), false, "rules additionalProperties must be rejected");
|
||
|
||
const bogusJinze = structuredClone(vectorsDoc);
|
||
const jinzeIndex = bogusJinze.vectors.findIndex((vector) => vector.category === "jinze");
|
||
assert.notEqual(jinzeIndex, -1, "valid fixture must contain a jinze vector");
|
||
bogusJinze.vectors[jinzeIndex] = {
|
||
id: "bogus-jinze",
|
||
category: "jinze",
|
||
input: {},
|
||
expected: { reason: "bogus" },
|
||
};
|
||
assert.equal(validateVectors(bogusJinze), false, "malformed jinze vector must be rejected");
|
||
assert.equal(
|
||
validateVectors.errors?.some((error) => error.keyword === "minItems"),
|
||
false,
|
||
"bogus jinze rejection must not rely on document-wide vector count",
|
||
);
|
||
|
||
const extraVectorField = structuredClone(vectorsDoc);
|
||
extraVectorField.vectors[0].input.unexpected = true;
|
||
assert.equal(
|
||
validateVectors(extraVectorField),
|
||
false,
|
||
"category input additionalProperties must be rejected",
|
||
);
|
||
|
||
assert.ok(vectors.length >= 20, `expected at least 20 vectors, got ${vectors.length}`);
|
||
const ids = vectors.map((vector) => vector.id);
|
||
assert.equal(new Set(ids).size, ids.length, "golden vector ids must be unique");
|
||
for (const category of [ "jinze", "orphan" ])
|
||
{
|
||
assert.ok(
|
||
vectors.some((vector) => vector.category === category && vector.expected.atoms),
|
||
`${category} must retain at least one cross-feature atom vector`,
|
||
);
|
||
}
|
||
const requiredCategories = [
|
||
"language-classification",
|
||
"punctuation-classification",
|
||
"punctuation-style",
|
||
"punctuation-adjacency",
|
||
"spacing",
|
||
"ivs-atom",
|
||
"jinze",
|
||
"orphan",
|
||
];
|
||
for (const category of requiredCategories)
|
||
{
|
||
assert.ok(vectorsOf(category).length > 0, `missing vector category ${category}`);
|
||
}
|
||
|
||
assert.deepEqual(packageJson.exports["./rules"], "./rules/juzhen-rules.json");
|
||
assert.deepEqual(packageJson.exports["./rules/schema"], "./rules/juzhen-rules.schema.json");
|
||
assert.deepEqual(packageJson.exports["./rules/vectors"], "./rules/golden-vectors.json");
|
||
assert.deepEqual(
|
||
packageJson.exports["./rules/vectors-schema"],
|
||
"./rules/golden-vectors.schema.json",
|
||
);
|
||
assert.ok(packageJson.files.includes("rules"));
|
||
});
|
||
|
||
test("shared rules: canonical character classes equal the Web source constants", () =>
|
||
{
|
||
const canonical = rules.characterClasses;
|
||
assert.equal(canonical.representation, "unicode-code-point-ranges");
|
||
assertEquivalentMembership("cjk", canonical.cjk, adapter.CJK);
|
||
assertEquivalentMembership("ansAfterCjk", canonical.ansAfterCjk, adapter.ANS_AFTER_CJK);
|
||
assertEquivalentMembership("ansBeforeCjk", canonical.ansBeforeCjk, adapter.ANS_BEFORE_CJK);
|
||
assertEquivalentMembership(
|
||
"variationSelectors",
|
||
canonical.variationSelectors,
|
||
adapter.VARIATION_SELECTORS,
|
||
);
|
||
assert.deepEqual(canonical.biaodian.classificationOrder, [
|
||
"stop", "pause", "open", "close", "middle", "liga",
|
||
]);
|
||
for (const punctuationClass of canonical.biaodian.classificationOrder)
|
||
{
|
||
const entries = canonical.biaodian.classes[punctuationClass];
|
||
assertOrderedCodePointSet("biaodian." + punctuationClass, entries);
|
||
assert.deepEqual(
|
||
canonicalCodePoints(entries),
|
||
Array.from(adapter.DEFAULT_RULESET.biaodian[punctuationClass], (character) =>
|
||
character.codePointAt(0)).sort((a, b) => a - b),
|
||
"biaodian." + punctuationClass,
|
||
);
|
||
}
|
||
|
||
const extended = adapter.resolveRuleset({
|
||
cjk: "〇",
|
||
western: "★",
|
||
biaodian: { stop: "‼" },
|
||
});
|
||
assert.equal(extended.cjk, adapter.CJK + "〇");
|
||
assert.equal(extended.ansAfter, adapter.ANS_AFTER_CJK + "★");
|
||
assert.equal(extended.ansBefore, adapter.ANS_BEFORE_CJK + "★");
|
||
assert.equal(extended.biaodian.stop, adapter.DEFAULT_RULESET.biaodian.stop + "‼");
|
||
assert.equal(canonical.overrideSemantics.mode, "append");
|
||
assert.equal(canonical.overrideSemantics.removalAllowed, false);
|
||
assert.equal(canonical.overrideSemantics.reclassificationAllowed, false);
|
||
});
|
||
|
||
test("shared rules: explicit script identifiers match whole subtags", () =>
|
||
{
|
||
for (const [ lang, expected ] of [
|
||
[ "zh-CN-x-shant", "zh-Hans" ],
|
||
[ "zh-TW-x-hansford", "zh-Hant" ],
|
||
[ "ZH-CN-X-EHANT", "zh-Hans" ],
|
||
[ "zh-Hant-CN", "zh-Hant" ],
|
||
[ "zh-Hans-TW", "zh-Hans" ],
|
||
])
|
||
{
|
||
assert.equal(adapter.classify(lang), expected, lang);
|
||
}
|
||
});
|
||
|
||
test("shared rules: locale defaults and all language vectors match the Web adapter", () =>
|
||
{
|
||
const config = { default: "zh-Hant", policy: {} };
|
||
for (const [ langClass, expectedStyle ] of Object.entries(rules.locales.defaultPolicies))
|
||
{
|
||
assert.equal(adapter.styleFor(langClass, config), expectedStyle, langClass);
|
||
}
|
||
for (const vector of vectorsOf("language-classification"))
|
||
{
|
||
assert.equal(
|
||
adapter.classify(vector.input.lang),
|
||
vector.expected.langClass,
|
||
vector.id,
|
||
);
|
||
}
|
||
assert.equal(adapter.classify("zhx"), "other", "BCP-47 boundary must reject zhx");
|
||
});
|
||
|
||
test("shared rules: punctuation classification and style vectors match Web behavior", () =>
|
||
{
|
||
for (const vector of vectorsOf("punctuation-classification"))
|
||
{
|
||
const actual = adapter.classifyBiaodian(vector.input.character);
|
||
const expected = vector.expected.punctuationClass;
|
||
assert.equal(actual, expected === null ? null : "bd-" + expected, vector.id);
|
||
}
|
||
|
||
for (const vector of vectorsOf("punctuation-style"))
|
||
{
|
||
const halfClasses = rules.punctuationStyles.baselineHalfWidthClasses[vector.input.style];
|
||
assert.equal(
|
||
halfClasses.includes(vector.input.punctuationClass),
|
||
vector.expected.halfWidth,
|
||
vector.id + " canonical",
|
||
);
|
||
const character = punctuationCharacter(vector.input.punctuationClass);
|
||
const { paragraph } = renderText(character, {
|
||
jiya: true,
|
||
lang: { default: "zh-Hant", style: vector.input.style },
|
||
});
|
||
const char = paragraph.querySelector("jz-char");
|
||
assert.ok(char, vector.id + " must be classified by Web adapter");
|
||
assert.equal(char.classList.contains("jz-half"), vector.expected.halfWidth, vector.id);
|
||
}
|
||
});
|
||
|
||
test("shared rules: punctuation adjacency vectors match Web behavior", () =>
|
||
{
|
||
for (const vector of vectorsOf("punctuation-adjacency"))
|
||
{
|
||
const leftCharacter = punctuationCharacter(vector.input.left);
|
||
const rightCharacter = punctuationCharacter(vector.input.right);
|
||
const { paragraph } = renderText(leftCharacter + rightCharacter, {
|
||
jiya: true,
|
||
lang: { default: "zh-Hant", style: "quanjiao" },
|
||
});
|
||
const chars = Array.from(paragraph.querySelectorAll("jz-char"));
|
||
assert.equal(chars.length, 2, vector.id);
|
||
assert.equal(
|
||
chars[0].classList.contains("jz-half"),
|
||
vector.expected.squeezeLeft,
|
||
vector.id + " left",
|
||
);
|
||
assert.equal(
|
||
chars[1].classList.contains("jz-half"),
|
||
vector.expected.squeezeRight,
|
||
vector.id + " right",
|
||
);
|
||
}
|
||
});
|
||
|
||
test("shared rules: spacing vectors match Web decisions and author-space normalization", () =>
|
||
{
|
||
assert.equal(rules.spacing.gapEm, 0.25);
|
||
const cssGap = /jz-hws\s*\{[^}]*margin-right:\s*([0-9.]+)em\s*;/s.exec(spacingCss);
|
||
assert.ok(cssGap, "Web adapter spacing CSS must declare jz-hws margin-right in em");
|
||
assert.equal(Number.parseFloat(cssGap[1]), rules.spacing.gapEm, "Web CSS gap drift");
|
||
for (const vector of vectorsOf("spacing"))
|
||
{
|
||
const { paragraph } = renderText(vector.input.text, { spacing: true });
|
||
assert.equal(
|
||
paragraph.querySelectorAll("jz-hws").length,
|
||
vector.expected.managedGaps,
|
||
vector.id + " gaps",
|
||
);
|
||
assert.equal(paragraph.textContent, vector.expected.normalizedText, vector.id + " text");
|
||
if (Object.hasOwn(vector.expected, "gapEm"))
|
||
{
|
||
assert.equal(vector.expected.gapEm, rules.spacing.gapEm, vector.id + " gap width");
|
||
}
|
||
}
|
||
});
|
||
|
||
test("shared rules: IVS vectors enforce base plus at most one selector", () =>
|
||
{
|
||
assert.equal(rules.atoms.maxVariationSelectors, 1);
|
||
for (const vector of vectorsOf("ivs-atom"))
|
||
{
|
||
assert.deepEqual(
|
||
adapter.splitVariationSequences(vector.input.text),
|
||
vector.expected.atoms,
|
||
vector.id,
|
||
);
|
||
}
|
||
});
|
||
|
||
test("Web adapter only: pass groups, orders, levels, and compatibility no-ops do not drift", () =>
|
||
{
|
||
const actual = {
|
||
charify: [ adapter.jiyaPass, adapter.longWordPass ],
|
||
standalone: [
|
||
adapter.levelMarkPass,
|
||
adapter.jiyaAdjacencyPass,
|
||
adapter.jinzePass,
|
||
adapter.spacingPass,
|
||
adapter.orphanPass,
|
||
adapter.lineEdgePass,
|
||
adapter.gapTrimPass,
|
||
],
|
||
};
|
||
assert.deepEqual(
|
||
actual.charify.map(({ name, order, level }) => ({ name, order, level })),
|
||
[
|
||
{ name: "jiya", order: 20, level: "text" },
|
||
{ name: "longWord", order: 60, level: "paragraph" },
|
||
],
|
||
"Web charify group",
|
||
);
|
||
assert.deepEqual(
|
||
actual.standalone.map(({ name, order, level }) => ({ name, order, level })),
|
||
[
|
||
{ name: "levelMark", order: 10, level: "text" },
|
||
{ name: "jiyaAdjacency", order: 25, level: "text" },
|
||
{ name: "jinze", order: 40, level: "paragraph" },
|
||
{ name: "spacing", order: 50, level: "text" },
|
||
{ name: "orphan", order: 70, level: "paragraph" },
|
||
{ name: "jiyaLineEdge", order: 90, level: "paragraph" },
|
||
{ name: "spacingGapTrim", order: 92, level: "paragraph" },
|
||
],
|
||
"Web standalone group",
|
||
);
|
||
const passNames = new Set([ ...actual.charify, ...actual.standalone ].map((pass) => pass.name));
|
||
for (const option of [ "hanging", "biaodian", "emphasis", "ruby" ])
|
||
{
|
||
assert.equal(passNames.has(option), false, option + " must remain unregistered");
|
||
}
|
||
});
|
||
|
||
test("shared rules: jinze and orphan semantic vectors match Web behavior", () =>
|
||
{
|
||
for (const vector of vectorsOf("jinze"))
|
||
{
|
||
if (vector.expected.atoms)
|
||
{
|
||
assert.deepEqual(
|
||
adapter.splitVariationSequences(vector.input.left + vector.input.right),
|
||
vector.expected.atoms,
|
||
vector.id + " atoms",
|
||
);
|
||
}
|
||
const { paragraph } = renderText(vector.input.left + vector.input.right, {
|
||
jiya: true,
|
||
jinze: true,
|
||
lang: { default: "zh-Hant", style: "quanjiao" },
|
||
});
|
||
assert.equal(
|
||
paragraph.querySelector("jz-jinze") !== null,
|
||
vector.expected.prohibitedBreak,
|
||
vector.id,
|
||
);
|
||
}
|
||
|
||
const anyCjk = new RegExp(
|
||
"[" + rangesToWebFragment(rules.characterClasses.cjk) + "]",
|
||
"u",
|
||
);
|
||
const punctuation = new Set(
|
||
Object.values(rules.characterClasses.biaodian.classes)
|
||
.flatMap((entries) => canonicalCodePoints(entries)),
|
||
);
|
||
for (const vector of vectorsOf("orphan"))
|
||
{
|
||
const atoms = adapter.splitVariationSequences(vector.input.text);
|
||
if (vector.expected.atoms)
|
||
{
|
||
assert.deepEqual(atoms, vector.expected.atoms, vector.id + " atoms");
|
||
}
|
||
const meaningful = atoms.filter((atom) =>
|
||
anyCjk.test(atom) && !punctuation.has(atom.codePointAt(0)));
|
||
assert.equal(meaningful.length, vector.expected.meaningfulCjkCount, vector.id + " count");
|
||
|
||
const { paragraph } = renderText(vector.input.text, {
|
||
orphan: { chars: vector.input.minimum },
|
||
});
|
||
const orphan = paragraph.querySelector("jz-orphan");
|
||
assert.equal(orphan ? orphan.textContent : null, vector.expected.protectedSuffix, vector.id);
|
||
}
|
||
});
|