Files
cjk-autospace/test/spacing-boundaries.test.mjs

520 lines
21 KiB
JavaScript

import assert from "node:assert/strict";
import { test } from "node:test";
import { createJuzhen } from "../dist/juzhen.mjs";
import { setupDom } from "./helpers.mjs";
const TEXT_ONLY = { jiya: false, jinze: false, orphan: false, longWord: false };
for (const cp of [ 0xE000, 0xE001, 0xE002, 0xE003 ])
{
test(`spacing source: U+${cp.toString(16).toUpperCase()} remains author text`, () =>
{
const text = "漢" + String.fromCodePoint(cp) + "A中";
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
paragraph.textContent = text;
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, text);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "A" ]);
const rendered = paragraph.innerHTML;
jz.render(paragraph);
assert.equal(paragraph.innerHTML, rendered, "repeat render stays idempotent");
jz.revert(paragraph);
assert.equal(paragraph.textContent, text);
assert.equal(paragraph.querySelector("[data-jz]"), null);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
for (const fakeIndex of [ "0", "9" ])
{
test(`spacing source: fake compound placeholder ${fakeIndex} does not collide with GPT-4`, () =>
{
const text = "漢\uE001" + fakeIndex + "\uE002 GPT-4中";
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
paragraph.textContent = text;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(paragraph);
assert.equal(paragraph.textContent, text);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "4" ]);
jz.revert(paragraph);
assert.equal(paragraph.textContent, text);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
test("spacing source: reserved characters across transparent nodes preserve IVS and space normalization", () =>
{
const { dom, document } = setupDom('<p><strong>漢\uE000\uE001</strong>'
+ '<em>0\uE002\uE003\uE004</em> GPT-4<span>\u{20000}</span>'
+ '<i>\u{E0100}</i> A<span>甲</span> <b>乙</b></p>');
const paragraph = document.querySelector("p");
const source = paragraph.textContent;
const expected = source.replace("\u{E0100} A", "\u{E0100}A").replace("甲 乙", "甲乙");
const authors = Array.from(paragraph.querySelectorAll("strong,em,span,i,b"));
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, expected, "only contracted author spaces are removed");
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent),
[ "4", "\u{20000}\u{E0100}", "A" ]);
jz.revert(paragraph);
assert.equal(paragraph.textContent, expected);
assert.equal(paragraph.querySelector("[data-jz]"), null);
assert.deepEqual(Array.from(paragraph.querySelectorAll("strong,em,span,i,b")), authors);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing source: candidate markers respect custom CJK and western classes", () =>
{
const { dom, document } = setupDom("<p>漢A中</p>");
const paragraph = document.querySelector("p");
const jz = createJuzhen({ ...TEXT_ONLY, charClass: {
cjk: "\\uE000-\\uE003", western: "\\uE004-\\uE007",
} });
try
{
jz.render(paragraph);
assert.equal(paragraph.textContent, "漢A中");
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "漢", "A" ]);
jz.revert(paragraph);
assert.equal(paragraph.textContent, "漢A中");
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing source: supplementary markers keep source PUA and DOM atom offsets intact", () =>
{
const { dom, document } = setupDom("<p>漢\u{F0000}A中</p>");
const paragraph = document.querySelector("p");
const text = paragraph.textContent;
const jz = createJuzhen({ ...TEXT_ONLY, charClass: { cjk: "\\uE000-\\uF8FF" } });
try
{
jz.render(paragraph);
assert.equal(paragraph.textContent, text);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "A" ]);
jz.revert(paragraph);
assert.equal(paragraph.textContent, text);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing source: exhausted marker candidates fail before author text playback", () =>
{
const { dom, document } = setupDom("<p>漢 A中</p>");
const paragraph = document.querySelector("p");
const before = paragraph.innerHTML;
const jz = createJuzhen({ ...TEXT_ONLY, charClass: { cjk: "\\u0000-\\u{10FFFF}" } });
try
{
assert.throws(() => jz.render(paragraph), /已佔用全部間隙標記候選/);
assert.equal(paragraph.innerHTML, before, "no source atom or author space was replayed");
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
for (const selector of [ "code", "code span" ])
{
test(`spacing root: directly rendering ${selector} preserves pill contents`, () =>
{
const { dom, document } = setupDom('<p>外<code><span>漢A</span> 字B</code>文</p>');
const root = document.querySelector(selector);
const before = document.body.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(root);
assert.equal(root.querySelector("jz-hws"), null);
jz.render(root);
jz.revert(root);
assert.equal(document.body.innerHTML, before);
}
finally { jz.revert(root); dom.window.close(); }
});
}
test("spacing pill: a single author space across transparent nodes normalizes on both sides", () =>
{
const { dom, document } = setupDom('<p><span>漢</span><em> </em><code>x</code>'
+ '<i> </i><strong>文</strong></p>');
const paragraph = document.querySelector("p");
const authors = Array.from(paragraph.querySelectorAll("span,em,code,i,strong"));
const expected = paragraph.innerHTML.replace("<em> </em>", "<em></em>").replace("<i> </i>", "<i></i>");
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, "漢x文");
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "漢", "x" ]);
const rendered = paragraph.innerHTML;
jz.render(paragraph);
assert.equal(paragraph.innerHTML, rendered);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, expected);
assert.deepEqual(Array.from(paragraph.querySelectorAll("span,em,code,i,strong")), authors);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
for (const base of [ "漢", "\u{20000}" ])
{
for (const withSpace of [ false, true ])
{
test(`spacing pill: split ${base.codePointAt(0).toString(16)} IVS with space=${withSpace} stays one atom`, () =>
{
const vs = "\u{E0100}";
const space = withSpace ? " " : "";
const { dom, document } = setupDom(`<p><strong>${base}</strong><em>${vs}</em>`
+ `<span>${space}</span><code>x</code><i>${space}</i><b>${base}</b><u>${vs}</u></p>`);
const paragraph = document.querySelector("p");
const expected = paragraph.innerHTML.replace("<span> </span>", "<span></span>").replace("<i> </i>", "<i></i>");
const authors = Array.from(paragraph.children);
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, base + vs + "x" + base + vs);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ base + vs, "x" ]);
const rendered = paragraph.innerHTML;
jz.render(paragraph);
assert.equal(paragraph.innerHTML, rendered);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, expected, "cross-node selector returns to its author element");
assert.deepEqual(Array.from(paragraph.children), authors);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
}
for (const whitespace of [ " ", "\n", "\t", " \n" ])
{
test(`spacing pill: whitespace ${JSON.stringify(whitespace)} is a boundary, not a single removable space`, () =>
{
const { dom, document } = setupDom(`<p><span>漢</span><i>${whitespace}</i>`
+ `<code>x</code><em>${whitespace}</em><span>文</span></p>`);
const paragraph = document.querySelector("p");
const before = paragraph.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(paragraph);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
for (const barrier of [
'<mark data-juzhen-off="spacing"></mark>',
'<code data-juzhen-off="spacing"></code>',
'<span data-jz-skip></span>',
"<br>",
"<x-opaque></x-opaque>",
])
{
test(`spacing pill: empty ${barrier} stops lookup beyond one space on both sides`, () =>
{
const { dom, document } = setupDom(`<p><span>漢</span>${barrier}<i> </i>`
+ `<code>x</code><em> </em>${barrier}<span>文</span></p>`);
const paragraph = document.querySelector("p");
const before = paragraph.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(paragraph);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
test("spacing pill: out-of-scope neighbors stop space normalization", () =>
{
const { dom, document } = setupDom('<p><span>漢</span><i class="prose"> </i>'
+ '<code class="prose">x</code><em class="prose"> </em><span>文</span></p>');
const paragraph = document.querySelector("p");
const before = paragraph.innerHTML;
const jz = createJuzhen({ ...TEXT_ONLY, scope: { include: ".prose" } });
try
{
jz.render(paragraph);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing pill: local render cannot move an outside IVS base or delete an inside trailing space", () =>
{
const { dom, document } = setupDom('<p><strong>\u{20000}</strong><span id="root">'
+ '<em>\u{E0100}</em><code>x</code><i> </i></span><b>文</b></p>');
const root = document.getElementById("root");
const before = document.body.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(root);
assert.equal(document.body.innerHTML, before);
jz.render(root);
jz.revert(root);
assert.equal(document.body.innerHTML, before);
}
finally { jz.revert(root); dom.window.close(); }
});
test("spacing pill: a local transparent root encloses only its own neighbor", () =>
{
const { dom, document } = setupDom('<p>外<span id="root"><strong>漢</strong>'
+ '<code>x</code></span> 文</p>');
const root = document.getElementById("root");
const before = document.body.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(root);
assert.deepEqual(Array.from(root.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "漢" ]);
assert.equal(document.querySelector("p").lastChild.data, " 文", "root-external space remains author text");
jz.revert(root);
assert.equal(document.body.innerHTML, before);
}
finally { jz.revert(root); dom.window.close(); }
});
function appendTextChunks(parent, chunks)
{
const tags = [ "strong", "em", "i", "span", "b", "u" ];
return chunks.map((chunk, index) =>
{
const element = parent.ownerDocument.createElement(tags[index % tags.length]);
element.textContent = chunk;
parent.appendChild(element);
return element;
});
}
const SPLIT_CODEPOINTS = [
[ "\uD840", "\uDC00" ],
[ "\uD840", "\uDC00", "\uDB40", "\uDD00" ],
[ "\uD840", "\uDC00\uDB40", "\uDD00" ],
[ "\u{20000}", "\uDB40", "\uDD00" ],
[ "漢", "\uDB40", "\uDD00" ],
];
for (const [index, chunks] of SPLIT_CODEPOINTS.entries())
{
test(`spacing source: split surrogate codepoint case ${index} maps to one complete gap`, () =>
{
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
const authors = appendTextChunks(paragraph, [ ...chunks, "A" ]);
const before = paragraph.innerHTML;
const atom = chunks.join("");
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, atom + "A");
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ atom ]);
const rendered = paragraph.innerHTML;
jz.render(paragraph);
assert.equal(paragraph.innerHTML, rendered);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
assert.deepEqual(Array.from(paragraph.children), authors);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
for (const withSpace of [ false, true ])
{
test(`spacing pill: split surrogate case ${index}, space=${withSpace} remains complete on both sides`, () =>
{
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
appendTextChunks(paragraph, chunks);
if (withSpace) { appendTextChunks(paragraph, [ " " ]); }
const code = document.createElement("code");
code.textContent = "x";
paragraph.appendChild(code);
if (withSpace) { appendTextChunks(paragraph, [ " " ]); }
appendTextChunks(paragraph, chunks);
const authors = Array.from(paragraph.children);
const before = paragraph.innerHTML.replaceAll("<strong> </strong>", "<strong></strong>");
const atom = chunks.join("");
const jz = createJuzhen(TEXT_ONLY);
try
{
for (let cycle = 0; cycle < 3; cycle += 1)
{
jz.render(paragraph);
assert.equal(paragraph.textContent, atom + "x" + atom);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ atom, "x" ]);
const rendered = paragraph.innerHTML;
jz.render(paragraph);
assert.equal(paragraph.innerHTML, rendered);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
assert.deepEqual(Array.from(paragraph.children), authors);
}
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
}
for (const barrier of [
'<mark data-juzhen-off="spacing"></mark>',
'<code data-juzhen-off="spacing"></code>',
'<span data-jz-skip></span>',
"<br>",
"<x-opaque></x-opaque>",
])
{
test(`spacing surrogate: ${barrier} prevents joining adjacent high and low halves`, () =>
{
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
appendTextChunks(paragraph, [ "\uD840" ]);
paragraph.insertAdjacentHTML("beforeend", barrier);
appendTextChunks(paragraph, [ "\uDC00" ]);
const code = document.createElement("code");
code.textContent = "x";
paragraph.appendChild(code);
appendTextChunks(paragraph, [ "\uD840" ]);
paragraph.insertAdjacentHTML("beforeend", barrier);
appendTextChunks(paragraph, [ "\uDC00", "A" ]);
const before = paragraph.innerHTML;
const text = paragraph.textContent;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(paragraph);
assert.equal(paragraph.textContent, text);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
}
test("spacing surrogate: actual isolated UTF-16 halves stay author text", () =>
{
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
appendTextChunks(paragraph, [ "\uD840", "A" ]);
const code = document.createElement("code");
code.textContent = "x";
paragraph.appendChild(code);
appendTextChunks(paragraph, [ "\uDC00" ]);
const text = paragraph.textContent;
const before = paragraph.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(paragraph);
assert.equal(paragraph.textContent, text);
assert.deepEqual(Array.from(paragraph.querySelectorAll("jz-hws"), (gap) => gap.textContent), [ "A" ]);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing surrogate: a local root cannot join either outside surrogate half", () =>
{
const { dom, document } = setupDom('<p><strong></strong><span id="root">'
+ '<em></em><code>x</code><i></i></span><b></b></p>');
document.querySelector("strong").textContent = "\uD840";
document.querySelector("em").textContent = "\uDC00";
document.querySelector("i").textContent = "\uD840";
document.querySelector("b").textContent = "\uDC00";
const root = document.getElementById("root");
const before = document.body.innerHTML;
const jz = createJuzhen(TEXT_ONLY);
try
{
jz.render(root);
assert.equal(document.body.innerHTML, before);
jz.revert(root);
assert.equal(document.body.innerHTML, before);
}
finally { jz.revert(root); dom.window.close(); }
});
test("spacing surrogate: an excluded Text is a run boundary even when empty", () =>
{
const { dom, document } = setupDom("<p></p>");
const paragraph = document.querySelector("p");
appendTextChunks(paragraph, [ "\uD840" ]);
const excluded = document.createTextNode("");
paragraph.appendChild(excluded);
appendTextChunks(paragraph, [ "\uDC00", "A" ]);
const before = paragraph.innerHTML;
const jz = createJuzhen({ ...TEXT_ONLY, isSkipped: (node) => node === excluded });
try
{
jz.render(paragraph);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});
test("spacing surrogate: complete codepoints across source scope boundaries are not joined", () =>
{
const { dom, document } = setupDom('<p><strong></strong><em class="prose"></em>'
+ '<code class="prose">x</code><i class="prose"></i><b></b></p>');
document.querySelector("strong").textContent = "\uD840";
document.querySelector("em").textContent = "\uDC00";
document.querySelector("i").textContent = "\uD840";
document.querySelector("b").textContent = "\uDC00";
const paragraph = document.querySelector("p");
const before = paragraph.innerHTML;
const jz = createJuzhen({ ...TEXT_ONLY, scope: { include: ".prose" } });
try
{
jz.render(paragraph);
assert.equal(paragraph.querySelector("jz-hws"), null);
jz.revert(paragraph);
assert.equal(paragraph.innerHTML, before);
}
finally { jz.revert(paragraph); dom.window.close(); }
});