diff --git a/src/Extension/AsciiHeadingIdsExtension.php b/src/Extension/AsciiHeadingIdsExtension.php new file mode 100644 index 00000000..3747c26e --- /dev/null +++ b/src/Extension/AsciiHeadingIdsExtension.php @@ -0,0 +1,59 @@ + `über-uns`), per carve spec #73. + * That is the GitHub/SSG convention and keeps `` / `[Heading][]` + * cross-references case-insensitive. + * + * Add this extension when you need share-safe ASCII fragment ids - e.g. + * URLs passed through auto-linkers that truncate or mis-encode non-ASCII. + * It transliterates the slug to ASCII before the final lowercase step + * (`# Über uns` -> `uber-uns`). Unmapped scripts (CJK, Arabic, Greek) + * still pass through unchanged; attach an explicit `{#id}` for those. + * + * The same transform is applied to the parse-time tracker so implicit + * `[Heading][]` references resolve to the folded ids. + * + * Example: + * ```php + * $converter = new CarveConverter(); + * $converter->addExtension(new AsciiHeadingIdsExtension()); + * ``` + */ +class AsciiHeadingIdsExtension implements ExtensionInterface +{ + protected AsciiTransliterator $transliterator; + + public function __construct(?AsciiTransliterator $transliterator = null) + { + $this->transliterator = $transliterator ?? new AsciiTransliterator(); + } + + public function register(CarveConverter $converter): void + { + $transliterator = $this->transliterator; + $transform = static fn (string $slug): string => $transliterator->transliterate($slug); + + // Parse-time tracker (implicit [Heading][] references). + $converter->getParser()->setHeadingIdTransformer($transform); + + // Render-time tracker (the ids emitted in HTML). Only meaningful + // with HtmlRenderer; silently skip otherwise. + if (!$converter->getRenderer() instanceof HtmlRenderer) { + return; + } + + $converter->getHeadingIdTracker()->setIdTransformer($transform); + } +} diff --git a/src/Parser/BlockParser.php b/src/Parser/BlockParser.php index f619056c..621b5bd1 100644 --- a/src/Parser/BlockParser.php +++ b/src/Parser/BlockParser.php @@ -176,6 +176,18 @@ class BlockParser */ protected bool $blocksInterruptParagraphs = false; + /** + * Optional slug transform mirrored onto the parse-time heading-id + * tracker so implicit `[Heading][]` references agree with the + * render-time ids (set by AsciiHeadingIdsExtension). + */ + protected ?Closure $headingIdTransformer = null; + + public function setHeadingIdTransformer(?Closure $headingIdTransformer): void + { + $this->headingIdTransformer = $headingIdTransformer; + } + public function __construct( bool $collectWarnings = false, bool $strictMode = false, @@ -754,6 +766,7 @@ protected function extractAbbreviations(array $lines): void protected function extractHeadingReferences(array $lines): void { $headingIdTracker = new HeadingIdTracker(); + $headingIdTracker->setIdTransformer($this->headingIdTransformer); $pendingId = null; $count = count($lines); diff --git a/src/Renderer/HeadingIdTracker.php b/src/Renderer/HeadingIdTracker.php index d979f300..cde7b589 100644 --- a/src/Renderer/HeadingIdTracker.php +++ b/src/Renderer/HeadingIdTracker.php @@ -13,6 +13,8 @@ use Carve\Node\Inline\Symbol; use Carve\Node\Inline\Text; use Carve\Node\Node; +use Closure; +use Normalizer; /** * Shared service for generating and deduplicating heading IDs @@ -62,11 +64,16 @@ class HeadingIdTracker */ protected array $textById = []; - protected AsciiTransliterator $transliterator; + /** + * Optional transform applied to the base slug (e.g. ASCII + * transliteration). Set by AsciiHeadingIdsExtension; null leaves + * non-ASCII characters in the id verbatim (the default). + */ + protected ?Closure $idTransformer = null; - public function __construct(?AsciiTransliterator $transliterator = null) + public function setIdTransformer(?Closure $idTransformer): void { - $this->transliterator = $transliterator ?? new AsciiTransliterator(); + $this->idTransformer = $idTransformer; } /** @@ -114,46 +121,53 @@ public function trackId(string $id): void /** * Normalize text to a Carve heading identifier (the normative - * "Automatic Identifiers" algorithm): + * "Automatic Identifiers" algorithm, carve spec #73): * - * 1. Lowercase, Unicode-aware. - * 2. Trim whitespace. - * 3. Delete the CSS-unsafe punctuation ' " ; : (so "What's New" - * becomes "whats-new", not "what-s-new"). - * 4. Replace every maximal run of characters that are not Unicode - * letters/digits/_/- (spaces included) with a single '-'. - * 5. Collapse runs of '-', then trim leading/trailing '-'. - * 6. If the result starts with a digit, prefix 'section-' (a CSS + * 1. NFC-normalize (so a decomposed `résumé` slugs identically to + * its precomposed form). + * 2. Replace each maximal run of non-alphanumeric ASCII with a + * single '-' and trim; non-ASCII characters are preserved. + * 3. If an id transformer is set (AsciiHeadingIdsExtension), apply + * it to the slug and re-run step 2 (opt-in ASCII transliteration). + * 4. Lowercase, Unicode-aware: GitHub-style anchors that make ids + * and `` / `[Heading][]` cross-references case-insensitive + * with no special lookup logic. + * 5. If the result starts with a digit, prefix 's-' (a CSS * identifier may not start with a digit). - * 7. If the result is empty, the identifier is 'section'. + * 6. If the result is empty, the identifier is 's'. * * Deduplication against the document namespace (shared by explicit * {#id} and generated ids) is applied by the caller. */ public function normalizeId(string $text): string { - // 1. Transliterate to ASCII so the id survives being shared as a - // URL fragment through auto-linkers (which routinely truncate or - // mis-encode non-ASCII). Latin/Cyrillic/Greek/punctuation become - // byte-identical with or without ext-intl; unmapped scripts (CJK, - // …) are romanized when intl is present and otherwise drop, so - // the empty-result branch below yields a stable `section` id. - $text = $this->transliterator->transliterate($text); - - // Carve "Automatic Identifiers" algorithm (normative). - $id = mb_strtolower($text, 'UTF-8'); // 2. lowercase - $id = trim($id); // 3. trim - $id = str_replace(["'", '"', ';', ':'], '', $id); // 4. drop CSS-unsafe punct - // 5/6. non letter/digit/_/- runs (incl. spaces) -> single '-' - $id = preg_replace('/[^\p{L}\p{N}_-]+/u', '-', $id) ?? $id; - $id = preg_replace('/-{2,}/', '-', $id) ?? $id; // 7. collapse - $id = trim($id, '-'); // 7. trim '-' + if (class_exists(Normalizer::class)) { + $text = Normalizer::normalize($text, Normalizer::FORM_C) ?: $text; + } + $id = $this->slugRun($text); + if ($this->idTransformer !== null) { + $id = $this->slugRun(($this->idTransformer)($id)); + } + + $id = mb_strtolower($id, 'UTF-8'); if ($id !== '' && preg_match('/^\p{N}/u', $id)) { - $id = 'section-' . $id; // 8. digit-leading + $id = 's-' . $id; } - return $id !== '' ? $id : 'section'; // 9. empty -> 'section' + return $id !== '' ? $id : 's'; + } + + /** + * jgm/djot#393 slug step: replace each maximal run of + * non-alphanumeric ASCII with a single '-' and trim. Non-ASCII + * characters and letter case are preserved. + */ + protected function slugRun(string $text): string + { + $text = preg_replace('/[^0-9A-Za-z\x{80}-\x{10FFFF}]+/u', '-', $text) ?? $text; + + return trim($text, '-'); } /** diff --git a/src/Renderer/HtmlRenderer.php b/src/Renderer/HtmlRenderer.php index 58ecc3ed..ced657d2 100644 --- a/src/Renderer/HtmlRenderer.php +++ b/src/Renderer/HtmlRenderer.php @@ -1271,7 +1271,12 @@ protected function renderInsert(Insert $node): string protected function renderHeadingRef(HeadingRef $node): string { $id = $node->getTargetId(); - $label = $this->getRenderContext()->headingIdTracker->getTextForId($id) ?? $id; + $label = $this->getRenderContext()->headingIdTracker->getTextForId($id); + if ($label === null) { + // An unresolved renders as its literal source text, + // not a dangling self-link (matches the spec and carve-js). + return $this->escape(''); + } return '' . $this->escape($label) . ''; diff --git a/tests/CarveCorpusTest.php b/tests/CarveCorpusTest.php index 5c808bbe..1af9281b 100644 --- a/tests/CarveCorpusTest.php +++ b/tests/CarveCorpusTest.php @@ -116,6 +116,7 @@ class CarveCorpusTest extends TestCase '78-fenced-code-language-with-punctuation', '79-multi-line-headings', '80-blockquote-lazy-continuation-stops-at-a-fenced-block', + '81-list-lazy-continuation', '82-compact-list-blocks', '83-list-continuation-marker', ]; diff --git a/tests/InlineCommentTest.php b/tests/InlineCommentTest.php index 9b098782..5621bfff 100644 --- a/tests/InlineCommentTest.php +++ b/tests/InlineCommentTest.php @@ -52,7 +52,7 @@ public function testCommentAtStartOfInlineRun(): void // Heading text "%% all" reaches the inline parser at offset 0, so the // start-of-run branch fires and the whole title is a comment. $this->assertSame( - "
\n

\n
", + "
\n

\n
", $this->html('# %% all'), ); } diff --git a/tests/TestCase/CarveConverterTest.php b/tests/TestCase/CarveConverterTest.php index e724ddef..6c89ba19 100644 --- a/tests/TestCase/CarveConverterTest.php +++ b/tests/TestCase/CarveConverterTest.php @@ -18,7 +18,6 @@ use PHPUnit\Framework\Attributes\DataProvider; use PHPUnit\Framework\TestCase; use RuntimeException; -use Transliterator; class CarveConverterTest extends TestCase { @@ -1808,18 +1807,12 @@ public function testUnicodeInHeading(): void $djot = '# 日本語の見出し'; $result = $this->converter->convert($djot); - // The visible heading text is unchanged; only the ID is made - // ASCII-safe so it survives being shared as a URL fragment. The id - // lives on the section wrapper, not the heading. + // By default the id preserves non-ASCII characters (only case is + // folded), per carve spec #73. The id lives on the section + // wrapper, not the heading. Opt into ASCII via + // AsciiHeadingIdsExtension (see its dedicated test). $this->assertStringContainsString('>日本語の見出し', $result); - $this->assertStringNotContainsString('id="日本語の見出し"', $result); - $this->assertMatchesRegularExpression('/
/', $result); - - if (class_exists(Transliterator::class)) { - // With ext-intl the CJK heading is romanized (lowercased per - // Carve's normative algorithm) rather than dropped. - $this->assertStringContainsString('
', $result); - } + $this->assertStringContainsString('
', $result); } public function testUnicodeInLink(): void diff --git a/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php b/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php new file mode 100644 index 00000000..f150ccea --- /dev/null +++ b/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php @@ -0,0 +1,86 @@ +convert('# Über uns'); + + // No extension: id keeps non-ASCII (only case folded). + $this->assertStringContainsString('
', $html); + } + + public function testExtensionFoldsLatinDiacriticsToAscii(): void + { + $converter = new CarveConverter(); + $converter->addExtension(new AsciiHeadingIdsExtension()); + + $html = $converter->convert('# Über uns'); + + $this->assertStringContainsString('
', $html); + $this->assertStringContainsString('>Über uns', $html); + } + + public function testExtensionFoldsCyrillic(): void + { + $converter = new CarveConverter(); + $converter->addExtension(new AsciiHeadingIdsExtension()); + + $html = $converter->convert('# Привет мир'); + + $this->assertStringContainsString('
', $html); + } + + public function testDigitLeadingSlugKeepsThePrefixAfterFold(): void + { + $converter = new CarveConverter(); + $converter->addExtension(new AsciiHeadingIdsExtension()); + + $html = $converter->convert('# 2024 Recap'); + + $this->assertStringContainsString('
', $html); + } + + public function testImplicitReferenceResolvesToTheFoldedId(): void + { + $converter = new CarveConverter(); + $converter->addExtension(new HeadingReferenceExtension()); + $converter->addExtension(new AsciiHeadingIdsExtension()); + + $html = $converter->convert(<<<'DJOT' +See [[Über uns]]. + +# Über uns +DJOT); + + // The parse-time tracker must apply the same fold, otherwise the + // implicit [[...]] reference would point at the unfolded id. + $this->assertStringContainsString('href="#uber-uns"', $html); + } + + public function testCjkIsRomanizedWithIntl(): void + { + if (!class_exists(Transliterator::class)) { + $this->markTestSkipped('ext-intl not available'); + } + + $converter = new CarveConverter(); + $converter->addExtension(new AsciiHeadingIdsExtension()); + + $html = $converter->convert('# 日本語の見出し'); + + $this->assertStringContainsString('
', $html); + $this->assertStringContainsString('>日本語の見出し', $html); + } +} diff --git a/tests/TestCase/Extension/HeadingReferenceExtensionTest.php b/tests/TestCase/Extension/HeadingReferenceExtensionTest.php index 6d567eec..8eb075b0 100644 --- a/tests/TestCase/Extension/HeadingReferenceExtensionTest.php +++ b/tests/TestCase/Extension/HeadingReferenceExtensionTest.php @@ -184,7 +184,10 @@ public function testHeadingWithSmartQuotesMatchesStraightQuoteReference(): void # Say "Hello" DJOT); - $this->assertStringContainsString('href="#say-hello"', $html); + // Both the heading and the reference smart-convert the quotes, and + // ids now keep non-ASCII verbatim (carve spec #73), so they share + // the smart-quoted id and still resolve. + $this->assertStringContainsString('href="#say-“hello”"', $html); $this->assertStringNotContainsString('[[Say "Hello"]]', $html); } @@ -213,12 +216,11 @@ public function testHeadingWithApostropheResolvesCorrectly(): void # Bob's Guide DJOT); - // Smart-punctuation turns the straight apostrophe into U+2019, then - // ASCII transliteration folds it back to a straight `'`, which the - // normalize step drops along with other CSS-unsafe punctuation: - // `Bob's Guide` -> `bobs-guide`. The href must match the heading id. - $this->assertStringContainsString('id="bobs-guide"', $html); - $this->assertStringContainsString('href="#bobs-guide"', $html); + // Smart-punctuation turns the straight apostrophe into U+2019, which + // is non-ASCII and now kept verbatim in the id (carve spec #73): + // `Bob's Guide` -> `bob’s-guide`. The href must match the heading id. + $this->assertStringContainsString('id="bob’s-guide"', $html); + $this->assertStringContainsString('href="#bob’s-guide"', $html); $this->assertStringNotContainsString('data-heading-ref=', $html); $this->assertStringNotContainsString('[[Bob\'s Guide]]', $html); } diff --git a/tests/TestCase/Renderer/HeadingIdTrackerTest.php b/tests/TestCase/Renderer/HeadingIdTrackerTest.php index aa62cc74..f0e31937 100644 --- a/tests/TestCase/Renderer/HeadingIdTrackerTest.php +++ b/tests/TestCase/Renderer/HeadingIdTrackerTest.php @@ -102,8 +102,8 @@ public function testEmptyHeadingGetsFallbackId(): void $id1 = $this->tracker->getIdForHeading($heading1); $id2 = $this->tracker->getIdForHeading($heading2); - $this->assertSame('section', $id1); - $this->assertSame('section-2', $id2); + $this->assertSame('s', $id1); + $this->assertSame('s-2', $id2); } public function testResetClearsState(): void @@ -157,14 +157,14 @@ public function testNormalizeId(): void $this->assertSame('multiple-spaces', $this->tracker->normalizeId('Multiple Spaces')); $this->assertSame('this-t-key-params-fallback', $this->tracker->normalizeId("\$this->t(\$key, \$params = [], \$fallback = '')")); $this->assertSame('my-title', $this->tracker->normalizeId('My --- title')); - // Non-ASCII is transliterated for link-safety; Latin and Cyrillic - // are byte-identical with or without ext-intl. - $this->assertSame('uber-uns', $this->tracker->normalizeId('Über uns')); - $this->assertSame('cafe-resume', $this->tracker->normalizeId('café résumé')); - $this->assertSame('privet-mir', $this->tracker->normalizeId('Привет мир')); - $this->assertSame('section', $this->tracker->normalizeId('###')); - $this->assertSame('section-123-things', $this->tracker->normalizeId('123 Things')); - $this->assertSame('section-1-introduction', $this->tracker->normalizeId('1. Introduction')); + // Non-ASCII is preserved by default (only case folded); see + // AsciiHeadingIdsExtension for the opt-in ASCII fold. + $this->assertSame('über-uns', $this->tracker->normalizeId('Über uns')); + $this->assertSame('café-résumé', $this->tracker->normalizeId('café résumé')); + $this->assertSame('привет-мир', $this->tracker->normalizeId('Привет мир')); + $this->assertSame('s', $this->tracker->normalizeId('###')); + $this->assertSame('s-123-things', $this->tracker->normalizeId('123 Things')); + $this->assertSame('s-1-introduction', $this->tracker->normalizeId('1. Introduction')); } public function testGetPlainText(): void diff --git a/tests/spec b/tests/spec index a50fdad0..c086db96 160000 --- a/tests/spec +++ b/tests/spec @@ -1 +1 @@ -Subproject commit a50fdad00ee30d7c0ed09ff6a6d430da0ea12b40 +Subproject commit c086db962239f2acecf7de6fbb4b42a09bf501fc