Heading ids: lowercase (GitHub-style) default + opt-in ASCII fold

src/Extension/AsciiHeadingIdsExtension.php

@@ -0,0 +1,59 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Carve\Extension;
+
+use Carve\CarveConverter;
+use Carve\Renderer\AsciiTransliterator;
+use Carve\Renderer\HtmlRenderer;
+
+/**
+ * Fold auto-generated heading ids to ASCII (opt-in)
+ *
+ * By default Carve heading ids are lowercased but keep non-ASCII
+ * characters verbatim (`# Über uns` -> `über-uns`), per carve spec #73.
+ * That is the GitHub/SSG convention and keeps `</#id>` / `[Heading][]`
+ * cross-references case-insensitive.
+ *
+ * Add this extension when you need share-safe ASCII fragment ids - e.g.
+ * URLs passed through auto-linkers that truncate or mis-encode non-ASCII.
+ * It transliterates the slug to ASCII before the final lowercase step
+ * (`# Über uns` -> `uber-uns`). Unmapped scripts (CJK, Arabic, Greek)
+ * still pass through unchanged; attach an explicit `{#id}` for those.
+ *
+ * The same transform is applied to the parse-time tracker so implicit
+ * `[Heading][]` references resolve to the folded ids.
+ *
+ * Example:
+ * ```php
+ * $converter = new CarveConverter();
+ * $converter->addExtension(new AsciiHeadingIdsExtension());
+ * ```
+ */
+class AsciiHeadingIdsExtension implements ExtensionInterface
+{
+    protected AsciiTransliterator $transliterator;
+
+    public function __construct(?AsciiTransliterator $transliterator = null)
+    {
+        $this->transliterator = $transliterator ?? new AsciiTransliterator();
+    }
+
+    public function register(CarveConverter $converter): void
+    {
+        $transliterator = $this->transliterator;
+        $transform = static fn (string $slug): string => $transliterator->transliterate($slug);
+
+        // Parse-time tracker (implicit [Heading][] references).
+        $converter->getParser()->setHeadingIdTransformer($transform);
+
+        // Render-time tracker (the ids emitted in HTML). Only meaningful
+        // with HtmlRenderer; silently skip otherwise.
+        if (!$converter->getRenderer() instanceof HtmlRenderer) {
+            return;
+        }
+
+        $converter->getHeadingIdTracker()->setIdTransformer($transform);
+    }
+}

src/Parser/BlockParser.php

@@ -176,6 +176,18 @@ class BlockParser
      */
     protected bool $blocksInterruptParagraphs = false;
 
+    /**
+     * Optional slug transform mirrored onto the parse-time heading-id
+     * tracker so implicit `[Heading][]` references agree with the
+     * render-time ids (set by AsciiHeadingIdsExtension).
+     */
+    protected ?Closure $headingIdTransformer = null;
+
+    public function setHeadingIdTransformer(?Closure $headingIdTransformer): void
+    {
+        $this->headingIdTransformer = $headingIdTransformer;
+    }
+
     public function __construct(
         bool $collectWarnings = false,
         bool $strictMode = false,
@@ -754,6 +766,7 @@ protected function extractAbbreviations(array $lines): void
     protected function extractHeadingReferences(array $lines): void
     {
         $headingIdTracker = new HeadingIdTracker();
+        $headingIdTracker->setIdTransformer($this->headingIdTransformer);
         $pendingId = null;
         $count = count($lines);
 

src/Renderer/HeadingIdTracker.php

@@ -13,6 +13,8 @@
 use Carve\Node\Inline\Symbol;
 use Carve\Node\Inline\Text;
 use Carve\Node\Node;
+use Closure;
+use Normalizer;
 
 /**
  * Shared service for generating and deduplicating heading IDs
@@ -62,11 +64,16 @@ class HeadingIdTracker
      */
     protected array $textById = [];
 
-    protected AsciiTransliterator $transliterator;
+    /**
+     * Optional transform applied to the base slug (e.g. ASCII
+     * transliteration). Set by AsciiHeadingIdsExtension; null leaves
+     * non-ASCII characters in the id verbatim (the default).
+     */
+    protected ?Closure $idTransformer = null;
 
-    public function __construct(?AsciiTransliterator $transliterator = null)
+    public function setIdTransformer(?Closure $idTransformer): void
     {
-        $this->transliterator = $transliterator ?? new AsciiTransliterator();
+        $this->idTransformer = $idTransformer;
     }
 
     /**
@@ -114,46 +121,53 @@ public function trackId(string $id): void
 
     /**
      * Normalize text to a Carve heading identifier (the normative
-     * "Automatic Identifiers" algorithm):
+     * "Automatic Identifiers" algorithm, carve spec #73):
      *
-     * 1. Lowercase, Unicode-aware.
-     * 2. Trim whitespace.
-     * 3. Delete the CSS-unsafe punctuation ' " ; : (so "What's New"
-     *    becomes "whats-new", not "what-s-new").
-     * 4. Replace every maximal run of characters that are not Unicode
-     *    letters/digits/_/- (spaces included) with a single '-'.
-     * 5. Collapse runs of '-', then trim leading/trailing '-'.
-     * 6. If the result starts with a digit, prefix 'section-' (a CSS
+     * 1. NFC-normalize (so a decomposed `résumé` slugs identically to
+     *    its precomposed form).
+     * 2. Replace each maximal run of non-alphanumeric ASCII with a
+     *    single '-' and trim; non-ASCII characters are preserved.
+     * 3. If an id transformer is set (AsciiHeadingIdsExtension), apply
+     *    it to the slug and re-run step 2 (opt-in ASCII transliteration).
+     * 4. Lowercase, Unicode-aware: GitHub-style anchors that make ids
+     *    and `</#id>` / `[Heading][]` cross-references case-insensitive
+     *    with no special lookup logic.
+     * 5. If the result starts with a digit, prefix 's-' (a CSS
      *    identifier may not start with a digit).
-     * 7. If the result is empty, the identifier is 'section'.
+     * 6. If the result is empty, the identifier is 's'.
      *
      * Deduplication against the document namespace (shared by explicit
      * {#id} and generated ids) is applied by the caller.
      */
     public function normalizeId(string $text): string
     {
-        // 1. Transliterate to ASCII so the id survives being shared as a
-        // URL fragment through auto-linkers (which routinely truncate or
-        // mis-encode non-ASCII). Latin/Cyrillic/Greek/punctuation become
-        // byte-identical with or without ext-intl; unmapped scripts (CJK,
-        // …) are romanized when intl is present and otherwise drop, so
-        // the empty-result branch below yields a stable `section` id.
-        $text = $this->transliterator->transliterate($text);
-
-        // Carve "Automatic Identifiers" algorithm (normative).
-        $id = mb_strtolower($text, 'UTF-8'); // 2. lowercase
-        $id = trim($id); // 3. trim
-        $id = str_replace(["'", '"', ';', ':'], '', $id); // 4. drop CSS-unsafe punct
-        // 5/6. non letter/digit/_/- runs (incl. spaces) -> single '-'
-        $id = preg_replace('/[^\p{L}\p{N}_-]+/u', '-', $id) ?? $id;
-        $id = preg_replace('/-{2,}/', '-', $id) ?? $id; // 7. collapse
-        $id = trim($id, '-'); // 7. trim '-'
+        if (class_exists(Normalizer::class)) {
+            $text = Normalizer::normalize($text, Normalizer::FORM_C) ?: $text;
+        }
 
+        $id = $this->slugRun($text);
+        if ($this->idTransformer !== null) {
+            $id = $this->slugRun(($this->idTransformer)($id));
+        }
+
+        $id = mb_strtolower($id, 'UTF-8');
         if ($id !== '' && preg_match('/^\p{N}/u', $id)) {
-            $id = 'section-' . $id; // 8. digit-leading
+            $id = 's-' . $id;
         }
 
-        return $id !== '' ? $id : 'section'; // 9. empty -> 'section'
+        return $id !== '' ? $id : 's';
+    }
+
+    /**
+     * jgm/djot#393 slug step: replace each maximal run of
+     * non-alphanumeric ASCII with a single '-' and trim. Non-ASCII
+     * characters and letter case are preserved.
+     */
+    protected function slugRun(string $text): string
+    {
+        $text = preg_replace('/[^0-9A-Za-z\x{80}-\x{10FFFF}]+/u', '-', $text) ?? $text;
+
+        return trim($text, '-');
     }
 
     /**

src/Renderer/HtmlRenderer.php

@@ -1271,7 +1271,12 @@ protected function renderInsert(Insert $node): string
     protected function renderHeadingRef(HeadingRef $node): string
     {
         $id = $node->getTargetId();
-        $label = $this->getRenderContext()->headingIdTracker->getTextForId($id) ?? $id;
+        $label = $this->getRenderContext()->headingIdTracker->getTextForId($id);
+        if ($label === null) {
+            // An unresolved </#id> renders as its literal source text,
+            // not a dangling self-link (matches the spec and carve-js).
+            return $this->escape('</#' . $id . '>');
+        }
 
         return '<a href="#' . $this->escapeAttribute($id) . '">'
             . $this->escape($label) . '</a>';

tests/CarveCorpusTest.php

@@ -116,6 +116,7 @@ class CarveCorpusTest extends TestCase
         '78-fenced-code-language-with-punctuation',
         '79-multi-line-headings',
         '80-blockquote-lazy-continuation-stops-at-a-fenced-block',
+        '81-list-lazy-continuation',
         '82-compact-list-blocks',
         '83-list-continuation-marker',
     ];

tests/InlineCommentTest.php

@@ -52,7 +52,7 @@ public function testCommentAtStartOfInlineRun(): void
         // Heading text "%% all" reaches the inline parser at offset 0, so the
         // start-of-run branch fires and the whole title is a comment.
         $this->assertSame(
-            "<section id=\"section\">\n  <h1></h1>\n</section>",
+            "<section id=\"s\">\n  <h1></h1>\n</section>",
             $this->html('# %% all'),
         );
     }

tests/TestCase/CarveConverterTest.php

@@ -18,7 +18,6 @@
 use PHPUnit\Framework\Attributes\DataProvider;
 use PHPUnit\Framework\TestCase;
 use RuntimeException;
-use Transliterator;
 
 class CarveConverterTest extends TestCase
 {
@@ -1808,18 +1807,12 @@ public function testUnicodeInHeading(): void
         $djot = '# 日本語の見出し';
         $result = $this->converter->convert($djot);
 
-        // The visible heading text is unchanged; only the ID is made
-        // ASCII-safe so it survives being shared as a URL fragment. The id
-        // lives on the section wrapper, not the heading.
+        // By default the id preserves non-ASCII characters (only case is
+        // folded), per carve spec #73. The id lives on the section
+        // wrapper, not the heading. Opt into ASCII via
+        // AsciiHeadingIdsExtension (see its dedicated test).
         $this->assertStringContainsString('>日本語の見出し</h1>', $result);
-        $this->assertStringNotContainsString('id="日本語の見出し"', $result);
-        $this->assertMatchesRegularExpression('/<section id="[\x21-\x7E]+">/', $result);
-
-        if (class_exists(Transliterator::class)) {
-            // With ext-intl the CJK heading is romanized (lowercased per
-            // Carve's normative algorithm) rather than dropped.
-            $this->assertStringContainsString('<section id="ri-ben-yuno-jian-chushi">', $result);
-        }
+        $this->assertStringContainsString('<section id="日本語の見出し">', $result);
     }
 
     public function testUnicodeInLink(): void

tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php

@@ -0,0 +1,86 @@
+<?php
+
+declare(strict_types=1);
+
+namespace Carve\Test\TestCase\Extension;
+
+use Carve\CarveConverter;
+use Carve\Extension\AsciiHeadingIdsExtension;
+use Carve\Extension\HeadingReferenceExtension;
+use PHPUnit\Framework\TestCase;
+use Transliterator;
+use function class_exists;
+
+class AsciiHeadingIdsExtensionTest extends TestCase
+{
+    public function testDefaultKeepsNonAsciiVerbatim(): void
+    {
+        $html = (new CarveConverter())->convert('# Über uns');
+
+        // No extension: id keeps non-ASCII (only case folded).
+        $this->assertStringContainsString('<section id="über-uns">', $html);
+    }
+
+    public function testExtensionFoldsLatinDiacriticsToAscii(): void
+    {
+        $converter = new CarveConverter();
+        $converter->addExtension(new AsciiHeadingIdsExtension());
+
+        $html = $converter->convert('# Über uns');
+
+        $this->assertStringContainsString('<section id="uber-uns">', $html);
+        $this->assertStringContainsString('>Über uns</h1>', $html);
+    }
+
+    public function testExtensionFoldsCyrillic(): void
+    {
+        $converter = new CarveConverter();
+        $converter->addExtension(new AsciiHeadingIdsExtension());
+
+        $html = $converter->convert('# Привет мир');
+
+        $this->assertStringContainsString('<section id="privet-mir">', $html);
+    }
+
+    public function testDigitLeadingSlugKeepsThePrefixAfterFold(): void
+    {
+        $converter = new CarveConverter();
+        $converter->addExtension(new AsciiHeadingIdsExtension());
+
+        $html = $converter->convert('# 2024 Recap');
+
+        $this->assertStringContainsString('<section id="s-2024-recap">', $html);
+    }
+
+    public function testImplicitReferenceResolvesToTheFoldedId(): void
+    {
+        $converter = new CarveConverter();
+        $converter->addExtension(new HeadingReferenceExtension());
+        $converter->addExtension(new AsciiHeadingIdsExtension());
+
+        $html = $converter->convert(<<<'DJOT'
+See [[Über uns]].
+
+# Über uns
+DJOT);
+
+        // The parse-time tracker must apply the same fold, otherwise the
+        // implicit [[...]] reference would point at the unfolded id.
+        $this->assertStringContainsString('href="#uber-uns"', $html);
+    }
+
+    public function testCjkIsRomanizedWithIntl(): void
+    {
+        if (!class_exists(Transliterator::class)) {
+            $this->markTestSkipped('ext-intl not available');
+        }
+
+        $converter = new CarveConverter();
+        $converter->addExtension(new AsciiHeadingIdsExtension());
+
+        $html = $converter->convert('# 日本語の見出し');
+
+        $this->assertStringContainsString('<section id="ri-ben-yuno-jian-chushi">', $html);
+        $this->assertStringContainsString('>日本語の見出し</h1>', $html);
+    }
+}

tests/TestCase/Extension/HeadingReferenceExtensionTest.php

@@ -184,7 +184,10 @@ public function testHeadingWithSmartQuotesMatchesStraightQuoteReference(): void
 # Say "Hello"
 DJOT);
 
-        $this->assertStringContainsString('href="#say-hello"', $html);
+        // Both the heading and the reference smart-convert the quotes, and
+        // ids now keep non-ASCII verbatim (carve spec #73), so they share
+        // the smart-quoted id and still resolve.
+        $this->assertStringContainsString('href="#say-“hello”"', $html);
         $this->assertStringNotContainsString('[[Say "Hello"]]', $html);
     }
 
@@ -213,12 +216,11 @@ public function testHeadingWithApostropheResolvesCorrectly(): void
 # Bob's Guide
 DJOT);
 
-        // Smart-punctuation turns the straight apostrophe into U+2019, then
-        // ASCII transliteration folds it back to a straight `'`, which the
-        // normalize step drops along with other CSS-unsafe punctuation:
-        // `Bob's Guide` -> `bobs-guide`. The href must match the heading id.
-        $this->assertStringContainsString('id="bobs-guide"', $html);
-        $this->assertStringContainsString('href="#bobs-guide"', $html);
+        // Smart-punctuation turns the straight apostrophe into U+2019, which
+        // is non-ASCII and now kept verbatim in the id (carve spec #73):
+        // `Bob's Guide` -> `bob’s-guide`. The href must match the heading id.
+        $this->assertStringContainsString('id="bob’s-guide"', $html);
+        $this->assertStringContainsString('href="#bob’s-guide"', $html);
         $this->assertStringNotContainsString('data-heading-ref=', $html);
         $this->assertStringNotContainsString('[[Bob\'s Guide]]', $html);
     }