diff --git a/src/Extension/AsciiHeadingIdsExtension.php b/src/Extension/AsciiHeadingIdsExtension.php
new file mode 100644
index 00000000..3747c26e
--- /dev/null
+++ b/src/Extension/AsciiHeadingIdsExtension.php
@@ -0,0 +1,59 @@
+ `über-uns`), per carve spec #73.
+ * That is the GitHub/SSG convention and keeps `#id>` / `[Heading][]`
+ * cross-references case-insensitive.
+ *
+ * Add this extension when you need share-safe ASCII fragment ids - e.g.
+ * URLs passed through auto-linkers that truncate or mis-encode non-ASCII.
+ * It transliterates the slug to ASCII before the final lowercase step
+ * (`# Über uns` -> `uber-uns`). Unmapped scripts (CJK, Arabic, Greek)
+ * still pass through unchanged; attach an explicit `{#id}` for those.
+ *
+ * The same transform is applied to the parse-time tracker so implicit
+ * `[Heading][]` references resolve to the folded ids.
+ *
+ * Example:
+ * ```php
+ * $converter = new CarveConverter();
+ * $converter->addExtension(new AsciiHeadingIdsExtension());
+ * ```
+ */
+class AsciiHeadingIdsExtension implements ExtensionInterface
+{
+ protected AsciiTransliterator $transliterator;
+
+ public function __construct(?AsciiTransliterator $transliterator = null)
+ {
+ $this->transliterator = $transliterator ?? new AsciiTransliterator();
+ }
+
+ public function register(CarveConverter $converter): void
+ {
+ $transliterator = $this->transliterator;
+ $transform = static fn (string $slug): string => $transliterator->transliterate($slug);
+
+ // Parse-time tracker (implicit [Heading][] references).
+ $converter->getParser()->setHeadingIdTransformer($transform);
+
+ // Render-time tracker (the ids emitted in HTML). Only meaningful
+ // with HtmlRenderer; silently skip otherwise.
+ if (!$converter->getRenderer() instanceof HtmlRenderer) {
+ return;
+ }
+
+ $converter->getHeadingIdTracker()->setIdTransformer($transform);
+ }
+}
diff --git a/src/Parser/BlockParser.php b/src/Parser/BlockParser.php
index f619056c..621b5bd1 100644
--- a/src/Parser/BlockParser.php
+++ b/src/Parser/BlockParser.php
@@ -176,6 +176,18 @@ class BlockParser
*/
protected bool $blocksInterruptParagraphs = false;
+ /**
+ * Optional slug transform mirrored onto the parse-time heading-id
+ * tracker so implicit `[Heading][]` references agree with the
+ * render-time ids (set by AsciiHeadingIdsExtension).
+ */
+ protected ?Closure $headingIdTransformer = null;
+
+ public function setHeadingIdTransformer(?Closure $headingIdTransformer): void
+ {
+ $this->headingIdTransformer = $headingIdTransformer;
+ }
+
public function __construct(
bool $collectWarnings = false,
bool $strictMode = false,
@@ -754,6 +766,7 @@ protected function extractAbbreviations(array $lines): void
protected function extractHeadingReferences(array $lines): void
{
$headingIdTracker = new HeadingIdTracker();
+ $headingIdTracker->setIdTransformer($this->headingIdTransformer);
$pendingId = null;
$count = count($lines);
diff --git a/src/Renderer/HeadingIdTracker.php b/src/Renderer/HeadingIdTracker.php
index d979f300..cde7b589 100644
--- a/src/Renderer/HeadingIdTracker.php
+++ b/src/Renderer/HeadingIdTracker.php
@@ -13,6 +13,8 @@
use Carve\Node\Inline\Symbol;
use Carve\Node\Inline\Text;
use Carve\Node\Node;
+use Closure;
+use Normalizer;
/**
* Shared service for generating and deduplicating heading IDs
@@ -62,11 +64,16 @@ class HeadingIdTracker
*/
protected array $textById = [];
- protected AsciiTransliterator $transliterator;
+ /**
+ * Optional transform applied to the base slug (e.g. ASCII
+ * transliteration). Set by AsciiHeadingIdsExtension; null leaves
+ * non-ASCII characters in the id verbatim (the default).
+ */
+ protected ?Closure $idTransformer = null;
- public function __construct(?AsciiTransliterator $transliterator = null)
+ public function setIdTransformer(?Closure $idTransformer): void
{
- $this->transliterator = $transliterator ?? new AsciiTransliterator();
+ $this->idTransformer = $idTransformer;
}
/**
@@ -114,46 +121,53 @@ public function trackId(string $id): void
/**
* Normalize text to a Carve heading identifier (the normative
- * "Automatic Identifiers" algorithm):
+ * "Automatic Identifiers" algorithm, carve spec #73):
*
- * 1. Lowercase, Unicode-aware.
- * 2. Trim whitespace.
- * 3. Delete the CSS-unsafe punctuation ' " ; : (so "What's New"
- * becomes "whats-new", not "what-s-new").
- * 4. Replace every maximal run of characters that are not Unicode
- * letters/digits/_/- (spaces included) with a single '-'.
- * 5. Collapse runs of '-', then trim leading/trailing '-'.
- * 6. If the result starts with a digit, prefix 'section-' (a CSS
+ * 1. NFC-normalize (so a decomposed `résumé` slugs identically to
+ * its precomposed form).
+ * 2. Replace each maximal run of non-alphanumeric ASCII with a
+ * single '-' and trim; non-ASCII characters are preserved.
+ * 3. If an id transformer is set (AsciiHeadingIdsExtension), apply
+ * it to the slug and re-run step 2 (opt-in ASCII transliteration).
+ * 4. Lowercase, Unicode-aware: GitHub-style anchors that make ids
+ * and `#id>` / `[Heading][]` cross-references case-insensitive
+ * with no special lookup logic.
+ * 5. If the result starts with a digit, prefix 's-' (a CSS
* identifier may not start with a digit).
- * 7. If the result is empty, the identifier is 'section'.
+ * 6. If the result is empty, the identifier is 's'.
*
* Deduplication against the document namespace (shared by explicit
* {#id} and generated ids) is applied by the caller.
*/
public function normalizeId(string $text): string
{
- // 1. Transliterate to ASCII so the id survives being shared as a
- // URL fragment through auto-linkers (which routinely truncate or
- // mis-encode non-ASCII). Latin/Cyrillic/Greek/punctuation become
- // byte-identical with or without ext-intl; unmapped scripts (CJK,
- // …) are romanized when intl is present and otherwise drop, so
- // the empty-result branch below yields a stable `section` id.
- $text = $this->transliterator->transliterate($text);
-
- // Carve "Automatic Identifiers" algorithm (normative).
- $id = mb_strtolower($text, 'UTF-8'); // 2. lowercase
- $id = trim($id); // 3. trim
- $id = str_replace(["'", '"', ';', ':'], '', $id); // 4. drop CSS-unsafe punct
- // 5/6. non letter/digit/_/- runs (incl. spaces) -> single '-'
- $id = preg_replace('/[^\p{L}\p{N}_-]+/u', '-', $id) ?? $id;
- $id = preg_replace('/-{2,}/', '-', $id) ?? $id; // 7. collapse
- $id = trim($id, '-'); // 7. trim '-'
+ if (class_exists(Normalizer::class)) {
+ $text = Normalizer::normalize($text, Normalizer::FORM_C) ?: $text;
+ }
+ $id = $this->slugRun($text);
+ if ($this->idTransformer !== null) {
+ $id = $this->slugRun(($this->idTransformer)($id));
+ }
+
+ $id = mb_strtolower($id, 'UTF-8');
if ($id !== '' && preg_match('/^\p{N}/u', $id)) {
- $id = 'section-' . $id; // 8. digit-leading
+ $id = 's-' . $id;
}
- return $id !== '' ? $id : 'section'; // 9. empty -> 'section'
+ return $id !== '' ? $id : 's';
+ }
+
+ /**
+ * jgm/djot#393 slug step: replace each maximal run of
+ * non-alphanumeric ASCII with a single '-' and trim. Non-ASCII
+ * characters and letter case are preserved.
+ */
+ protected function slugRun(string $text): string
+ {
+ $text = preg_replace('/[^0-9A-Za-z\x{80}-\x{10FFFF}]+/u', '-', $text) ?? $text;
+
+ return trim($text, '-');
}
/**
diff --git a/src/Renderer/HtmlRenderer.php b/src/Renderer/HtmlRenderer.php
index 58ecc3ed..ced657d2 100644
--- a/src/Renderer/HtmlRenderer.php
+++ b/src/Renderer/HtmlRenderer.php
@@ -1271,7 +1271,12 @@ protected function renderInsert(Insert $node): string
protected function renderHeadingRef(HeadingRef $node): string
{
$id = $node->getTargetId();
- $label = $this->getRenderContext()->headingIdTracker->getTextForId($id) ?? $id;
+ $label = $this->getRenderContext()->headingIdTracker->getTextForId($id);
+ if ($label === null) {
+ // An unresolved #id> renders as its literal source text,
+ // not a dangling self-link (matches the spec and carve-js).
+ return $this->escape('#' . $id . '>');
+ }
return ''
. $this->escape($label) . '';
diff --git a/tests/CarveCorpusTest.php b/tests/CarveCorpusTest.php
index 5c808bbe..1af9281b 100644
--- a/tests/CarveCorpusTest.php
+++ b/tests/CarveCorpusTest.php
@@ -116,6 +116,7 @@ class CarveCorpusTest extends TestCase
'78-fenced-code-language-with-punctuation',
'79-multi-line-headings',
'80-blockquote-lazy-continuation-stops-at-a-fenced-block',
+ '81-list-lazy-continuation',
'82-compact-list-blocks',
'83-list-continuation-marker',
];
diff --git a/tests/InlineCommentTest.php b/tests/InlineCommentTest.php
index 9b098782..5621bfff 100644
--- a/tests/InlineCommentTest.php
+++ b/tests/InlineCommentTest.php
@@ -52,7 +52,7 @@ public function testCommentAtStartOfInlineRun(): void
// Heading text "%% all" reaches the inline parser at offset 0, so the
// start-of-run branch fires and the whole title is a comment.
$this->assertSame(
- "",
+ "",
$this->html('# %% all'),
);
}
diff --git a/tests/TestCase/CarveConverterTest.php b/tests/TestCase/CarveConverterTest.php
index e724ddef..6c89ba19 100644
--- a/tests/TestCase/CarveConverterTest.php
+++ b/tests/TestCase/CarveConverterTest.php
@@ -18,7 +18,6 @@
use PHPUnit\Framework\Attributes\DataProvider;
use PHPUnit\Framework\TestCase;
use RuntimeException;
-use Transliterator;
class CarveConverterTest extends TestCase
{
@@ -1808,18 +1807,12 @@ public function testUnicodeInHeading(): void
$djot = '# 日本語の見出し';
$result = $this->converter->convert($djot);
- // The visible heading text is unchanged; only the ID is made
- // ASCII-safe so it survives being shared as a URL fragment. The id
- // lives on the section wrapper, not the heading.
+ // By default the id preserves non-ASCII characters (only case is
+ // folded), per carve spec #73. The id lives on the section
+ // wrapper, not the heading. Opt into ASCII via
+ // AsciiHeadingIdsExtension (see its dedicated test).
$this->assertStringContainsString('>日本語の見出し', $result);
- $this->assertStringNotContainsString('id="日本語の見出し"', $result);
- $this->assertMatchesRegularExpression('//', $result);
-
- if (class_exists(Transliterator::class)) {
- // With ext-intl the CJK heading is romanized (lowercased per
- // Carve's normative algorithm) rather than dropped.
- $this->assertStringContainsString('', $result);
- }
+ $this->assertStringContainsString('', $result);
}
public function testUnicodeInLink(): void
diff --git a/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php b/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php
new file mode 100644
index 00000000..f150ccea
--- /dev/null
+++ b/tests/TestCase/Extension/AsciiHeadingIdsExtensionTest.php
@@ -0,0 +1,86 @@
+convert('# Über uns');
+
+ // No extension: id keeps non-ASCII (only case folded).
+ $this->assertStringContainsString('', $html);
+ }
+
+ public function testExtensionFoldsLatinDiacriticsToAscii(): void
+ {
+ $converter = new CarveConverter();
+ $converter->addExtension(new AsciiHeadingIdsExtension());
+
+ $html = $converter->convert('# Über uns');
+
+ $this->assertStringContainsString('', $html);
+ $this->assertStringContainsString('>Über uns', $html);
+ }
+
+ public function testExtensionFoldsCyrillic(): void
+ {
+ $converter = new CarveConverter();
+ $converter->addExtension(new AsciiHeadingIdsExtension());
+
+ $html = $converter->convert('# Привет мир');
+
+ $this->assertStringContainsString('', $html);
+ }
+
+ public function testDigitLeadingSlugKeepsThePrefixAfterFold(): void
+ {
+ $converter = new CarveConverter();
+ $converter->addExtension(new AsciiHeadingIdsExtension());
+
+ $html = $converter->convert('# 2024 Recap');
+
+ $this->assertStringContainsString('', $html);
+ }
+
+ public function testImplicitReferenceResolvesToTheFoldedId(): void
+ {
+ $converter = new CarveConverter();
+ $converter->addExtension(new HeadingReferenceExtension());
+ $converter->addExtension(new AsciiHeadingIdsExtension());
+
+ $html = $converter->convert(<<<'DJOT'
+See [[Über uns]].
+
+# Über uns
+DJOT);
+
+ // The parse-time tracker must apply the same fold, otherwise the
+ // implicit [[...]] reference would point at the unfolded id.
+ $this->assertStringContainsString('href="#uber-uns"', $html);
+ }
+
+ public function testCjkIsRomanizedWithIntl(): void
+ {
+ if (!class_exists(Transliterator::class)) {
+ $this->markTestSkipped('ext-intl not available');
+ }
+
+ $converter = new CarveConverter();
+ $converter->addExtension(new AsciiHeadingIdsExtension());
+
+ $html = $converter->convert('# 日本語の見出し');
+
+ $this->assertStringContainsString('', $html);
+ $this->assertStringContainsString('>日本語の見出し', $html);
+ }
+}
diff --git a/tests/TestCase/Extension/HeadingReferenceExtensionTest.php b/tests/TestCase/Extension/HeadingReferenceExtensionTest.php
index 6d567eec..8eb075b0 100644
--- a/tests/TestCase/Extension/HeadingReferenceExtensionTest.php
+++ b/tests/TestCase/Extension/HeadingReferenceExtensionTest.php
@@ -184,7 +184,10 @@ public function testHeadingWithSmartQuotesMatchesStraightQuoteReference(): void
# Say "Hello"
DJOT);
- $this->assertStringContainsString('href="#say-hello"', $html);
+ // Both the heading and the reference smart-convert the quotes, and
+ // ids now keep non-ASCII verbatim (carve spec #73), so they share
+ // the smart-quoted id and still resolve.
+ $this->assertStringContainsString('href="#say-“hello”"', $html);
$this->assertStringNotContainsString('[[Say "Hello"]]', $html);
}
@@ -213,12 +216,11 @@ public function testHeadingWithApostropheResolvesCorrectly(): void
# Bob's Guide
DJOT);
- // Smart-punctuation turns the straight apostrophe into U+2019, then
- // ASCII transliteration folds it back to a straight `'`, which the
- // normalize step drops along with other CSS-unsafe punctuation:
- // `Bob's Guide` -> `bobs-guide`. The href must match the heading id.
- $this->assertStringContainsString('id="bobs-guide"', $html);
- $this->assertStringContainsString('href="#bobs-guide"', $html);
+ // Smart-punctuation turns the straight apostrophe into U+2019, which
+ // is non-ASCII and now kept verbatim in the id (carve spec #73):
+ // `Bob's Guide` -> `bob’s-guide`. The href must match the heading id.
+ $this->assertStringContainsString('id="bob’s-guide"', $html);
+ $this->assertStringContainsString('href="#bob’s-guide"', $html);
$this->assertStringNotContainsString('data-heading-ref=', $html);
$this->assertStringNotContainsString('[[Bob\'s Guide]]', $html);
}
diff --git a/tests/TestCase/Renderer/HeadingIdTrackerTest.php b/tests/TestCase/Renderer/HeadingIdTrackerTest.php
index aa62cc74..f0e31937 100644
--- a/tests/TestCase/Renderer/HeadingIdTrackerTest.php
+++ b/tests/TestCase/Renderer/HeadingIdTrackerTest.php
@@ -102,8 +102,8 @@ public function testEmptyHeadingGetsFallbackId(): void
$id1 = $this->tracker->getIdForHeading($heading1);
$id2 = $this->tracker->getIdForHeading($heading2);
- $this->assertSame('section', $id1);
- $this->assertSame('section-2', $id2);
+ $this->assertSame('s', $id1);
+ $this->assertSame('s-2', $id2);
}
public function testResetClearsState(): void
@@ -157,14 +157,14 @@ public function testNormalizeId(): void
$this->assertSame('multiple-spaces', $this->tracker->normalizeId('Multiple Spaces'));
$this->assertSame('this-t-key-params-fallback', $this->tracker->normalizeId("\$this->t(\$key, \$params = [], \$fallback = '')"));
$this->assertSame('my-title', $this->tracker->normalizeId('My --- title'));
- // Non-ASCII is transliterated for link-safety; Latin and Cyrillic
- // are byte-identical with or without ext-intl.
- $this->assertSame('uber-uns', $this->tracker->normalizeId('Über uns'));
- $this->assertSame('cafe-resume', $this->tracker->normalizeId('café résumé'));
- $this->assertSame('privet-mir', $this->tracker->normalizeId('Привет мир'));
- $this->assertSame('section', $this->tracker->normalizeId('###'));
- $this->assertSame('section-123-things', $this->tracker->normalizeId('123 Things'));
- $this->assertSame('section-1-introduction', $this->tracker->normalizeId('1. Introduction'));
+ // Non-ASCII is preserved by default (only case folded); see
+ // AsciiHeadingIdsExtension for the opt-in ASCII fold.
+ $this->assertSame('über-uns', $this->tracker->normalizeId('Über uns'));
+ $this->assertSame('café-résumé', $this->tracker->normalizeId('café résumé'));
+ $this->assertSame('привет-мир', $this->tracker->normalizeId('Привет мир'));
+ $this->assertSame('s', $this->tracker->normalizeId('###'));
+ $this->assertSame('s-123-things', $this->tracker->normalizeId('123 Things'));
+ $this->assertSame('s-1-introduction', $this->tracker->normalizeId('1. Introduction'));
}
public function testGetPlainText(): void
diff --git a/tests/spec b/tests/spec
index a50fdad0..c086db96 160000
--- a/tests/spec
+++ b/tests/spec
@@ -1 +1 @@
-Subproject commit a50fdad00ee30d7c0ed09ff6a6d430da0ea12b40
+Subproject commit c086db962239f2acecf7de6fbb4b42a09bf501fc