diff --git a/Lib/glyphsLib/builder/features.py b/Lib/glyphsLib/builder/features.py index cf92e9375..da3a70449 100644 --- a/Lib/glyphsLib/builder/features.py +++ b/Lib/glyphsLib/builder/features.py @@ -34,6 +34,7 @@ INSERT_FEATURE_MARKER_COMMENT, ) from .tokens import TokenExpander, PassThruExpander +from .multi_language import expand_multi_language_statements from .variable_features import VariableFeatureConverter if TYPE_CHECKING: @@ -198,6 +199,12 @@ def _to_ufo_features( # noqa: C901 if master is not None: full_text = VariableFeatureConverter(font).convert(full_text) + # Expand Glyphs' multiple languages syntax (`language AZE CRT;`) into the + # one-tag-per-statement form the FEA spec allows. After the conversion + # above, so that `#ifndef VARIABLE` blocks are already resolved and no rule + # is ever replayed out of one. + full_text = expand_multi_language_statements(full_text) + if not full_text or not expand_includes: return full_text diff --git a/Lib/glyphsLib/builder/multi_language.py b/Lib/glyphsLib/builder/multi_language.py new file mode 100644 index 000000000..f349b70d0 --- /dev/null +++ b/Lib/glyphsLib/builder/multi_language.py @@ -0,0 +1,404 @@ +# +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Expand Glyphs' multiple languages syntax into plain FEA. + +Glyphs lets a single ``language`` statement carry several tags and applies +everything that follows to each of them:: + + language AZE CRT KAZ TAT TRK; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + +The FEA spec allows exactly one tag per statement, so feaLib rejects this with +``Expected ';'`` at the second tag. See +https://handbook.glyphsapp.com/layout/multiple-languages-syntax/, +https://github.com/googlefonts/glyphsLib/issues/1109 and, for the extension +written up as a spec proposal, +https://github.com/adobe-type-tools/feature_file_workshops/pull/8. + +The scan runs on feaLib's own ``Lexer`` rather than on the text, because FEA +treats a line ending as ordinary whitespace: statements may share a line or +span several, and a brace or a semicolon inside a comment or a string is not +code. Tokenizing is what tells the three apart. +""" + +import collections +import logging +import re + +from fontTools.feaLib.error import FeatureLibError +from fontTools.feaLib.lexer import Lexer + +logger = logging.getLogger(__name__) + +# Keywords the FEA spec allows after the tag of a `language` statement. +_LANGUAGE_KEYWORDS = frozenset(("exclude_dflt", "include_dflt", "required")) + +# A `language` or `script` statement closes the block opened by the previous one. +_BLOCK_DELIMITERS = frozenset(("language", "script")) + +# The language every other one inherits from. It has to be specified alone -- +# see the spec proposal linked above -- so a statement listing it beside other +# tags is not the shorthand, and Glyphs rejects it too. +_DFLT = "dflt" + +# An OpenType language system tag. Unlike a feature tag it may be shorter than +# four characters (`ROM`, `AZE`). +_TAG_LENGTH = 4 + +# What a statement inside the shorthand's scope is, and what happens to it when +# the scope is replayed for the second and every further tag: +# +# _LOOKUP `lookup NAME [useExtension] { ... } NAME;` +# emitted once, replayed as the reference `lookup NAME;` +# _DEFINITION `@Class = ...;` or `markClass ...;` +# emitted once, dropped from the replay +# _INCLUDE `include(...);` +# unknown contents, so the shorthand is left for feaLib to reject +# _PLAIN anything else, `lookup NAME;` references included +# repeated verbatim, which is what the shorthand means +_LOOKUP = "lookup" +_DEFINITION = "definition" +_INCLUDE = "include" +_PLAIN = "plain" + +# `language` as a whole word. `languagesystem` does not match, which matters: +# the guard below would otherwise fire for nearly every font. +_language_re = re.compile(r"\blanguage\b") + +# What feaLib treats as one line ending. `\r\n` has to come first so that the +# pair is not read as two. +_line_ending_re = re.compile(r"\r\n|\r|\n") + +_Token = collections.namedtuple("_Token", "kind value start") + +_Item = collections.namedtuple("_Item", "kind name first last") + + +def expand_multi_language_statements(fea): + """Rewrite multi-tag ``language`` statements into one tag per statement. + + The scope of the shorthand is emitted once for the first tag and then + replayed for every remaining one, statement by statement; see the table + above for what happens to each kind of statement. Named lookups cannot + simply be duplicated -- that would redefine them -- so the replays + reference the lookup defined under the first tag:: + + language AZE; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + language CRT; + lookup idotaccent; + ... + + The emitted text is sliced out of ``fea`` by offset, so comments, + indentation and blank lines survive the round trip untouched. + + A statement that is already spec-compliant, or that cannot be classified + with confidence, is left exactly as it is: the shorthand is invalid FEA, so + leaving it alone means feaLib reports it rather than glyphsLib emitting a + silently wrong language mapping. So is one listing ``dflt`` beside other + tags, which is not the shorthand at all, one sitting outside any block, + where FEA does not allow ``language`` in the first place, and one whose + scope holds a statement with no closing semicolon. + """ + # Lexing is not free -- feaLib takes a few microseconds per token -- and + # most feature code has no `language` statement at all. The keyword has to + # be there as a word for the shorthand to exist, so this cannot skip work + # that was needed; a match inside a comment only costs the lexing. + if not _language_re.search(fea): + return fea + + tokens = _tokenize(fea) + if tokens is None: + return fea + + out = [] + expanded = False + # Offset in `fea` up to which `out` has been filled. + consumed = 0 + i = 0 + while i < len(tokens): + if not _is_name(tokens[i], "language"): + i += 1 + continue + + end = _statement_end(tokens, i) + if end is None: + # No semicolon closes it, so where the statement stops is anyone's + # guess. Skip the keyword and let feaLib report the code. + i += 1 + continue + + tags, keywords = _split_language_tokens(tokens, i + 1, end) + if not tags or len(tags) < 2: + i = end + continue + + if _depth_at(tokens, i) < 1: + # FEA only allows `language` inside a feature block, so a + # shorthand at the top level is invalid whatever it was meant to + # say. There is no enclosing brace for its scope to end at either, + # so replaying it would swallow whatever block follows. + logger.warning( + "'language %s;' sits outside any block and was left unexpanded", + " ".join(tags), + ) + i = end + continue + + if _DFLT in tags: + # Expanding this one would compile: `language AZE;` implies + # `include_dflt`, so AZE would carry the rules twice, once + # inherited from `dflt` and once replayed. Leaving it alone keeps + # the failure loud instead. + logger.warning( + "'language %s;' lists dflt beside other tags; dflt has to be " + "specified alone, so the statement was left unexpanded", + " ".join(tags), + ) + i = end + continue + + items, scope_end, reason = _scan_scope(tokens, end) + if items is None: + logger.warning( + "'language %s;' %s and was left unexpanded", " ".join(tags), reason + ) + i = end + continue + + # The semicolon of the shorthand, and the end of the scope it governs. + semicolon = tokens[end - 1].start + scope = _scope_end_offset(tokens, items, semicolon) + indent = _indent(fea, tokens[i].start) + + out.append(fea[consumed : tokens[i].start]) + out.append(_language_statement(tags[0], keywords)) + # Everything from just after the semicolon (a trailing comment) to the + # end of the scope is kept verbatim. + out.append(fea[semicolon + 1 : scope]) + for tag in tags[1:]: + out.append("\n" + indent + _language_statement(tag, keywords)) + out.extend(_replay(fea, tokens, items, indent)) + + logger.debug("Expanded 'language %s;'", " ".join(tags)) + expanded = True + consumed = scope + i = scope_end + + if not expanded: + return fea + out.append(fea[consumed:]) + return "".join(out) + + +def _tokenize(fea): + """Return ``fea`` as a list of ``_Token``s carrying absolute offsets. + + Comments are dropped. Returns ``None`` if feaLib cannot even tokenize the + code -- there is nothing to expand with confidence then, and feaLib reports + the problem itself once it compiles the UFO. Feature text still holding + unexpanded Glyphs tokens (``$[...]``, ``${...}``) lands here as well, and + is likewise left alone. + """ + starts = _line_starts(fea) + tokens = [] + try: + for kind, value, (_, line, column) in Lexer(fea, ""): + if kind == Lexer.COMMENT: + continue + tokens.append(_Token(kind, value, starts[line - 1] + column - 1)) + except FeatureLibError: + return None + return tokens + + +def _line_starts(fea): + """Offset of the first character of every line. + + feaLib ends a line at ``\\n``, at ``\\r`` and at the pair ``\\r\\n``, each + counting as one. Splitting on ``\\n`` alone would leave every offset after + a lone carriage return short by one per line, and the whole module slices + ``fea`` by offset. + """ + starts = [0] + for line_ending in _line_ending_re.finditer(fea): + starts.append(line_ending.end()) + return starts + + +def _is_name(token, value): + return token.kind == Lexer.NAME and token.value == value + + +def _is_symbol(token, value): + return token.kind == Lexer.SYMBOL and token.value == value + + +def _statement_end(tokens, i): + """Index just past the token that ends the statement starting at ``i``. + + That is the first semicolon at nesting depth zero. A block statement + (``lookup X { ... } X;``) therefore ends at the semicolon following its + closing brace, wherever the braces happen to sit on the page. + + Returns ``None`` if the statement does not end that way: the tokens run + out, or the brace closing the block the statement sits in comes first. + Both mean a semicolon is missing, and the code cannot then be split into + statements with confidence. + """ + depth = 0 + while i < len(tokens): + token = tokens[i] + if token.kind == Lexer.SYMBOL: + if token.value == "{": + depth += 1 + elif token.value == "}": + depth -= 1 + if depth < 0: + return None + elif token.value == ";" and depth == 0: + return i + 1 + i += 1 + return None + + +def _depth_at(tokens, i): + """How many blocks are open just before the token at ``i``.""" + depth = 0 + for token in tokens[:i]: + if token.kind == Lexer.SYMBOL: + if token.value == "{": + depth += 1 + elif token.value == "}": + depth -= 1 + return depth + + +def _split_language_tokens(tokens, start, end): + """Split the inside of a ``language ...;`` statement. + + ``end`` is what ``_statement_end`` returned, so the closing semicolon is + already known to be there. Returns ``(tags, keywords)``, or + ``(None, None)`` for anything unexpected, so that the caller leaves the + statement alone rather than guessing. + """ + tags = [] + keywords = [] + for token in tokens[start : end - 1]: + if token.kind != Lexer.NAME: + return None, None + if token.value in _LANGUAGE_KEYWORDS: + keywords.append(token.value) + elif keywords or len(token.value) > _TAG_LENGTH: + # A tag after a keyword, or a token that is not a tag at all. + return None, None + else: + tags.append(token.value) + return tags, keywords + + +def _scan_scope(tokens, start): + """Split the statements governed by the shorthand into ``_Item``s. + + The scope runs to the next ``language``/``script`` statement or to the end + of the enclosing block, whichever comes first. Returns + ``(None, start, reason)`` when the scope cannot be replayed at all. + """ + items = [] + i = start + while i < len(tokens): + token = tokens[i] + if _is_symbol(token, "}"): + # Closes the block the `language` statement itself sits in. + break + if token.kind == Lexer.NAME and token.value in _BLOCK_DELIMITERS: + break + end = _statement_end(tokens, i) + if end is None: + return None, i, "governs a statement that does not end in ';'" + kind, name = _classify(tokens, i, end) + if kind == _INCLUDE: + return None, i, "governs an include()" + items.append(_Item(kind, name, i, end)) + i = end + return items, i, None + + +def _classify(tokens, i, end): + """Return ``(kind, name)`` for the statement between ``i`` and ``end``.""" + head = tokens[i] + if head.kind == Lexer.GLYPHCLASS: + # `@Class = ...` defines; a rule may start with a class as well. + if i + 1 < end and _is_symbol(tokens[i + 1], "="): + return _DEFINITION, None + return _PLAIN, None + if head.kind != Lexer.NAME: + return _PLAIN, None + if head.value == "markClass": + return _DEFINITION, None + if head.value == "include": + return _INCLUDE, None + if head.value == "lookup" and i + 1 < end: + # A definition opens a block; `lookup NAME;` is a reference and is + # repeated as it stands. + if any(_is_symbol(token, "{") for token in tokens[i:end]): + return _LOOKUP, tokens[i + 1].value + return _PLAIN, None + + +def _scope_end_offset(tokens, items, semicolon): + """Offset just past the last statement of the scope. + + Deliberately not the start of the next one: whatever sits between them + (newlines, a comment) belongs to the untouched remainder of the code. + """ + if not items: + return semicolon + 1 + return tokens[items[-1].last - 1].start + 1 + + +def _replay(fea, tokens, items, indent): + """Render the scope again for a repeated tag. + + Each replayed statement is laid out at ``indent``, the indentation of the + ``language`` statement itself, rather than its own. + """ + out = [] + for item in items: + if item.kind == _LOOKUP: + out.append(f"\n{indent}lookup {item.name};") + elif item.kind != _DEFINITION: + statement = fea[tokens[item.first].start : tokens[item.last - 1].start + 1] + out.append("\n" + indent + statement) + return out + + +def _language_statement(tag, keywords): + return "language %s;" % " ".join([tag] + keywords) + + +def _indent(fea, offset): + """The whitespace the statement at ``offset`` is indented by. + + ``""`` if the statement is not the first thing on its line. + """ + line_start = fea.rfind("\n", 0, offset) + 1 + indent = fea[line_start:offset] + return indent if not indent.strip() else "" diff --git a/tests/builder/multi_language_test.py b/tests/builder/multi_language_test.py new file mode 100644 index 000000000..ed5c27d2d --- /dev/null +++ b/tests/builder/multi_language_test.py @@ -0,0 +1,859 @@ +# +# Copyright 2026 Google Inc. All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import io +from textwrap import dedent + +from fontTools.feaLib.parser import Parser + +from glyphsLib import classes, to_glyphs, to_ufos +from glyphsLib.builder.multi_language import expand_multi_language_statements + +GLYPH_NAMES = ["i", "idotaccent", "Scedilla", "Scommaaccent", "Lcommaaccent"] + +LANGUAGE_SYSTEMS = "languagesystem DFLT dflt;\nlanguagesystem latn dflt;\n" + "".join( + f"languagesystem latn {tag};\n" + for tag in ("AZE", "CRT", "KAZ", "TAT", "TRK", "ROM", "MOL") +) + + +def parse(fea): + """Parse the feature text, raising on anything feaLib does not accept.""" + feature_file = io.StringIO(LANGUAGE_SYSTEMS + fea) + return Parser(feature_file, glyphNames=GLYPH_NAMES).parse() + + +def normalize(fea): + """The text with every kind of line ending written as ``\\n``.""" + return fea.replace("\r\n", "\n").replace("\r", "\n") + + +# Every shape the expansion has to survive, well formed or not, gathered so +# that the invariants below run over all of them rather than over one example +# each. The individual tests above pin down what each one is expected to do. +AWKWARD_SOURCES = ( + # Well formed, and expanded. + "feature locl {\nlanguage AZE CRT;\nsub i by idotaccent;\n} locl;\n", + "feature locl {\nlanguage ROM MOL;\nlookup l {\nsub i by idotaccent;\n} l;\n" + "@Ced = [Scedilla];\nsub @Ced by Scommaaccent;\n} locl;\n", + # Line endings feaLib counts but a `\n` split does not. + "feature locl {\r\nlanguage AZE CRT;\r\nsub i by idotaccent;\r\n} locl;\r\n", + "feature locl {\rlanguage AZE CRT;\rsub i by idotaccent;\r} locl;\r", + "feature locl {\nlanguage AZE CRT;\rsub i by idotaccent;\n} locl;\n", + # Not the shorthand, or not classifiable: left untouched. + "feature locl {\nlanguage dflt AZE;\nsub i by idotaccent;\n} locl;\n", + "feature locl {\nlanguage TOOLONGTAG OTHER;\nsub i by idotaccent;\n} locl;\n", + "feature locl {\nlanguage AZE CRT;\ninclude(other.fea);\n} locl;\n", + "feature locl {\nlanguage AZE CRT;\nsub $[name] by idotaccent;\n} locl;\n", + # A missing semicolon, at the end of the code and before a closing brace. + "feature locl {\nlanguage AZE CRT;\nsub i by idotaccent\n", + "feature locl {\nlanguage AZE CRT;\nsub i by idotaccent\n} locl;\n\n" + "feature ccmp {\nsub Scedilla by Scommaaccent;\n} ccmp;\n", + # Outside any block, where FEA does not allow `language` at all. + "language AZE CRT;\nsub i by idotaccent;\n\nfeature locl {\n" + "sub Scedilla by Scommaaccent;\n} locl;\n", +) + + +def check(source, expected): + """Expand the source, assert the whole output, then parse it.""" + fea = expand_multi_language_statements(dedent(source)) + assert fea == dedent(expected) + parse(fea) + + +def test_bare_rules_are_repeated(): + check( + """\ + feature locl { + script latn; + language ROM MOL; + sub Scedilla by Scommaaccent; + } locl; + """, + """\ + feature locl { + script latn; + language ROM; + sub Scedilla by Scommaaccent; + language MOL; + sub Scedilla by Scommaaccent; + } locl; + """, + ) + + +def test_named_lookup_is_defined_once_and_referenced(): + check( + """\ + feature locl { + script latn; + language AZE CRT KAZ; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + language CRT; + lookup idotaccent; + language KAZ; + lookup idotaccent; + } locl; + """, + ) + + +def test_lookup_with_use_extension_is_referenced(): + check( + """\ + feature locl { + language AZE CRT; + lookup idot useExtension { + sub i by idotaccent; + } idot; + } locl; + """, + """\ + feature locl { + language AZE; + lookup idot useExtension { + sub i by idotaccent; + } idot; + language CRT; + lookup idot; + } locl; + """, + ) + + +def test_lookup_with_brace_on_the_next_line_is_referenced(): + check( + """\ + feature locl { + language AZE CRT; + lookup idot + { + sub i by idotaccent; + } idot; + } locl; + """, + """\ + feature locl { + language AZE; + lookup idot + { + sub i by idotaccent; + } idot; + language CRT; + lookup idot; + } locl; + """, + ) + + +def test_glyph_class_is_not_redefined(): + check( + """\ + feature locl { + language ROM MOL; + @Cedillas = [Scedilla]; + sub @Cedillas by Scommaaccent; + } locl; + """, + """\ + feature locl { + language ROM; + @Cedillas = [Scedilla]; + sub @Cedillas by Scommaaccent; + language MOL; + sub @Cedillas by Scommaaccent; + } locl; + """, + ) + + +def test_multi_line_glyph_class_is_dropped_whole(): + check( + """\ + feature locl { + language ROM MOL; + @Ced = [Scedilla + Lcommaaccent]; + sub @Ced by Scommaaccent; + } locl; + """, + """\ + feature locl { + language ROM; + @Ced = [Scedilla + Lcommaaccent]; + sub @Ced by Scommaaccent; + language MOL; + sub @Ced by Scommaaccent; + } locl; + """, + ) + + +def test_mark_class_is_not_redefined(): + check( + """\ + feature test { + language ROM MOL; + markClass [Scedilla] @MC; + pos base i mark @MC; + } test; + """, + """\ + feature test { + language ROM; + markClass [Scedilla] @MC; + pos base i mark @MC; + language MOL; + pos base i mark @MC; + } test; + """, + ) + + +def test_closing_brace_in_a_comment_does_not_truncate_the_body(): + """It would otherwise bind the rules to the last tag only. + + And parse cleanly while doing so. The comment itself is not a statement, + so it stays where the author put it instead of being repeated. + """ + check( + """\ + feature locl { + language AZE CRT; + # see the } sign + sub i by idotaccent; + } locl; + """, + """\ + feature locl { + language AZE; + # see the } sign + sub i by idotaccent; + language CRT; + sub i by idotaccent; + } locl; + """, + ) + + +def test_opening_brace_in_a_comment_does_not_swallow_the_closing_brace(): + check( + """\ + feature locl { + language AZE CRT; + # an opening { sign + sub i by idotaccent; + } locl; + """, + """\ + feature locl { + language AZE; + # an opening { sign + sub i by idotaccent; + language CRT; + sub i by idotaccent; + } locl; + """, + ) + + +def test_trailing_comment_on_a_delimiter_ends_the_body(): + check( + """\ + feature locl { + language AZE CRT; + sub i by idotaccent; + language TRK; # Turkish + sub Scedilla by Scommaaccent; + } locl; + """, + """\ + feature locl { + language AZE; + sub i by idotaccent; + language CRT; + sub i by idotaccent; + language TRK; # Turkish + sub Scedilla by Scommaaccent; + } locl; + """, + ) + + +def test_trailing_comment_on_the_statement_is_kept(): + check( + """\ + feature locl { + script latn; + language AZE CRT; # Turkic + sub i by idotaccent; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; # Turkic + sub i by idotaccent; + language CRT; + sub i by idotaccent; + } locl; + """, + ) + + +def test_keywords_are_kept_on_every_tag(): + check( + """\ + feature locl { + script latn; + language ROM MOL exclude_dflt; + sub Scedilla by Scommaaccent; + } locl; + """, + """\ + feature locl { + script latn; + language ROM exclude_dflt; + sub Scedilla by Scommaaccent; + language MOL exclude_dflt; + sub Scedilla by Scommaaccent; + } locl; + """, + ) + + +def test_single_tag_is_untouched(): + original = "script latn;\nlanguage TRK;\nsub i by idotaccent;\n" + + assert expand_multi_language_statements(original) == original + + +def test_unparseable_statement_is_untouched(): + """Leave it for feaLib to report rather than guess at it.""" + original = "language TOOLONGTAG OTHER;\nsub i by idotaccent;\n" + + assert expand_multi_language_statements(original) == original + + +def test_statement_sharing_the_line_with_the_shorthand_is_expanded(): + check( + """\ + feature locl { + script latn; + language AZE CRT; sub i by idotaccent; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; sub i by idotaccent; + language CRT; + sub i by idotaccent; + } locl; + """, + ) + + +def test_lookup_closed_on_one_line_does_not_swallow_what_follows(): + """The braces open and close within the line. + + A line-wise nesting depth therefore never rises above zero, and the lookup + used to be taken as running to the end of the scope -- dropping the rule + after it from every repeat. + """ + check( + """\ + feature locl { + script latn; + language AZE CRT; + lookup idot { sub i by idotaccent; } idot; + sub Scedilla by Scommaaccent; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; + lookup idot { sub i by idotaccent; } idot; + sub Scedilla by Scommaaccent; + language CRT; + lookup idot; + sub Scedilla by Scommaaccent; + } locl; + """, + ) + + +def test_definition_sharing_a_line_with_a_rule_replays_the_rule(): + """The whole line used to count as one definition. + + The rule on it was dropped from the repeats instead of being replayed. + """ + check( + """\ + feature locl { + script latn; + language AZE CRT; + @Ced = [Scedilla]; sub @Ced by Scommaaccent; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; + @Ced = [Scedilla]; sub @Ced by Scommaaccent; + language CRT; + sub @Ced by Scommaaccent; + } locl; + """, + ) + + +def test_line_break_after_lookup_keyword_is_still_a_definition(): + """A definition broken after the keyword used to go unrecognised. + + It was then repeated whole, which redefines the lookup. + """ + check( + """\ + feature locl { + script latn; + language AZE CRT; + lookup + idot { sub i by idotaccent; } idot; + } locl; + """, + """\ + feature locl { + script latn; + language AZE; + lookup + idot { sub i by idotaccent; } idot; + language CRT; + lookup idot; + } locl; + """, + ) + + +def test_tab_indentation_survives_the_replay(): + """Statements are sliced out of the original by offset. + + The offsets come from feaLib's line/column locations. Glyphs indents + feature code with tabs, so this pins down that a tab counts as one column + and the slices do not drift. + """ + check( + "feature locl {\nscript latn;\nlanguage AZE CRT;\n" + "\tsub i by idotaccent;\n} locl;\n", + "feature locl {\nscript latn;\nlanguage AZE;\n" + "\tsub i by idotaccent;\nlanguage CRT;\n" + "sub i by idotaccent;\n} locl;\n", + ) + + +def test_include_in_the_scope_leaves_the_shorthand_alone(): + """An include cannot be replayed, because its contents are not visible. + + The shorthand is invalid FEA, so leaving it alone makes feaLib report it + rather than glyphsLib emitting a silently wrong language mapping. + """ + original = dedent("""\ + feature locl { + language AZE CRT; + include(other.fea); + sub i by idotaccent; + } locl; + """) + + assert expand_multi_language_statements(original) == original + + +def test_dflt_alongside_other_tags_is_left_alone(): + """`dflt` has to be specified alone, so this is not the shorthand. + + Glyphs rejects it as well -- 4.0.1 (4004) and 3.4.1 both report + `Expected ";" after language statement`. Expanding it would compile, but + `language AZE;` implies `include_dflt`, so AZE would end up with the + substitution twice: once inherited from `dflt` and once replayed, as two + distinct lookups with identical content. + """ + original = dedent("""\ + feature locl { + script latn; + language dflt AZE; + sub i by idotaccent; + } locl; + """) + + assert expand_multi_language_statements(original) == original + + +def test_statement_without_a_semicolon_is_left_alone(): + """A missing semicolon leaves no way to tell where a statement stops. + + The scan would run to the end of the token stream and slice the first copy + off mid-token. The code is invalid FEA either way, so it is handed to + feaLib as it stands. + """ + original = dedent("""\ + feature locl { + language AZE CRT; + sub i by idotaccent + """) + + assert expand_multi_language_statements(original) == original + + +def test_statement_without_a_semicolon_does_not_escape_the_block(): + """The scan must not walk out through the brace that closes the block. + + Looking for the next semicolon regardless of depth would find the one + ending `} locl;`, and the replay would then repeat the whole `ccmp` + feature that follows. + """ + original = dedent("""\ + feature locl { + language AZE CRT; + sub i by idotaccent + } locl; + + feature ccmp { + sub Scedilla by Scommaaccent; + } ccmp; + """) + + assert expand_multi_language_statements(original) == original + + +def test_shorthand_outside_a_block_is_left_alone(): + """FEA only allows `language` inside a feature block. + + Such a statement is invalid wherever it came from, and its scope has no + enclosing brace to end at, so replaying it would swallow the block that + follows. + """ + original = dedent("""\ + language AZE CRT; + sub i by idotaccent; + + feature locl { + sub Scedilla by Scommaaccent; + } locl; + """) + + assert expand_multi_language_statements(original) == original + + +def test_glyphs_4_output_matches_what_glyphs_3_wrote(): + """The same source, saved by both versions, ends up as the same FEA. + + Modelled on a production source opened and saved in Glyphs 4: the + automatic `locl` code nobody edited came back with the shorthand where + Glyphs 3 had written one statement per tag. Expanding what Glyphs 4 saved + gives back what Glyphs 3 saved, tabs and blank line included. + """ + glyphs_4 = ( + "feature locl {\n" + "script latn;\n" + "language AZE CRT KAZ TAT TRK;\n" + "lookup locl_latn_0 {\n" + "\tsub i by idotaccent;\n" + "} locl_latn_0;\n" + "\n" + "script latn;\n" + "language ROM MOL;\n" + "lookup locl_latn_1 {\n" + "\tsub Scedilla by Scommaaccent;\n" + "} locl_latn_1;\n" + "} locl;\n" + ) + glyphs_3 = ( + "feature locl {\n" + "script latn;\n" + "language AZE;\n" + "lookup locl_latn_0 {\n" + "\tsub i by idotaccent;\n" + "} locl_latn_0;\n" + "language CRT;\n" + "lookup locl_latn_0;\n" + "language KAZ;\n" + "lookup locl_latn_0;\n" + "language TAT;\n" + "lookup locl_latn_0;\n" + "language TRK;\n" + "lookup locl_latn_0;\n" + "\n" + "script latn;\n" + "language ROM;\n" + "lookup locl_latn_1 {\n" + "\tsub Scedilla by Scommaaccent;\n" + "} locl_latn_1;\n" + "language MOL;\n" + "lookup locl_latn_1;\n" + "} locl;\n" + ) + + assert expand_multi_language_statements(glyphs_4) == glyphs_3 + parse(glyphs_3) + + +def test_shorthand_inside_a_lookup_block_repeats_the_rules(): + """Glyphs also writes the language statements inside the lookup block. + + The scope then ends at the block's own closing brace, and the rules are + repeated where a reference would be wrong -- the lookup is the thing being + defined. The single-tag form here is what Glyphs 3 wrote for a production + source; the shorthand is that form collapsed by hand, since this is not a + shape Glyphs 4 has been seen to emit. + """ + glyphs_4 = ( + "feature locl {\n" + "lookup locl_latn_0 {\n" + "\tscript latn;\n" + "\tlanguage AZE CRT KAZ;\n" + "\tsub i by idotaccent;\n" + "} locl_latn_0;\n" + "} locl;\n" + ) + glyphs_3 = ( + "feature locl {\n" + "lookup locl_latn_0 {\n" + "\tscript latn;\n" + "\tlanguage AZE;\n" + "\tsub i by idotaccent;\n" + "\tlanguage CRT;\n" + "\tsub i by idotaccent;\n" + "\tlanguage KAZ;\n" + "\tsub i by idotaccent;\n" + "} locl_latn_0;\n" + "} locl;\n" + ) + + assert expand_multi_language_statements(glyphs_4) == glyphs_3 + parse(glyphs_3) + + +def test_crlf_line_endings_do_not_shift_the_slices(): + """Offsets come from feaLib's line/column locations. + + feaLib counts a ``\r\n`` as one line ending, so the replayed slices have to + stay in step with it on Windows-authored feature code. + """ + source = ( + "feature locl {\r\nlanguage AZE CRT;\r\nsub i by idotaccent;\r\n} locl;\r\n" + ) + + assert expand_multi_language_statements(source) == ( + "feature locl {\r\nlanguage AZE;\r\nsub i by idotaccent;\n" + "language CRT;\nsub i by idotaccent;\r\n} locl;\r\n" + ) + + +def test_lone_carriage_return_does_not_shift_the_slices(): + """feaLib ends a line at a bare ``\r`` too, not only at ``\r\n``. + + Counting lines by splitting on ``\n`` alone would leave every offset after + the first carriage return one short per line, and since the emitted text + is sliced out of the source by offset, the replay would silently lose the + rules while still reading as valid FEA. + """ + source = "feature locl {\rlanguage AZE CRT;\rsub i by idotaccent;\r} locl;\r" + + assert expand_multi_language_statements(source) == ( + "feature locl {\rlanguage AZE;\rsub i by idotaccent;\n" + "language CRT;\nsub i by idotaccent;\r} locl;\r" + ) + + +def test_line_endings_do_not_change_what_is_expanded(): + """The same code expands the same way whichever line ending it uses. + + The replays are emitted with ``\n`` whatever went in, so the comparison is + on the normalised text; what must not differ is which rules end up under + which tag. + """ + source = dedent("""\ + feature locl { + script latn; + language AZE CRT; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + sub Scedilla by Scommaaccent; + } locl; + """) + expected = normalize(expand_multi_language_statements(source)) + + for line_ending in ("\r\n", "\r"): + rewritten = source.replace("\n", line_ending) + + assert normalize(expand_multi_language_statements(rewritten)) == expected + + +def test_code_that_does_not_lex_is_left_alone(): + """Feature text still holding Glyphs tokens cannot be classified. + + ``PassThruExpander`` leaves ``$[...]`` in place, and feaLib's lexer stops + at the ``$``. Leaving the text untouched keeps the failure with feaLib + rather than guessing at what the token stands for. + """ + original = "language AZE CRT;\nsub $[name] by idotaccent;\n" + + assert expand_multi_language_statements(original) == original + + +def test_expanding_twice_changes_nothing(): + """The output holds one tag per statement, so a second pass is a no-op.""" + once = expand_multi_language_statements(dedent("""\ + feature locl { + script latn; + language AZE CRT; + lookup idot { + sub i by idotaccent; + } idot; + } locl; + """)) + + assert expand_multi_language_statements(once) == once + + +def make_font(): + font = classes.GSFont() + font.masters.append(classes.GSFontMaster()) + for name in ("i", "idotaccent"): + glyph = classes.GSGlyph(name) + glyph.layers.append(classes.GSLayer()) + glyph.layers[0].layerId = font.masters[0].id + font.glyphs.append(glyph) + + prefix = classes.GSFeaturePrefix() + prefix.name = "Languagesystems" + prefix.code = "languagesystem latn AZE;\nlanguagesystem latn TRK;" + font.featurePrefixes.append(prefix) + + feature = classes.GSFeature("locl") + feature.code = dedent("""\ + script latn; + language AZE TRK; + lookup idotaccent { + sub i by idotaccent; + } idotaccent;""") + font.features.append(feature) + return font + + +def test_output_is_the_input_or_valid_fea(): + """The invariant the whole module rests on. + + Either a source is left exactly as it was, for feaLib to report, or what + comes back compiles. Anything in between -- text sliced mid-token, a block + replayed past its closing brace -- is the failure mode worth guarding + against as a class, rather than one example at a time. + """ + for source in AWKWARD_SOURCES: + expanded = expand_multi_language_statements(source) + if expanded == source: + continue + parse(expanded) + + +def test_expanding_anything_twice_changes_nothing(): + """The output holds one tag per statement, so a second pass is a no-op.""" + for source in AWKWARD_SOURCES: + once = expand_multi_language_statements(source) + + assert expand_multi_language_statements(once) == once + + +def test_to_ufos_expands_feature_code(ufo_module): + (ufo,) = to_ufos(make_font(), ufo_module=ufo_module) + + assert ufo.features.text == dedent("""\ + # Prefix: Languagesystems + languagesystem latn AZE; + languagesystem latn TRK; + + feature locl { + script latn; + language AZE; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + language TRK; + lookup idotaccent; + } locl; + """) + Parser(io.StringIO(ufo.features.text), glyphNames=GLYPH_NAMES).parse() + + +def test_conditional_block_is_resolved_before_the_expansion(ufo_module): + """`#ifndef VARIABLE` blocks are stripped before the shorthand is expanded. + + The markers are comments, so they are not statements and are not replayed. + Expanding first would therefore put the repeats *inside* the block -- the + strip that follows would then take the second `language` statement with it + and leave the rules bound to the first tag alone. + """ + font = make_font() + font.features[0].code = dedent("""\ + script latn; + language AZE TRK; + #ifndef VARIABLE + sub i by idotaccent; + #endif""") + + (ufo,) = to_ufos(font, ufo_module=ufo_module) + + assert ufo.features.text.endswith(dedent("""\ + feature locl { + script latn; + language AZE; + language TRK; + } locl; + """)) + + +def test_expansion_does_not_round_trip(ufo_module): + """The expansion is one-way: the shorthand is not restored. + + Going back to Glyphs yields the expanded form rather than the original + statement. UFO -> Glyphs -> UFO is unaffected, because the original + feature text is recovered from ORIGINAL_FEATURE_CODE_KEY. + """ + (ufo,) = to_ufos(make_font(), ufo_module=ufo_module) + + (feature,) = to_glyphs([ufo]).features + assert feature.code == dedent("""\ + script latn; + language AZE; + lookup idotaccent { + sub i by idotaccent; + } idotaccent; + language TRK; + lookup idotaccent;""")