Cleanup/manual review of feature/cagen-v1 part 1 + 2 minor improvements

.github/workflows/end-to-end-tests.yaml

@@ -3,7 +3,7 @@ name: capgen end-to-end tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [develop]
+    branches: [develop, feature/capgen-v1]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

.github/workflows/unit-tests.yaml

@@ -3,7 +3,7 @@ name: capgen unit tests
 on:
   workflow_dispatch:
   pull_request:
-    branches: [develop]
+    branches: [develop, feature/capgen-v1]
 
 concurrency:
   group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }}

capgen/ccpp_datafile.py

@@ -19,6 +19,8 @@
   * ``--suite-list``
   * ``--required-variables`` / ``--input-variables`` /
     ``--output-variables`` / ``--host-variables``
+  * ``--suite-variables`` — the suite-owned (interstitial) variables
+    promoted into ``ccpp_<suite>_data.F90`` (a capgen addition)
   * ``--show`` (pretty-print)
   * ``--separator``, ``--exclude-protected``, ``--line-wrap``, ``--indent``
 
@@ -93,6 +95,10 @@
     {"report": "host_variables", "type": bool,
      "help": ("Return a list of required host model variable "
               "standard names")},
+    {"report": "suite_variables", "type": str,
+     "help": ("Return a list of suite-owned (interstitial) variable "
+              "standard names for suite, <SUITE_NAME>"),
+     "metavar": "SUITE_NAME"},
     {"report": "show", "type": bool,
      "help":
      "Pretty print the database contents to the screen"},
@@ -724,6 +730,37 @@ def _retrieve_variable_list(table, suite_name,
     return sorted(var_set)
 
 
+def _retrieve_suite_variable_list(table, suite_name):
+    """Find and return the sorted standard names of the suite-owned
+    (interstitial) variables promoted for suite <suite_name>.
+
+    Suite-owned variables are those no host table declares: first written
+    by a scheme with ``intent(out)`` and stored in ``ccpp_<suite>_data.F90``.
+    Returns an empty list if <suite_name> has no suite dictionary.
+
+    >>> table = ET.fromstring("<ccpp_datatable version='1.0'><var_dictionaries>"\
+            "<var_dictionary name='fruit' type='suite'><variables>"\
+            "<var name='var_b' local_name='vb'></var>"\
+            "<var name='var_a' local_name='va'></var>"\
+            "</variables></var_dictionary></var_dictionaries></ccpp_datatable>")
+    >>> _retrieve_suite_variable_list(table, 'fruit')
+    ['var_a', 'var_b']
+    >>> _retrieve_suite_variable_list(table, 'veggie')
+    []
+    """
+    result = set()
+    suite_dict = _find_var_dictionary(table, dict_name=suite_name,
+                                      dict_type="suite")
+    if suite_dict is not None:
+        svars = suite_dict.find("variables")
+        if svars is not None:
+            for var in svars:
+                name = var.get("name")
+                if name:
+                    result.add(name)
+    return sorted(result)
+
+
 def datatable_report(datatable, action, sep, exclude_protected=False):
     """Perform a lookup <action> on <datatable> and return the result."""
     if not action:
@@ -770,6 +807,8 @@ def datatable_report(datatable, action, sep, exclude_protected=False):
         result = _retrieve_variable_list(table, "host",
                                          exclude_protected=exclude_protected,
                                          intent_type="host")
+    elif action.action_is("suite_variables"):
+        result = _retrieve_suite_variable_list(table, action.value)
     else:
         result = ''
     if isinstance(result, list):

capgen/generator/datatable.py

@@ -340,7 +340,31 @@ def _build_var_dictionaries(
         suite_d.set('name', suite_resolution.suite_name)
         suite_d.set('type', 'suite')
         suite_d.set('parent', _API_DICT_NAME)
-        ET.SubElement(suite_d, 'variables')
+        s_vars = ET.SubElement(suite_d, 'variables')
+        # Suite-owned (interstitial) variables: promoted from a scheme's
+        # intent(out) that no host table declares, and stored in
+        # ccpp_<suite>_data.F90.  Recorded here so ccpp_datafile.py's
+        # --suite-variables report can enumerate them.
+        for std_name in sorted(suite_resolution.suite_vars):
+            svar = suite_resolution.suite_vars[std_name]
+            v = ET.SubElement(s_vars, 'var')
+            v.set('name', std_name)
+            if svar.local_name:
+                v.set('local_name', svar.local_name)
+            if svar.units:
+                v.set('units', svar.units)
+            if svar.type_:
+                v.set('type', svar.type_)
+            if svar.kind:
+                v.set('kind', svar.kind)
+            if svar.dimensions:
+                v.set('dimensions', ', '.join(svar.dimensions))
+            if svar.source_scheme:
+                v.set('source_scheme', svar.source_scheme)
+            if svar.source_phase:
+                v.set('source_phase', svar.source_phase)
+            if getattr(svar, 'allocatable', False):
+                v.set('allocatable', 'True')
 
         for resolved_group in suite_resolution.groups:
             group_d = ET.SubElement(dicts, 'var_dictionary')

capgen/generator/group_cap.py

@@ -517,14 +517,24 @@ def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str:
         else:
             is_identity = (arg.unit_forward == arg.call_expr)
         if not is_identity:
-            if reverse:
-                bits.append('unit conversion: {} to {}'.format(
-                    arg.kind_scheme or '', arg.kind_host or '',
-                ))
-            else:
-                bits.append('unit conversion: {} to {}'.format(
-                    arg.kind_host or '', arg.kind_scheme or '',
-                ))
+            if (arg.kind_scheme != arg.kind_host):
+                if reverse:
+                    bits.append('type conversion: {} to {}'.format(
+                        arg.kind_scheme or '', arg.kind_host or '',
+                    ))
+                else:
+                    bits.append('type conversion: {} to {}'.format(
+                        arg.kind_host or '', arg.kind_scheme or '',
+                    ))
+            if (arg.unit_scheme != arg.unit_host):
+                if reverse:
+                    bits.append('unit conversion: {} to {}'.format(
+                        arg.unit_scheme or '', arg.unit_host or '',
+                    ))
+                else:
+                    bits.append('unit conversion: {} to {}'.format(
+                        arg.unit_host or '', arg.unit_scheme or '',
+                    ))
     if arg.needs_vert_flip:
         bits.append('vertical flip (top_at_one mismatch)')
     if not bits:

capgen/generator/host_constituents.py

@@ -418,7 +418,7 @@ def _wrap_method_subs(host_dict) -> List[str]:
         'ccpp_const_get_index', 'const_index',
         [
             ('stdname', 'character(len=*), intent(in) :: stdname',
-             'standard_name=stdname'),
+             'standard_name=to_lower(stdname)'),
             ('const_index', 'integer, intent(out) :: const_index',
              'index=const_index'),
         ],
@@ -580,6 +580,7 @@ def _generate_host_constituents(
     lines.append('module {}'.format(_HOST_CONST_MOD))
     lines.append('')
     lines.append('{}use ccpp_kinds, only: kind_phys'.format(_INDENT))
+    lines.append('{}use ccpp_constituent_prop_mod, only: to_lower'.format(_INDENT))
     lines.append('{}use {}, only: &'.format(_INDENT, _CONST_PROP_MOD))
     lines.append('{}{}, &'.format(_INDENT * 2, _CONST_DDT))
     lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_TYPE))

capgen/generator/suite_resolver.py

@@ -1059,6 +1059,10 @@ class ResolvedArg:
         Kind declared in the scheme metadata.
     kind_host : str
         Kind of the host/suite variable.
+    unit_scheme : str
+        Unit declared in the scheme metadata.
+    unit_host : str
+        Unit of the host/suite variable.
     temp_name : str
         Name for the transformation temporary (``local_name + '_l'``).
     ptr_name : str
@@ -1085,6 +1089,8 @@ class ResolvedArg:
     unit_backward: str
     kind_scheme: str
     kind_host: str
+    unit_scheme: str
+    unit_host: str
     temp_name: str
     ptr_name: str
     transform_case: int
@@ -1559,6 +1565,8 @@ def _resolve_one_arg(
             unit_backward='',
             kind_scheme=scheme_var.kind,
             kind_host='',
+            unit_scheme=scheme_var.units,
+            unit_host='',
             temp_name='',
             ptr_name='',
             transform_case=1,
@@ -1599,6 +1607,8 @@ def _resolve_one_arg(
             unit_backward='',
             kind_scheme=scheme_var.kind,
             kind_host='',
+            unit_scheme=scheme_var.units,
+            unit_host='',
             temp_name='',
             ptr_name='',
             transform_case=1,
@@ -1965,6 +1975,8 @@ def _resolve_one_arg(
         unit_backward=unit_backward,
         kind_scheme=scheme_kind,
         kind_host=host_kind,
+        unit_scheme=scheme_units,
+        unit_host=host_units,
         temp_name=temp_name,
         ptr_name=ptr_name,
         transform_case=transform_case,
@@ -2185,6 +2197,8 @@ def _common_kwargs(base_expr, subscript, call_expr,
             unit_backward='',
             kind_scheme=scheme_var.kind,
             kind_host='',
+            unit_scheme=scheme_var.units,
+            unit_host='',
             temp_name='',
             ptr_name='',
             transform_case=1,

capgen/generator/suite_xml.py

@@ -583,7 +583,7 @@ def parse_suite_xml(
 
     # ---- schema validation (pre-expansion) --------------------------------
     if not skip_validation:
-        validate_xml_file(suite_file, 'suite', version, log, schema_path=sdir)
+        validate_xml_file(suite_file, version, log, schema_path=sdir)
 
     # ---- expand nested suites (v2 only) -----------------------------------
     if version[0] >= 2:
@@ -605,7 +605,7 @@ def parse_suite_xml(
 
     # ---- re-validate the expanded XML (catches duplicate xs:ID errors) ----
     if not skip_validation:
-        validate_xml_file(expanded_path, 'suite', version, log, schema_path=sdir)
+        validate_xml_file(expanded_path, version, log, schema_path=sdir)
 
     return suite
 

capgen/metadata/metadata_table.py

@@ -88,7 +88,7 @@
     ParseSyntaxError,
     check_cf_standard_name,
     check_units,
-    check_dimensions,
+    check_dimension,
     check_diagnostic_fixed,
     check_diagnostic_id,
     check_fortran_id,
@@ -317,7 +317,7 @@ def _parse_dimensions(value: str, context: ParseContext) -> List[str]:
                 "empty dimension entry in '{}'".format(value),
                 context=context
             )
-        check_dimensions([part], None, error=True)
+        check_dimension(part)
         # Lowercase every non-integer token so the resolver's
         # host_dict lookups succeed regardless of the user's metadata
         # casing.  Range form ``lower:upper`` lowercases each half;
@@ -636,7 +636,7 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None:
                 # check_cf_standard_name (which lowercases) so mixed-case
                 # legacy spellings are captured.  No-op otherwise.
                 self.standard_name = legacy_compat.translate(
-                    check_cf_standard_name(value, None, error=True))
+                    check_cf_standard_name(value))
             elif key == 'long_name':
                 self.long_name = value
             elif key == 'units':

capgen/metadata/parse_tools/__init__.py

@@ -3,20 +3,17 @@
 from .parse_source import (
     CCPPError,
     ParseSyntaxError,
-    ParseInternalError,
     ParseContext,
-    context_string,
 )
-from .parse_log import init_log, set_log_level, set_log_to_null, set_log_to_stdout
+from .parse_log import init_log, set_log_level, set_log_to_null
 from .parse_checkers import (
     check_units,
-    check_dimensions,
+    check_dimension,
     check_cf_standard_name,
     check_diagnostic_fixed,
     check_diagnostic_id,
     check_fortran_id,
     check_fortran_ref,
-    check_fortran_type,
     check_fortran_intrinsic,
     check_molar_mass,
     # auto-clone-constituents: legacy-shim checkers exported here so

capgen/metadata/parse_tools/fortran_conditional.py

@@ -7,7 +7,28 @@
 
 import re
 
+# Every Fortran token in a conditional that is NOT an operand: whitespace,
+# parentheses, the comparison operators in both symbolic (==, /=, <=, >=, <, >)
+# and F77 dotted (.lt., .le., .eq., .ge., .gt., .ne.) spellings, the logical
+# literals (.true., .false.) and the logical operators (.eqv., .neqv., .not.,
+# .and., .or., .xor.).  These are the delimiters the tokenizer below splits on.
+# Multi-character operators are listed before their single-character prefixes
+# so, e.g., '<=' is matched whole rather than as '<' followed by '='.
 FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.',
                                    '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.',
                                    '.not.', '.and.', '.or.', '.xor.']
-FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(',r'\(').replace(')', r'\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS]))
+# Tokenizer for a Fortran conditional: re.findall() returns an ordered list of
+# every operand and operator in the expression.  Each match is EITHER
+#   [\w']+ -- one operand: a run of identifier characters (a standard name or a
+#             number), with ' included so a quoted literal like 'active' stays a
+#             single token instead of being split -- OR
+#   one of the operator/delimiter strings from the list above, each passed
+#   through re.escape() -- so its regex metacharacters (the parentheses and the
+#   dots in .and./.eq./...) match literally -- and joined with '|'.
+# The caller then walks the tokens and swaps each operand's standard name for
+# its local name, leaving the operators untouched.
+# Known limitation: decimal literals are not supported.  An operand is a run of
+# word characters, so '1.5' tokenizes as '1' and '5' -- the '.' matches no token
+# and is dropped.  Metadata conditionals compare standard names against integers
+# or .true./.false., so this has not mattered in practice.
+FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([re.escape(word) for word in FORTRAN_CONDITIONAL_REGEX_WORDS]))