diff --git a/.github/workflows/end-to-end-tests.yaml b/.github/workflows/end-to-end-tests.yaml index 3d64ef76..d789aaa0 100644 --- a/.github/workflows/end-to-end-tests.yaml +++ b/.github/workflows/end-to-end-tests.yaml @@ -3,7 +3,7 @@ name: capgen end-to-end tests on: workflow_dispatch: pull_request: - branches: [develop] + branches: [develop, feature/capgen-v1] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/.github/workflows/unit-tests.yaml b/.github/workflows/unit-tests.yaml index 7a87f6d7..2d3d48bd 100644 --- a/.github/workflows/unit-tests.yaml +++ b/.github/workflows/unit-tests.yaml @@ -3,7 +3,7 @@ name: capgen unit tests on: workflow_dispatch: pull_request: - branches: [develop] + branches: [develop, feature/capgen-v1] concurrency: group: ${{ github.workflow }}-${{ github.event.pull_request.number || github.ref }} diff --git a/capgen/ccpp_datafile.py b/capgen/ccpp_datafile.py index 4ebfcd62..473f1df9 100755 --- a/capgen/ccpp_datafile.py +++ b/capgen/ccpp_datafile.py @@ -19,6 +19,8 @@ * ``--suite-list`` * ``--required-variables`` / ``--input-variables`` / ``--output-variables`` / ``--host-variables`` + * ``--suite-variables`` — the suite-owned (interstitial) variables + promoted into ``ccpp__data.F90`` (a capgen addition) * ``--show`` (pretty-print) * ``--separator``, ``--exclude-protected``, ``--line-wrap``, ``--indent`` @@ -93,6 +95,10 @@ {"report": "host_variables", "type": bool, "help": ("Return a list of required host model variable " "standard names")}, + {"report": "suite_variables", "type": str, + "help": ("Return a list of suite-owned (interstitial) variable " + "standard names for suite, "), + "metavar": "SUITE_NAME"}, {"report": "show", "type": bool, "help": "Pretty print the database contents to the screen"}, @@ -724,6 +730,37 @@ def _retrieve_variable_list(table, suite_name, return sorted(var_set) +def _retrieve_suite_variable_list(table, suite_name): + """Find and return the sorted standard names of the suite-owned + (interstitial) variables promoted for suite . + + Suite-owned variables are those no host table declares: first written + by a scheme with ``intent(out)`` and stored in ``ccpp__data.F90``. + Returns an empty list if has no suite dictionary. + + >>> table = ET.fromstring(""\ + ""\ + ""\ + ""\ + "") + >>> _retrieve_suite_variable_list(table, 'fruit') + ['var_a', 'var_b'] + >>> _retrieve_suite_variable_list(table, 'veggie') + [] + """ + result = set() + suite_dict = _find_var_dictionary(table, dict_name=suite_name, + dict_type="suite") + if suite_dict is not None: + svars = suite_dict.find("variables") + if svars is not None: + for var in svars: + name = var.get("name") + if name: + result.add(name) + return sorted(result) + + def datatable_report(datatable, action, sep, exclude_protected=False): """Perform a lookup on and return the result.""" if not action: @@ -770,6 +807,8 @@ def datatable_report(datatable, action, sep, exclude_protected=False): result = _retrieve_variable_list(table, "host", exclude_protected=exclude_protected, intent_type="host") + elif action.action_is("suite_variables"): + result = _retrieve_suite_variable_list(table, action.value) else: result = '' if isinstance(result, list): diff --git a/capgen/generator/datatable.py b/capgen/generator/datatable.py index 97235ed9..cdec7ad5 100644 --- a/capgen/generator/datatable.py +++ b/capgen/generator/datatable.py @@ -340,7 +340,31 @@ def _build_var_dictionaries( suite_d.set('name', suite_resolution.suite_name) suite_d.set('type', 'suite') suite_d.set('parent', _API_DICT_NAME) - ET.SubElement(suite_d, 'variables') + s_vars = ET.SubElement(suite_d, 'variables') + # Suite-owned (interstitial) variables: promoted from a scheme's + # intent(out) that no host table declares, and stored in + # ccpp__data.F90. Recorded here so ccpp_datafile.py's + # --suite-variables report can enumerate them. + for std_name in sorted(suite_resolution.suite_vars): + svar = suite_resolution.suite_vars[std_name] + v = ET.SubElement(s_vars, 'var') + v.set('name', std_name) + if svar.local_name: + v.set('local_name', svar.local_name) + if svar.units: + v.set('units', svar.units) + if svar.type_: + v.set('type', svar.type_) + if svar.kind: + v.set('kind', svar.kind) + if svar.dimensions: + v.set('dimensions', ', '.join(svar.dimensions)) + if svar.source_scheme: + v.set('source_scheme', svar.source_scheme) + if svar.source_phase: + v.set('source_phase', svar.source_phase) + if getattr(svar, 'allocatable', False): + v.set('allocatable', 'True') for resolved_group in suite_resolution.groups: group_d = ET.SubElement(dicts, 'var_dictionary') diff --git a/capgen/generator/group_cap.py b/capgen/generator/group_cap.py index 53927a7c..9dc8acea 100644 --- a/capgen/generator/group_cap.py +++ b/capgen/generator/group_cap.py @@ -517,14 +517,24 @@ def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str: else: is_identity = (arg.unit_forward == arg.call_expr) if not is_identity: - if reverse: - bits.append('unit conversion: {} to {}'.format( - arg.kind_scheme or '', arg.kind_host or '', - )) - else: - bits.append('unit conversion: {} to {}'.format( - arg.kind_host or '', arg.kind_scheme or '', - )) + if (arg.kind_scheme != arg.kind_host): + if reverse: + bits.append('type conversion: {} to {}'.format( + arg.kind_scheme or '', arg.kind_host or '', + )) + else: + bits.append('type conversion: {} to {}'.format( + arg.kind_host or '', arg.kind_scheme or '', + )) + if (arg.unit_scheme != arg.unit_host): + if reverse: + bits.append('unit conversion: {} to {}'.format( + arg.unit_scheme or '', arg.unit_host or '', + )) + else: + bits.append('unit conversion: {} to {}'.format( + arg.unit_host or '', arg.unit_scheme or '', + )) if arg.needs_vert_flip: bits.append('vertical flip (top_at_one mismatch)') if not bits: @@ -532,6 +542,29 @@ def _transform_comment(arg: ResolvedArg, reverse: bool = False) -> str: return '! ' + '; '.join(bits) +def _log_one_transform(logger, suite_name, group_name, phase, scheme_name, + arg: ResolvedArg, reverse: bool) -> None: + """Emit one log line for a value-changing transform on *arg*. + + Reuses :func:`_transform_comment` so the log text is *identical* to the + inline comment written into the generated cap. A suppressed/identity + transform (comment == '') logs nothing, matching the cap. + + TEMPORARY level choice: logged at WARNING so transforms show up in a + default capgen run (capgen's default log level is WARNING). + """ + comment = _transform_comment(arg, reverse=reverse) + if not comment: + return + desc = comment[1:].strip() # drop the leading '! ' + when = 'post-call' if reverse else 'pre-call' + logger.warning( + "CCPP transform: %s/%s/%s %s: %s (%s) [%s] %s", + suite_name, group_name, phase, scheme_name, + arg.standard_name, arg.scheme_local_name, when, desc, + ) + + def _active_required_guard_lines( arg: ResolvedArg, scheme_name: str, @@ -688,6 +721,9 @@ def _emit_phase_items( phase: str = '', errflg_local: Optional[str] = None, errmsg_local: Optional[str] = None, + logger=None, + suite_name: str = '', + group_name: str = '', ) -> None: """Recursively emit Fortran for a list of :data:`PhaseItem` objects. @@ -700,6 +736,7 @@ def _emit_phase_items( if isinstance(item, ResolvedCall): _emit_one_call( item, indent, lines, phase, errflg_local, errmsg_local, + logger=logger, suite_name=suite_name, group_name=group_name, ) elif isinstance(item, ResolvedSubcycle): counter = _loop_counter_name(depth) @@ -711,6 +748,7 @@ def _emit_phase_items( phase=phase, errflg_local=errflg_local, errmsg_local=errmsg_local, + logger=logger, suite_name=suite_name, group_name=group_name, ) lines.append('{}end do'.format(indent)) lines.append('') @@ -723,8 +761,18 @@ def _emit_one_call( phase: str = '', errflg_local: Optional[str] = None, errmsg_local: Optional[str] = None, + logger=None, + suite_name: str = '', + group_name: str = '', ) -> None: - """Append Fortran lines for a single scheme call (with transforms + errcheck).""" + """Append Fortran lines for a single scheme call (with transforms + errcheck). + + When *logger* is given, each emitted value-changing transform is also + logged (via :func:`_log_one_transform`) so a capgen run documents on + stdout/stderr the same conversions it writes as inline cap comments. + This covers group-cap calls AND the suite-level ````/```` + hooks, which share this emitter (see ``suite_cap.py``). + """ # Pre-call: runtime guard for any non-optional arg whose host declares # ``active = (...)``. Emitted before transforms so an inactive-but-required # var bails out with a clear error rather than reading host memory through @@ -735,9 +783,13 @@ def _emit_one_call( errflg_local, errmsg_local, indent, )) - # Pre-call transformations. + # Pre-call transformations (and log each on capgen's stdout/stderr). for arg in resolved_call.args: lines.extend(_pre_call_lines(arg)) + if logger is not None and arg.unit_forward: + _log_one_transform(logger, suite_name, group_name, + resolved_call.phase, resolved_call.scheme_name, + arg, reverse=False) call_args_exprs = [ '{}={}'.format(a.scheme_local_name, _call_arg_expr(a)) @@ -761,6 +813,10 @@ def _emit_one_call( for arg in resolved_call.args: lines.extend(_post_call_lines(arg)) + if logger is not None and arg.unit_backward: + _log_one_transform(logger, suite_name, group_name, + resolved_call.phase, resolved_call.scheme_name, + arg, reverse=True) lines.append('') @@ -851,6 +907,7 @@ def _generate_phase_subroutine( phase_items, ctrl_entries, host_dict, + logger=None, ) -> List[str]: """Generate one phase subroutine for a group cap. @@ -978,11 +1035,15 @@ def _generate_phase_subroutine( ) # ---- scheme calls --------------------------------------------------- + # Transforms are logged inside _emit_one_call (same text as the inline + # cap comments), so group caps and suite / hooks share one + # code path. _emit_phase_items( phase_items, call_indent, lines, depth=1, phase=phase, errflg_local=errflg_local, errmsg_local=errmsg_local, + logger=logger, suite_name=suite_name, group_name=group_name, ) # ---- post-call state transitions ------------------------------------ @@ -1132,6 +1193,7 @@ def _generate_group_cap( resolved_group: ResolvedGroup, host_dict, trace: bool = False, + logger=None, ) -> List[str]: """Generate the full group cap module source lines. @@ -1240,7 +1302,8 @@ def _generate_group_cap( for phase in _GROUP_PHASE_ORDER: phase_items = resolved_group.phase_calls.get(phase, []) sub_lines = _generate_phase_subroutine( - suite_name, group_name, phase, phase_items, ctrl_sig_entries, host_dict + suite_name, group_name, phase, phase_items, ctrl_sig_entries, + host_dict, logger=logger, ) lines.extend(sub_lines) lines.append('') @@ -1302,6 +1365,7 @@ def write_group_cap( lines = _generate_group_cap( suite_name, group_name, resolved_group, host_dict, trace=trace, + logger=logger, ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') diff --git a/capgen/generator/host_constituents.py b/capgen/generator/host_constituents.py index 6109bb10..72ce6db6 100644 --- a/capgen/generator/host_constituents.py +++ b/capgen/generator/host_constituents.py @@ -418,7 +418,7 @@ def _wrap_method_subs(host_dict) -> List[str]: 'ccpp_const_get_index', 'const_index', [ ('stdname', 'character(len=*), intent(in) :: stdname', - 'standard_name=stdname'), + 'standard_name=to_lower(stdname)'), ('const_index', 'integer, intent(out) :: const_index', 'index=const_index'), ], @@ -580,6 +580,7 @@ def _generate_host_constituents( lines.append('module {}'.format(_HOST_CONST_MOD)) lines.append('') lines.append('{}use ccpp_kinds, only: kind_phys'.format(_INDENT)) + lines.append('{}use ccpp_constituent_prop_mod, only: to_lower'.format(_INDENT)) lines.append('{}use {}, only: &'.format(_INDENT, _CONST_PROP_MOD)) lines.append('{}{}, &'.format(_INDENT * 2, _CONST_DDT)) lines.append('{}{}, &'.format(_INDENT * 2, _CONST_PROP_TYPE)) diff --git a/capgen/generator/suite_cap.py b/capgen/generator/suite_cap.py index af3a8ddd..3ac958d6 100644 --- a/capgen/generator/suite_cap.py +++ b/capgen/generator/suite_cap.py @@ -633,6 +633,7 @@ def _init_lines( suite_name: str, suite_res: SuiteResolution, host_dict=None, + logger=None, ) -> List[str]: """Generate the ``_init`` framework-setup subroutine lines. @@ -769,7 +770,9 @@ def _init_lines( if suite_res.suite_init_call is not None: lines.append('') from generator.group_cap import _emit_one_call - _emit_one_call(suite_res.suite_init_call, i2, lines) + _emit_one_call(suite_res.suite_init_call, i2, lines, + logger=logger, suite_name=suite_name, + group_name='(suite init)') lines += [ '', @@ -786,6 +789,7 @@ def _final_lines( suite_name: str, suite_res: SuiteResolution, host_dict=None, + logger=None, ) -> List[str]: """Generate the ``_final`` framework-teardown subroutine lines. @@ -905,7 +909,9 @@ def _final_lines( # transition to UNREGISTERED. Errflg check follows the call. if suite_res.suite_final_call is not None: from generator.group_cap import _emit_one_call - _emit_one_call(suite_res.suite_final_call, i2, lines) + _emit_one_call(suite_res.suite_final_call, i2, lines, + logger=logger, suite_name=suite_name, + group_name='(suite final)') lines.append( '{}ccpp_suite_state({}) = CCPP_SUITE_UNREGISTERED'.format(i2, inst_idx) @@ -1215,6 +1221,7 @@ def _generate_suite_cap( scheme_store: SchemeStore, host_dict=None, trace: bool = False, + logger=None, ) -> List[str]: """Generate the full ``ccpp__cap.F90`` module source lines. @@ -1296,10 +1303,10 @@ def _generate_suite_cap( # Subroutines. Order: register, init, physics_*, final, state_alloc/dealloc. lines.extend(_register_lines(suite_name, suite_res, host_dict)) - lines.extend(_init_lines(suite_name, suite_res, host_dict)) + lines.extend(_init_lines(suite_name, suite_res, host_dict, logger=logger)) for phase in _PHYSICS_PHASES: lines.extend(_physics_dispatch_lines(suite_name, phase, suite_res, host_dict)) - lines.extend(_final_lines(suite_name, suite_res, host_dict)) + lines.extend(_final_lines(suite_name, suite_res, host_dict, logger=logger)) has_suite_vars = bool(suite_res.suite_vars) lines.extend(_suite_state_alloc_lines(suite_name, has_suite_vars)) @@ -1346,6 +1353,7 @@ def write_suite_cap( lines = _generate_suite_cap( suite_name, suite_res, scheme_store, host_dict, trace=trace, + logger=logger, ) with open_if_changed(out_path, logger=logger) as fh: fh.write('\n'.join(lines) + '\n') diff --git a/capgen/generator/suite_resolver.py b/capgen/generator/suite_resolver.py index 006dc7d2..30aefab6 100644 --- a/capgen/generator/suite_resolver.py +++ b/capgen/generator/suite_resolver.py @@ -1059,6 +1059,10 @@ class ResolvedArg: Kind declared in the scheme metadata. kind_host : str Kind of the host/suite variable. + unit_scheme : str + Unit declared in the scheme metadata. + unit_host : str + Unit of the host/suite variable. temp_name : str Name for the transformation temporary (``local_name + '_l'``). ptr_name : str @@ -1085,6 +1089,8 @@ class ResolvedArg: unit_backward: str kind_scheme: str kind_host: str + unit_scheme: str + unit_host: str temp_name: str ptr_name: str transform_case: int @@ -1559,6 +1565,8 @@ def _resolve_one_arg( unit_backward='', kind_scheme=scheme_var.kind, kind_host='', + unit_scheme=scheme_var.units, + unit_host='', temp_name='', ptr_name='', transform_case=1, @@ -1599,6 +1607,8 @@ def _resolve_one_arg( unit_backward='', kind_scheme=scheme_var.kind, kind_host='', + unit_scheme=scheme_var.units, + unit_host='', temp_name='', ptr_name='', transform_case=1, @@ -1965,6 +1975,8 @@ def _resolve_one_arg( unit_backward=unit_backward, kind_scheme=scheme_kind, kind_host=host_kind, + unit_scheme=scheme_units, + unit_host=host_units, temp_name=temp_name, ptr_name=ptr_name, transform_case=transform_case, @@ -2185,6 +2197,8 @@ def _common_kwargs(base_expr, subscript, call_expr, unit_backward='', kind_scheme=scheme_var.kind, kind_host='', + unit_scheme=scheme_var.units, + unit_host='', temp_name='', ptr_name='', transform_case=1, diff --git a/capgen/generator/suite_xml.py b/capgen/generator/suite_xml.py index fb8f50e7..bd29a351 100644 --- a/capgen/generator/suite_xml.py +++ b/capgen/generator/suite_xml.py @@ -583,7 +583,7 @@ def parse_suite_xml( # ---- schema validation (pre-expansion) -------------------------------- if not skip_validation: - validate_xml_file(suite_file, 'suite', version, log, schema_path=sdir) + validate_xml_file(suite_file, version, log, schema_path=sdir) # ---- expand nested suites (v2 only) ----------------------------------- if version[0] >= 2: @@ -605,7 +605,7 @@ def parse_suite_xml( # ---- re-validate the expanded XML (catches duplicate xs:ID errors) ---- if not skip_validation: - validate_xml_file(expanded_path, 'suite', version, log, schema_path=sdir) + validate_xml_file(expanded_path, version, log, schema_path=sdir) return suite diff --git a/capgen/metadata/metadata_table.py b/capgen/metadata/metadata_table.py index 995ca7c6..2db48f18 100644 --- a/capgen/metadata/metadata_table.py +++ b/capgen/metadata/metadata_table.py @@ -88,7 +88,7 @@ ParseSyntaxError, check_cf_standard_name, check_units, - check_dimensions, + check_dimension, check_diagnostic_fixed, check_diagnostic_id, check_fortran_id, @@ -317,7 +317,7 @@ def _parse_dimensions(value: str, context: ParseContext) -> List[str]: "empty dimension entry in '{}'".format(value), context=context ) - check_dimensions([part], None, error=True) + check_dimension(part) # Lowercase every non-integer token so the resolver's # host_dict lookups succeed regardless of the user's metadata # casing. Range form ``lower:upper`` lowercases each half; @@ -636,7 +636,7 @@ def set_attr(self, key: str, value: str, context: ParseContext) -> None: # check_cf_standard_name (which lowercases) so mixed-case # legacy spellings are captured. No-op otherwise. self.standard_name = legacy_compat.translate( - check_cf_standard_name(value, None, error=True)) + check_cf_standard_name(value)) elif key == 'long_name': self.long_name = value elif key == 'units': diff --git a/capgen/metadata/parse_tools/__init__.py b/capgen/metadata/parse_tools/__init__.py index 2bb38592..6994db2e 100644 --- a/capgen/metadata/parse_tools/__init__.py +++ b/capgen/metadata/parse_tools/__init__.py @@ -3,20 +3,17 @@ from .parse_source import ( CCPPError, ParseSyntaxError, - ParseInternalError, ParseContext, - context_string, ) -from .parse_log import init_log, set_log_level, set_log_to_null, set_log_to_stdout +from .parse_log import init_log, set_log_level, set_log_to_null from .parse_checkers import ( check_units, - check_dimensions, + check_dimension, check_cf_standard_name, check_diagnostic_fixed, check_diagnostic_id, check_fortran_id, check_fortran_ref, - check_fortran_type, check_fortran_intrinsic, check_molar_mass, # auto-clone-constituents: legacy-shim checkers exported here so diff --git a/capgen/metadata/parse_tools/fortran_conditional.py b/capgen/metadata/parse_tools/fortran_conditional.py index 17ae6859..4c50b00d 100755 --- a/capgen/metadata/parse_tools/fortran_conditional.py +++ b/capgen/metadata/parse_tools/fortran_conditional.py @@ -7,7 +7,28 @@ import re +# Every Fortran token in a conditional that is NOT an operand: whitespace, +# parentheses, the comparison operators in both symbolic (==, /=, <=, >=, <, >) +# and F77 dotted (.lt., .le., .eq., .ge., .gt., .ne.) spellings, the logical +# literals (.true., .false.) and the logical operators (.eqv., .neqv., .not., +# .and., .or., .xor.). These are the delimiters the tokenizer below splits on. +# Multi-character operators are listed before their single-character prefixes +# so, e.g., '<=' is matched whole rather than as '<' followed by '='. FORTRAN_CONDITIONAL_REGEX_WORDS = [' ', '(', ')', '==', '/=', '<=', '>=', '<', '>', '.eqv.', '.neqv.', '.true.', '.false.', '.lt.', '.le.', '.eq.', '.ge.', '.gt.', '.ne.', '.not.', '.and.', '.or.', '.xor.'] -FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([word.replace('(',r'\(').replace(')', r'\)') for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) +# Tokenizer for a Fortran conditional: re.findall() returns an ordered list of +# every operand and operator in the expression. Each match is EITHER +# [\w']+ -- one operand: a run of identifier characters (a standard name or a +# number), with ' included so a quoted literal like 'active' stays a +# single token instead of being split -- OR +# one of the operator/delimiter strings from the list above, each passed +# through re.escape() -- so its regex metacharacters (the parentheses and the +# dots in .and./.eq./...) match literally -- and joined with '|'. +# The caller then walks the tokens and swaps each operand's standard name for +# its local name, leaving the operators untouched. +# Known limitation: decimal literals are not supported. An operand is a run of +# word characters, so '1.5' tokenizes as '1' and '5' -- the '.' matches no token +# and is dropped. Metadata conditionals compare standard names against integers +# or .true./.false., so this has not mattered in practice. +FORTRAN_CONDITIONAL_REGEX = re.compile(r"[\w']+|" + "|".join([re.escape(word) for word in FORTRAN_CONDITIONAL_REGEX_WORDS])) diff --git a/capgen/metadata/parse_tools/io_helpers.py b/capgen/metadata/parse_tools/io_helpers.py index 44108fee..d9c73ab5 100644 --- a/capgen/metadata/parse_tools/io_helpers.py +++ b/capgen/metadata/parse_tools/io_helpers.py @@ -1,25 +1,28 @@ """File-write helpers with no-op-if-unchanged semantics. -The original ``ccpp-prebuild`` and ``ccpp-capgen`` both avoided rewriting -generated cap files when their content was unchanged — preserving each -file's mtime so downstream build systems (CMake, Make, Ninja) don't -trigger unnecessary recompilation cascades. This module reproduces that -behaviour for ``capgen``. - -Staging strategy ----------------- -A naive ``open(path, 'w')`` always touches the mtime, even when the -content is identical. Instead each writer builds the file's content in -memory and calls :func:`write_if_changed`, which: - -1. Reads the existing file at *file_path* (if any). -2. If the existing content matches the new content byte-for-byte, returns - ``False`` without touching the filesystem. -3. Otherwise writes the new content to a sibling temp file (in the same - directory, which sits **under the generator's output root** — never - ``/tmp``, so this works on systems that disallow ``/tmp`` writes) and - then ``os.replace``s it over the target. Same-directory replace is - atomic on POSIX and Windows; no partial writes. +This module implements the same no-op-if-unchanged semantics as the original +``ccpp-prebuild`` and ``ccpp-capgen`` — avoiding rewriting generated cap files +when their content was unchanged — preserving each file's mtime so downstream +build systems (CMake, Make, Ninja) don't trigger unnecessary recompilation +cascades. + +Two separate concerns +--------------------- +:func:`write_if_changed` handles *whether* to write and *how* to write +independently — they are different problems and solved by different means: + +- **Whether** — the new content is compared in memory against the existing + file. If it is byte-for-byte identical the file is left untouched (mtime + preserved), so CMake/Make/Ninja don't see a spurious change and trigger a + needless recompilation cascade. Because the comparison is in memory, + nothing is written to disk in the common unchanged case. + +- **How** — when the content *does* differ it is written to a sibling temp + file which is then ``os.replace``d over the target. A same-directory + replace is atomic on POSIX and Windows, so an interrupted or failed write + can never leave a partially written / corrupt cap in place for the build + to compile. The temp file lives in the target's parent directory (under + the generator's output root) and is written with the default umask. For writers that already produce content via a ``with open(...) as fh`` pattern, use :func:`open_if_changed` as a drop-in replacement — it yields @@ -50,31 +53,18 @@ def write_if_changed( Full file content to write. encoding : str Encoding passed to :func:`open` for both the read-back comparison - and the staged write. Defaults to ``'utf-8'`` to match every - capgen writer. + and the staged write. Defaults to ``'utf-8'``. logger : logging.Logger, optional When supplied, the helper logs an ``info``-level message after each call: ``"Wrote "`` if the file was newly written or rewritten, or ``"Unchanged: "`` if the existing content - matched and the filesystem was left untouched. Callers want - this so end users can tell at a glance which generated files - actually changed on a rerun (the original ccpp-prebuild / - ccpp-capgen output distinguished the two cases too). + matched and the filesystem was left untouched. Returns ------- bool ``True`` if the file was written or replaced; ``False`` if the existing content already matched. - - Notes - ----- - The temp file is created in the target's parent directory via - :func:`tempfile.mkstemp` (which generates a unique name and opens it - with ``O_EXCL`` semantics). On any exception, the temp file is - removed so we never leak ``.capgen_tmp_*`` artifacts. Crucially the - staging directory is the target's parent — which is under the - generator's output root — so no ``/tmp`` access is required. """ parent = os.path.dirname(os.path.abspath(file_path)) or '.' os.makedirs(parent, exist_ok=True) @@ -100,6 +90,10 @@ def write_if_changed( try: with os.fdopen(tmp_fd, 'w', encoding=encoding) as fh: fh.write(content) + # mkstemp opens the temp file 0600; restore the umask-based default + umask = os.umask(0) + os.umask(umask) + os.chmod(tmp_path, 0o666 & ~umask) os.replace(tmp_path, file_path) except BaseException: try: diff --git a/capgen/metadata/parse_tools/parse_checkers.py b/capgen/metadata/parse_tools/parse_checkers.py index 63bed4ab..502fc349 100644 --- a/capgen/metadata/parse_tools/parse_checkers.py +++ b/capgen/metadata/parse_tools/parse_checkers.py @@ -52,63 +52,39 @@ def check_units(test_val, prop_dict, error): return test_val -def check_dimensions(test_val, prop_dict, error, max_len=0): - """Return if a valid dimensions list, otherwise, None - If > 0, each string in must not be longer than - . - if is True, raise an Exception if is not valid. - >>> check_dimensions(["dim1", "dim2name"], None, False) - ['dim1', 'dim2name'] - >>> check_dimensions([":", ":"], None, False) - [':', ':'] - >>> check_dimensions(["8", "::"], None, False) - ['8', '::'] - >>> check_dimensions(['start1:end1', 'start2:end2'], None, False) - ['start1:end1', 'start2:end2'] - >>> check_dimensions(['size(foo)'], None, False) - ['size(foo)'] - >>> check_dimensions(['size(foo,1'], None, False) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: Invalid dimension component, size(foo,1 - >>> check_dimensions(["dim1", "dim2name"], None, True, max_len=5) #doctest: +IGNORE_EXCEPTION_DETAIL +def check_dimension(test_val): + """Return if a valid single dimension entry, else raise CCPPError. + + A dimension entry is a colon-separated range (``lower``, ``lower:upper``, + or ``lower:upper:stride``) whose non-empty bounds are each an integer + literal or a Fortran identifier. Integer literals are valid in any bound + position; semantic restrictions (e.g. horizontal_dimension lower bound + must be 1) are enforced by the resolver, not here. + >>> check_dimension("dim2name") + 'dim2name' + >>> check_dimension("8") + '8' + >>> check_dimension(":") + ':' + >>> check_dimension("start:end") + 'start:end' + >>> check_dimension("ccpp_constant_one:1") + 'ccpp_constant_one:1' + >>> check_dimension("a:b:c:d") #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): - CCPPError: 'dim2name' is too long (> 5 chars) - >>> check_dimensions("hi_mom", None, True) #doctest: +IGNORE_EXCEPTION_DETAIL + CCPPError: 'a:b:c:d' is an invalid dimension range + >>> check_dimension("hi mom") #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): - CCPPError: 'hi_mom' is invalid; not a list - >>> check_dimensions(["ccpp_constant_one:1", "dim2name"], None, True) - ['ccpp_constant_one:1', 'dim2name'] + CCPPError: 'hi mom' is not a valid Fortran identifier """ - if not isinstance(test_val, list): - if error: - raise CCPPError("'{}' is invalid; not a list".format(test_val)) - return None - for item in test_val: - isplit = item.split(':') - if len(isplit) > 3: - if error: - raise CCPPError("'{}' is an invalid dimension range".format(item)) - return None - # Integer literals are valid in any bound position; semantic - # restrictions (e.g. horizontal_dimension lower bound must be 1) - # are enforced by the resolver, not here. - tdims = [x.strip() for x in isplit if len(x) > 0] - for tdim in tdims: - try: - int(tdim) - valid = True - except ValueError: - valid = check_fortran_id(tdim, None, error, - max_len=max_len) is not None - if not valid and tdim.strip().lower()[0:4] == 'size': - if -1 in check_balanced_paren(tdim[4:]): - raise CCPPError( - 'Invalid dimension component, {}'.format(tdim)) - valid = True - if not valid: - if error: - raise CCPPError(f"'{item}' is an invalid dimension name") - return None + isplit = test_val.split(':') + if len(isplit) > 3: + raise CCPPError("'{}' is an invalid dimension range".format(test_val)) + for tdim in [x.strip() for x in isplit if len(x) > 0]: + try: + int(tdim) + except ValueError: + check_fortran_id(tdim, None, error=True) return test_val @@ -116,27 +92,25 @@ def check_dimensions(test_val, prop_dict, error, max_len=0): __CFID_RE = re.compile(CF_ID + r"$") -def check_cf_standard_name(test_val, prop_dict, error): - """Return if a valid CF Standard Name, otherwise, None. +def check_cf_standard_name(test_val): + """Return the lowercased if a valid CCPP Standard Name, + otherwise raise CCPPError. http://cfconventions.org/Data/cf-standard-names/docs/guidelines.html - if is True, raise an Exception if is not valid. - >>> check_cf_standard_name("hi_mom", None, False) + >>> check_cf_standard_name("hi_mom") 'hi_mom' - >>> check_cf_standard_name("hi mom", None, False) - - >>> check_cf_standard_name("", None, False) #doctest: +IGNORE_EXCEPTION_DETAIL + >>> check_cf_standard_name("Agood4tranID") + 'agood4tranid' + >>> check_cf_standard_name("") #doctest: +IGNORE_EXCEPTION_DETAIL Traceback (most recent call last): CCPPError: CCPP Standard Name cannot be blank - >>> check_cf_standard_name("Agood4tranID", None, False) - 'agood4tranid' + >>> check_cf_standard_name("hi mom") #doctest: +IGNORE_EXCEPTION_DETAIL + Traceback (most recent call last): + CCPPError: 'hi mom' is not a valid CCPP Standard Name """ if len(test_val) == 0: raise CCPPError("CCPP Standard Name cannot be blank") if __CFID_RE.match(test_val) is None: - if error: - raise CCPPError( - "'{}' is not a valid CCPP Standard Name".format(test_val)) - return None + raise CCPPError("'{}' is not a valid CCPP Standard Name".format(test_val)) return test_val.lower() @@ -156,8 +130,6 @@ def check_cf_standard_name(test_val, prop_dict, error): "double precision", "character"] FORTRAN_DP_RE = re.compile(r"(?i)double\s*precision") -_REGISTERED_FORTRAN_DDT_NAMES = ["ccpp_constituent_prop_ptr_t"] - def check_fortran_id(test_val, prop_dict, error, max_len=0): """Return if a valid Fortran identifier, otherwise, None @@ -261,31 +233,6 @@ def check_fortran_intrinsic(typestr, error=False): return typestr -def check_fortran_type(typestr, prop_dict, error): - """Return if a valid Fortran type, otherwise, None - if is True, raise an Exception if is not valid. - >>> check_fortran_type("real", None, False) - 'real' - >>> check_fortran_type("char", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'char' is not a valid Fortran type - >>> check_fortran_type("type", {}, True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: 'type' is not a valid derived Fortran type - """ - dt = "" - match = check_fortran_intrinsic(typestr, error=False) - if match is None: - match = registered_fortran_ddt_name(typestr) - dt = " derived" - if match is None: - if error: - raise CCPPError( - "'{}' is not a valid{} Fortran type".format(typestr, dt)) - return None - return typestr - - def check_diagnostic_fixed(test_val, prop_dict, error): """Return if a valid descriptor for a CCPP diagnostic, otherwise, None. @@ -601,62 +548,3 @@ def check_mixing_ratio_type(test_val, prop_dict, error): return None # auto-clone-constituents: END legacy-shim checkers. - - -def check_balanced_paren(string, start=0, error=False): - """Return indices delineating a balance set of parentheses. - Parentheses in character context do not count. - Left parenthesis search begins at . - Return start and end indices if found - If no parentheses are found, return (-1, -1). - If a left parenthesis is found but no balancing right, return (begin, -1) - where begin is the index where the left parenthesis was found. - If error is True, raise a CCPPError. - >>> check_balanced_paren("foo") - (-1, -1) - >>> check_balanced_paren("(foo, bar)") - (0, 9) - >>> check_balanced_paren("(size(foo,1), qux)") - (0, 17) - >>> check_balanced_paren("(foo('bar()'))") - (0, 13) - >>> check_balanced_paren("(foo('bar()')") - (0, -1) - >>> check_balanced_paren("(foo('bar()')", error=True) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - CCPPError: ERROR: Unbalanced parenthesis in '(foo('bar()')' - """ - index = start - begin = -1 - end = -1 - depth = 0 - inchar = None - str_len = len(string) - while index < str_len: - c = string[index] - if c in ('"', "'"): - if inchar == c: - inchar = None - elif inchar is None: - inchar = c - elif inchar is not None: - pass - elif c == '(': - if depth == 0: - begin = index - depth += 1 - elif c == ')': - depth -= 1 - if depth == 0: - end = index - break - index += 1 - if begin >= 0 and end < 0 and error: - raise CCPPError("ERROR: Unbalanced parenthesis in '{}'".format(string)) - return begin, end - - -def registered_fortran_ddt_name(name): - if name in _REGISTERED_FORTRAN_DDT_NAMES: - return name - return None diff --git a/capgen/metadata/parse_tools/parse_log.py b/capgen/metadata/parse_tools/parse_log.py index fe080cc6..43cf8d83 100644 --- a/capgen/metadata/parse_tools/parse_log.py +++ b/capgen/metadata/parse_tools/parse_log.py @@ -4,20 +4,13 @@ def init_log(name, level=None): - """Initialize and return a named logger. + """Initialize and return a named logger writing to stdout. - Defaults to WARNING level when *level* is not specified and the logger - has no existing level set. - - >>> logger = init_log('test_logger') - >>> logger.name - 'test_logger' + When *level* is given it is applied; otherwise the logger inherits the + root default (WARNING). """ logger = logging.getLogger(name) - llevel = logger.getEffectiveLevel() - if level is None and llevel == logging.NOTSET: - logger.setLevel(logging.WARNING) - elif level: + if level: logger.setLevel(level) set_log_to_stdout(logger) return logger diff --git a/capgen/metadata/parse_tools/parse_source.py b/capgen/metadata/parse_tools/parse_source.py index 6648f59d..6d68e291 100644 --- a/capgen/metadata/parse_tools/parse_source.py +++ b/capgen/metadata/parse_tools/parse_source.py @@ -1,59 +1,25 @@ """Parsing primitives: parse context and exception types.""" -import logging -import os.path - - -def context_string(context=None, with_comma=True, nodir=False): - """Return a human-readable location string from *context*. - - Parameters - ---------- - context : ParseContext or None - Parsing location. ``None`` returns an empty string. - with_comma : bool - Prepend ``', at '`` or ``', in '`` when *context* is given. - nodir : bool - Strip the directory portion of the filename. - - >>> context_string() - '' - >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False) - 'dir/source.F90:33' - >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=True) - ', at dir/source.F90:33' - >>> context_string(context=ParseContext(filename="dir/source.F90"), with_comma=False) - 'dir/source.F90' - >>> context_string(context=ParseContext(linenum=32, filename="dir/source.F90"), with_comma=False, nodir=True) - 'source.F90:33' - """ - if context is None: - return '' - if context.line_num < 0: - where_str = 'in ' - else: - where_str = 'at ' - comma = ', ' if with_comma else '' - if not with_comma: - where_str = '' - spec = '{ctx:nodir}' if nodir else '{ctx}' - return ('{comma}{where_str}' + spec).format(comma=comma, where_str=where_str, ctx=context) - class CCPPError(ValueError): """User-facing error with a plain message and no traceback noise.""" - def __init__(self, message): - logging.shutdown() - super().__init__(message) - class ParseSyntaxError(CCPPError): - """Syntax error that includes parsing context in the message.""" + """Syntax error that includes parsing context in the message. + + >>> str(ParseSyntaxError("dimension", token="foo", context=ParseContext(32, "s.F90"))) + "Invalid dimension, 'foo', at s.F90:33" + >>> str(ParseSyntaxError("End of file", context=ParseContext(filename="s.F90"))) + 'End of file, in s.F90' + """ def __init__(self, token_type, token=None, context=None): - logging.shutdown() - cstr = context_string(context) + if context is None: + cstr = '' + else: + where_str = 'at' if context.line_num >= 0 else 'in' + cstr = ", {} {}".format(where_str, context) if token is None: message = "{}{}".format(token_type, cstr) else: @@ -61,20 +27,11 @@ def __init__(self, token_type, token=None, context=None): super().__init__(message) -class ParseInternalError(Exception): - """Internal parser logic error — not caught by normal user-error handlers.""" - - def __init__(self, errmsg, context=None): - logging.shutdown() - message = "{}{}".format(errmsg, context_string(context)) - super().__init__(message) - - class ParseContext: """File-position record used as the location anchor for parse errors. Holds a filename and a zero-based line number (negative means «file - level, no specific line»); formats as ``filename:line``. + level, no specific line»); formats as ``filename:line`` (1-based). >>> str(ParseContext(linenum=0, filename="foo.F90")) 'foo.F90:1' @@ -97,22 +54,10 @@ def __init__(self, linenum=None, filename=None): elif not isinstance(filename, str): raise CCPPError('ParseContext filename must be a string') - self.__linenum = linenum - self.__filename = filename - - @property - def line_num(self): - return self.__linenum - - @property - def filename(self): - return self.__filename - - def __format__(self, spec): - fname = os.path.basename(self.__filename) if spec == 'nodir' else self.__filename - if self.__linenum >= 0: - return "{}:{}".format(fname, self.__linenum + 1) - return fname + self.line_num = linenum + self.filename = filename def __str__(self): - return format(self) + if self.line_num >= 0: + return "{}:{}".format(self.filename, self.line_num + 1) + return self.filename diff --git a/capgen/metadata/parse_tools/xml_tools.py b/capgen/metadata/parse_tools/xml_tools.py index 52f80c12..c691d61f 100644 --- a/capgen/metadata/parse_tools/xml_tools.py +++ b/capgen/metadata/parse_tools/xml_tools.py @@ -5,28 +5,14 @@ """ import os -import re import shutil import subprocess -import sys import xml.etree.ElementTree as ET import xml.dom.minidom from .parse_source import CCPPError from .parse_log import init_log, set_log_to_null -_INDENT_STR = " " -beg_tag_re = re.compile(r"([<][^/][^<>]*[^/][>])") -end_tag_re = re.compile(r"([<][/][^<>/]+[>])") -simple_tag_re = re.compile(r"([<][^/][^<>/]+[/][>])") - -PYSUBVER = sys.version_info[1] -_LOGGER = None - - -class XMLToolsInternalError(ValueError): - """Internal error raised by helpers in this module.""" - def find_schema_version(root): """Return the schema version as ``[major, minor]`` from the *root*'s @@ -84,26 +70,23 @@ def find_schema_file(schema_root, version, schema_path=None): return None -def validate_xml_file(filename, schema_root, version, logger, schema_path=None): - """Validate *filename* against the matching schema using xmllint.""" +def validate_xml_file(filename, version, logger, schema_path=None): + """Validate *filename* against the suite schema for *version* using xmllint.""" if not os.path.isfile(filename): raise CCPPError("validate_xml_file: Filename, '{}', does not exist".format(filename)) if not os.access(filename, os.R_OK): raise CCPPError("validate_xml_file: Cannot open '{}'".format(filename)) - if os.path.isfile(schema_root): - schema_file = schema_root - else: - if not schema_path: - thispath = os.path.abspath(__file__) - pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) - schema_path = os.path.join(pdir, 'schema') - schema_file = find_schema_file(schema_root, version, schema_path) - if not (schema_file and os.path.isfile(schema_file)): - verstring = '.'.join([str(x) for x in version]) - raise CCPPError( - f"validate_xml_file: Cannot find schema for version {verstring},\n" - f" {schema_file} does not exist" - ) + if not schema_path: + thispath = os.path.abspath(__file__) + pdir = os.path.dirname(os.path.dirname(os.path.dirname(thispath))) + schema_path = os.path.join(pdir, 'schema') + schema_file = find_schema_file('suite', version, schema_path) + if not (schema_file and os.path.isfile(schema_file)): + verstring = '.'.join([str(x) for x in version]) + raise CCPPError( + f"validate_xml_file: Cannot find suite schema for version {verstring},\n" + f" {schema_file} does not exist" + ) if not os.access(schema_file, os.R_OK): raise CCPPError( "validate_xml_file: Cannot open schema, '{}'".format(schema_file)) @@ -177,35 +160,6 @@ def load_suite_by_name(suite_name, group_name, file, logger=None): ------- xml.etree.ElementTree.Element The matching suite or group element. - - Examples - -------- - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... - ... - ... ''') - >>> load_suite_by_name("physics_suite", None, file1_path, logger).tag - 'suite' - >>> load_suite_by_name("physics_suite", "dynamics", file1_path, logger).attrib['name'] - 'dynamics' - >>> load_suite_by_name("physics_suite", "missing_group", file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite physics_suite, group missing_group, not found - >>> load_suite_by_name("missing_suite", None, file1_path, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Nested suite missing_suite not found - >>> tmpdir.cleanup() """ _, root = read_xml_file(file, logger) try: @@ -214,7 +168,7 @@ def load_suite_by_name(suite_name, group_name, file, logger=None): raise CCPPError( f"{verr} in nested suite XML file '{file}'" ) from verr - if not validate_xml_file(file, 'suite', schema_version, logger): + if not validate_xml_file(file, schema_version, logger): raise CCPPError(f"Invalid suite definition file, '{file}'") if root.attrib.get("name") == suite_name: if group_name: @@ -246,66 +200,6 @@ def replace_nested_suite(element, nested_suite, default_path, logger): ------- str Name of the suite that was substituted in. - - Examples - -------- - >>> import tempfile - >>> import xml.etree.ElementTree as ET - >>> logger = init_log('xml_tools') - >>> set_log_to_null(logger) - >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... my_scheme - ... - ... - ... ''') - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> xml = f''' - ... - ... - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> top_group = top_suite.find("group") - >>> nested = top_group.find("nested_suite") - >>> replace_nested_suite(top_group, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> xml = f''' - ... - ... - ... - ... ''' - >>> top_suite = ET.fromstring(xml) - >>> nested = top_suite.find("nested_suite") - >>> replace_nested_suite(top_suite, nested, tmpdir.name, logger) - 'my_suite' - >>> [child.tag for child in top_suite] - ['group'] - >>> top_suite.find("group").find("scheme").text - 'my_scheme' - >>> tmpdir.cleanup() """ suite_name = nested_suite.attrib.get("name") group_name = nested_suite.attrib.get("group") @@ -342,90 +236,25 @@ def expand_nested_suites(suite, default_path, logger=None): Examples -------- + Expand a single group-less ```` reference in place (error + paths — missing suites/groups, cycle detection — are covered by + ``test_suite_xml.py``): + >>> import tempfile >>> import xml.etree.ElementTree as ET >>> logger = init_log('xml_tools') >>> set_log_to_null(logger) >>> tmpdir = tempfile.TemporaryDirectory() - >>> file1_path = os.path.join(tmpdir.name, "file1.xml") - >>> file2_path = os.path.join(tmpdir.name, "file2.xml") - >>> file3_path = os.path.join(tmpdir.name, "file3.xml") - >>> file4_path = os.path.join(tmpdir.name, "file4.xml") - >>> file5_path = os.path.join(tmpdir.name, "file5.xml") - >>> with open(file1_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... cloud_scheme - ... - ... - ... ''') - >>> with open(file2_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... pbl_scheme - ... - ... - ... ''') - >>> with open(file3_path, "w") as f: - ... _ = f.write(''' - ... - ... - ... rrtmg_lw_scheme - ... - ... - ... rrtmg_sw_scheme - ... - ... - ... ''') - >>> with open(file4_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> with open(file5_path, "w") as f: - ... _ = f.write(f''' - ... - ... - ... - ... ''') - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) + >>> ref = os.path.join(tmpdir.name, "pbl.xml") + >>> with open(ref, "w") as f: + ... _ = f.write('' + ... 'pbl_scheme') + >>> suite = ET.fromstring( + ... f'' + ... f'') >>> expand_nested_suites(suite, tmpdir.name, logger) - >>> ET.dump(suite) - - - cloud_scheme - - pbl_scheme - - rrtmg_lw_scheme - - rrtmg_sw_scheme - - >>> xml_content = f''' - ... - ... - ... - ... - ... - ... - ... ''' - >>> suite = ET.fromstring(xml_content) - >>> expand_nested_suites(suite, tmpdir.name, logger) #doctest: +IGNORE_EXCEPTION_DETAIL - Traceback (most recent call last): - ... - CCPPError: Exceeded number of iterations while expanding nested suites + >>> [g.attrib["name"] for g in suite.findall("group")] + ['pbl'] >>> tmpdir.cleanup() """ max_iterations = 10 diff --git a/capgen/src/ccpp_constituent_prop_mod.F90 b/capgen/src/ccpp_constituent_prop_mod.F90 index dbe33f84..060277d2 100644 --- a/capgen/src/ccpp_constituent_prop_mod.F90 +++ b/capgen/src/ccpp_constituent_prop_mod.F90 @@ -211,6 +211,9 @@ module ccpp_constituent_prop_mod procedure :: constituent_props_ptr => ccp_constituent_props_ptr end type ccpp_model_constituents_t + ! Public interfaces + public to_lower + ! Private interfaces private to_str private initialize_errvars @@ -416,7 +419,7 @@ subroutine ccp_instantiate(this, std_name, long_name, diag_name, units, & else errcode = 0 errmsg = '' - this%var_std_name = trim(std_name) + this%var_std_name = trim(to_lower(std_name)) end if if (errcode == 0) then this%var_long_name = trim(long_name) @@ -2676,4 +2679,29 @@ subroutine ccpt_set_water_species(this, water_flag, errcode, errmsg) end subroutine ccpt_set_water_species + !####################################################################### + + function to_lower(str) + + character(len=*), intent(in) :: str ! String to convert to lower case + character(len=len(str)) :: to_lower + + ! Local variables + integer :: i ! Index + integer :: aseq ! ascii collating sequence + integer :: upper_to_lower ! integer to convert case + character(len=1) :: ctmp ! Character temporary + + upper_to_lower = iachar("a") - iachar("A") + + do i = 1, len(str) + ctmp = str(i:i) + aseq = iachar(ctmp) + if (aseq >= iachar("A") .and. aseq <= iachar("Z")) & + ctmp = achar(aseq + upper_to_lower) + to_lower(i:i) = ctmp + end do + + end function to_lower + end module ccpp_constituent_prop_mod diff --git a/capgen/src/ccpp_scheme_utils.F90 b/capgen/src/ccpp_scheme_utils.F90 index d4de6499..d91f0580 100644 --- a/capgen/src/ccpp_scheme_utils.F90 +++ b/capgen/src/ccpp_scheme_utils.F90 @@ -3,7 +3,7 @@ module ccpp_scheme_utils ! Module of utilities available to CCPP schemes use ccpp_constituent_prop_mod, only: ccpp_model_constituents_t, & - int_unassigned + int_unassigned, to_lower implicit none private @@ -12,6 +12,7 @@ module ccpp_scheme_utils public :: ccpp_initialize_constituent_ptr ! Used by framework to initialize public :: ccpp_constituent_index ! Lookup index constituent by name public :: ccpp_constituent_indices ! Lookup indices of consitutents by name + public :: to_lower ! Utility to convert string to lowercase !! Private module variables & interfaces @@ -81,7 +82,7 @@ subroutine ccpp_constituent_index(standard_name, const_index, errcode, errmsg) call check_initialization(caller=subname, errcode=errcode, errmsg=errmsg) if (status_ok(errcode)) then - call constituent_obj%const_index(const_index, standard_name, & + call constituent_obj%const_index(const_index, to_lower(standard_name), & errcode, errmsg) else const_index = int_unassigned @@ -110,7 +111,7 @@ subroutine ccpp_constituent_indices(standard_names, const_inds, errcode, errmsg) do indx = 1, size(standard_names) ! For each std name in , find the const. index call constituent_obj%const_index(const_inds(indx), & - standard_names(indx), errcode, errmsg) + to_lower(standard_names(indx)), errcode, errmsg) if (errcode /= 0) then exit end if diff --git a/doc/code_walkthrough_DRAFT.md b/doc/code_walkthrough_DRAFT.md index 49760b37..7b2ab702 100644 --- a/doc/code_walkthrough_DRAFT.md +++ b/doc/code_walkthrough_DRAFT.md @@ -30,6 +30,36 @@ The single sentence to keep in mind: --- +## 0a. Feature overview — the three generators side by side + +The one-slide summary a prebuild or original-capgen developer wants before reading further. +**v1** is the redesign documented in this walkthrough (formerly `ccpp-capgen-ng`); **v0** is +the original `ccpp-capgen` it replaces; **prebuild** is the legacy `ccpp-prebuild`. Fuller +two-way detail lives in `doc/redesign_analysis.md` §5 (prebuild vs. v0) and +`doc/briefing.md` §4/§5 (prebuild/v0 vs. v1). + +| Feature | `ccpp-prebuild` | `ccpp-capgen` v0 | `ccpp-capgen` v1 | +|---|---|---|---| +| **Cap generation model** | Python-templated monolithic cap (`ccpp_prebuild_config.py`) | metadata → OO scope-chain resolution → emit | metadata → flat `ResolvedArg` resolution → emit | +| **Generator code style** | template-driven Python | deep OO class hierarchy | flat data classes + procedural resolver | +| **Group-cap argument shape** | DDT references | flat fields (1200+ dummy args at UFS scale) | DDT references (restored) | +| **Variable resolution** | flat metadata dict | five-layer scope-chain promotion | flat host+control dict + suite-owned discovery | +| **Host metadata mechanism** | hard-coded Python dict (`TYPEDEFS_NEW_METADATA`) | `type = module` tables | `type = host` / `type = ddt` tables | +| **Fortran ↔ metadata validation** | none (trusts metadata) | embedded in generator | standalone tool (`ccpp_validator.py`) | +| **Multi-instance / ensemble** | logical `initialized(200)` array (ad-hoc) | not supported | first-class, paired opt-in (per-instance state + constituents) | +| **Suite state runtime check** | logical `initialized` flag | string comparison | integer named-parameter state machine | +| **Dispatch / API surface** | `ccpp_static_api.F90` (runtime `select case`) | host-cap model (no static API) | host cap, introspection folded in | +| **Constituent handling** | hand-rolled, host-specific glue | generator auto-clone (`ConstituentVarDict`) | explicit `register`-phase opt-in (`ccpp_constituent_properties_t`) | +| **Build file lists** | hand-maintained `*.cmake` snippets | queryable datatable XML | `ccpp_datafile.py` + datatable (derived, see §10) | +| **Doc generation (HTML / LaTeX)** | yes | stub (unimplemented) | stub (unimplemented) | + +**Bottom line:** v1 keeps v0's metadata-driven pipeline but returns to prebuild's DDT +call interface (v0's flat-field explosion at UFS scale was the primary reason it was +abandoned), adds first-class multi-instance support, and splits Fortran↔metadata checking +out into a standalone validator. + +--- + ## 1. The pipeline at a glance Everything is orchestrated by `capgen()` in **`ccpp_capgen.py:863`**. @@ -56,6 +86,10 @@ flowchart TD | 6 | **Emit calls** | `write_group_cap` (`group_cap.py:1272`) | `ccpp___cap.F90` | | 7 | Emit rest | `write_suite_data/_types/_cap`, `write_host_cap`, `write_datatable` | suite data module, host cap, datatable | +> The seven stages above are **one invocation** of `ccpp_capgen.py`. For how a host's +> build system *drives* that invocation — alongside the validator and the datatable +> query — see **§10 (build-system integration)**. + --- ## 2. The dictionaries — what exists *before* matching @@ -508,6 +542,160 @@ to §8.1–8.4. Use it only when the audience needs the multi-instance constitue --- +## 10. Build-system integration — the three-utility contract in practice + +Sections 1–8 are the *inside* of a single `ccpp_capgen.py` run. This section is the +**outside**: how a host model's build system drives capgen. The governing rule — + +> **capgen has exactly one public interface: three standalone command-line scripts.** +> `ccpp_validator.py`, `ccpp_capgen.py`, and `ccpp_datafile.py`. A host build invokes +> them as subprocesses and reads their stdout / exit code. **Nothing imports a capgen +> module, and nothing depends on a capgen internal.** (A host may add its *own* tooling +> — e.g. to post-process `datatable.xml` — but that lives in the host repo, not here.) + +Everything below uses **CCPP-SCM's CMake** as the worked example, but the workflow is +build-system-agnostic: Make, Meson, or a shell script would wire up the same three steps. +The SCM binding lives in two files: + +- `cmake/ccpp_capgen.cmake` — three thin wrapper functions, one per utility + (`ccpp_validator()`, `ccpp_capgen()`, `ccpp_datafile()`), each of which just marshals + arguments and `execute_process()`es the script. +- `ccpp/CMakeLists.txt` — the driver: assembles the input file lists, calls the three + functions in order, and feeds the results into an `add_library()` target. + +### 10.1 The workflow — validate → generate → query + +```mermaid +flowchart TD + subgraph inputs["Build-system inputs (host repo)"] + HM[".meta
host files"] + SM[".meta
scheme files"] + SDF["suite XML
(SDFs)"] + SRC["Fortran
sources"] + end + + HM & SM & SRC --> V["ccpp_validator.py
--host-files / --scheme-files
+ --source-files
(build gate: exit≠0 → stop)"] + V -->|pass| G + + HM --> G["ccpp_capgen.py
--host-files --scheme-files --suites
--host-name --output-root --kind-type"] + SM --> G + SDF --> G + + G --> OUT["generated caps (.F90)
+ datatable.xml
under --output-root"] + + OUT --> Q["ccpp_datafile.py <report> datatable.xml"] + Q -->|--capgen-files| L1["generated caps"] + Q -->|--scheme-files| L2["used-only
scheme sources"] + Q -->|--dependencies| L3["co-located deps"] + + L1 & L2 & L3 --> LIB["compile target
(add_library / Make rule)"] + HSRC["host sources"] --> LIB +``` + +| Step | Utility | Role | Key inputs | Output the build consumes | +|------|---------|------|-----------|---------------------------| +| 1 | `ccpp_validator.py` | **Gate** — check every `.meta` against its `.F90` (types, ranks, intents, kinds) | `--source-files` + either `--host-files` **or** `--scheme-files` (one run each) | exit code only (nonzero → build stops) | +| 2 | `ccpp_capgen.py` | **Generate** the caps + `datatable.xml` | `--host-files`, `--scheme-files`, `--suites`, `--host-name`, `--output-root`, `--kind-type` | files under `--output-root` (incl. `datatable.xml`) | +| 3 | `ccpp_datafile.py` | **Query** `datatable.xml` for the file lists the compile needs | positional `datatable.xml` + one report flag | comma-separated file list on stdout | + +The validator is a **gate**, not a producer: it writes nothing capgen consumes, it just +fails the build early on a metadata↔Fortran mismatch. Steps 2 and 3 are the load-bearing +pair — and the reason step 3 exists at all is the next point. + +### 10.2 Why the datatable query exists — closing the loop at *configure* time + +A build system has to know **which files to compile** before it can define a target. But +capgen *decides* that set — it emits caps whose names depend on the host name and the +suites, and it filters the scheme sources down to only those a loaded suite actually uses. +So there is a genuine chicken-and-egg: the list of sources to compile is an **output** of +capgen. + +`datatable.xml` + `ccpp_datafile.py` close that loop. capgen records everything it did in +`datatable.xml`; the datatable query reads it back and hands the build system exactly the +lists it needs. This is capgen's answer to prebuild's hand-maintained `*.cmake`/Makefile +snippets — the file list is **derived**, never curated. + +The corollary for CMake: capgen must run at **configure time** (`execute_process()` inside +`CMakeLists.txt`), not build time (`add_custom_command`), because its output *defines* the +`add_library()` target. SCM assembles four query results plus the host sources into one +static library (`ccpp/CMakeLists.txt`): + +```cmake +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--dependencies") +set(CAPGEN_DEPENDENCIES ${CCPP_FILES}) # co-located deps from metadata 'dependencies =' +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--scheme-files") +set(SCHEME_FORTRAN_FILES ${CCPP_FILES}) # scheme .F90s a loaded suite actually uses +ccpp_datafile(DATATABLE "${OUTPUT_ROOT}/datatable.xml" REPORT_NAME "--capgen-files") +set(CAPGEN_FILES ${CCPP_FILES}) # the generated caps themselves + +add_library(scm-ccpp STATIC + ${EXTRA_FILES} # hand-added; do not survive the suite filter (see 10.4) + ${CAPGEN_DEPENDENCIES} + ${SCHEME_FORTRAN_FILES} + ${HOST_FORTRAN_FILES} # host sources, gathered host-side (see 10.4) + ${CAPGEN_FILES}) +``` + +The three relevant reports (of the full menu in `ccpp_datafile.py`): + +| Report | Returns | Why the build needs it | +|--------|---------|------------------------| +| `--capgen-files` | the generated caps (`.F90`) — union of host/suite/utility caps | these must be compiled | +| `--scheme-files` | the **used-only** scheme sources (group phases + suite `/` hooks) | compile only schemes a suite references, not the whole superset handed to capgen | +| `--dependencies` | co-located sources named by a metadata `dependencies =` attribute | pull in helper `.F90`s that have no metadata of their own | + +(Other reports — `--host-files`, `--suite-files`, `--utility-files`, `--module-list`, +`--suite-list`, `--{required,input,output,host}-variables` — exist for host tooling and +introspection but aren't needed to build.) + +### 10.3 The general shape (any build system) + +Strip away the CMake and the workflow is three subprocess calls with a file passed between +the last two: + +``` +1. for each (metadata set, source set): ccpp_validator.py --{host,scheme}-files … --source-files … # or fail +2. ccpp_capgen.py --host-files … --scheme-files … --suites … --host-name H --output-root R --kind-type … +3. for report in {--capgen-files, --scheme-files, --dependencies}: ccpp_datafile.py $report R/datatable.xml +4. compile (host sources) + (the three lists from step 3) into the CCPP library +``` + +Make would express step 3 as `$(shell …)` and step 4 as a normal rule; the substance is +identical. The only capgen-specific knowledge the build needs is **the argv of three +scripts and the name `datatable.xml`** — nothing else crosses the boundary. + +### 10.4 Two things that legitimately live host-side + +The contract permits host-specific glue *around* the three utilities — SCM has two worth +calling out, both in the host repo (not capgen): + +- **`.meta → .F90` mapping** (`ccpp_source_files()` in `cmake/ccpp_capgen.cmake`). The + *validator* needs Fortran source paths, but the build lists only metadata. This helper + reads each `.meta`'s optional `source_path` and probes for the sibling `.F90` by suffix. + It's a convenience for feeding `--source-files`; capgen itself never needs it. +- **`EXTRA_FILES`.** A source that no loaded suite references won't appear in + `--scheme-files`, so if the host still needs it linked (e.g. `module_ccpp_suite_simulator.F90`) + it is added by hand. This is the escape hatch for "compile it anyway," kept explicit and + separate from the derived lists. + +### 10.5 Transient migration flags (not part of the stable contract) + +SCM's wrappers currently append a few flags that are **temporary shims**, each tracked for +removal once the host metadata is cleaned up. They are on the `ccpp_capgen.py` / +`ccpp_validator.py` command lines today but should *not* be read as part of the durable +interface: + +| Flag | What it does | Status | +|------|--------------|--------| +| `--legacy-mode` | rewrites `horizontal_loop_extent → horizontal_dimension` and `number_of_openmp_threads → number_of_threads` at parse time (loud warning) | transient; drop when scheme metadata is updated | +| `--gfs-dim-aliases` | treats a few GFS-specific dimension names as equal inside dimension canonicalisation only | transient; GFS-tree-specific | +| `--no-host-introspection` | stubs the host cap's five introspection routines (shrinks the generated `_ccpp_cap.F90` dramatically) | transient; pending the runtime-listing redesign | + +They're isolated by design so the day SCM's metadata no longer needs them, deleting the +three `list(APPEND … )` lines in `ccpp_capgen.cmake` is the whole change. + +--- + ## Appendix — `file → routine → line` quick reference | Concept | Routine | File:line | diff --git a/doc/constituents_overhaul.md b/doc/constituents_overhaul.md index 6343c6b7..d0c73202 100644 --- a/doc/constituents_overhaul.md +++ b/doc/constituents_overhaul.md @@ -518,6 +518,10 @@ added 2026-05-12). Stronger options: calls and cross-check. - (c) Keep runtime check as authoritative, document the gap. +**See also §4.16** — the same blind spot (register-phase `%instantiate` names are +invisible to codegen) seen from the *resolution* side; options (a)/(b) above close +both. + ### 4.10 Capgen: scheme-metadata `diagnostic_name` for is_constituent args is host-specific (OPEN) Same issue as §4.4 but in capgen's metadata layer. Today's @@ -759,6 +763,77 @@ shim. Remove the rewrite once known consumers are migrated. - **Position relative to Proposals A/B/C**: orthogonal — a host-adapter bug exposed by rule b, not a framework constituent-model change. +### 4.16 Capgen: register-phase constituents are invisible to codegen — the *accessing* scheme's flag is load-bearing, not the registration (OPEN) + +**The gap.** Intuitively, once a register-phase scheme (`cld_ice_register`) +`%instantiate`s a constituent, *any* scheme that references that standard name +should resolve to it. It does not — at code-generation time. capgen decides +whether a scheme arg is a constituent solely from **run-phase metadata flags** +and **host declarations**; the register-phase `%instantiate(std_name=…)` calls +are Fortran that capgen never reads, so the registered names are unknown to the +generator. + +**Code evidence.** + +- The constituent-name set is built *only* from `is_constituent` metadata flags + (`advected` / `constituent` / `molar_mass`): `SchemeStore.constituent_stdnames()` + (`metadata/variable_resolver.py:870`) scans scheme-arg metadata and adds + `var.standard_name` iff `var.is_constituent`. Register-phase args + (`type = ccpp_constituent_properties_t`) carry no per-constituent standard + names in metadata — the names live only in the Fortran `%instantiate` calls — + so registration contributes nothing to this set. +- Consumer inference gates on that same set: `inferred_constituent_consumer` + requires `std_name in const_stds` (`generator/suite_resolver.py:2129`, with + `const_stds = scheme_store.constituent_stdnames()` at `:2834`). +- With neither an explicit flag nor an inferred hit, `_resolve_constituent_arg` + returns `None` (`generator/suite_resolver.py:2247`); the arg then falls through + to ordinary host/suite resolution and — for an `intent=in/inout` name absent + from `host_dict` and `suite_vars` — becomes a **hard error** ("nobody produces + it"). + +**Runtime vs. codegen.** The constituent *does* exist at runtime the moment +`%instantiate` runs (it is in `ccpp_model_constituents_obj` with an index). The +gap is purely at codegen: the generator has no compile-time list of registered +names, so it cannot wire `%vars_layer(:,:, index_of_)` accesses on the +strength of registration alone. + +**Concrete failure.** A constituent registered by scheme A but read only through +**unflagged** args, where no scheme anywhere flags the name and the host does not +declare it → codegen error, despite being correctly registered. In practice this +is masked because the registering scheme's run phase (e.g. `cld_ice_run`) usually +reads it *with* `advected=.true.`, which is what actually seeds +`constituent_stdnames()`. The **flag, not the registration, is load-bearing**. +(Once *some* scheme flags the name, others may read it unflagged — the "rule b" +inference in §2.2.) + +**Why it matters.** The scheme-author mental model is "register it, then use it by +standard name." Today that holds only if at least one *accessing* scheme flags the +name (or the host declares it). Registration is *necessary for existence* but *not +sufficient for resolution* — a comprehension trap, and a portability hazard: a +scheme that registers-and-reads-unflagged works only when co-loaded with some +other scheme that flags the same name. + +**Fix options** (the positive framing of §4.9 — the same two mechanisms close +both this gap and the missing cross-check): + +- **(a)** A register-phase metadata attribute enumerating the registered standard + names (e.g. `registers_std_names = a, b, c`), folded into + `constituent_stdnames()` so registration becomes codegen-visible and + authoritative. +- **(b)** Parse each scheme's `_register` Fortran for `%instantiate(std_name=…)` + and feed the names into `constituent_stdnames()` (heavier; capgen otherwise + never parses scheme bodies). +- **(c)** Keep the flag-on-consumer contract as-is and document the gap + (status quo). + +**Position relative to Proposals A/B/C.** Proposal A leaves the gap. Options +(a)/(b) resolve it directly under the current scheme-register model. **Proposal C +(host-only registration) dissolves it**: the host's constituent enumeration is +already codegen-visible metadata, so "any `advected=true` scheme arg whose +std_name is not in the host's enumeration → codegen error" (§8, Proposal C) +becomes the single source of truth, and registration is authoritative by +construction. + --- ## 5. Property classification (Class A vs Class B) diff --git a/doc/redesign_analysis_original_202060505T2044.md b/doc/redesign_analysis_original_202060505T2044.md deleted file mode 100644 index 2248f1c0..00000000 --- a/doc/redesign_analysis_original_202060505T2044.md +++ /dev/null @@ -1,2489 +0,0 @@ -# CCPP Framework Code Generator — Technical Analysis for Redesign - -*Analysis date: 2026-05-04. Clarifications added: 2026-05-05.* - -This document is a deep-dive technical analysis of the two existing CCPP Framework code generators — -`ccpp-prebuild` and `ccpp-capgen` — produced as input to a planned complete redesign. -It covers execution flow, data structures, feature sets, build system integration, and -key architectural differences. - ---- - -## Table of Contents - -1. [Background and motivation](#1-background-and-motivation) -2. [ccpp-prebuild — detailed analysis](#2-ccpp-prebuild--detailed-analysis) -3. [ccpp-capgen — detailed analysis](#3-ccpp-capgen--detailed-analysis) -4. [Shared infrastructure](#4-shared-infrastructure) -5. [Feature comparison](#5-feature-comparison) -6. [Build system integration](#6-build-system-integration) -7. [Key architectural differences](#7-key-architectural-differences) -8. [Design considerations for the redesign](#8-design-considerations-for-the-redesign) -9. [Real-world example: CCPP Single Column Model (SCM)](#9-real-world-example-ccpp-single-column-model-scm) -10. [Real-world example: CAM-SIMA (capgen)](#10-real-world-example-cam-sima-capgen) -11. [Real-world example: UFS Weather Model (prebuild)](#11-real-world-example-ufs-weather-model-prebuild) -12. [Real-world example: Navy NEPTUNE (prebuild, restricted)](#12-real-world-example-navy-neptune-prebuild-restricted) -13. [Cross-cutting design decision: how host data enters the cap chain](#13-cross-cutting-design-decision-how-host-data-enters-the-cap-chain) - ---- - -## 1. Background and motivation - -The CCPP Framework is a code generator that analyzes metadata describing variables required -by physical parameterizations in numerical weather prediction (NWP) models, compares them -against metadata provided by a host model, and generates Fortran interface ("cap") code that -connects the two. - -There are two generations of the generator: - -**`ccpp-prebuild`** (`scripts/ccpp_prebuild.py`): -- Simple, mostly procedural Python -- Used in: NOAA UFS Weather Model, Navy NEPTUNE, CCPP-SCM -- Extremely reliable in research, development, and operations -- Fewer capabilities; simpler design - -**`ccpp-capgen`** (`scripts/ccpp_capgen.py`): -- Highly complex, object-oriented Python taken to the extreme -- Used in: NCAR CAM-SIMA (still mostly a research/development model) -- Many advanced features designed but never implemented (funding/priority gaps) -- Notoriously difficult to develop; no remaining team member fully understands it - -**The original plan** was to update `ccpp-capgen` with missing features from `ccpp-prebuild` -and transition all models to it. **This plan has been abandoned** in favor of a complete -redesign that draws the best lessons from both generations. - -The immediate trigger for abandoning capgen was the failure — after considerable effort by -three developers — to make capgen pass DDT arguments to group caps the way prebuild does. -This is the root cause of capgen's severe performance problem (seconds for prebuild, -10+ minutes for capgen on the same suite set) and of its broken handling of optional -variables under Fortran compiler debugging flags. - ---- - -## 2. ccpp-prebuild — detailed analysis - -### 2.1 Command-line arguments and configuration - -Entry point: `scripts/ccpp_prebuild.py`, `main()`. - -Arguments parsed by `argparse`: - -| Argument | Required | Purpose | -|---|---|---| -| `--config` | yes | Path to host-model Python config module | -| `--suites` | no | Comma-separated suite names (without `.xml`) | -| `--builddir` | no | Override build directory from config | -| `--namespace` | no | Appended to static API module name | -| `--debug` | no | Insert Fortran array-size checks in generated caps | -| `--clean` | no | Remove generated files and exit | -| `--verbose` | no | Set logging to DEBUG | - -The `--config` file is a plain Python module imported dynamically via `importlib`. -Key variables it must define: - -| Config variable | Purpose | -|---|---| -| `VARIABLE_DEFINITION_FILES` | List of host-model Fortran sources with metadata hooks | -| `SCHEME_FILES` | List of physics scheme Fortran sources | -| `CAPS_DIR` | Output directory for generated cap `.F90` files | -| `SUITES_DIR` | Directory containing suite definition XML files | -| `STATIC_API_DIR` | Output directory for `ccpp_static_api.F90` | -| `TYPEDEFS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for typedef build snippets | -| `SCHEMES_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for scheme build snippets | -| `CAPS_MAKEFILE/CMAKEFILE/SOURCEFILE` | Paths for cap build snippets | -| `HTML_VARTABLE_FILE`, `LATEX_VARTABLE_FILE` | Documentation output paths | -| `TYPEDEFS_NEW_METADATA` | Optional: dict enabling DDT member name translation bridge | - -The config file can contain arbitrary Python expressions — computed file lists, -conditional logic, environment-variable lookups — making it very flexible. - -### 2.2 Step-by-step execution pipeline - -``` -1. Import config module dynamically via importlib - -2. gather_variable_definitions() - for each file in VARIABLE_DEFINITION_FILES: - parse_variable_tables(file) [metadata_parser.py] - → metadata_define: OrderedDict[standard_name → [mkcap.Var]] - -3. collect_physics_subroutines() - for each file in SCHEME_FILES: - parse_scheme_tables(file) [metadata_parser.py] - → metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] - → arguments_request: OrderedDict[scheme → OrderedDict[subroutine → [std_names]]] - → dependencies_request: OrderedDict[scheme → [abs_paths]] - → schemes_in_files: OrderedDict[scheme → abs_path] - -4. compare_metadata() [batch matching] - for each std_name in metadata_request: - check exists in metadata_define - check type/kind/rank compatibility - register unit conversions in var.actions - copy local_name as var.target - → metadata: OrderedDict[std_name → [Var]] (targets and actions set) - -5. check_optional_arguments() [warnings only] - -6. For each requested suite XML: - Suite.parse(xml) [mkstatic.py] → Suite + Group objects - Group.write() → ccpp___cap.F90 - Suite.write() → ccpp__cap.F90 - -7. API.write() [mkstatic.py] - → ccpp_static_api[_].F90 - -8. Write build-system snippets [mkcap.py writers] - → CCPP_CAPS.cmake/mk/sh - → CCPP_SCHEMES.cmake/mk/sh - → CCPP_TYPEDEFS.cmake/mk/sh - → CCPP_API.cmake/sh - -9. mkdoc.metadata_to_html() → HTML variable table - mkdoc.metadata_to_latex() → LaTeX variable table -``` - -### 2.3 Data structures — the "flat dict" model - -Everything in prebuild lives in flat Python `OrderedDict` structures. There is no object -hierarchy; variables are simple Python objects with plain attributes. - -```python -# Top-level data containers -metadata_define: OrderedDict[standard_name → [mkcap.Var]] # 1 Var per std_name -metadata_request: OrderedDict[standard_name → [mkcap.Var, ...]] # N Vars (one per scheme×subroutine) -arguments_request: OrderedDict[scheme_name → OrderedDict[subroutine_name → [std_names]]] -dependencies_request: OrderedDict[scheme_name → [abs_paths]] -schemes_in_files: OrderedDict[scheme_name → abs_path] -``` - -`mkcap.Var` attributes: - -| Attribute | Type | Description | -|---|---|---| -| `standard_name` | str | CF-convention unique identifier | -| `long_name` | str | Human-readable description | -| `units` | str | Physical units | -| `local_name` | str | Fortran local name (may be DDT member reference) | -| `type` | str | Fortran type (real, integer, logical, or DDT name) | -| `kind` | str | Fortran kind parameter | -| `dimensions` | list[str] | Dimension standard names | -| `intent` | str | in / out / inout | -| `active` | str | `'T'`, `'F'`, or expression string | -| `optional` | str | `'T'` or `'F'` | -| `pointer` | bool | Whether Fortran POINTER attribute needed | -| `target` | str | Set during matching: the host model local_name | -| `actions` | dict | `{'in': fn, 'out': fn}` for unit conversions | -| `container` | str | Encoded provenance: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | - -**Performance note on `container` and `target`**: these two attributes act as a lookup -cache computed once during the `compare_metadata()` batch step. The `container` string -encodes where each variable lives in the host model (module and, if applicable, the -DDT member chain). The `target` records the resolved Fortran local name. Both are -computed once and then used directly during Fortran cap generation — no further dictionary -lookups are needed. This is a major contributor to prebuild's speed advantage. - -### 2.4 Metadata parsing and the bridge to capgen - -`metadata_parser.py` is a shared module that acts as a bridge. It detects whether a -metadata section in a Fortran source file uses the old pipe-delimited format (deprecated, -warning emitted) or the new `.meta` format (triggered by `!! \htmlinclude .html` -in the Fortran source comment hook). - -For `.meta` files, `read_new_metadata()` in `metadata_parser.py`: -1. Calls capgen's `metadata_table.parse_metadata_file()` → `[MetadataTable]` -2. Converts each `metavar.Var` to a `mkcap.Var` -3. Normalizes `active` to `'T'`/`'F'`/expression, `optional` to `'T'`/`'F'` - -The `TYPEDEFS_NEW_METADATA` config variable (when provided) triggers an additional -pass via `convert_local_name_from_new_metadata()` which translates flat -standard-name-style local names into DDT member references such as -`Atm(blk_no)%q(:,:,:,graupel_index)`. This is the bridge that makes the newer -`.meta` format work with the older DDT-heavy host model code. - -### 2.5 Variable matching — `compare_metadata()` - -A single batch function processes all matching. For each standard name in `metadata_request`: - -1. Check it exists in `metadata_define` — error if missing -2. Check there is exactly one definition — error if ambiguous -3. Call `var.compatible(other_var)` — checks equality of `standard_name`, `type`, `kind`, and rank -4. Register unit conversions: if units differ, `var.convert_from()` / `var.convert_to()` - stores a conversion function in `var.actions` -5. Check `active` attribute: if host variable is conditionally allocated and scheme variable - is not `optional`, issue a warning (not an error) -6. Copy `local_name` from the define side as `var.target` -7. Build module use list from container strings - -Result: `metadata` dict where each `Var` has `.target` set to the host model local name -and `.actions` populated with any needed unit conversion functions. - -### 2.6 Generated Fortran files - -#### Group cap: `ccpp___cap.F90` - -One module per group. For each CCPP stage (tsinit, init, run, tsfinal, finalize), a subroutine: - -```fortran -module ccpp_suite_A_physics_cap - use scheme_module, only: scheme_run - use host_module_A, only: ddt_A ! DDT, not flat fields - use host_module_B, only: ddt_B - implicit none - contains - - subroutine suite_A_physics_run_cap(ddt_A, ddt_B, im, iaend, ierr, ...) - type(ddt_A_type), intent(inout), target :: ddt_A ! entire DDT passed - type(ddt_B_type), intent(inout), target :: ddt_B - integer, intent(in) :: im, iaend ! loop bounds - integer, intent(out) :: ierr - logical, save :: initialized(200) = .false. - ! optional variable: local pointer, conditionally associated - real(kind_phys), pointer :: opt_var(:) => null() - if (ddt_A%active_flag) then - opt_var => ddt_A%opt_field - end if - ! unit conversion: local variable - real(kind_phys) :: converted_var(im) - converted_var(:) = ddt_B%field(:im) * conversion_factor - ! fixed-index extraction: local pointer for a specific tracer - real(kind_phys), pointer :: q_water_vapor(:,:) => null() - q_water_vapor => ddt_A%q(:,:,ntqv) ! ntqv = water vapor index in tracer array - ! call scheme with loop-bound application and extracted variables at the call site - call scheme_run( & - arg1 = ddt_A%field1(1:im), & ! horizontal loop-bound applied here - arg2 = ddt_A%field2(1:im,:), & ! loop-bound + all levels - qv = q_water_vapor(1:im,:), & ! specific tracer, loop-bound applied - arg3 = converted_var, & ! unit-converted local var - opt_arg = opt_var, & ! optional pointer - ...) - if (ierr /= 0) return - end subroutine -end module -``` - -Key points: -- **DDTs are passed as arguments, not flat fields.** Hundreds of variables arrive as - one or a small number of DDT arguments. This is the fundamental architectural choice - that makes prebuild fast and safe with compiler debugging flags. -- **Two distinct "subsetting" operations happen at the scheme call site:** - 1. *Loop-bound application*: horizontal range `1:im` (or `im` for scalar extents) - applied in the scheme call argument expressions. - 2. *Fixed-index extraction*: a specific element along one dimension is selected, - e.g. `q_water_vapor => ddt%q(:,:,ntqv)` extracts the water vapor tracer from the - full tracer array. A local pointer (or local variable for unit conversions) is - declared just before the scheme call and passed as the scheme argument. The group - cap always receives the full data; these extractions are local to the group cap. -- **Optional variables** are handled by declaring a local `pointer` variable and - conditionally associating it with the DDT field based on the `active` expression. - An unassociated pointer is passed to the scheme if the variable is inactive. This - avoids compiler exceptions when mandatory debugging flags are enabled, because the - unallocated field is never directly referenced — only the already-null pointer is. -- `logical :: initialized(200), save` — per-instance initialization tracking. The - 200 is the maximum number of complete model instances that can coexist in memory - simultaneously (used in ensemble approaches where multiple copies of the full model - state live in memory at once). Each instance has its own initialization flag. -- For the `run` phase, `im` and `iaend` (or similar) carry `horizontal_loop_begin` - and `horizontal_loop_end`, enabling OpenMP thread-level parallelism where each - thread processes a horizontal slice. -- Explicit keyword argument passing in scheme calls. -- Unit conversion: a local variable is declared and populated before the call; the - local variable is then passed to the scheme. -- Error check after each scheme call; returns immediately on error. -- `--debug` flag inserts Fortran array-size assertions. - -#### Suite cap: `ccpp__cap.F90` - -Imports all group cap functions and exposes one function per stage that chains group calls. - -#### Static API: `ccpp_static_api[_].F90` - -A single Fortran module `ccpp_static_api` with one subroutine per stage: - -```fortran -subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) - character(len=*), intent(in) :: suite_name, group_name - select case(trim(suite_name)) - case('suite_A') - select case(trim(group_name)) - case('physics') - call suite_A_physics_run_cap(cdata, ierr) - ... - end select - ... - end select -end subroutine -``` - -This is the **single entry point** the host model calls. The host model passes `suite_name` -and `group_name` at runtime; the static API dispatches to the appropriate cap function. - -### 2.7 Build system snippet files generated - -Six output files (Makefile, CMakefile, shell source) for three variable sets: - -| File | Content | -|---|---| -| `CCPP_CAPS.cmake` | `set(CAPS /abs/path/cap1.F90 /abs/path/cap2.F90 ...)` | -| `CCPP_SCHEMES.cmake` | `set(SCHEMES /abs/path/scheme1.F90 ...)` | -| `CCPP_TYPEDEFS.cmake` | `set(TYPEDEFS module1 module2 ...)` (module names, not paths) | -| `CCPP_API.cmake` | `set(API /abs/path/ccpp_static_api.F90)` | - -All files are written as `.tmp` first and compared against the existing version; they are -replaced only if the content changed, which avoids unnecessary recompilation of downstream -Fortran targets. - -### 2.8 What `mkcap.py`, `mkstatic.py`, and `mkdoc.py` each do - -**`mkcap.py`**: -- Defines the `mkcap.Var` class (prebuild's variable data class) -- Defines six file-writer classes: `CapsMakefile`, `CapsCMakefile`, `CapsSourcefile`, - `SchemesMakefile`, `SchemesCMakefile`, `SchemesSourcefile`, `TypedefsMakefile`, - `TypedefsCMakefile`, `TypedefsSourcefile` -- Each writer has a `write(file_list)` method that produces a formatted include file -- Does NOT generate any Fortran - -**`mkstatic.py`**: -- Defines `Suite`, `Group`, `Subcycle` classes that parse suite definition XML and - generate Fortran caps -- `Suite.parse()`: reads SDF XML via `xml.etree.ElementTree`, builds `Group` and - `Subcycle` objects -- `Suite.write()`: drives cap generation for all groups and the suite-level cap -- `Group.write()`: generates the group cap Fortran — argument list construction, - module `use` statements, unit conversion code, scheme calls, error handling -- Defines `API` class: generates the static API Fortran module (suite_name/group_name - dispatch switch) -- `CCPP_SUITE_VARIABLES` dict: mandatory variables always included (error message, - error code, loop counter, loop extent) -- Helper functions `extract_parents_and_indices_from_local_name()` and - `extract_dimensions_from_local_name()` handle complex DDT member access like - `Atm(blk_no)%q(:,:,:,graupel_index)` — these are critical for DDT-heavy host models - -**`mkdoc.py`**: -- `metadata_to_html()`: produces an HTML table of all host-model provided variables - (standard_name, long_name, units, rank, type, kind, source, local_name) -- `metadata_to_latex()`: produces a LaTeX table combining host-defined and scheme-requested - variables, annotating which schemes use each variable and whether unit conversion is needed -- Informational outputs only; do not affect the build - ---- - -## 3. ccpp-capgen — detailed analysis - -### 3.1 Command-line arguments - -Entry point: `scripts/ccpp_capgen.py`, `_main_func()`. -Arguments parsed via `framework_env.parse_command_line()` into a `CCPPFrameworkEnv` object: - -| Argument | Required | Purpose | -|---|---|---| -| `--host-files` | yes | `.meta` files or `.txt` indirect file lists | -| `--scheme-files` | yes | Same format | -| `--suites` | yes | `.xml` SDF files or `.txt` lists | -| `--output-root` | no | Directory for generated files | -| `--host-name` | no | If given, generates a host cap | -| `--ccpp-datafile` | no | Path for datatable XML (default: `datatable.xml`) | -| `--kind-type` | no (repeatable) | Fortran kind mappings, e.g. `kind_phys=REAL64` | -| `--preproc-directives` | no | Fortran preprocessor macros | -| `--use-error-obj` | no | Use error object instead of scalar error variables | -| `--force-overwrite` | no | Always regenerate output | -| `--clean` | no | Remove files listed in datatable and exit | -| `--verbose` | no (repeatable) | Increase log verbosity | - -`CCPPFrameworkEnv` (defined in `framework_env.py`) consolidates all settings into typed -properties and stores a `kind_dict` mapping CCPP kind names to `[kind_spec, module]` pairs. - -### 3.2 Step-by-step execution pipeline - -``` -1. create_file_list() - expand .txt indirect file lists, validate .meta extensions - -2. register_ddts(scheme_files) - pre-scan all scheme .meta files - register DDT type names via register_fortran_ddt_name() - (so the host parser can recognize them as non-intrinsic types) - -3. parse_host_model_files() - for each host .meta file: - metadata_table.parse_metadata_file() → [MetadataTable] - find_associated_fortran_file() → matching .F90 path - parse_fortran_file() → Fortran declarations (via fortran_tools) - check_fortran_against_metadata() → cross-validation (type, kind, rank, intent) - accumulate MetadataSection headers: DDT, module, host types - -4. HostModel(table_dict, host_name, run_env) - process DDT headers: → DDTLibrary + ddt_dict (VarDictionary) - process module/host headers: → main VarDictionary + __var_locations - add ConstituentVarDict synthetically for ccpp_model_constituents_t - -5. API(sdfs, host_model, scheme_headers, run_env) - for each SDF XML: - Suite construction: - auto-create 5 phase groups: register, initialize, timestep_initial, - timestep_final, finalize - parse elements → Group objects (RUN_PHASE_NAME) - parse / tags → Scheme objects in full-phase groups - Suite.analyze(host_model, scheme_library, ddt_library, run_env): - Group.analyze() → Scheme.analyze(): - for each scheme argument: - VarDictionary.find_variable() [scope chain search] - Var.compatible() [→ VarCompatObj with transformations] - loop dim substitution for _run phase - register constituent if constituent=True - variable promotion: group outputs → suite level if needed by later group - -6. ccpp_api.write(outdir, run_env) - suite cap .F90 per suite - group caps (embedded or separate) - host cap .F90 (if --host-name given) - ccpp_kinds.F90 - -7. generate_ccpp_datatable() → datatable.xml -``` - -### 3.3 Object hierarchy - -``` -API (ccpp_suite.py) - └── Suite (extends VarDictionary) [one per SDF XML] - parent → ConstituentVarDict (extends VarDictionary) - parent → API - ├── Group (suite_objects.py, extends VarDictionary) [one per ] - │ call_list: CallList (extends VarDictionary) - │ ├── Subcycle (suite_objects.py) - │ │ └── Scheme (suite_objects.py, extends SuiteObject) - │ └── Scheme (for full-phase groups: init, register, etc.) - └── (auto groups: register, initialize, timestep_initial, - timestep_final, finalize) - -HostModel (host_model.py, extends VarDictionary) - ├── ddt_lib: DDTLibrary - │ └── {ddt_name → MetadataSection} - ├── ddt_dict: VarDictionary (all DDT field variables, expanded) - └── loop_vars: VarDictionary (run-time dimension variables) - -VarDictionary (metavar.py) - ├── {standard_name → Var} - └── parent_dict → VarDictionary ← scope chain for find_variable() - -Var (metavar.py) - └── __prop_dict: {property_name → validated_value} - -VarDDT (ddt_library.py, extends Var) - └── __field: Var | VarDDT ← recursive DDT traversal chain -``` - -### 3.4 Variable matching — scope-chain and VarCompatObj - -Unlike prebuild's single batch `compare_metadata()`, capgen performs incremental, -scope-aware matching during the suite analysis phase. - -For each scheme argument in `Scheme.analyze()`: -1. `VarDictionary.find_variable(standard_name)` — searches scope chain: - local group dict → suite dict → ConstituentVarDict → host model dict -2. `Var.compatible(other, run_env)` returns a `VarCompatObj` — not a bool. - `VarCompatObj` carries: - - Whether the variables are equivalent (no transformation needed) - - Whether they are compatible with transformations (unit conversion, dimension - substitution, `top_at_one` flip) - - The reason for any incompatibility (for error messages) -3. For `_run` phase: `horizontal_dimension` is automatically substituted with - `horizontal_loop_begin:horizontal_loop_end` -4. For `constituent = True` variables: auto-registered in `ConstituentVarDict`; - allocation/management code is generated -5. Variable promotion: if a Group produces a variable needed by a later Group, it is - promoted to Suite-level scope - -`VarCompatObj` compatibility considers: -- Type equality -- Kind equality (with ISO kind aliases) -- Units compatibility (triggers unit conversion if compatible) -- Dimension substitutability (horizontal loop vs. full dimension, vertical extent) -- `top_at_one` orientation (triggers flip if needed) -- `protected` status (cannot be an output if protected) -- `CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` - from `var_props.py` - -### 3.5 `metavar.Var` properties - -`metavar.Var` stores all properties in a validated `__prop_dict`. Properties: - -**Specification properties** (all metadata contexts): - -| Property | Type | Notes | -|---|---|---| -| `local_name` | str | Valid Fortran identifier | -| `standard_name` | str | CF-convention, lowercase+underscores | -| `long_name` | str | Human-readable description | -| `units` | str | Physical units string | -| `dimensions` | list | Dimension standard names or `()` | -| `type` | str | Intrinsic or registered DDT name | -| `kind` | str | Fortran kind parameter | -| `active` | str | Conditional allocation expression | -| `optional` | bool | Whether scheme can handle missing var | -| `protected` | bool | Cannot be written by schemes | -| `allocatable` | bool | Has ALLOCATABLE attribute | -| `state_variable` | bool | Persists across timesteps | -| `persistence` | str | `timestep` or `run` | -| `default_value` | str | Fortran expression | -| `diagnostic_name` | str | Diagnostic output name | -| `target` | bool | Has TARGET attribute | -| `polymorphic` | bool | CLASS(*) type | -| `top_at_one` | bool | Vertical ordering: top at index 1 | - -**Scheme-only properties**: - -| Property | Type | Notes | -|---|---|---| -| `intent` | str | in / out / inout | - -**Constituent properties**: - -| Property | Type | Notes | -|---|---|---| -| `constituent` | bool | Is a CCPP-managed constituent (tracer) | -| `advected` | bool | Is advected by the dynamical core | -| `molar_mass` | float | Molecular weight (positive) | - -### 3.6 Capgen-only features - -**Fortran cross-validation** (`check_fortran_against_metadata()`): -- Parses the actual `.F90` file alongside the `.meta` file -- Checks that every metadata entry matches the real Fortran declaration: - variable count, local_name, type, kind, intent (for schemes), dimension rank/names -- Catches bugs where metadata was updated but the Fortran source was not (or vice versa) - -**State machine** (`ccpp_state_machine.py`, `state_machine.py`): -- `CCPP_STATE_MACH`: a `StateMachine` instance with 6 transitions -- Valid state sequence: `register → uninitialized → initialized → in_time_step` -- Suite caps include a `character(len=16) :: ccpp_suite_state` variable -- State-checking code at the start of each phase function enforces correct call ordering -- `CCPP_STATE_MACH.function_match()` uses compiled regex to identify which CCPP phase - a subroutine name belongs to - -**Constituent variable support** (`constituents.py`): -- `ConstituentVarDict` (extends `VarDictionary`) manages traceable species (tracers) -- When a scheme declares `constituent = True`, `find_variable()` auto-creates the variable -- Allocation code for the constituent array is auto-generated -- Constants: `CONST_DDT_NAME = "ccpp_model_constituents_t"`, - `CONST_PROP_TYPE = "ccpp_constituent_properties_t"` - -**DDT library** (`ddt_library.py`): -- `VarDDT(Var)`: represents a DDT field variable at any nesting level -- Traversal chain: `VarDDT → VarDDT → ... → Var` (innermost is the actual leaf field) -- `DDTLibrary`: dictionary of DDT `MetadataSection` objects -- `collect_ddt_fields()` expands DDT variables into component fields in `ddt_dict` - -**Host cap generation** (`host_cap.py`): -- Generated only when `--host-name` is given -- Produces `_ccpp_cap.F90` -- Subroutines: `_ccpp_physics_(api_vars)` - that call into suite cap functions - -**`ccpp_kinds.F90`**: -- Simple Fortran module `ccpp_kinds` containing `use` statements that import all kind - parameters specified via `--kind-type` -- Makes kind parameters available to both schemes and caps without circular dependencies - -**Datatable XML** (`ccpp_datafile.py`): -- Produced after generation; lists all generated files, scheme entries, variable properties, - suite configurations -- Queryable by the build system via `ccpp_datafile.py --suite-files` etc. -- Supports `--clean` workflow: reads the file list, removes all generated files, deletes itself -- `DatatableReport` class provides a programmatic query API - -**In-memory database** (`ccpp_database_obj.py`): -- `CCPPDatabaseObj`: wraps `HostModel` and `API` for programmatic access to capgen results -- Returned when `capgen()` is called with `return_db=True` -- Provides `host_model_dict()`, `suite_list()`, `constituent_dictionary(suite)` - -**Variable tracking tool** (`ccpp_track_variables.py`): -- Standalone diagnostic: traces a specific variable through a suite, showing which schemes - use it and with what intent -- Uses prebuild's `import_config` and capgen's `Suite`/`parse_metadata_file` together - -**Fortran-to-metadata tool** (`ccpp_fortran_to_metadata.py`): -- Standalone utility: parses annotated Fortran source files and generates skeleton `.meta` - files — used to bootstrap new scheme metadata - ---- - -## 4. Shared infrastructure - -### 4.1 Module sharing map - -| Module | Used by prebuild | Used by capgen | Notes | -|---|---|---|---| -| `metadata_parser.py` | yes | partial | **Bridge module**: calls capgen's parser, returns mkcap.Var | -| `metadata_table.py` | via bridge | yes (primary) | Native `.meta` format parser | -| `metavar.py` | no | yes | Primary `Var` class, `VarDictionary` | -| `var_props.py` | no | yes | `VariableProperty`, `VarCompatObj`, dimension constants | -| `mkcap.py` | yes | no | `mkcap.Var` class + build-snippet writers | -| `mkstatic.py` | yes | no | Suite/Group/API Fortran generators | -| `mkdoc.py` | yes | no | HTML/LaTeX documentation generators | -| `common.py` | yes | partial | `CCPP_STAGES`, container encoding | -| `framework_env.py` | dummy instance | yes (primary) | `CCPPFrameworkEnv` | -| `file_utils.py` | no | yes | `create_file_list`, `move_modified_files` | -| `code_block.py` | no | yes | Structured Fortran output | -| `ddt_library.py` | no | yes | `DDTLibrary`, `VarDDT` | -| `host_model.py` | no | yes | `HostModel` class | -| `host_cap.py` | no | yes | Host cap generation | -| `ccpp_suite.py` | no | yes | `Suite`, `API` classes | -| `suite_objects.py` | no | yes | `Scheme`, `Group`, `Subcycle`, `CallList` | -| `constituents.py` | no | yes | `ConstituentVarDict` | -| `ccpp_datafile.py` | no | yes | Datatable XML | -| `ccpp_database_obj.py` | no | yes | `CCPPDatabaseObj` | -| `ccpp_state_machine.py` | no | yes | `CCPP_STATE_MACH` | -| `state_machine.py` | no | yes | `StateMachine` base class | -| `ccpp_fortran_to_metadata.py` | no | yes | Fortran→metadata bootstrap tool | -| `ccpp_track_variables.py` | partial | partial | Uses both worlds | - -**The key architectural debt**: `metadata_parser.py` is a prebuild module that internally -calls capgen's `metadata_table.parse_metadata_file()` and converts results to `mkcap.Var` -objects. This creates a one-way dependency (prebuild → capgen's parser infrastructure) -while presenting a prebuild-style API to `ccpp_prebuild.py`. It exists only because -prebuild predates the `.meta` format. - -### 4.2 The `.meta` file format - -The `.meta` format is the native format for capgen and the expected format for all new -scheme development. The Fortran source file contains a comment hook pointing to the `.meta` -file: - -```fortran -!! \section arg_table_scheme_name_run Argument Table -!! \htmlinclude scheme_name_run.html -``` - -The `.meta` file itself uses an INI-style format: - -```ini -[ccpp-table-properties] - name = scheme_name - type = scheme - dependencies_path = ../some/path - dependencies = utility_module.F90, another.F90 - -[ccpp-arg-table] - name = scheme_name_run - type = scheme -[ im ] - standard_name = horizontal_loop_extent - long_name = horizontal loop extent - units = count - type = integer - dimensions = () - intent = in -[ dz ] - standard_name = layer_thickness - long_name = thickness of each model layer - units = m - type = real - kind = kind_phys - dimensions = (horizontal_loop_extent, vertical_layer_dimension) - intent = in -``` - -Multiple `[ccpp-arg-table]` sections are allowed in a scheme file (one per phase: -`_init`, `_run`, `_finalize`, `_timestep_init`, `_timestep_finalize`). -Singleton tables (DDT, module, host) allow only one section. - -### 4.3 Variable property validation (`var_props.py`) - -`VariableProperty` encapsulates a single metadata property with its name, Python type, -optionality, default, valid-value constraints, and a check function. Check functions used: - -| Checker | What it validates | -|---|---| -| `check_local_name` | Valid Fortran identifier | -| `check_cf_standard_name` | Lowercase, underscores, alphanumeric only | -| `check_fortran_type` | Intrinsic type or registered DDT name | -| `check_units` | Valid unit string (normalizes `+` in exponents) | -| `check_dimensions` | Valid dimension specification | -| `check_default_value` | Valid Fortran expression | -| `check_molar_mass` | Positive float (for constituents) | - -`CCPP_HORIZONTAL_DIMENSIONS`, `CCPP_VERTICAL_DIMENSIONS`, `CCPP_LOOP_DIM_SUBSTS` -in `var_props.py` define the recognized dimension forms and the run-time substitution -map (e.g., `horizontal_dimension → horizontal_loop_begin:horizontal_loop_end`). - ---- - -## 5. Feature comparison - -| Feature | prebuild | capgen | Notes | -|---|---|---|---| -| **Input formats** | | | | -| Native `.meta` format | via bridge | yes | | -| Old pipe-delimited format | deprecated warn | not supported | | -| **Parsing and validation** | | | | -| Fortran source cross-validation | no | yes | capgen parses actual .F90 to cross-check | -| Preprocessor directive support | no | yes | `--preproc-directives` | -| **Variable handling** | | | | -| Variable data class | `mkcap.Var` (flat attrs) | `metavar.Var` (validated prop dict) | | -| Scope-chain variable search | no | yes | group→suite→constituent→host | -| Variable promotion group→suite | no | yes | | -| Unit conversion | yes | yes | | -| Optional/active variables | yes (fully) | yes | both: local pointer + conditional ASSOCIATE | -| DDT library (first-class) | no | yes | `VarDDT` recursive chain | -| **Suite and cap generation** | | | | -| Suite definition (SDF XML) | yes | yes | Same XML format | -| Subcycle loops | yes | yes | | -| State machine in generated caps | no | yes | Runtime state enforcement | -| Static API module (dispatch switch) | yes | no | `ccpp_static_api.F90` | -| Host cap generation | no | yes | `_ccpp_cap.F90` | -| `ccpp_kinds.F90` | no | yes | | -| **Constituent/tracer support** | | | | -| Constituent variable management | no | yes | Auto-allocation, `ConstituentVarDict` | -| **Build system output** | | | | -| CMake/Makefile file-list snippets | yes | no | Six snippet files | -| Datatable XML (queryable) | no | yes | `ccpp_datafile.py` | -| Clean via datatable | no | yes | | -| **Documentation** | | | | -| HTML variable table | yes | no (stub, raises error) | `mkdoc.metadata_to_html` | -| LaTeX variable table | yes | no | `mkdoc.metadata_to_latex` | -| **Developer tools** | | | | -| Variable tracking diagnostic | yes | no | `ccpp_track_variables.py` | -| Fortran-to-metadata bootstrap | no | yes | `ccpp_fortran_to_metadata.py` | -| **Runtime API** | | | | -| In-memory database object | no | yes | `CCPPDatabaseObj` | -| **Debug / developer aids** | | | | -| Debug array-size checks in caps | yes (`--debug`) | no | | -| Namespace suffix for API name | yes (`--namespace`) | no | | -| **Configuration** | | | | -| Config mechanism | Python module (flexible) | CLI args only | | - -**Known gaps and corrections:** - -- Capgen's `--generate-docfiles` is declared in the CLI but raises - `CCPPError("not yet supported")` — documentation generation is unimplemented. -- Prebuild handles `TYPEDEFS_NEW_METADATA` for mixed old/new metadata deployments; - capgen has no equivalent because it only accepts the new format. -- Capgen validates Fortran source against metadata; prebuild trusts metadata and never - reads Fortran code. -- Capgen has no `--namespace` equivalent for the generated API module name. -- Capgen's `CCPPDatabaseObj` and datatable XML allow programmatic querying; prebuild - has no equivalent. -- Prebuild's static API pattern (single Fortran module with runtime dispatch) is absent - from capgen, which uses a different host-cap integration model. -- **Capgen cannot pass DDTs to group caps** — it passes everything as flat fields. - Despite considerable effort by multiple developers, this has not been fixed. This is - the primary reason capgen is being abandoned. -- **Capgen does not support multiple model instances in memory** (ensemble approach). - Prebuild's `initialized(200)` array handles this correctly. -- **Capgen does not own or allocate any data.** Wait — this is a prebuild characteristic. - Capgen *does* allocate data for physics-internal variables (variables used only within - the physics, not provided by the host model) at the suite level. Prebuild requires the - host model to provide and own all data, including any physics-internal scratch space. - ---- - -## 6. Build system integration - -### 6.1 How a host model invokes ccpp-prebuild - -Direct call (as in the test suite): -```bash -python ../../scripts/ccpp_prebuild.py \ - --config=ccpp_prebuild_config.py \ - --builddir=build \ - --suites=suite_A,suite_B \ - [--debug] [--namespace mymodel] -``` - -Typical CMake integration: -```cmake -# Run prebuild at configure time -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_prebuild.py - --config=${HOST_CCPP_PREBUILD_CONFIG} - --builddir=${CMAKE_CURRENT_BINARY_DIR} - --suites=${CCPP_SUITES} - WORKING_DIRECTORY ${CMAKE_SOURCE_DIR} - RESULT_VARIABLE PREBUILD_RESULT -) -if(NOT PREBUILD_RESULT EQUAL 0) - message(FATAL_ERROR "ccpp_prebuild.py failed") -endif() - -# Consume the generated snippet files -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_CAPS.cmake) # → ${CAPS} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_SCHEMES.cmake) # → ${SCHEMES} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} -include(${CMAKE_CURRENT_BINARY_DIR}/CCPP_API.cmake) # → ${API} - -add_library(ccpp_physics OBJECT ${CAPS} ${SCHEMES} ${API}) -``` - -### 6.2 How a host model invokes ccpp-capgen - -Direct call: -```bash -python scripts/ccpp_capgen.py \ - --host-files host_data.meta,host_model.meta \ - --scheme-files scheme1.meta,scheme2.meta \ - --suites suite_A.xml,suite_B.xml \ - --output-root ${BUILD_DIR}/ccpp \ - --host-name my_host \ - --kind-type kind_phys=REAL64 \ - --ccpp-datafile ${BUILD_DIR}/ccpp/datatable.xml -``` - -Typical CMake integration: -```cmake -# Run capgen at configure time -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_capgen.py - --host-files ${HOST_META_FILES} - --scheme-files ${SCHEME_META_FILES} - --suites ${SUITE_SDFS} - --output-root ${CMAKE_CURRENT_BINARY_DIR}/ccpp - --host-name ${HOST_MODEL_NAME} - --ccpp-datafile ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - RESULT_VARIABLE CAPGEN_RESULT -) - -# Query the datatable for generated file lists -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py - ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - --suite-files - OUTPUT_VARIABLE SUITE_CAPS OUTPUT_STRIP_TRAILING_WHITESPACE -) -execute_process( - COMMAND ${Python3_EXECUTABLE} - ${CCPP_FRAMEWORK}/scripts/ccpp_datafile.py - ${CMAKE_CURRENT_BINARY_DIR}/ccpp/datatable.xml - --host-files - OUTPUT_VARIABLE HOST_CAP OUTPUT_STRIP_TRAILING_WHITESPACE -) - -add_library(ccpp_physics OBJECT ${SUITE_CAPS} ${HOST_CAP}) -``` - -### 6.3 Available datatable query flags - -``` ---host-files → generated host cap .F90 files ---suite-files → generated suite cap .F90 files ---utility-files → generated utility .F90 files (e.g. ccpp_kinds.F90) ---ccpp-files → all generated .F90 files ---process-list → physics process types in the suite ---module-list → Fortran module names needed ---dependencies → scheme dependency files ---suite-list → configured suite names ---required-variables → variables required by all suites ---input-variables → input-only variables for a suite ---output-variables → output variables for a suite ---host-variables → variables provided by the host model -``` - ---- - -## 7. Key architectural differences - -### 7.1 Data model - -| Dimension | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| Variable representation | `mkcap.Var` with plain Python attributes | `metavar.Var` with validated `__prop_dict` | -| Variable storage | Two flat `OrderedDict`s | Scope-chain `VarDictionary` tree | -| Container encoding | Encoded string: `MODULE_foo SCHEME_bar SUBROUTINE_baz` | Explicit class hierarchy | -| DDT handling | Encoded as string in `local_name`; helper regexes to extract | First-class `VarDDT` recursive chain | -| Variable matching | One batch `compare_metadata()` call | Incremental during suite analysis | -| Matching result | `bool` + side effects on `.target` / `.actions` | Rich `VarCompatObj` with transformation info | -| **Cap argument style** | **DDTs passed to group caps** | **Flat fields passed to group caps** | -| Subsetting location | At the scheme call site inside the group cap | Done at a higher level, before group cap | -| Data ownership | Host model owns all data including physics-internal | Capgen allocates physics-internal suite-level data | -| Multiple model instances | Yes — `initialized(200)` array, one flag per instance | No | -| Optional variable handling | Local pointer, conditionally associated | Same mechanism, but blocked by flat-field issue | - -### 7.2 Error handling - -| Aspect | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| Style | `(success, result)` tuples + `logging.error()` | `CCPPError` / `ParseInternalError` exceptions | -| Collection | Errors accumulate via `logging`; `main()` checks success | Raised immediately at point of detection | -| Location info | Filename from context; line numbers sometimes | `ParseContext` objects with file + line number | -| User errors vs bugs | Not distinguished | `CCPPError` (user) vs `ParseInternalError` (programmer) | - -### 7.3 Extensibility - -| Aspect | ccpp-prebuild | ccpp-capgen | -|---|---|---| -| New metadata property | Add to `VALID_ITEMS` dict + `mkcap.Var` attribute | Add one `VariableProperty` entry + checker fn | -| New CCPP phase | Update `CCPP_STAGES` + regenerate static API template | Add one transition tuple to `CCPP_STATE_MACH` | -| New compatibility rule | Modify `var.compatible()` in `mkcap.py` | Extend `VarCompatObj` in `var_props.py` | -| New host model | Write a new Python config file | New `.meta` files + CLI invocation | - -### 7.4 Performance - -Prebuild generates caps for multiple suites in seconds. Capgen, on the same suite set -with the same physics, takes more than 10 minutes. Two independent causes: - -**Cause 1 — Repeated scope-chain traversal.** Every variable lookup in capgen traverses -a five-level `VarDictionary` parent chain (group → suite → constituent dict → host model -→ DDT dict) for every scheme argument in every group in every suite. Prebuild's -`compare_metadata()` does one flat dict lookup per standard name, once, and caches the -result in `var.container` and `var.target`. All subsequent use during Fortran generation -reads these cached attributes directly. - -**Cause 2 — Flat-field cap arguments.** This is likely the dominant cost. Capgen resolves -every scheme argument down to its individual flat field, generates a `use` statement and -an explicit argument for each one, and emits them in the generated Fortran. A DDT with -200 fields becomes 200 individual argument declarations, 200 `use` statements, and 200 -argument positions in the scheme call. Prebuild passes the DDT itself — one argument, -one `use` statement — and then subsets at the call site. - -**Consequence for correctness.** Passing flat fields in capgen also breaks optional -variable handling under Fortran compiler debugging flags. When a field inside a DDT is -conditionally allocated (optional), passing it as a flat field requires dereferencing the -DDT to extract the field — which the compiler will flag as an error if debugging is on -and the field happens to be unallocated. Prebuild avoids this entirely by passing the -DDT and using a local pointer at the scheme call site. - -### 7.5 Team comprehension and maintainability - -This is the critical real-world difference. `ccpp-prebuild` is understood by the whole -team because it is procedural Python: you can read `ccpp_prebuild.py` top-to-bottom and -follow what happens. The data structures are flat dicts; the control flow is linear. - -`ccpp-capgen` has a five-level class hierarchy, scope-chain dictionary lookups, -`VarCompatObj` carrying transformation state, `ConstituentVarDict` as a pluggable -scope-chain node, and a `StateMachine` with regex-based dispatch. No remaining team -member fully understands all of it. Development is extremely slow and risky. - -The failed effort to make capgen pass DDTs instead of flat fields is the concrete proof -point: three developers spent considerable time and could not fix it without fully -understanding the interplay between `VarDDT`, `DDTLibrary`, `VarDictionary` scope chains, -and the Fortran writer. This is the proximate reason for the redesign. - ---- - -## 8. Design considerations for the redesign - -The following observations from this analysis should inform the redesign: - -### 8.1 What to keep from prebuild -- Procedural, top-down control flow — easy to read and debug -- Config file as a Python module — extremely flexible without adding CLI arguments -- The static API pattern (`ccpp_static_api.F90` with runtime suite/group dispatch) — - proven, simple integration for the host model -- **DDT arguments in group caps** — pass DDTs, not flat fields; this is the core correctness - and performance requirement -- **Subsetting at the scheme call site** — group caps always receive full data; loop-bound - application and fixed-index extraction happen in the individual scheme call expressions - or via a local variable/pointer declared just before the call -- **Optional variable pattern** — local pointer declared in the group cap, conditionally - associated based on the `active` expression, then passed to the scheme; this is safe - under all compiler debugging modes -- The `initialized(N)` per-instance tracking — handles multiple simultaneous model - instances in memory (ensemble approach); `N` is the max number of instances -- **Framework-owned data needs a simpler design** — capgen's variable promotion and - `ConstituentVarDict` scope-chain approach is too complex; a cleaner mechanism for - framework-allocated physics-internal data is needed (to be designed) -- HTML and LaTeX documentation generation -- The six CMake/Makefile/shell snippet output files — simple and direct (can be revisited) - -### 8.2 What to keep from capgen -- Native `.meta` file parsing (eliminate the `metadata_parser.py` bridge entirely) -- Fortran source cross-validation (`check_fortran_against_metadata()`) — catches real bugs -- Rich compatibility reporting (`VarCompatObj`-style) — better error messages -- `ccpp_kinds.F90` generation — important for portability -- Datatable XML as output accounting (strictly better than six include files) -- `--preproc-directives` support -- Constituent variable support (needed for CAM-SIMA) -- State machine enforcement (optional feature, but architecturally clean) - -### 8.3 What to eliminate -- The `mkcap.Var` / `metavar.Var` duality — one variable class, natively reading `.meta` -- The `metadata_parser.py` bridge module — it exists only because of the old format -- The scope-chain `VarDictionary` hierarchy — replace with flat, explicit lookup: - one host dict, one scheme dict; no parent-chain traversal -- The five-level class inheritance (Suite → VarDictionary → ParseSource → ...) -- `ConstituentVarDict` as a scope-chain node — a simple explicit constituent registry suffices -- Capgen's variable promotion (group → suite level) — this complexity exists only because - capgen allocates physics-internal data; if the host always owns all data, promotion - is unnecessary -- Capgen's flat-field cap generation — DDT arguments must be the foundation - -### 8.4 Framework-owned data — open design question - -Capgen's variable promotion mechanism (promoting a variable from group scope to suite scope -when a later group needs it) and the `ConstituentVarDict` complexity exist because capgen -allocates and manages physics-internal data — variables used only within the physics, -not visible to the host model. This capability is **wanted** in the redesign: the host -model should not have to declare and own scratch variables that are purely internal to the -physics. - -The problem is not the concept but the implementation. Capgen's approach — weaving -framework-allocated variables into the `VarDictionary` scope chain and promoting them -upward — produces the complexity that made capgen unmaintainable. - -**Open question for the redesign:** What is a simpler mechanism for the framework to -allocate, own, and pass physics-internal variables? Candidate approaches (to be evaluated -with real-world examples): - -- A completely separate, flat "framework data" dictionary, distinct from the host variable - lookup, populated during analysis and passed explicitly to the caps as a dedicated - argument (e.g., a framework-managed DDT or allocatable array container). -- A simplified promotion concept: variables are statically promoted to the widest scope - that needs them during the analysis phase, but stored in a simple flat dict rather than - via a scope-chain lookup. -- Constituent variables (tracers) as a special sub-case with their own well-defined - allocation interface, separate from generic physics-internal data. - -This question will be revisited once real-world examples clarify how many and what kind of -physics-internal variables actually need to be managed. - -### 8.5 Critical design decisions for the redesign prompt - -1. **DDT cap arguments are non-negotiable.** Group caps must receive DDTs. The entire - subsetting, optional-variable, and performance story depends on this. - -2. **Data ownership**: host-owns-all (prebuild model) vs. generator-allocates-internals - (capgen model). This single decision determines whether variable promotion and - suite-level allocation are needed. - -3. **Integration pattern**: static API (prebuild style, `suite_name` + `group_name` dispatch) - vs. host cap (capgen style, separate host-side Fortran glue). Models currently using - each pattern depend on it. - -4. **Config mechanism**: Python module (prebuild style, flexible) vs. pure CLI + file lists - (capgen style, scriptable). The Python module config is very powerful for complex models. - -5. **DDT member access parsing**: `extract_parents_and_indices_from_local_name()` and - `extract_dimensions_from_local_name()` in `mkstatic.py` handle expressions like - `Atm(blk_no)%q(:,:,:,graupel_index)`. The redesign needs a clean, explicit design for - parsing and emitting these — not an afterthought regex patch. - -6. **Output accounting**: datatable XML (capgen) is the right answer. The six CMake snippet - files (prebuild) are redundant and harder to extend. - -7. **Multiple model instances**: the redesign must preserve the `initialized(N)` pattern - or an equivalent. The value of `N` may need to be configurable. - -8. **Backward compatibility of generated Fortran interfaces**: real-world model examples - will define exactly which naming conventions, argument orders, and module structures the - host models depend on. - ---- - -## 9. Real-world example: CCPP Single Column Model (SCM) - -*Source:* `EXT/ccpp-scm/` — uses `ccpp-prebuild`. - -The SCM is a horizontally degenerate model (always `im = 1`, no OpenMP threading) but -it compiles the largest set of suites in the CCPP ecosystem, making it the most complete -real-world picture of what prebuild must handle. - -**Scale:** 63 suites, 257 scheme files (137 scheme entries in config, many containing -multiple modules), 300 generated cap files, ~1,200+ host model variables, ~550 optional -(conditionally active) variables. - ---- - -### 9.1 The `TYPEDEFS_NEW_METADATA` bridge — the DDT accessor map - -This is the most important SCM-specific configuration. It maps each DDT type name to the -Fortran expression used to access an instance of that type from the host model's top-level -scope. It is what allows the code generator to convert a `local_name` like `tgrs` (declared -inside `GFS_statein_type`) into the cap argument expression -`physics%Statein%tgrs(...)`. - -```python -TYPEDEFS_NEW_METADATA = { - 'GFS_typedefs': { - 'GFS_diag_type' : 'physics%Diag', - 'GFS_control_type' : 'physics%Model', - 'GFS_cldprop_type' : 'physics%Cldprop', - 'GFS_tbd_type' : 'physics%Tbd', - 'GFS_sfcprop_type' : 'physics%Sfcprop', - 'GFS_coupling_type': 'physics%Coupling', - 'GFS_statein_type' : 'physics%Statein', - 'GFS_radtend_type' : 'physics%Radtend', - 'GFS_grid_type' : 'physics%Grid', - 'GFS_stateout_type': 'physics%Stateout', - 'GFS_typedefs' : '', - }, - 'CCPP_typedefs': { - 'GFS_interstitial_type': 'physics%Interstitial(cdata%thrd_no)', - 'CCPP_typedefs' : '', - }, - 'scm_type_defs': { - 'physics_type': 'physics', - 'scm_type_defs': '', - }, - 'ccpp_types': { - 'ccpp_t' : 'cdata', - 'ccpp_types': '', - 'MPI_Comm': '', - }, - # ... plus 8 more entries for physics-side modules (machine, radsw_param, etc.) -} -``` - -**How it works:** For a variable with `local_name = tgrs` declared in `GFS_statein_type`, -the generator looks up `'GFS_statein_type'` in the map, finds `'physics%Statein'`, and -constructs the target as `physics%Statein%tgrs`. For the thread-indexed interstitial DDT, -`physics%Interstitial(cdata%thrd_no)%` is produced automatically. - -This dictionary is the **entire** mechanism by which the prebuild bridge converts flat -metadata into correct DDT-member accessor expressions. It is a hand-maintained workaround -that the redesigned generator must **eliminate**: all information needed to derive these -accessor expressions is already present in the CCPP metadata, provided the metadata storage -model is designed correctly to capture the DDT hierarchy and instance/thread indexing. - ---- - -### 9.2 Host model DDT structure - -``` -! Module-level variables accessible globally: -physics (type physics_type, from module scm_type_defs) -cdata (type ccpp_t, from module ccpp_types) -one (integer parameter = 1, from module ccpp_types) - -! physics_type contains: -physics%Model → GFS_control_type (control parameters: integers, logicals, 1D arrays) -physics%Statein → GFS_statein_type (input atmospheric state: 2D/3D real arrays) -physics%Stateout → GFS_stateout_type (output tendencies) -physics%Sfcprop → GFS_sfcprop_type (surface properties: 2D real arrays) -physics%Coupling → GFS_coupling_type (coupling fields) -physics%Grid → GFS_grid_type (grid geometry) -physics%Tbd → GFS_tbd_type (to-be-determined / miscellaneous) -physics%Cldprop → GFS_cldprop_type (cloud microphysics properties) -physics%Radtend → GFS_radtend_type (radiation tendencies) -physics%Diag → GFS_diag_type (diagnostic output arrays) -physics%Interstitial(1:thrd_cnt) → GFS_interstitial_type (per-thread scratch space) -``` - -The interstitial DDT is an array indexed by thread number. Even though the SCM is -single-threaded, all caps use `physics%Interstitial(cdata%thrd_no)` (i.e., index 1). -This is the pattern that enables OpenMP parallelism in the full UFS models. - ---- - -### 9.3 The horizontal dimension in the SCM - -The SCM uses a **chunked** horizontal loop even though `im = 1`. The chunk mechanism is: - -```fortran -chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) -chunk_end = physics%Model%chunk_end(cdata%chunk_no) -``` - -All 2D and 3D array slice expressions in caps use this pattern: -```fortran -physics%Statein%tgrs(chunk_begin:chunk_end, one:levs) -``` - -In the SCM, `chunk_begin = chunk_end = 1` always, but the pattern is general enough for -multi-column models. The `one` lower bound (a named integer constant = 1) is a framework -convention used consistently throughout all caps. - ---- - -### 9.4 Four categories of local variables in group caps - -Every group cap generates four categories of local variable declarations before its scheme -calls: - -**Category 1 — Loop bounds and scalars (always present):** -```fortran -integer :: chunk_begin, chunk_end -integer :: levs -chunk_begin = physics%Model%chunk_begin(cdata%chunk_no) -chunk_end = physics%Model%chunk_end(cdata%chunk_no) -levs = physics%Model%levs -``` - -**Category 2 — Fixed-index extractions (tracer indices, surface-level slices):** - -For a tracer `qgrs(:,:,ntqv)`: -```fortran -! No local variable declared — the expression is used inline at the call site: -call scheme_run(qv = physics%Statein%qgrs(chunk_begin:chunk_end, one:levs, physics%Model%ntqv), ...) -``` - -For a surface-level slice `prsi(:,1)`: -```fortran -call scheme_run(prsi_sfc = physics%Statein%prsi(chunk_begin:chunk_end, 1), ...) -``` - -The fixed index may be a literal integer (`1`) or a runtime scalar variable from a DDT -field (`physics%Model%ntqv`). Both are inlined at the call site. - -**Category 3 — Optional variable pointer arrays:** - -One pointer-array type and one pointer-array variable are declared for each optional -variable. They are dimensioned by thread count: -```fortran -type :: real_kind_phys_rank2_ptr_arr_type - real(kind_phys), dimension(:,:), pointer :: p => null() -end type real_kind_phys_rank2_ptr_arr_type -type(real_kind_phys_rank2_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array -``` - -Before each scheme call that uses the variable, the condition is evaluated and the pointer -either associated or left null: -```fortran -if (physics%Model%lndp_type /= 0) then - sfc_wts_1_ptr_array(cdata%thrd_no)%p => & - physics%Coupling%sfc_wts(chunk_begin:chunk_end, one:physics%Model%n_var_lndp) -end if -``` - -Passed to the scheme as a keyword argument: -```fortran -call gfs_surface_generic_pre_run(..., sfc_wts=sfc_wts_1_ptr_array(cdata%thrd_no)%p, ...) -``` - -After the call, the pointer is nullified: -```fortran -if (physics%Model%lndp_type /= 0) then - nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) -end if -``` - -**Category 4 — Unit conversion local variables:** - -Not present in the SCM (GFS uses consistent SI units throughout). When present in other -models, a local array is declared, populated before the call, and passed as the argument: -```fortran -real(kind_phys) :: converted_var(chunk_begin:chunk_end) -converted_var(:) = physics%Statein%source_field(chunk_begin:chunk_end) * conversion_factor -call scheme_run(..., target_arg=converted_var, ...) -``` - ---- - -### 9.5 Array size checks - -Every array argument — mandatory or optional — has a size check immediately before the -scheme call. The check uses `size()` and computes the expected size from dimension variables: - -```fortran -! Mandatory variable — outer condition is always .true. -if (.true.) then - if (size(physics%Statein%tgrs(chunk_begin:chunk_end, one:levs)) /= & - (chunk_end-chunk_begin+1)*(levs-one+1)) then - write(cdata%errmsg, '(a,i8,a,i8)') & - 'Detected size mismatch for variable tgrs: expected ', expected, ' but got ', actual - ierr = 1 - return - end if -end if - -! Optional variable — outer condition mirrors the active= expression -if (physics%Model%lndp_type /= 0) then - if (associated(sfc_wts_1_ptr_array(cdata%thrd_no)%p)) then - if (size(sfc_wts_1_ptr_array(cdata%thrd_no)%p) /= expected_size) then - ...error... - end if - end if -end if -``` - ---- - -### 9.6 The `initialized(200)` array and instance management - -```fortran -logical, dimension(200), save :: initialized = .false. -``` - -`cdata%ccpp_instance` is a 1-based integer assigned to each independent CCPP state object. -In an ensemble, each ensemble member gets a different instance number (1–200). The `init_cap` -sets `initialized(cdata%ccpp_instance) = .true.` at the end of successful init. The -`run_cap` checks `if (.not. initialized(cdata%ccpp_instance))` and aborts with an error -if init was never called for that instance. The `final_cap` resets the flag to `.false.`. - -The value 200 is hardcoded — it is the maximum supported number of simultaneous model -instances. This could be made configurable. - ---- - -### 9.7 Suite and group cap hierarchy - -Three-level cap hierarchy: - -``` -ccpp_static_api.F90 (module ccpp_static_api) - → dispatches by suite_name + optional group_name - → owns physics, cdata, constants via module use - → calls suite-level caps: - -ccpp_scm_gfs_v16_cap.F90 (module ccpp_scm_gfs_v16_cap) - → aggregates arguments from all groups - → calls group caps in order per phase: - -ccpp_scm_gfs_v16_time_vary_cap.F90 (module ccpp_scm_gfs_v16_time_vary_cap) -ccpp_scm_gfs_v16_radiation_cap.F90 (module ccpp_scm_gfs_v16_radiation_cap) -ccpp_scm_gfs_v16_phys_ps_cap.F90 (module ccpp_scm_gfs_v16_phys_ps_cap) -ccpp_scm_gfs_v16_phys_ts_cap.F90 (module ccpp_scm_gfs_v16_phys_ts_cap) -``` - -Each level is a pure Fortran module. Argument passing is explicit keyword-argument style -at every level; no implicit global data (except in the static API, which uses `use`). - ---- - -### 9.8 Static API: module-level variable ownership - -The static API module uses all host-model modules and accesses their variables at module -scope. It does **not** take host data as subroutine arguments — instead it fills the -group cap arguments from its own module-use-associated variables: - -```fortran -module ccpp_static_api - use scm_type_defs, only: physics - use ccpp_types, only: cdata, one - use scm_physical_constants, only: con_g, con_pi, con_t0c, ... - use gfs_typedefs, only: ltp - use ccpp_scm_gfs_v16_cap, only: scm_gfs_v16_run_cap, ... - ... -contains - subroutine ccpp_physics_run(cdata, suite_name, group_name, ierr) - ! cdata passed in, others accessed from module scope - select case (to_lower(trim(suite_name))) - case ('scm_gfs_v16') - if (present(group_name)) then - select case (to_lower(trim(group_name))) - case ('phys_ps') - ierr = scm_gfs_v16_phys_ps_run_cap(one=one, physics=physics, cdata=cdata, ...) - ... - end select - else - ierr = scm_gfs_v16_run_cap(one=one, physics=physics, cdata=cdata, ...) - end if - case ('scm_gfs_v17_p8') - ... - end select - end subroutine -end module -``` - -This design means the static API file must be recompiled whenever any host-model module -changes (because it `use`s them), and it must be regenerated whenever suites change. -Its location in the **source tree** (not build tree) is a deliberate SCM design choice: -the file is committed to the repository as a generated artifact. - ---- - -### 9.9 Build system - -Prebuild runs at **cmake configure time** via `execute_process()`, before any compilation -starts. This is unusual but simplifies the cmake dependency graph. - -```cmake -execute_process( - COMMAND ccpp/framework/scripts/ccpp_prebuild.py - --config=ccpp/config/ccpp_prebuild_config.py - --suites=${CCPP_SUITES} - --builddir=${CMAKE_CURRENT_BINARY_DIR} - WORKING_DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/../.. - OUTPUT_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.out - ERROR_FILE ${PROJECT_BINARY_DIR}/ccpp_prebuild.err -) -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_CAPS.cmake) # → ${CAPS} -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_SCHEMES.cmake) # → ${SCHEMES} -include(${CMAKE_CURRENT_BINARY_DIR}/ccpp/physics/CCPP_TYPEDEFS.cmake) # → ${TYPEDEFS} -include(scm/src/CCPP_STATIC_API.cmake) # → ${API} -``` - -**Suite selection:** If `CCPP_SUITES` is not set by the user, a helper script -`suite_info.py` selects a compiler-appropriate subset. The full set of 63 suites is -used for production; subsets speed up development builds. - ---- - -### 9.10 Observations relevant to the redesign - -1. **`TYPEDEFS_NEW_METADATA` is a workaround that the redesign must eliminate.** The - DDT accessor information (which type lives at which accessor path) can be fully derived - from the CCPP metadata itself, given a well-designed metadata storage model. The - redesign must derive DDT accessor expressions automatically from the metadata rather - than requiring a separate hand-maintained dictionary. This is one of the primary - motivations for the new metadata storage design. - -2. **Three-level cap hierarchy (group → suite → static API) should be preserved.** - It provides clean separation: group caps are independently testable, suite caps - aggregate phases, the static API is the single host-callable entry point. - -3. **The static API's module-level `use` of host data is model-specific.** In models - where host data is not module-level (e.g., passed as subroutine arguments), the - static API pattern changes. The SCM is the simplest case because `physics` and `cdata` - are global module variables. - -4. **Instance and thread indexing are two orthogonal dimensions of host data access.** - Host model data uses two distinct indexing patterns that must be handled correctly: - - - **Regular state data** (Statein, Stateout, Sfcprop, etc.): dimensioned by instance - number — `physics%Statein(ccpp_instance_number)%array(1:horizontal_dimension, 1:vertical_dimension, ...)`. - In models supporting multiple in-memory model instances (ensemble), the top-level - DDT is an array indexed by `cdata%ccpp_instance`. - - - **Interstitial (per-thread scratch) data**: dimensioned by both instance and thread — - `physics%Interstitial(ccpp_instance_number, ccpp_thread_number)%array(1:horizontal_loop_extent, ...)`. - Critically, the horizontal dimension of interstitial arrays is sized to - `horizontal_loop_extent` (one OpenMP thread's chunk), not `horizontal_dimension` - (the full column count). `max_number_of_threads` instances are allocated per model - instance. Interstitial data can only be used during the **run phase** — this is a - known limitation of ccpp-prebuild that the redesign should address or at minimum - preserve explicitly. - -5. **Optional variable pointer arrays dimensioned by thread count** are the current - solution to thread-safe optional variable handling. This pattern is verbose (one - derived type + one array per optional variable per cap function) but correct. - The redesign could simplify this. - -6. **~550 optional variables in this model.** Optional/conditional variables are not - a corner case — they are a first-class feature. The redesign must handle them - efficiently and correctly. - -7. **Array size checks are debug-only and should not appear in the redesign by default.** - In prebuild they are only generated when the `--debug` flag is passed. The redesigned - generator should not produce them in normal mode — out-of-bounds access is caught at - runtime by compiler flags (e.g., `-fcheck=bounds` with gfortran, `-check bounds` with - ifort). The 12,991-line group cap is partly a consequence of generating these checks - unconditionally in the debug mode artifact examined here. - -8. **No unit conversions appear in GFS/SCM.** Unit conversion infrastructure must be - present in the redesign but the GFS physics package is self-consistent in units. - Unit conversions are more relevant for other host models. - -9. **The `one` constant** (integer parameter = 1) is passed as an explicit argument - everywhere and used as the lower bound in all array slices. This is a framework - convention. The redesign should decide whether this convention is preserved or - whether array lower bounds are handled differently. - -10. **Subcycles produce actual Fortran `do` loops inside the generated group cap.** - The loop from `1` to `cdata%loop_max` is generated directly in the cap function, - not left to the host model: - ```fortran - cdata%loop_max = 2 - do cdata%loop_cnt = 1, cdata%loop_max - call scheme_A_run(...) - if (ierr /= 0) return - call scheme_B_run(...) - if (ierr /= 0) return - end do - ``` - `cdata%loop_max` is set at the start of the subcycle block (from the `loop=` attribute - in the SDF XML) and `cdata%loop_cnt` is the current iteration counter, both visible - to schemes via the `ccpp_t` DDT. - ---- - -## 10. Real-world example: CAM-SIMA (capgen) - -*Source:* `EXT/cam-sima/` — uses `ccpp-capgen`. - -CAM-SIMA is the only model currently using capgen. It is still primarily a research model. -Unlike the SCM it uses a full 3D grid with OpenMP parallelism, but exposes host model -data as flat module variables rather than DDTs in the metadata layer. This example reveals -both what capgen can do and where it fundamentally fails. - -**Scale:** 1 suite (`cam7`), 2 run groups (`physics_before_coupler`, `physics_after_coupler`), -~75 scheme calls, 18 host `.meta` files, 893-line host cap, 2865-line suite cap. - ---- - -### 10.1 Suite structure - -`suite_cam7.xml` has two groups and no subcycles: - -| Group | Schemes (approx.) | Purpose | -|---|---|---| -| `physics_before_coupler` | 52 scheme calls | Cloud fraction, energy checks, dry adiabatic adjustment, Zhang-McFarlane deep convection full cycle, constituent tendency application | -| `physics_after_coupler` | ~20 scheme calls | Tropopause diagnostics, gravity wave drag (7 parameterizations + diagnostics), tendency application, energy consistency | - -CCPP phases in use: register, initialize, timestep_initial, run (per group), timestep_final, finalize. - ---- - -### 10.2 Host model variable structure - -**18 host `.meta` files, all of type `module`.** There are no `host` or `ddt` table types -anywhere. All host variables are flat scalars or arrays in Fortran modules. - -CAM-SIMA does **not** expose its physics DDTs (`phys_state`, `phys_tend`, `cam_in`, etc.) -through metadata. These types exist in `physics_types.F90` but have no `.meta` file. -The generated host cap accesses them directly via `use physics_types, only: phys_state, ...` -and passes individual DDT members as flat keyword arguments: -```fortran -! In cam_ccpp_cap.F90 — direct access to non-metadataized DDT members: -call cam7_physics_before_coupler(..., pint=phys_state%pint, t=phys_state%t, & - dtdt_total=phys_tend%dtdt_total, landfrac=cam_in%landfrac, ...) -``` - -This means capgen has no knowledge of how host data is structured. The host cap is -partly machine-generated and partly depends on manually wiring non-metadataized sources. -**This is a fundamental architectural gap** — changes to `physics_types` are invisible -to the framework. - -**Key host variables by module:** - -| Module | Key variables | -|---|---| -| `physics_grid` | `columns_on_task` (horizontal_dimension), `col_start`, `col_end`, lat, lon, area | -| `vert_coord` | `pver` (vertical_layer_dimension), `pverp` (vertical_interface_dimension) | -| `physconst` | ~35 physical constants, all `protected = True` | -| `cam_constituents` | `num_advected` (count of advected tracers) | -| `spmd_utils` | `mpicom`, `masterproc`, `npes`, `iam` | - -No instance indexing (`physics(1)`) and no thread-indexed DDTs appear — CAM-SIMA uses -a fundamentally different data model from the GFS/SCM stack. - ---- - -### 10.3 The two-cap architecture - -Capgen generates two distinct Fortran files: - -**`cam_ccpp_cap.F90` — the host cap (893 lines)** -- Module `cam_ccpp_cap` -- Imports non-metadataized host variables directly via `use physics_types`, `use physconst`, etc. -- Manages the constituent object (`ccpp_model_constituents_t`) — registration, initialization, gather/scatter, index lookup -- Public subroutines: `cam_ccpp_physics_run`, `cam_ccpp_physics_initialize`, etc. — the entry points the host model calls -- Dispatches to the suite cap, passing ~61–76 flat keyword arguments - -**`ccpp_cam7_cap.F90` — the suite cap (2865 lines)** -- Module `ccpp_cam7_cap` -- No host-specific imports — knows nothing about `physics_types`, `phys_state`, etc. -- All arguments are flat scalars and arrays, fully matched to metadata standard_names -- Contains all scheme calls, suite-level persistent variables, local temporaries, state machine -- The suite cap could in principle be used with any host model that provides the same standard names - -This two-cap split is **architecturally correct**: it separates host-specific binding -from physics-neutral dispatch. The redesign should preserve this separation. - ---- - -### 10.4 The flat-field argument problem — concrete evidence - -The run-phase subroutines expose the core problem with capgen's approach directly: - -```fortran -subroutine cam7_physics_before_coupler(errflg, errmsg, col_start, col_end, pver, dtime, & - gravit, pint, te_ini_dyn, teout, amiroot, iulog, ptend_s, temp, dtdt_total, cpair, & - lagrang, layer_surf, layer_toa, interface_surf, interface_toa, ncnst, piln, pmid, pdel, & - rpdel, qv, carr, cprops, rair, zvir, zi, zm, cp_or_cv_dycore, u, v, pintdry, phis, & - te_cur_phys, te_cur_dyn, tw_cur, latice, latvap, energy_formula_physics, & - energy_formula_dycore, cappa, q_tend, const_tend, qmin, pverp, cpwv, cpliq, rh2o, lat, & - long, pblh, mcon, tpert, dlf, rprd, ql, rliq, landfrac, cpair3, ttend_dp, tmelt, & - top_lev, ke, ke_lnd, cldfrc, domomtran, momcu, momcd, il1g, nstep, & - dudt_total, dvdt_total, fracis, dpdry, ps) -``` - -**61 dummy arguments for one group cap.** `physics_after_coupler` has 76. These are -individual flat arrays and scalars — no DDT in sight. This is exactly the problem that -three developers failed to fix: in the GFS/UFS context, this would be 1,200+ arguments. -The GFS physics stack simply cannot be connected to capgen in its current form. - -In contrast, the prebuild equivalent for the same data would pass `physics` (one DDT argument) -and `cdata` — two arguments covering hundreds of variables. - ---- - -### 10.5 Suite-level persistent variables — the framework-owned data pattern - -The suite cap allocates and owns arrays that persist across group calls within a timestep. -These are allocated in `cam7_initialize` and deallocated in `cam7_finalize`: - -```fortran -! Suite-level persistent (allocated in initialize, freed in finalize): -real(kind_phys), allocatable :: windu_tend(:,:) ! GW drag u-tendency accumulator -real(kind_phys), allocatable :: windv_tend(:,:) ! GW drag v-tendency accumulator -real(kind_phys), allocatable :: scaling_dycore(:,:) ! energy scaling factor -real(kind_phys), allocatable :: tend_te_tnd(:) ! energy tendency accumulator -real(kind_phys), allocatable :: tend_tw_tnd(:) ! water tendency accumulator -real(kind_phys), allocatable :: temp_ini(:,:) ! temperature saved at timestep start -real(kind_phys), allocatable :: z_ini(:,:) ! height saved at timestep start -real(kind_phys), allocatable :: flx_vap(:), flx_cnd(:), flx_ice(:), flx_sen(:) -logical, allocatable :: doconvtran(:) ! per-constituent convection flag -type(coords1d) :: p ! pressure coordinate DDT for GW drag -``` - -These are physics-internal variables — the host model does not know about them, does not -own them, and does not need to. This is the capgen "data ownership" model: the suite cap -is the data owner for variables that only matter within the physics. - -**This pattern is correct and desirable.** The complexity in capgen comes not from the -concept but from how these variables are discovered during analysis (scope-chain promotion) -and passed around (via VarDictionary). The redesign needs a simpler mechanism to achieve -the same result: statically enumerate physics-internal variables during analysis and have -the suite cap own them as named allocatables. - -During the run phase, suite-level persistent arrays are subsetted when passed to schemes: -```fortran -call gw_common_run(..., windu_tend=windu_tend(col_start:col_end, 1:pver), ...) -``` - -Run-phase local temporaries (e.g., `cape`, `cme`, `mu`, `md`) are allocated at function -entry and deallocated at exit: -```fortran -allocate(cape(col_start:col_end)) -... -call zm_convr_run(..., cape=cape, ...) -... -deallocate(cape) -``` - -These temporaries use `col_start` as the lower bound so that assumed-shape dummy arguments -in schemes see a 1-based array — a subtle but important detail. - ---- - -### 10.6 Horizontal chunking model - -CAM-SIMA uses `col_start`/`col_end` (passed as arguments to every run subroutine) to -define the current horizontal chunk: - -```fortran -ncol = col_end - col_start + 1 -``` - -Schemes declare `horizontal_loop_extent` and receive `ncol`. The horizontal dimension -in the host (storage dimension) is `columns_on_task`. The subsetting from storage to -loop extent happens at the boundary between host cap and suite cap — the host cap -passes the right subsections: - -```fortran -! In cam_ccpp_cap.F90: -call cam7_physics_before_coupler(..., col_start=col_start, col_end=col_end, & - pmid=phys_state%pmid, ...) ! full arrays passed; suite cap subsets internally -``` - -Inside the suite cap, persistent arrays are subsetted explicitly when passed to schemes: -```fortran -windu_tend(col_start:col_end, 1:pver) -``` -Local temporaries allocated as `allocate(cape(col_start:col_end))` are already -correctly sized and passed as assumed-shape `(:)`. - ---- - -### 10.7 State machine - -The suite cap has a character module variable tracking lifecycle state: - -```fortran -character(len=16) :: ccpp_suite_state = 'uninitialized' -``` - -Transitions: `uninitialized` → register → `uninitialized` → initialize → `initialized` -→ timestep_initial → `in_time_step` → (run, no state change) → timestep_final → -`initialized` → finalize → `uninitialized`. - -Each phase entry point checks the expected prior state: -```fortran -if (trim(ccpp_suite_state) /= 'in_time_step') then - errflg = 1 - write(errmsg, '(3a)') "Invalid initial CCPP state, '", trim(ccpp_suite_state), & - "' in cam7_physics_before_coupler" - return -end if -``` - -Non-run phases also include an OpenMP thread guard: -```fortran -#ifdef _OPENMP - if (omp_get_thread_num() > 1) then - errflg = 1 - errmsg = "Cannot call initialize routine from a threaded region" - return - end if -#endif -``` - -The state machine is simple, complete, and useful. The redesign should preserve it. - ---- - -### 10.8 Constituent variable handling - -CAM-SIMA demonstrates the full constituent lifecycle: - -```fortran -! In cam_ccpp_cap.F90: -type(ccpp_model_constituents_t), target :: cam_constituents_obj - -! Registration (scheme-declared constituents): -call suite_cam7_constituents_num_consts(num_consts) -call suite_cam7_constituents_const_name(iconst, const_name) -call cam_constituents_obj%new_field(const_name, ...) - -! Initialization (host-declared constituents like water vapor): -cam_model_const_stdnames(1) = "water_vapor_mixing_ratio_wrt_moist_air_and_condensed_water" -call cam_constituents_obj%new_field(cam_model_const_stdnames(1), ...) - -! Per-timestep gather from host: -call cam_ccpp_gather_constituents(phys_state%q, ...) - -! Passing to suite cap: -call cam7_physics_before_coupler(..., - qv = cam_constituents_obj%vars_layer(:, :, cam_model_const_indices(1)), - carr = cam_constituents_obj%vars_layer, - cprops = cam_constituents_obj%const_metadata, ...) - -! Per-timestep scatter back to host: -call cam_ccpp_update_constituents(phys_state%q, ...) -``` - -The suite cap sees constituents as: -- `carr(:,:,:)` — the full rank-3 constituent array (ncol, nlev, ncnst) -- `qv(:,:)` — water vapor slice extracted in the host cap: `cam_constituents_obj%vars_layer(:,:,cam_model_const_indices(1))` -- `cprops(:)` — array of `ccpp_constituent_prop_ptr_t` metadata objects -- `doconvtran(1:ncnst)` — suite-level logical array set by scheme init indicating which constituents are convected - -This constituent API is sophisticated and worth preserving or improving in the redesign. - ---- - -### 10.9 Known defects in the capgen output - -**Repeated scheme init/final calls.** Capgen generates one init call per occurrence of -a scheme name in the XML, without deduplication: -- `qneg_init` called 5 times (once per `qneg` entry in the suite XML) -- `qneg_timestep_final` called 5 times -- `check_energy_chng_init` called twice -- `save_ttend_from_convect_deep_timestep_init` called 3 times - -If these routines have internal state, allocations, or side effects, this is a correctness -defect. The redesign must deduplicate init/final calls by unique scheme name. - -**Unit conversion embedded silently in the cap.** Before `zm_conv_convtran_run`: -```fortran -dpdry_local(:,1:pver) = 1.0E-2_kind_phys * dpdry(:,1:pver) ! Pa → hPa -``` -This is generated from the metadata units mismatch but appears as an opaque transform -in the cap. The redesign should make this visible (e.g., a comment naming the standard -name, the source units, and the target units). - ---- - -### 10.10 Build system — capgen invocation - -Capgen is invoked from Python (`cam_autogen.py`), not from cmake: - -```python -from ccpp_capgen import capgen -capgen_db = capgen(run_env, return_db=True) -``` - -This is a programmatic API call, not a subprocess. The `CCPPDatabaseObj` returned -(`capgen_db`) is then used directly in Python to query scheme lists, constituent names, -and file paths — avoiding the datatable XML query step that cmake-based invocations need. - -Output files consumed by the build: -- `cam_ccpp_cap.F90` — compiled into the atmosphere component -- `ccpp_cam7_cap.F90` — compiled into the atmosphere component -- `ccpp_kinds.F90` — compiled into the atmosphere component -- Utility files from `ccpp_framework/src/` (copied to build dir) -- `ccpp_datatable.xml` — queried by the build system for file lists - ---- - -### 10.11 Observations relevant to the redesign - -1. **The two-cap split (host cap + suite cap) is the right architecture.** It cleanly - separates host-specific binding from physics-neutral dispatch. The redesign must - preserve this. - -2. **Flat-field arguments in the suite cap are the critical failure.** 61–76 dummy - arguments per run subroutine is already large for a research model; for UFS/GFS - with 1,200+ variables it is completely infeasible. The redesign must pass DDTs. - -3. **The CAM-SIMA host does not use DDTs in metadata.** All host variables are flat - module variables. This is a fundamentally different host model architecture from - GFS/SCM. The redesign must support both styles: flat-module hosts (CAM-SIMA) and - deep-DDT hosts (GFS/SCM). - -4. **Non-metadataized variables hardwired into the host cap is a serious gap.** - `phys_state`, `phys_tend`, `cam_in` from `physics_types` have no `.meta` files. - The host cap accesses them directly. This means the framework cannot verify or - track these variables. The redesign should either require full metadata coverage - or have an explicit mechanism for declaring non-metadataized pass-through variables. - -5. **Suite-level persistent variables (framework-owned data) work well in practice.** - `windu_tend`, `scaling_dycore`, `temp_ini`, etc. are owned by the suite cap, invisible - to the host, and persist across group calls. This is the right pattern for - physics-internal state. The redesign needs this but with a simpler discovery mechanism - than capgen's scope-chain promotion. - -6. **Deduplicate init/final calls.** The redesign must deduplicate `_init`, `_finalize`, - `_timestep_init`, and `_timestep_final` calls by unique scheme name (not by occurrence - in the XML). - -7. **The constituent API in the host cap is comprehensive.** The `ccpp_model_constituents_t` - object with its register/init/gather/scatter/index API is sophisticated and should be - preserved or improved. - -8. **The suite-variables introspection subroutine** (`ccpp_physics_suite_variables`, - enumerating 83 standard names as inputs/outputs) is a useful capability for build - system integration and should be in the redesign. - -9. **The programmatic Python API** (`capgen(run_env, return_db=True)`) is valuable - for hosts like CAM-SIMA that invoke the generator from Python. The redesign should - support both CLI and programmatic invocation. - -10. **Unit conversions must be annotated in the generated cap**, not silently embedded - as magic-number multiplications. A comment with source units, target units, and the - standard name involved is the minimum. - -11. **The horizontal chunking model** (`col_start`/`col_end` as explicit arguments, - `ncol = col_end - col_start + 1` computed at entry) works and is clean. Suite-level - persistent arrays are allocated full-size and subsetted at call sites. - -12. **No optional variables in this model.** CAM-SIMA does not exercise optional/active - variable handling. This feature must be in the redesign but is not demonstrated here. - ---- - -## 11. Real-world example: UFS Weather Model (prebuild) - -The UFS Weather Model is the most complex and production-critical of the three examples. -It is a fully-coupled, 3-D operational NWP model. The CCPP physics is used in the -atmospheric component (`UFSATM`). Unlike the SCM (column model, process-split only) and -CAM-SIMA (capgen, flat-field arguments), UFS uses prebuild in a 3-D blocked/threaded -configuration that is architecturally distinct from both prior examples. - -The two suites analyzed here are: -- `FV3_GFS_v17_coupled_p8` — the primary operational GFS suite -- `FV3_GFS_v17_coupled_p8_ugwpv1` — a variant replacing `unified_ugwp` with `ugwpv1` - -The ugwpv1 suite is structurally identical to the base suite except for the `phys_ps` -group (4 extra scheme calls), so all observations below apply to both. - ---- - -### 11.1 Suite structure - -The primary suite has 5 groups: - -| Group | Subcycles | Scheme calls | Phase called | -|-------|-----------|-------------|--------------| -| `time_vary` | 1 | 4 | timestep_init (domain-level, no blocking) | -| `radiation` | 1 | 8 | run (block/thread loop) | -| `phys_ps` | 3 (loop=1, loop=2, loop=1) | 21 | run (block/thread loop) | -| `phys_ts` | 3 (loop=1, loop=1, loop=1) | 12 | run (block/thread loop) | -| `stochastics` | 1 | 2 | run (block/thread loop) | - -The `time_vary` group is the only one called at timestep_init/finalize. All other groups -are called from the run phase via the OpenMP blocked loop. This is a fundamentally -different usage pattern from SCM (which runs everything sequentially) and CAM-SIMA -(which has no run phase at all for the groups analyzed). - -The `phys_ps` group has a surface iteration subcycle with `loop="2"`, which generates an -actual Fortran `do` loop in the cap body: -```fortran -! Start of next subcycle -cdata%loop_max = 2 -do cdata%loop_cnt = 1, cdata%loop_max - ! ... sfc_diff, sfc_nst, noahmpdrv, sfc_land, sfc_cice, sfc_sice ... -end do -``` - ---- - -### 11.2 Cap hierarchy and scale - -The three-level hierarchy is preserved from prebuild: - -``` -ccpp_static_api.F90 (627 lines) ← suite+group name dispatch - ↓ -ccpp_fv3_gfs_v17_coupled_p8_cap.F90 (363 lines) ← calls all group caps in order - ↓ -ccpp_fv3_gfs_v17_coupled_p8_time_vary_cap.F90 (1404 lines) -ccpp_fv3_gfs_v17_coupled_p8_radiation_cap.F90 (967 lines) -ccpp_fv3_gfs_v17_coupled_p8_phys_ps_cap.F90 (4226 lines) ← 200 optional ptr arrays -ccpp_fv3_gfs_v17_coupled_p8_phys_ts_cap.F90 (1953 lines) -ccpp_fv3_gfs_v17_coupled_p8_stochastics_cap.F90 (443 lines) -``` - -The ugwpv1 variant generates another 10,220 lines of largely redundant code (identical -caps with one suite-name prefix change and minor scheme-list differences). Total for both -suites: 18,333 lines of generated Fortran. - -This redundancy is a key motivation for the redesign: suite variants that share groups -should not regenerate identical cap code. The redesign should support group-level cap -sharing across suite variants. - ---- - -### 11.3 Host model DDT structure - -All host data lives in `CCPP_data.F90` as module-level `save, target` variables: - -```fortran -type(GFS_control_type) :: GFS_control ! config/control -type(GFS_statein_type) :: GFS_statein ! atmospheric state in -type(GFS_stateout_type) :: GFS_stateout ! atmospheric state out -type(GFS_grid_type) :: GFS_grid ! grid geometry -type(GFS_tbd_type) :: GFS_tbd ! temporal interp data -type(GFS_cldprop_type) :: GFS_cldprop ! cloud properties -type(GFS_sfcprop_type) :: GFS_sfcprop ! surface properties -type(GFS_radtend_type) :: GFS_radtend ! radiation tendencies -type(GFS_coupling_type) :: GFS_coupling ! coupling fields -type(GFS_diag_type) :: GFS_intdiag ! diagnostics -type(GFS_interstitial_type), allocatable (:) :: GFS_interstitial ! scratch, per thread -``` - -Plus three `ccpp_t` instances for different levels of parallelism (see §11.5). - -This is structurally similar to the SCM's `physics` DDT hierarchy, but with one key -difference: all DDTs are at the same flat level rather than nested (no `physics%Statein`, -only `GFS_statein`). Each DDT maps to a distinct functional role. - -The `GFS_typedefs.F90` file (not auto-generated) defines all DDT types along with ~30 -physical constants (`con_pi`, `con_g`, `con_rd`, etc.) that also appear in the metadata. - ---- - -### 11.4 DDT arguments in the cap chain - -The static API imports all DDTs and physical constants from `CCPP_data` and `GFS_typedefs` -via `use` statements, then passes them as named arguments to group cap functions. This is -the full DDT-argument pattern that prebuild implements: - -```fortran -! In ccpp_static_api.F90: -use ccpp_data, only: gfs_control, gfs_statein, gfs_sfcprop, ... -use gfs_typedefs, only: con_pi, con_g, con_rd, ... - -ierr = fv3_gfs_v17_coupled_p8_phys_ps_run_cap( & - one=one, gfs_control=gfs_control, cdata=cdata, & - gfs_statein=gfs_statein, gfs_sfcprop=gfs_sfcprop, & - con_g=con_g, con_pi=con_pi, ... & - gfs_interstitial=gfs_interstitial) -``` - -The group cap receives these as typed `intent(*), target` dummy arguments and uses them -directly to construct call-site subsections. This means **the group cap is fully portable -— it does not use any host module directly**, only what it receives as arguments. - -The `target` attribute is required because the cap creates pointer sections of these DDTs -(array subsections via pointer assignment) when handling optional variables. - ---- - -### 11.5 The dual cdata architecture - -UFS uses two distinct sets of `ccpp_t` handles with different scopes: - -**Domain-level (`cdata_domain`)**: Used for non-run phases (init, finalize, time_vary -timestep_init/finalize). Called once per step, no blocking: -```fortran -cdata_domain%blk_no = 1; cdata_domain%chunk_no = 1 -cdata_domain%thrd_no = 1; cdata_domain%thrd_cnt = 1 -``` - -**Block/thread-level (`cdata_block(nb, nt)`)**: Used for run phase (radiation, phys_ps, -phys_ts, stochastics). Allocated as a 2-D array `(1:nblks, 1:nthrdsX)` where `nthrdsX` -accounts for non-uniform last-block sizing: -```fortran -cdata_block(nb,nt)%blk_no = nb -cdata_block(nb,nt)%chunk_no = nb ! block number = chunk number -cdata_block(nb,nt)%thrd_no = nt -cdata_block(nb,nt)%thrd_cnt = nthrdsX -``` - -The redesign must support this dual cdata usage: a single `cdata` handle for domain-level -phases and a 2-D array of handles for blocked run phases. - ---- - -### 11.6 OpenMP threading model - -Non-run phases allow internal threading in physics schemes: -```fortran -GFS_control%nthreads = nthrds ! all N threads available to physics -call ccpp_physics_timestep_init(cdata_domain, ...) -``` - -Run phase uses all threads for blocking, so physics must not spawn additional threads: -```fortran -GFS_control%nthreads = 1 ! no internal threading allowed -!$OMP parallel num_threads(nthrds) ... -!$OMP do schedule(dynamic,1) -do nb = 1, nblks - call GFS_Interstitial(nt)%create(ixs=chunk_begin(nb), ixe=chunk_end(nb), model=GFS_control) - call ccpp_physics_run(cdata_block(nb,nt), group_name="phys_ps", ...) - call GFS_Interstitial(nt)%destroy(GFS_control) -end do -!$OMP end do -!$OMP end parallel -``` - -The `nt = omp_get_thread_num()+1` pattern (1-based thread index) is used throughout. -Each thread owns one `GFS_Interstitial(nt)` and one `cdata_block(nb,nt)` per block -iteration. The dynamic schedule means different threads process different blocks at -different times, which is why the interstitial must be created/destroyed per-iteration -rather than pre-allocated per-thread. - ---- - -### 11.7 Horizontal dimension: the chunk_begin/chunk_end pattern - -For non-run phases, the full horizontal dimension is used at every call site: -```fortran -tgrs(one:gfs_control%ncols, one:gfs_control%levs) -``` - -For run phases, the chunk range is looked up from the control DDT using the block number: -```fortran -tgrs(gfs_control%chunk_begin(cdata%chunk_no) : gfs_control%chunk_end(cdata%chunk_no), & - one:gfs_control%levs) -``` - -The chunk size (horizontal extent `im`) is retrieved as: -```fortran -im = gfs_control%blksz(cdata%blk_no) -``` - -`blksz(nb)` handles **non-uniform block sizes**: the last block may be smaller than the -others if the domain size is not divisible by the number of blocks. The `chunk_begin`/ -`chunk_end` arrays (indexed by chunk number = block number) give the global offset range. - -This is a cleaner pattern than SCM's `chunk_begin`/`chunk_end` as explicit dummy -arguments, because UFS looks them up from the already-passed `gfs_control` DDT. - -**Critical implication for the redesign**: The subsetting pattern `(chunk_begin:chunk_end)` -appears at every single array call site in the run phase — literally hundreds of times in -the phys_ps cap alone. This boilerplate is generated by prebuild from the metadata. In -the redesign, this subsetting must remain at the call site (not higher up) to allow each -thread to process its own chunk independently. - ---- - -### 11.8 The GFS_interstitial — pointer-based scratch DDT - -`GFS_interstitial_type` (defined in `CCPP_typedefs.F90`) is a DDT where **every field is -a pointer**, initialized to null: -```fortran -type GFS_interstitial_type - real(kind_phys), pointer :: adjsfculw_land(:) => null() - real(kind_phys), pointer :: del(:,:) => null() - ! ... ~200+ pointer fields -end type -``` - -This is dramatically different from the SCM's interstitial (which is a regular allocatable -DDT allocated once per thread at startup). The UFS interstitial is: -1. **Created** (`GFS_Interstitial(nt)%create(ixs, ixe, model)`) before each block — this - allocates all required fields to the chunk size `ixe-ixs+1` -2. **Reset** (`GFS_Interstitial(nt)%reset(model)`) to zero before radiation and phys_ps -3. **Destroyed** (`GFS_Interstitial(nt)%destroy(model)`) after each block — deallocates - -This design exists because different blocks (especially the last block) can have different -sizes. Pre-allocating to the maximum size wastes memory at scale; per-block allocation -ensures exact sizing. The pointer-based design also allows the `create()` method to -selectively allocate only the fields needed for the current physics configuration. - -In the caps, the interstitial is accessed as: -```fortran -gfs_interstitial(cdata%thrd_no)%del(chunk_begin:chunk_end, one:levs) -``` - -The interstitial array is 1-D (indexed by thread, not by `(instance, thread)` as in SCM). -This works because UFS has only one model instance at runtime — no ensemble-in-memory. - ---- - -### 11.9 Optional variables — the pointer array pattern at scale - -The phys_ps run cap has **200 optional pointer arrays** in its local variable section. -Each looks like: -```fortran -type :: real_kind_phys_rank1_ptr_arr_type - real(kind_phys), dimension(:), pointer :: p => null() -end type real_kind_phys_rank1_ptr_arr_type -type(real_kind_phys_rank1_ptr_arr_type), dimension(1:cdata%thrd_cnt) :: sfc_wts_1_ptr_array -``` - -Usage pattern (consistent with SCM but with threading dimension): -```fortran -if (gfs_control%lndp_type /= 0) then - sfc_wts_1_ptr_array(cdata%thrd_no)%p => & - gfs_coupling%sfc_wts(chunk_begin:chunk_end, one:gfs_control%n_var_lndp) -end if -! ... scheme call ... -if (gfs_control%lndp_type /= 0) then - nullify(sfc_wts_1_ptr_array(cdata%thrd_no)%p) -end if -``` - -The array is dimensioned by `cdata%thrd_cnt` (total thread count) and indexed by -`cdata%thrd_no` (current thread number). This handles the threaded run phase where -multiple threads are simultaneously executing the same run cap function with different -chunk ranges. Each thread independently associates and nullifies its own pointer slot. - -200 optional variables in `phys_ps` alone. This is the regime for which the SCM had ~550 -total optional vars — confirming that operational 3-D GFS physics is heavily optional-var -driven. The design is sound but generates enormous boilerplate. - -A key observation: the type definition for each pointer wrapper (`integer_..._ptr_arr_type`, -`real_kind_phys_rank1_ptr_arr_type`, etc.) is **re-declared inside every single function -that needs it**. This results in duplicate type definitions across all group caps. The -redesign should define these wrapper types once in a shared module. - ---- - -### 11.10 Physical constants as metadata variables - -The UFS static API has an extensive USE list of physical constants from `gfs_typedefs`: -``` -con_pi, con_g, con_t0c, con_hfus, con_solr_2008, con_solr_2002, con_c, con_plnk, -con_boltz, con_rd, ltp, con_zero, con_rerth, con_p0, con_rv, con_cp, con_rgas, -con_amd, con_amw, con_avgd, con_hvap, con_eps, con_omega, con_fvirt, con_ttp, -con_thgni, con_epsm1, con_rog, con_rocp, con_tice, con_sbc, con_jcal, con_rhw0, -rlapse, rhowater, karman, con_1ovg, con_cliq, con_cvap, rainmin, con_epsm1 (30+ total) -``` - -These travel through the full chain: static API USE → suite cap argument → group cap -argument → scheme call argument. Each constant is declared as a separate scalar dummy -argument (`real(kind_phys), intent(in), target :: con_pi`) in every group cap that needs -it. - -This is correct but verbose. The redesign should consider whether constants should be -gathered into a dedicated DDT (e.g., `gfs_constants_type`) so the cap chain carries one -argument instead of 30. This would also eliminate the need to explicitly enumerate which -constants each group needs — they could all come along in the constants DDT. - ---- - -### 11.11 The `one` lower-bound anchor - -The integer constant `one = 1` (from `ccpp_types`) is passed as an explicit argument -throughout the UFS cap chain for the same reason as in SCM: it anchors lower array bounds -without triggering association-status issues: -```fortran -type(gfs_interstitial_type), intent(inout), target :: gfs_interstitial(one:) -tgrs(one:gfs_control%ncols, one:gfs_control%levs) -``` - -This pattern is ubiquitous and is a known prebuild idiom. - ---- - -### 11.12 No framework-owned persistent variables - -Unlike CAM-SIMA (which allocates scheme-persistent variables in the suite cap), the UFS -has no framework-owned persistent state in any cap. All persistent state lives in the host -DDTs (`GFS_tbd`, `GFS_sfcprop`, etc.). The interstitial DDT (`GFS_interstitial`) is purely -transient — created and destroyed each block. - -This is consistent with UFS's prebuild-based architecture. Whether framework-owned -persistent variables would be beneficial for UFS is an open question for the redesign. - ---- - -### 11.13 Build system and driver - -Prebuild is invoked from CMake (not programmatically) and generates: -- Group cap files (one per group × suites) -- Suite cap files (one per suite) -- `ccpp_static_api.F90` -- `CCPP_CAPS.cmake`, `CCPP_SCHEMES.cmake`, `CCPP_TYPEDEFS.cmake` — consumed by CMake to - enumerate files to compile - -The host driver (`CCPP_driver.F90`) is **hand-written**, not auto-generated. It owns the -OpenMP loop, the cdata allocation/setup, the interstitial create/destroy, and the -diagnostic bucket zeroing. This is a significant difference from CAM-SIMA where the -equivalent driver code is partially generated. In the redesign, this host driver code -should remain hand-written — it encodes model-specific threading and blocking decisions -that cannot be derived from metadata alone. - ---- - -### 11.14 Observations relevant to the redesign - -1. **The DDT-argument cap chain is fully validated at UFS scale.** Passing 10+ DDTs plus - 30+ scalar constants as named arguments through three cap levels works correctly in - production. The redesign must replicate this exactly. - -2. **The chunk_begin/chunk_end subsetting at call sites is non-negotiable.** Hundreds of - array sections per group cap. The generator must produce this from the metadata - `horizontal_dimension` standard name and the `active` flag for optional variables. - This is prebuild's core value at 3-D scale. - - *Design direction*: Rather than carrying `chunk_no` in cdata and having the cap look - up `gfs_control%chunk_begin(chunk_no)`, the redesign should pass - `horizontal_loop_begin` and `horizontal_loop_end` as explicit arguments directly to - `ccpp_physics_run()` (and analogous calls). This decouples the cap from knowing about - the host's internal chunk-lookup arrays. The host driver sets these for each block - iteration and passes them in; the cap uses them directly. - -3. **The domain-vs-block execution contexts must be supported, but the cdata object is - not necessarily the right mechanism.** The key information is: instance number, thread - number, horizontal_loop_begin, horizontal_loop_end, error flag/message. If all of - these are explicit named arguments to `ccpp_physics_*`, the cdata object becomes - redundant scaffolding. This is an open design question to be discussed separately, but - the UFS analysis shows that cdata carries exactly these values — the object is a - transport container, not a framework abstraction. - -4. **The `blksz` non-uniform block size is a first-class concern.** The generator must - produce `im = gfs_control%blksz(cdata%blk_no)` (or an equivalent `horizontal_loop_extent` - computed from the explicit begin/end) for the horizontal extent argument in run phases. - -5. **GFS_interstitial as a pointer-DDT is the correct design for 3-D models.** Creating - and destroying per block avoids memory waste from over-allocation to the maximum chunk - size. The pointer-based field design enables selective allocation. The redesign should - document this pattern and support it. (Whether the generator should emit the - `type(X_interstitial_type)` DDT definition itself or only the caps is TBD.) - -6. **200 optional pointer arrays in one group cap is manageable but the wrapper type - proliferation is not.** The 4 wrapper types (`integer_r1_ptr_arr_type`, - `real_r1_ptr_arr_type`, `real_r2_ptr_arr_type`, `character_len3_r1_ptr_arr_type`) - should be defined once in a shared module (e.g., `ccpp_types.F90`) and reused across - all caps, eliminating thousands of duplicate lines. - -7. **Physical constants as metadata variables must be gathered into a constants DDT.** - The redesign will collect all physics constants into a single `constants_type` DDT - (or equivalent), reducing 30+ individual scalar arguments in the cap chain to one - argument. This requires a metadata declaration mechanism for compound read-only - objects (i.e., constants do not need intent tracking the way state variables do). - -8. **No framework-owned persistent variables in UFS** confirms that this feature is - optional and model-specific. The redesign needs to support it (for CAM-SIMA-like - models) but should not force it on models that do not need it. - -9. **The host driver is correctly hand-written.** The OpenMP blocking, interstitial - lifecycle, diagnostic bucket management — these are model-specific decisions that - belong in the host driver, not in generated code. The redesign should not try to - generate the driver. - -10. **Suite variant cap redundancy is not a concern.** For research/development, multiple - suites are active simultaneously and generated code size doesn't matter. For - production, only one suite is compiled and used at a time. The redesign need not - prioritize eliminating redundant group cap code across suite variants. - ---- - -## 12. Real-world example: Navy NEPTUNE (prebuild, restricted) - -The NEPTUNE source code cannot be shared. The following is based on architectural -description provided by the lead developer. - -NEPTUNE uses `ccpp-prebuild` with the same GFS physics as UFS and nearly identical suites. -Its unique distinguishing feature is **multiple coexisting CCPP physics instances** — it -is the only model among the four examples that exercises this capability at runtime. - ---- - -### 12.1 Multiple instances — the N-dimensioned DDT array mechanism - -In NEPTUNE, the host model allocates N copies of all GFS DDTs as 1-D arrays indexed by -instance number: - -```fortran -type(GFS_sfcprop_type), allocatable :: gfs_sfcprop(1:N) -type(GFS_statein_type), allocatable :: gfs_statein(1:N) -type(GFS_stateout_type), allocatable :: gfs_stateout(1:N) -! ... all GFS DDTs dimensioned 1:N -type(GFS_control_type), allocatable :: gfs_control(1:N) -``` - -The static API imports these module-level arrays via `use` statements (same as UFS). -The instance selection happens at the call site inside the group cap, using -`cdata%ccpp_instance` as the array index: - -```fortran -call foo_run( & - tair = gfs_statein(cdata%ccpp_instance)%tair( & - gfs_control(cdata%ccpp_instance)%chunk_begin(cdata%chunk_no) : & - gfs_control(cdata%ccpp_instance)%chunk_end(cdata%chunk_no), & - 1:nvertical), & - ...) -``` - -Three things are happening simultaneously at each call-site array section: -1. **Instance selection**: `gfs_statein(cdata%ccpp_instance)` picks the correct DDT from - the N-element array -2. **Chunk subsetting**: `chunk_begin(chunk_no):chunk_end(chunk_no)` applies the run-phase - horizontal slice -3. **Vertical bound**: explicit `1:nvertical` - -This is the same pattern as UFS except the DDTs are 1-D arrays rather than scalars. -The generator must produce this instance-indexed subsetting when the host declares its -DDTs as arrays. - ---- - -### 12.2 What NEPTUNE tells us about `cdata%ccpp_instance` - -The `initialized(200)` array in every group cap (confirmed in both SCM and UFS caps) now -has its full motivation: it handles up to 200 simultaneous instances without requiring -per-instance cap code. The `cdata%ccpp_instance` value (1-based) is the runtime selector -into both the host DDT arrays and the `initialized` guard array. - -NEPTUNE is the reason `200` is not `1`. In single-instance models (UFS, SCM, CAM-SIMA) -`cdata%ccpp_instance` is always 1 and the N-dimensioned DDT arrays have `N=1`. - ---- - -### 12.3 Observations relevant to the redesign - -1. **Multiple instances require only one change at the call site**: inserting the instance - index at the correct dimension position. Everything else (chunking, optional variables, - threading) composes with this unchanged. - -2. **The instance dimension can appear anywhere in any host variable — not just as an - index into an array of DDTs.** A flat array `flat_field(1:ninstance, 1:nhoriz, 1:nvert)` - is equally valid; its call site becomes: - ```fortran - flat_field(instance_number, horiz_begin:horiz_end, 1:nvert) - ``` - The generator handles this by classifying each dimension by its declared standard name. - `instance_dimension` is a registered standard name (like `horizontal_dimension` and - `vertical_dimension`) — the generator knows its semantics regardless of where it - appears in the dimension list or whether the variable is a DDT array element or a - plain array. See §13.4 for the full dimension classification model. - -3. **No new cap-level mechanism is needed for multi-instance.** The instance number - (from the control layer, see §13) is sufficient. The cap code shape is the same; - only the call-site indexing expression differs based on the declared dimension roles. - ---- - -## 13. Cross-cutting design decision: how host data enters the cap chain - -Across all four models, two mechanisms are used for getting host model data into the -generated caps: - -| Mechanism | Models using it | Description | -|-----------|----------------|-------------| -| **Module USE** | UFS, SCM, CAM-SIMA, NEPTUNE | Static API has `use ccpp_data, only: gfs_statein, ...`. Data module name is known at generation time. | -| **Command-line arguments** | capgen (optional) | Generator accepts host variable access paths as CLI flags; generated caps receive data as explicit dummy arguments. | - -### 13.1 The capgen dual-mechanism problem - -Capgen supports both mechanisms, and this is a direct source of its complexity. The -variable-matching logic, VarDictionary scope chains, and `CCPPDatabaseObj` all exist -partly to handle the routing of variables that may arrive via either path. Maintaining -two entry points to the data layer doubles the surface area that must be tested and -reasoned about. - -### 13.2 The proposed single-mechanism approach - -The redesign will use **module USE exclusively** for all host data. The reasoning: - -- All four production models already use module USE, including CAM-SIMA (the capgen - model), which does not use capgen's CLI-argument path in practice. -- Module names are stable, known at generation time, and make the generated code - self-documenting (`use ccpp_data, only: gfs_statein` is unambiguous). -- Eliminating the CLI-argument entry path eliminates an entire class of generator - complexity. - -### 13.3 Runtime control variables — the thin explicit layer - -While all *data* enters via module USE, a set of *control* variables must be passed at -runtime because they change from call to call. These are not physics data; they tell the -cap *how* to index into the data it already has access to: - -| Variable | Purpose | When it matters | -|----------|---------|----------------| -| `ccpp_instance` | Select the instance dimension in host variables | NEPTUNE (N>1); others use 1 | -| `ccpp_thread_no` | Index optional pointer arrays per thread | Run phase with OpenMP | -| `horizontal_loop_begin` | Start of horizontal chunk to process | Run phase | -| `horizontal_loop_end` | End of horizontal chunk to process | Run phase | -| `ccpp_nthreads` | Max threads available for internal physics use | Non-run phases (currently `gfs_control%nthreads`) | -| `errmsg` / `errflg` | Error reporting return path | All phases | - -These are exactly the values that `cdata` carries in the current implementation. -Whether they are packaged as a `ccpp_t` struct or passed as individual named arguments to -`ccpp_physics_*` is an open design question for implementation. Either way, the generator -only needs to know about these variables and their standard names — it does not need to -accept host data paths on the command line. - -### 13.4 The dimension classification model - -A host variable's metadata declares the **standard name of each of its dimensions** in -order. The generator classifies every dimension into one of three categories and -constructs the call-site expression accordingly. - -**Category 1 — Registered dimensions.** The generator knows the semantics of these -standard names and generates special call-site expressions for them: - -| Standard name | Call-site expression | Notes | -|--------------|---------------------|-------| -| `instance_dimension` | `instance_number` (scalar index) | Omitted if variable has no instance dimension | -| `horizontal_dimension` | `1:horizontal_dimension` (non-run) or `horiz_begin:horiz_end` (run) | Phase-dependent | -| `vertical_dimension` | `1:vertical_dimension` | Fixed range | - -`instance_dimension` has the same registered status as `horizontal_dimension` and -`vertical_dimension`. Single-instance models simply do not declare any variables with -an `instance_dimension`, and the generator omits that index entirely. - -**Category 2 — Arbitrary host-declared dimensions.** Any dimension whose standard name -is not in the registered set. These are declared in host metadata pointing to a Fortran -expression accessible via module USE — either a flat module variable or a DDT member -(e.g. `gfs_control%ntrac`, `gfs_control%kice`). The generator emits `1:expression` -at the call site, resolved at generation time from the metadata. Fixed-index extractions -(e.g. `gfs_statein%qgrs(..., gfs_control%ntqv)`) are a special case: the dimension -value is a scalar index rather than a range upper bound, and the metadata must declare -which case applies. - -**Category 3 — Optional selector.** Not a dimension per se, but a boolean `active` -condition declared in variable metadata. Generates a pointer-association guard around -the call site (the pattern described in §9 and §11). - -This three-category model works uniformly regardless of host layout: -- `gfs_statein(instance)%tair(horiz, vert)` — registered instance + registered horizontal + registered vertical -- `flat_field(instance, horiz, vert, ntrac)` — registered + registered + registered + arbitrary -- `flat_field(horiz, vert)` — no instance dimension, single-instance model - -No special-casing per host model is needed in the generator. - -### 13.5 `type = control` — metadata declaration for runtime control variables - -The registered dimensions (§13.4 Category 1) are *dimension names* that appear in a -variable's `dimensions = (...)` list. Their actual *runtime values* are supplied by a -separate set of variables declared with `type = control` in host metadata. - -| `type = control` standard name | Fills in registered dimension / purpose | -|-------------------------------|----------------------------------------| -| `ccpp_instance` | `instance_dimension` — scalar index selecting the active instance | -| `ccpp_thread_no` | Not a dimension; indexes optional pointer arrays per thread | -| `horizontal_loop_begin` | Lower bound of `horizontal_dimension` in run phase | -| `horizontal_loop_end` | Upper bound of `horizontal_dimension` in run phase | -| `ccpp_nthreads` | Not a dimension; max threads available for internal physics use | -| `errmsg` / `errflg` | Error reporting return path | - -Variables declared `type = control` are: -- **Passed explicitly as runtime arguments** to `ccpp_physics_*` by the host driver - (not accessed via module USE, because their values change per call) -- **Used by the generator** to construct call-site indexing expressions for registered - dimensions, and to generate the `ccpp_nthreads` assignment before non-run scheme calls -- **Available to physics schemes** by standard name like any other variable — if a scheme - declares a variable with a matching standard name (e.g. `ccpp_nthreads`, - `horizontal_loop_begin`), the framework passes it as a scheme argument in the normal way - -This is similar in concept to capgen's `type = host` annotation but with a narrower, -well-defined scope. The name `control` is intentional: these variables *control* how -the cap indexes into the data, not what the data is. - -The set of recognized standard names for `type = control` variables is fixed and small. -Declaring them explicitly in metadata — rather than having the generator recognize magic -names — keeps the mechanism open and self-documenting. - -### 13.6 Consequences for the generator - -1. The generator reads host metadata to learn: - - Module names for all host data variables (emitted as `use` statements in the static API) - - The dimension standard names of each variable (for call-site expression construction) - - Which variables are `type = control` (for the runtime argument layer) -2. At cap generation time, the static API's `use` statements are emitted from the module - names — no runtime flexibility, no CLI data routing. -3. Call-site subsetting for every variable is constructed purely from its declared - dimension standard names: registered dimensions use the Category 1 rules; arbitrary - dimensions are resolved to Fortran expressions via the host metadata. -4. The only runtime inputs to the cap are the `type = control` variables. Their values - are supplied by the host driver for each `ccpp_physics_*` call. diff --git a/end-to-end-tests/advection/cld_liq.F90 b/end-to-end-tests/advection/cld_liq.F90 index c0d00a43..53cd68c1 100644 --- a/end-to-end-tests/advection/cld_liq.F90 +++ b/end-to-end-tests/advection/cld_liq.F90 @@ -30,7 +30,7 @@ subroutine cld_liq_register(dyn_const, errmsg, errcode) errmsg = 'Error allocating dyn_const in cld_liq_register' return end if - call dyn_const(1)%instantiate(std_name="dyn_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & + call dyn_const(1)%instantiate(std_name="DYN_const3_wrt_moist_air_and_condensed_water", long_name='dyn const3', & diag_name='DYNCONST3', units='kg kg-1', default_value=1._kind_phys, & vertical_dim='vertical_layer_dimension', advected=.true., & water_species=.true., mixing_ratio_type='dry', & diff --git a/end-to-end-tests/advection/test_host_data.F90 b/end-to-end-tests/advection/test_host_data.F90 index 4bcb753b..991a61ce 100644 --- a/end-to-end-tests/advection/test_host_data.F90 +++ b/end-to-end-tests/advection/test_host_data.F90 @@ -16,7 +16,7 @@ module test_host_data !! \htmlinclude arg_table_test_host_data.html integer, public, parameter :: num_consts = 3 character(len=32), public, parameter :: std_name_array(num_consts) = (/ & - 'specific_humidity ', & + 'SPECIFIC_HUMIDITY ', & 'cloud_liquid_dry_mixing_ratio', & 'cloud_ice_dry_mixing_ratio ' /) character(len=32), public, parameter :: const_std_name = std_name_array(1) diff --git a/unit-tests/test_auto_clone_constituents.py b/unit-tests/test_auto_clone_constituents.py index 5c378d2a..94b16fb3 100644 --- a/unit-tests/test_auto_clone_constituents.py +++ b/unit-tests/test_auto_clone_constituents.py @@ -592,6 +592,7 @@ def _resolved_arg(self, std_name): needs_kind_transform=False, unit_forward='', unit_backward='', kind_scheme='', kind_host='', + unit_scheme='', unit_host='', temp_name='', ptr_name='', transform_case=1, scheme_dimensions=[], diff --git a/unit-tests/test_ccpp_datafile.py b/unit-tests/test_ccpp_datafile.py index fe03ae98..0c1a44c1 100644 --- a/unit-tests/test_ccpp_datafile.py +++ b/unit-tests/test_ccpp_datafile.py @@ -1,6 +1,6 @@ """Tests for the ccpp_datafile query CLI. -Covers each of the 17 CLI flags end-to-end by: +Covers each of the 18 CLI flags end-to-end by: 1. building a real datatable.xml via the writer in generator.datatable, 2. invoking datatable_report / datatable_pretty_print on it, 3. asserting the textual output. @@ -12,6 +12,7 @@ import sys import tempfile import unittest +import xml.etree.ElementTree as ET _TESTS_DIR = os.path.dirname(os.path.abspath(__file__)) _CAPGEN_DIR = os.path.join(os.path.dirname(_TESTS_DIR), 'capgen') @@ -24,6 +25,7 @@ DatatableReport, datatable_pretty_print, datatable_report, + _retrieve_suite_variable_list, ) from generator.datatable import write_datatable @@ -32,7 +34,7 @@ _load_scheme_store, _parse_suite, ) -from generator.suite_resolver import resolve_suite +from generator.suite_resolver import resolve_suite, SuiteResolution, SuiteVar def _build_datatable(tmpdir, @@ -264,6 +266,70 @@ def test_unknown_suite_returns_empty(self): self.assertEqual(out, '') +class TestRetrieveSuiteVariableList(unittest.TestCase): + """_retrieve_suite_variable_list returns sorted suite-owned var names.""" + + def _table(self): + return ET.fromstring( + "" + "" + "" + "" + "" + "" + "" + "" + "") + + def test_returns_sorted_names(self): + self.assertEqual( + _retrieve_suite_variable_list(self._table(), 'fruit'), + ['alpha', 'beta']) + + def test_unknown_suite_returns_empty(self): + self.assertEqual( + _retrieve_suite_variable_list(self._table(), 'kumquat'), []) + + def test_non_suite_dict_not_matched(self): + # a dictionary of another type sharing the queried name is ignored + self.assertEqual( + _retrieve_suite_variable_list(self._table(), 'veg'), []) + + +class TestDatatableReportSuiteVariables(unittest.TestCase): + """--suite-variables end-to-end via datatable_report.""" + + @classmethod + def setUpClass(cls): + cls._tmpdir = tempfile.mkdtemp() + sv = SuiteVar(standard_name='promoted_x', local_name='px', + type_='real', kind='kind_phys', units='K', + dimensions=['horizontal_dimension'], + source_scheme='sch_a', source_phase='run') + sr = SuiteResolution(suite_name='test_simple', groups=[], + suite_vars={'promoted_x': sv}) + cls._datatable = write_datatable( + [sr], _load_scheme_store(), + ['/out/ccpp_kinds.F90'], ['/out/ccpp_test_simple_cap.F90'], + cls._tmpdir, host_dict=_load_full_host_dict()) + + @classmethod + def tearDownClass(cls): + shutil.rmtree(cls._tmpdir) + + def test_lists_the_interstitial(self): + out = datatable_report( + self._datatable, + DatatableReport('suite_variables', 'test_simple'), ',') + self.assertEqual(out, 'promoted_x') + + def test_unknown_suite_returns_empty(self): + out = datatable_report( + self._datatable, + DatatableReport('suite_variables', 'no_such_suite'), ',') + self.assertEqual(out, '') + + class TestExcludeProtected(unittest.TestCase): def setUp(self): diff --git a/unit-tests/test_datatable.py b/unit-tests/test_datatable.py index 5d70d7c8..e2b7f64e 100644 --- a/unit-tests/test_datatable.py +++ b/unit-tests/test_datatable.py @@ -6,7 +6,7 @@ import unittest import xml.etree.ElementTree as ET -from generator.suite_resolver import resolve_suite +from generator.suite_resolver import resolve_suite, SuiteResolution, SuiteVar import generator.datatable as dt_mod from generator.datatable import write_datatable from test_suite_resolver import ( @@ -556,6 +556,58 @@ def test_group_call_list_has_vars(self): # but every var must have a name +class TestSuiteOwnedVarDictionary(unittest.TestCase): + """The per-suite is populated from + SuiteResolution.suite_vars (promoted interstitials).""" + + def _suite_dict(self, tmpdir, suite_vars): + sr = SuiteResolution(suite_name='test_simple', groups=[], + suite_vars=suite_vars) + path = write_datatable( + [sr], _load_scheme_store(), + ['/out/ccpp_kinds.F90'], ['/out/ccpp_test_simple_cap.F90'], + tmpdir, host_dict=_load_full_host_dict(), + ) + root = ET.parse(path).getroot() + return next(vd for vd in root.find('var_dictionaries') + .findall('var_dictionary') + if vd.get('type') == 'suite') + + def test_suite_vars_recorded_with_attributes(self): + sv = SuiteVar(standard_name='promoted_interstitial', local_name='pi_l', + type_='real', kind='kind_phys', units='K', + dimensions=['horizontal_dimension'], + source_scheme='scheme_a', source_phase='run') + with tempfile.TemporaryDirectory() as d: + suite_d = self._suite_dict(d, {'promoted_interstitial': sv}) + vars_ = suite_d.find('variables').findall('var') + v = next(v for v in vars_ + if v.get('name') == 'promoted_interstitial') + self.assertEqual(v.get('local_name'), 'pi_l') + self.assertEqual(v.get('units'), 'K') + self.assertEqual(v.get('type'), 'real') + self.assertEqual(v.get('kind'), 'kind_phys') + self.assertEqual(v.get('dimensions'), 'horizontal_dimension') + self.assertEqual(v.get('source_scheme'), 'scheme_a') + self.assertEqual(v.get('source_phase'), 'run') + + def test_suite_vars_sorted_by_standard_name(self): + svs = {n: SuiteVar(standard_name=n, local_name=n + '_l', type_='real', + kind='', units='1', dimensions=[], + source_scheme='s', source_phase='run') + for n in ('zeta', 'alpha', 'mu')} + with tempfile.TemporaryDirectory() as d: + suite_d = self._suite_dict(d, svs) + names = [v.get('name') + for v in suite_d.find('variables').findall('var')] + self.assertEqual(names, ['alpha', 'mu', 'zeta']) + + def test_empty_when_no_suite_vars(self): + with tempfile.TemporaryDirectory() as d: + suite_d = self._suite_dict(d, {}) + self.assertEqual(len(suite_d.find('variables').findall('var')), 0) + + class TestVarDictionariesProtectedAttr(unittest.TestCase): """Protected host vars carry protected='True'; others omit the attr.""" diff --git a/unit-tests/test_host_constituents.py b/unit-tests/test_host_constituents.py index 0511c692..5f9fc867 100644 --- a/unit-tests/test_host_constituents.py +++ b/unit-tests/test_host_constituents.py @@ -628,10 +628,11 @@ def test_update_constituents(self): def test_const_get_index(self): # Keyword args ensure unambiguous mapping to the DDT's signature - # (index, standard_name, errcode, errmsg). + # (index, standard_name, errcode, errmsg). The query name is passed + # through to_lower() so constituent lookups are case-insensitive. self.assertIn( 'call ccpp_model_constituents_obj(inst_num)%const_index(' - 'standard_name=stdname, index=const_index, ' + 'standard_name=to_lower(stdname), index=const_index, ' 'errcode=errcode, errmsg=errmsg)', self.text, ) diff --git a/unit-tests/test_suite_resolver.py b/unit-tests/test_suite_resolver.py index c939fc3e..44e38b03 100644 --- a/unit-tests/test_suite_resolver.py +++ b/unit-tests/test_suite_resolver.py @@ -52,6 +52,8 @@ _generate_group_cap, _collect_kinds_used, _transform_comment, + _log_one_transform, + _emit_one_call, write_group_cap, ) @@ -2644,9 +2646,21 @@ def test_no_temp_name_skipped(self): class TestTransformComment(unittest.TestCase): - """The trailing inline comment must list every active transform, but - must suppress "unit conversion" when the rendered formula is the - identity (formula ``'{var}'`` for dimensionally-equivalent units). + """The trailing inline comment must list every active transform with the + correct label and payload: + + * a **unit conversion** reports the actual *units* (host → scheme), never + the Fortran kinds; + * a **type conversion** (kind change) is reported separately, using the + kinds; + * when both apply, both are listed (type first, then unit); + * an identity unit conversion (formula ``'{var}'`` for dimensionally- + equivalent spellings such as ``J kg-1`` ↔ ``m2 s-2``) is suppressed. + + The exact-string assertions below are deliberate: the earlier bug printed + the *kinds* under a "unit conversion" label (e.g. ``kind_phys to + kind_phys``) and the old substring-only checks (``assertIn('unit + conversion', ...)``) could not see it. """ def _arg(self, **kwargs): @@ -2661,6 +2675,8 @@ def _arg(self, **kwargs): a.temp_name = kwargs.get('temp_name', '') a.kind_host = kwargs.get('kind_host', '') a.kind_scheme = kwargs.get('kind_scheme', '') + a.unit_host = kwargs.get('unit_host', '') + a.unit_scheme = kwargs.get('unit_scheme', '') return a def test_no_transforms_returns_empty(self): @@ -2673,6 +2689,7 @@ def test_identity_forward_suppressed(self): unit_forward='gt0(lb:ub, 1:nlev)', call_expr='gt0(lb:ub, 1:nlev)', kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m2 s-2', unit_scheme='J kg-1', ) self.assertEqual(_transform_comment(a, reverse=False), '') @@ -2683,28 +2700,93 @@ def test_identity_backward_suppressed(self): unit_backward='foo_l', temp_name='foo_l', kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m2 s-2', unit_scheme='J kg-1', ) self.assertEqual(_transform_comment(a, reverse=True), '') - def test_non_identity_forward_emitted(self): - """Forward formula scales the call_expr → comment lists the - unit conversion.""" + def test_unit_conversion_forward_reports_units(self): + """Forward unit conversion (same kind) → the comment lists the + host→scheme *units*, and must NOT mention kinds.""" a = self._arg( needs_unit_transform=True, unit_forward='1.0E-3_kind_phys*gt0(lb:ub)', call_expr='gt0(lb:ub)', kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m', unit_scheme='km', ) - self.assertIn('unit conversion', _transform_comment(a, reverse=False)) + self.assertEqual(_transform_comment(a, reverse=False), + '! unit conversion: m to km') - def test_non_identity_backward_emitted(self): + def test_unit_conversion_backward_reports_reversed_units(self): + """Backward unit conversion → scheme→host units (reversed).""" a = self._arg( needs_unit_transform=True, unit_backward='1.0E+3_kind_phys*foo_l', temp_name='foo_l', kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m', unit_scheme='km', + ) + self.assertEqual(_transform_comment(a, reverse=True), + '! unit conversion: km to m') + + def test_unit_conversion_never_reports_kinds(self): + """Regression guard for the original bug: a pure unit conversion with + equal kinds must never emit ``kind_phys to kind_phys`` (or any kind) + under the "unit conversion" label.""" + a = self._arg( + needs_unit_transform=True, + unit_forward='1.0E-2_kind_phys*p(lb:ub)', + call_expr='p(lb:ub)', + kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='Pa', unit_scheme='hPa', + ) + comment = _transform_comment(a, reverse=False) + self.assertEqual(comment, '! unit conversion: Pa to hPa') + self.assertNotIn('kind_phys', comment) + self.assertNotIn('type conversion', comment) + + def test_type_conversion_forward_reports_kinds(self): + """Pure kind change (same units) → a *type* conversion listing the + host→scheme kinds, with no "unit conversion".""" + a = self._arg( + needs_kind_transform=True, + unit_forward='real(con_pi, kind=kind_phys)', + call_expr='con_pi', + kind_host='kind_dyn', kind_scheme='kind_phys', + unit_host='1', unit_scheme='1', + ) + comment = _transform_comment(a, reverse=False) + self.assertEqual(comment, '! type conversion: kind_dyn to kind_phys') + self.assertNotIn('unit conversion', comment) + + def test_type_conversion_backward_reports_reversed_kinds(self): + """Backward pure kind change → scheme→host kinds (reversed).""" + a = self._arg( + needs_kind_transform=True, + unit_backward='real(foo_l, kind=kind_dyn)', + temp_name='foo_l', + kind_host='kind_dyn', kind_scheme='kind_phys', + unit_host='1', unit_scheme='1', + ) + comment = _transform_comment(a, reverse=True) + self.assertEqual(comment, '! type conversion: kind_phys to kind_dyn') + self.assertNotIn('unit conversion', comment) + + def test_type_and_unit_both_listed(self): + """When kind AND units both differ, both conversions are listed, + type first then unit.""" + a = self._arg( + needs_unit_transform=True, + needs_kind_transform=True, + unit_forward='1.0E-3_kind_phys*real(gt0(lb:ub), kind=kind_phys)', + call_expr='gt0(lb:ub)', + kind_host='kind_dyn', kind_scheme='kind_phys', + unit_host='m', unit_scheme='km', + ) + self.assertEqual( + _transform_comment(a, reverse=False), + '! type conversion: kind_dyn to kind_phys; unit conversion: m to km', ) - self.assertIn('unit conversion', _transform_comment(a, reverse=True)) def test_vert_flip_alone_emits_flip_only(self): """A pure vertical flip (identity unit conversion, no kind change) @@ -2725,10 +2807,125 @@ def test_unit_and_flip_both_listed(self): unit_forward='1.0E-3_kind_phys*gt0(lb:ub, nlev:1:-1)', call_expr='gt0(lb:ub, nlev:1:-1)', kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m', unit_scheme='km', ) - comment = _transform_comment(a, reverse=False) - self.assertIn('unit conversion', comment) - self.assertIn('vertical flip', comment) + self.assertEqual( + _transform_comment(a, reverse=False), + '! unit conversion: m to km; vertical flip (top_at_one mismatch)', + ) + + +class TestTransformLogging(unittest.TestCase): + """capgen logs each value-changing transform to stdout/stderr (INFO), + with text identical to the inline cap comment (`_transform_comment`).""" + + def _arg(self, **kw): + from unittest.mock import MagicMock + a = MagicMock() + a.needs_unit_transform = kw.get('needs_unit_transform', False) + a.needs_kind_transform = kw.get('needs_kind_transform', False) + a.needs_vert_flip = kw.get('needs_vert_flip', False) + a.unit_forward = kw.get('unit_forward', '') + a.unit_backward = kw.get('unit_backward', '') + a.call_expr = kw.get('call_expr', '') + a.temp_name = kw.get('temp_name', '') + a.kind_host = kw.get('kind_host', '') + a.kind_scheme = kw.get('kind_scheme', '') + a.unit_host = kw.get('unit_host', '') + a.unit_scheme = kw.get('unit_scheme', '') + a.scheme_local_name = kw.get('scheme_local_name', 'x') + a.standard_name = kw.get('standard_name', 'some_std_name') + return a + + def _msg(self, logger): + # transforms are logged at WARNING (temporary — see group_cap comment) + call = logger.warning.call_args + return call.args[0] % call.args[1:] + + def _resolved_transform_arg(self): + """Build a REAL transformed ResolvedArg via the resolver (or skip).""" + from generator.suite_resolver import (_resolve_one_arg, + find_unit_conversion) + from metadata.metadata_table import MetaVar + hd = _load_full_host_dict() + for std, e in hd.items(): + if getattr(e, 'type', '') != 'real': + continue + for tgt in ('hPa', 'km', 'cm', 'mm', 'min', 'h', 'Pa', 'm'): + if tgt != e.units and find_unit_conversion(e.units, tgt): + dims = '()' if not e.dimensions else \ + '(' + ','.join(e.dimensions) + ')' + v = MetaVar('x', _ctx()) + v.set_attr('standard_name', std, _ctx()) + v.set_attr('units', tgt, _ctx()) + v.set_attr('dimensions', dims, _ctx()) + v.set_attr('type', 'real', _ctx()) + v.set_attr('intent', 'in', _ctx()) + if e.kind: + v.set_attr('kind', e.kind, _ctx()) + arg = _resolve_one_arg(v, 'run', hd, {}, 'demo', set()) + if arg.unit_forward: + return arg, std + self.skipTest("no convertible host var available in the fixture") + + def test_forward_unit_conversion_logged_pre_call(self): + from unittest.mock import MagicMock + a = self._arg(needs_unit_transform=True, + unit_forward='1.0E-2_kind_phys*p', call_expr='p', + kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='Pa', unit_scheme='hPa', + scheme_local_name='p', standard_name='air_pressure') + log = MagicMock() + _log_one_transform(log, 'sui', 'grp', 'run', 'radiation', a, + reverse=False) + log.warning.assert_called_once() + msg = self._msg(log) + self.assertIn('unit conversion: Pa to hPa', msg) + self.assertIn('pre-call', msg) + self.assertIn('air_pressure', msg) + self.assertIn('radiation', msg) + # the logged text is exactly the inline cap comment (minus '! ') + self.assertIn(_transform_comment(a)[1:].strip(), msg) + + def test_backward_is_post_call_and_direction_reversed(self): + from unittest.mock import MagicMock + a = self._arg(needs_unit_transform=True, + unit_backward='1.0E+2_kind_phys*p_l', temp_name='p_l', + kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='Pa', unit_scheme='hPa') + log = MagicMock() + _log_one_transform(log, 'sui', 'grp', 'run', 'radiation', a, + reverse=True) + msg = self._msg(log) + self.assertIn('unit conversion: hPa to Pa', msg) + self.assertIn('post-call', msg) + + def test_identity_transform_logs_nothing(self): + from unittest.mock import MagicMock + a = self._arg(needs_unit_transform=True, + unit_forward='gt0(lb:ub)', call_expr='gt0(lb:ub)', + kind_host='kind_phys', kind_scheme='kind_phys', + unit_host='m2 s-2', unit_scheme='J kg-1') + log = MagicMock() + _log_one_transform(log, 'sui', 'grp', 'run', 'sch', a, reverse=False) + log.warning.assert_not_called() + + def test_emit_one_call_logs_transform(self): + # End-to-end through the real emitter: a resolved transform is logged. + from unittest.mock import MagicMock + arg, std = self._resolved_transform_arg() + rc = ResolvedCall(scheme_name='demo', phase='run', args=[arg]) + log = MagicMock() + _emit_one_call(rc, ' ', [], phase='run', logger=log, + suite_name='sui', group_name='grp') + log.warning.assert_called() + self.assertIn('unit conversion', self._msg(log)) + self.assertIn(std, self._msg(log)) + + def test_emit_one_call_none_logger_is_noop(self): + arg, _ = self._resolved_transform_arg() + rc = ResolvedCall(scheme_name='demo', phase='run', args=[arg]) + _emit_one_call(rc, ' ', [], phase='run', logger=None) # no raise class TestFortranTypeStr(unittest.TestCase): diff --git a/unit-tests/test_suite_xml.py b/unit-tests/test_suite_xml.py index 11d94bac..987dae7f 100644 --- a/unit-tests/test_suite_xml.py +++ b/unit-tests/test_suite_xml.py @@ -651,7 +651,7 @@ def test_good_v2_suite_validates(self): _, root = read_xml_file(_sample('suite_good_v2_test01.xml'), self._log) version = find_schema_version(root) result = validate_xml_file( - _sample('suite_good_v2_test01.xml'), 'suite', version, self._log, + _sample('suite_good_v2_test01.xml'), version, self._log, schema_path=_SCHEMA_DIR ) self.assertTrue(result) @@ -662,7 +662,7 @@ def test_bad_suite_tag_rejected(self): version = find_schema_version(root) try: result = validate_xml_file( - _sample('suite_bad_v2_suite_tag.xml'), 'suite', version, + _sample('suite_bad_v2_suite_tag.xml'), version, self._log, schema_path=_SCHEMA_DIR ) # Some xmllint versions return True even on error @@ -677,7 +677,7 @@ def test_invalid_fortran_id_scheme_rejected(self): version = find_schema_version(root) with self.assertRaises(CCPPError) as cm: validate_xml_file( - _sample('suite_invalid_scheme_fortran_id.xml'), 'suite', + _sample('suite_invalid_scheme_fortran_id.xml'), version, self._log, schema_path=_SCHEMA_DIR ) self.assertIn("scheme-1", str(cm.exception)) @@ -689,7 +689,7 @@ def test_invalid_fortran_id_group_rejected(self): version = find_schema_version(root) with self.assertRaises(CCPPError) as cm: validate_xml_file( - _sample('suite_invalid_group_fortran_id.xml'), 'suite', + _sample('suite_invalid_group_fortran_id.xml'), version, self._log, schema_path=_SCHEMA_DIR ) self.assertIn("group-1", str(cm.exception)) @@ -701,7 +701,7 @@ def test_invalid_fortran_id_suite_rejected(self): version = find_schema_version(root) with self.assertRaises(CCPPError) as cm: validate_xml_file( - _sample('suite_invalid_suite_fortran_id.xml'), 'suite', + _sample('suite_invalid_suite_fortran_id.xml'), version, self._log, schema_path=_SCHEMA_DIR ) self.assertIn("ver-test-suite", str(cm.exception)) @@ -712,7 +712,7 @@ def test_duplicate_group_name_rejected_after_expansion(self): version = find_schema_version(root) # Initial file validates OK result = validate_xml_file( - _sample('suite_bad_v2_duplicate_group.xml'), 'suite', version, + _sample('suite_bad_v2_duplicate_group.xml'), version, self._log, schema_path=_SCHEMA_DIR ) self.assertTrue(result) @@ -721,7 +721,7 @@ def test_duplicate_group_name_rejected_after_expansion(self): expanded_path = os.path.join(self._tmp, 'dup_group_expanded.xml') write_xml_file(root, expanded_path, self._log) with self.assertRaises(CCPPError) as cm: - validate_xml_file(expanded_path, 'suite', version, self._log, + validate_xml_file(expanded_path, version, self._log, schema_path=_SCHEMA_DIR) self.assertIn('group1', str(cm.exception))