Feature/rocoto

docs/user_guide/drivers.rst

@@ -77,3 +77,27 @@ Since ``uwtools`` driver tasks are idempotent, now that ``runscript.inference``
 by subsequent driver invocations. It could now be manually edited, for example to add debugging statements, and 
 the ``run`` task then invoked to execute inference with the debugging statements in place. If ``runscript.inference`` 
 were manually deleted and the driver invoked again, the runscript would be recreated with its default contents.
+
+.. _ExecutingWithoutBatchSubmission:
+
+Executing Without Batch Submission
+------------------------------------------------------------------------------
+
+Some targets defined in ``src/Makefile`` invoke their drivers by default with the ``--batch`` flag set, which directs ``uwtools`` to provision a runscript that includes batch-system directives (e.g. ``#SBATCH`` directives for Slurm) and to submit that script to the batch system for execution. Alternatively, the optional ``batch=`` clause may be used to disable (or to explicitly enable) batch submission. YAML true and false `boolean values <https://yaml.org/type/bool.html>`_ are supported; true values enable batch submission and false values disable it.
+
+For example, the following ``make`` invocations would all execute inference with batch submission enabled:
+
+.. code-block:: text
+
+    $ make inference config=eagle.yaml
+    $ make inference config=eagle.yaml batch=true
+    $ make inference config=eagle.yaml batch=yes
+
+These invocations would execute inference without batch submission, i.e. on the current local machine:
+
+.. code-block:: text
+
+    $ make inference config=eagle.yaml batch=false
+    $ make inference config=eagle.yaml batch=no
+
+Invocation without batch submission can be useful when running on a system with hardware appropriate to the driver given its configuration, for example in a batch job created by a Rocoto workflow where it would be unnecessary for EAGLE to submit a new batch job from within an existing one.

src/Makefile

@@ -1,24 +1,24 @@
-BASHSRCS = $(wildcard ../.github/scripts/*) format setup
-HELPERS  = vx zarr
-MODULES  = data inference training visualization wxvx
-PACKAGE  = eagle
-STEPS    = data grids-and-meshes inference training vis-grid-global vis-grid-lam vis-obs-global vis-obs-lam vx-grid-global vx-grid-lam vx-obs-global vx-obs-lam zarr-gfs zarr-hrrr
-TOOLING  = config devenv env format lint realize shellcheck test typecheck unittest validate yamllint
-VERBOSE  = $(if $(filter undefined,$(origin DEBUG)),, --verbose)
-
 INSTALLDIR ?= conda
+
+ACTIVATE    = source $(INSTALLDIR)/etc/profile.d/conda.sh && conda activate
+BASHSRCS    = $(wildcard ../.github/scripts/*) format setup
+ENVS        = data inference training visualization wxvx
+HELPERS     = vx zarr
+PACKAGE     = eagle
+STEPS       = data grids-and-meshes inference training vis-grid-global vis-grid-lam vis-obs-global vis-obs-lam vx-grid-global vx-grid-lam vx-obs-global vx-obs-lam zarr-gfs zarr-hrrr
+TOOLING     = config devenv env format lint realize shellcheck test typecheck unittest validate workflow yamllint
+VERBOSE     = $(if $(filter undefined,$(origin DEBUG)),, --verbose)
+
+BATCH      := $(if $(filter True,$(shell $(ACTIVATE) base; python -c "import yaml; print(yaml.safe_load('$(batch)' or 'yes'))")), --batch)
 SHELL      := $(shell which bash)
 NOW        := $(shell date -uIseconds)
 
-export NOW
-
-activate = @source $(INSTALLDIR)/etc/profile.d/conda.sh && conda activate $(1)
-check    = @$(if $(1),,$(error $(2)= argument required))
-exec     = set -x && uw execute$(VERBOSE) --config-file $(config) --module $(PACKAGE)/$(1)/$(2).py --classname $(3) --task $(4)
-make     = $(MAKE) --no-print-directory
-modenv   = $(env4mod_$(1))
-modloop  = @set -e && for mod in $(PACKAGE) $(MODULES); do $(make) mod=$$mod $(1); done
-tasklist = @(set -x && uw execute$(VERBOSE) --module $(PACKAGE)/$(1)/$(2).py --classname $(3)) || true
+activate    = @$(ACTIVATE) $(1)
+check       = @$(if $(1),,$(error $(2)= argument required))
+envloop     = @set -e && for env in $(PACKAGE) $(ENVS); do $(make) env=$$env $(1); done
+exec        = set -x && uw execute$(VERBOSE) --config-file $(config) --module $(PACKAGE)/$(1)/$(2).py --classname $(3) --task $(4)
+make        = $(MAKE) --no-print-directory
+tasklist    = @(set -x && uw execute$(VERBOSE) --module $(PACKAGE)/$(1)/$(2).py --classname $(3)) || true
 
 env4mod_data          = data
 env4mod_$(PACKAGE)    = base
@@ -30,6 +30,8 @@ env4mod_wxvx          = wxvx
 .ONESHELL:
 .PHONY: $(HELPERS) $(STEPS) $(TOOLING)
 
+export NOW
+
 all:
 	@echo Available targets:
 	@echo
@@ -74,7 +76,7 @@ ifeq ($(task),?)
 	$(call tasklist,inference,inference,Inference)
 else
 	$(call check,$(config),config)
-	$(call exec,inference,inference,Inference,$(or $(task),run) --batch)
+	$(call exec,inference,inference,Inference,$(or $(task),run)$(BATCH))
 endif
 
 lint:
@@ -104,7 +106,7 @@ ifeq ($(task),?)
 	$(call tasklist,training,training,Training)
 else
 	$(call check,$(config),config)
-	$(call exec,training,training,Training,$(or $(task),run) --batch)
+	$(call exec,training,training,Training,$(or $(task),run)$(BATCH))
 endif
 
 typecheck:
@@ -161,7 +163,7 @@ else
 	$(call check,$(config),config)
 	$(call check,$(extent),extent)
 	$(call check,$(truth),truth)
-	$(call exec,wxvx,wxvx,WXVX,$(or $(task),run) --key-path vx.grid2$(truth).$(extent) --batch)
+	$(call exec,wxvx,wxvx,WXVX,$(or $(task),run) --key-path vx.grid2$(truth).$(extent)$(BATCH))
 endif
 
 vx-grid-global:
@@ -176,6 +178,12 @@ vx-obs-global:
 vx-obs-lam:
 	@$(make) vx truth=obs extent=lam
 
+workflow:
+	$(call check,$(config),config)
+	$(call activate,base)
+	@(set -x && uw rocoto realize -c $(config) -o eagle.xml)
+	@(set -x && uw rocoto iterate -w eagle.xml -d eagle.db --cycle 2022-02-03T12 --task vis_obs_lam)
+
 yamllint:
 	$(call activate,base)
 	@echo "=> Linting YAML configs"
@@ -188,7 +196,7 @@ ifeq ($(task),?)
 else
 	$(call check,$(config),config)
 	$(call check,$(source),source)
-	$(call exec,data,zarr,Zarr,$(or $(task),run) --key-path zarrs.$(source) --batch)
+	$(call exec,data,zarr,Zarr,$(or $(task),run) --key-path zarrs.$(source)$(BATCH))
 endif
 
 zarr-gfs:

src/config/quickstart.yaml

@@ -0,0 +1,85 @@
+submit_common: &submit_common
+  attrs:
+    maxtries: 3
+  nodes: '{{ app.quickstart.nodes }}'
+  walltime: '{{ app.quickstart.walltime }}'
+  account: '{{ app.platform.account }}'
+  partition: '{{ app.partitions.cpu }}'
+  queue: '{{ app.queues.cpu }}'
+  command: 'make {{ jobname }} config=eagle.yaml batch=false'
+  join: '{{ app.base }}/{{ app.platform.scheduler }}/{{ jobname }}.{{ app.platform.scheduler }}'
+submit_netaccess: &submit_netaccess
+  <<: *submit_common
+  partition: '{{ app.partitions.netaccess }}'
+submit_gpu: &submit_gpu
+  <<: *submit_common
+  partition: '{{ app.gpu.batchargs.partition }}'
+  queue: '{{ app.gpu.batchargs.queue }}'
+  native: '{{ app.quickstart.gpu.native }}'
+workflow:
+  attrs:
+    realtime: false
+    scheduler: '{{ app.platform.scheduler }}'
+  cycledef: 
+    - attrs: 
+        group: cyc
+      spec: '{{ app.time.inference_start.strftime("%Y%m%d%H%M") }} {{ app.time.inference_start.strftime("%Y%m%d%H%M") }} 06:00:00'
+  log:
+    value:
+      cyclestr:
+        value: '{{ app.base }}/{{ app.platform.scheduler }}/rocoto.@Y@m@d@H.{{ app.platform.scheduler }}'
+  tasks:
+    task_grids_and_meshes:
+      <<: *submit_netaccess
+      jobname: grids-and-meshes
+    task_zarr_gfs:
+      <<: *submit_netaccess
+      jobname: zarr-gfs
+      native: '{{ app.quickstart.zarr.native }}'
+      dependency:
+        taskdep_grids_and_meshes:
+          attrs:
+            task: grids_and_meshes
+    task_zarr_hrrr:
+      <<: *submit_netaccess
+      jobname: zarr-hrrr
+      native: '{{ app.quickstart.zarr.native }}'
+      dependency:
+        taskdep_grids_and_meshes:
+          attrs: 
+            task: grids_and_meshes
+    task_training:
+      <<: *submit_gpu
+      jobname: training
+      dependency:
+        and:
+          datadep_zarr_gfs:
+            value: 'run/{{ app.experiment_name }}/data/runscript.zarr-gfs.done'
+          datadep_zarr_hrrr:
+            value: 'run/{{ app.experiment_name }}/data/runscript.zarr-hrrr.done'
+    task_inference:
+      <<: *submit_gpu
+      jobname: inference
+      dependency:
+          datadep_training:
+            value: 'run/{{ app.experiment_name }}/training/runscript.training.done'
+    metatask_vx:
+      var:
+        extent: global global lam lam
+        metric: grid obs grid obs
+      task_vx_#metric#_#extent#:
+        <<: *submit_netaccess
+        jobname: vx-#metric#-#extent#
+        dependency:
+          datadep_inference:
+            value: 'run/{{ app.experiment_name }}/inference/runscript.inference.done'
+    metatask_vis:
+      var:
+        extent: global global lam lam
+        metric: grid obs grid obs 
+      task_vis_#metric#_#extent#:
+        <<: *submit_common
+        jobname: vis-#metric#-#extent#
+        dependency:
+          datadep_inference:
+            value: 'run/{{ app.experiment_name }}/inference/runscript.inference.done'

src/config/ursa.yaml

@@ -7,18 +7,31 @@ app:
     batchargs: &gpu-batchargs
       --gres: 'gpu:h100:1'
       nodes: 1
-      partition: u1-h100
-      queue: gpuwf
+      partition: '{{ app.partitions.gpu }}'
+      queue: '{{ app.queues.gpu }}'
   partitions:
     netaccess: u1-service
+    cpu: u1-compute
+    gpu: u1-h100
+  queues:
+    netaccess: batch
+    cpu: batch
+    gpu: gpuwf
   platform: &platform
     account: epic
     scheduler: slurm
+  quickstart:
+    walltime: 00:10:00
+    nodes: 1:ppn=1
+    gpu:
+      native: '--gres gpu:h100:1'
+    zarr:
+      native: '--mem-per-cpu 3G'
 zarrs:
   common:
     execution:
       batchargs:
         --mem-per-cpu: 3G
-      envcmds: !list '{{ ["source /etc/profile.d/slurm.sh"] + zarrs.common.envcmds }}'
+      envcmds: !list '{{ ["source /etc/profile.d/profile-slurm.sh"] + zarrs.common.envcmds }}'
       mpiargs:
         - "--export=ALL"

src/envs/base.yaml

@@ -1,6 +1,8 @@
 channels:
   - nodefaults
   - conda-forge
+  - oar-gsl
   - ufs-community
 dependencies:
   - uwtools 2.14.*
+  - rocoto 1.3.*