HTML listing and description of of provider processes

CHANGES.rst

@@ -12,6 +12,13 @@ Changes
 
 Changes:
 --------
+- Add `HTML` shortcut links to easily toggle contextually between full `Provider` or only local `Process` listing.
+- Add `HTML` support of ``/providers/{providerID}/processes/?f=html`` to list all `Process` references of a `Provider`.
+- Add `HTML` support of ``/processes/?providers=true`` to list all `Provider` and local `Process` references.
+- Add ``?provider={providerID}`` and ``?service={providerID}`` query parameters to ``/processes/{processID}`` request
+  as alias to resolve the corresponding ``/providers/{providerID}/processes/{processID}`` resource.
+- Add ``provider`` details embedded within the ``/processes/{processID}`` response if ``?provider={providerID}``
+  or ``?service={providerID}`` query parameter is specified and that the `Provider`'s `Process` can be resolved.
 - Add links and breadcrumbs including the `Process` ID within `HTML` pages of `Job` responses
   if referenced by the request.
 - Add the full URI definitions for ``Accept-Profile`` header that correspond to equivalent shorthand notation

tests/test_datatype.py

@@ -3,14 +3,15 @@
 from datetime import datetime, timedelta
 
 import pytest
-from visibility import Visibility
 
 from tests import resources
 from weaver.datatype import Authentication, AuthenticationTypes, DockerAuthentication, Job, Process, Service
 from weaver.execute import ExecuteControlOption, ExecuteMode, ExecuteResponse, ExecuteReturnPreference
 from weaver.formats import ContentType
+from weaver.processes.types import ProcessType
 from weaver.status import Status
 from weaver.utils import localize_datetime, now
+from weaver.visibility import Visibility
 
 TEST_UUID = uuid.uuid4()
 
@@ -215,6 +216,11 @@ def test_process_split_version(process_id, result):
     assert Process.split_version(process_id) == result
 
 
+def test_process_cleanup_keywords():
+    proc = Process(id="test", type=ProcessType.WPS_REMOTE, keywords=[" PyWPS ", " WPS", " OGC", " demo "], package={})
+    assert proc.keywords == ["PyWPS", "WPS", "OGC", "demo", ProcessType.WPS_REMOTE]
+
+
 @pytest.mark.parametrize(
     ["attribute", "value", "result"],
     [

tests/wps_restapi/test_processes.py

@@ -640,6 +640,7 @@ def test_get_processes_invalid_schemas_handled(self):
         assert resp.content_type == ContentType.APP_JSON
         assert process_name in resp.json.get("description")
 
+    @pytest.mark.html
     def test_get_processes_html_accept_header(self):
         path = "/processes"
         resp = self.app.get(path, headers=self.html_headers)
@@ -649,6 +650,7 @@ def test_get_processes_html_accept_header(self):
         assert "</body>" in resp.text
         assert "Processes" in resp.text
 
+    @pytest.mark.html
     def test_get_processes_html_format_query(self):
         path = "/processes"
         resp = self.app.get(path, params={"f": OutputFormat.HTML})
@@ -658,6 +660,7 @@ def test_get_processes_html_format_query(self):
         assert "</body>" in resp.text
         assert "Processes" in resp.text
 
+    @pytest.mark.html
     def test_describe_process_html_accept_header(self):
         path = f"/processes/{self.process_public.identifier}"
         resp = self.app.get(path, headers=self.html_headers)
@@ -668,6 +671,7 @@ def test_describe_process_html_accept_header(self):
         assert "Process:" in resp.text
         assert self.process_public.identifier in resp.text
 
+    @pytest.mark.html
     def test_describe_process_html_format_query(self):
         path = f"/processes/{self.process_public.identifier}"
         resp = self.app.get(path, params={"f": OutputFormat.HTML})
@@ -678,6 +682,7 @@ def test_describe_process_html_format_query(self):
         assert "Process:" in resp.text
         assert self.process_public.identifier in resp.text
 
+    @pytest.mark.html
     def test_get_processes_html_accept_header_user_agent_browser_disabled(self):
         path = "/processes"
         headers = copy.deepcopy(dict(self.html_headers))
@@ -689,6 +694,7 @@ def test_get_processes_html_accept_header_user_agent_browser_disabled(self):
         assert "</body>" in resp.text
         assert "Processes" in resp.text
 
+    @pytest.mark.html
     def test_get_processes_html_accept_header_user_agent_browser_override(self):
         path = "/processes"
         headers = copy.deepcopy(dict(self.html_headers))
@@ -2776,6 +2782,7 @@ def test_execute_process_nested_invalid_results_amount(self, test_outputs, mock_
 
 # pylint: disable=C0103,invalid-name
 @pytest.mark.functional
+@pytest.mark.html
 class WpsRestApiProcessesNoHTMLTest(WpsConfigBase):
     settings = {
         "weaver.url": "https://localhost",

tests/wps_restapi/test_providers.py

@@ -16,7 +16,7 @@
 from weaver.config import WeaverConfiguration
 from weaver.datatype import Service
 from weaver.execute import ExecuteControlOption, ExecuteTransmissionMode
-from weaver.formats import ContentType
+from weaver.formats import ContentType, OutputFormat
 from weaver.processes.constants import ProcessSchema
 
 
@@ -43,6 +43,7 @@ def setUpClass(cls):
         cls.config = setup_config_with_mongodb(settings=cls.settings)
         cls.app = get_test_weaver_app(config=cls.config)
         cls.json_headers = {"Accept": ContentType.APP_JSON, "Content-Type": ContentType.APP_JSON}
+        cls.html_headers = {"Accept": ContentType.TEXT_HTML}
 
     def setUp(self):
         # rebuild clean db on each test
@@ -287,6 +288,36 @@ def test_get_provider_processes(self):
             remote_processes.append(process["id"])
         assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in remote_processes
 
+    @mocked_remote_server_requests_wps1([
+        resources.TEST_REMOTE_SERVER_URL,
+        resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,
+        [resources.TEST_REMOTE_SERVER_WPS1_DESCRIBE_PROCESS_XML],
+    ])
+    def test_get_provider_processes_with_providers_query(self):
+        self.register_provider()
+
+        path = "/processes"
+        query = {"providers": True}
+        resp = self.app.get(path, headers=self.json_headers, params=query)
+        assert resp.status_code == 200
+        assert resp.content_type == ContentType.APP_JSON
+        assert "providers" in resp.json and isinstance(resp.json["providers"], list)
+        assert len(resp.json["providers"]) == 1
+        provider = resp.json["providers"][0]
+        assert "processes" in provider and isinstance(provider["processes"], list)
+        assert len(provider["processes"]) == 2
+        remote_processes = []
+        for process in resp.json["processes"]:
+            assert "id" in process and isinstance(process["id"], str)
+            assert "title" in process and isinstance(process["title"], str)
+            assert "version" in process and isinstance(process["version"], str)
+            assert "keywords" in process and isinstance(process["keywords"], list)
+            assert "metadata" in process and isinstance(process["metadata"], list)
+            assert len(process["jobControlOptions"]) == 1
+            assert ExecuteControlOption.ASYNC in process["jobControlOptions"]
+            remote_processes.append(process["id"])
+        assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in remote_processes
+
     @pytest.mark.xfail(condition=Version(owslib.__version__) <= Version("0.25.0"),
                        reason="OWSLib fix for retrieval of processVersion from DescribeProcess not yet available "
                               "(https://github.com/geopython/OWSLib/pull/794)")
@@ -390,6 +421,58 @@ def test_get_provider_process_description_ogc_schema(self):
         assert all(isinstance(p_io, str) and isinstance(process["outputs"][p_io], dict) for p_io in process["outputs"])
         assert all("id" not in process["outputs"][p_io] for p_io in process["outputs"])
 
+    @pytest.mark.html
+    @mocked_remote_server_requests_wps1([
+        resources.TEST_REMOTE_SERVER_URL,
+        resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,
+        [resources.TEST_REMOTE_SERVER_WPS1_DESCRIBE_PROCESS_XML],
+    ])
+    def test_get_provider_process_description_html(self):
+        self.register_provider()
+
+        path = f"/providers/{self.remote_provider_name}/processes/{resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID}"
+        resp = self.app.get(path, headers=self.html_headers)
+        assert resp.status_code == 200
+        assert resp.content_type == ContentType.TEXT_HTML
+        assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in resp.text
+        assert self.remote_provider_name in resp.text
+
+    @pytest.mark.html
+    @mocked_remote_server_requests_wps1([
+        resources.TEST_REMOTE_SERVER_URL,
+        resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,
+        [resources.TEST_REMOTE_SERVER_WPS1_DESCRIBE_PROCESS_XML],
+    ])
+    def test_get_provider_process_description_html_with_provider_query_ogc_schema(self):
+        self.register_provider()
+
+        path = f"/processes/{resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID}"
+        query = {"provider": self.remote_provider_name, "f": OutputFormat.HTML}
+        resp = self.app.get(path, params=query)
+        assert resp.status_code == 200
+        assert resp.content_type == ContentType.TEXT_HTML
+        assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in resp.text
+        assert self.remote_provider_name in resp.text
+        assert resources.TEST_REMOTE_SERVER_URL in resp.text
+
+    @pytest.mark.html
+    @mocked_remote_server_requests_wps1([
+        resources.TEST_REMOTE_SERVER_URL,
+        resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,
+        [resources.TEST_REMOTE_SERVER_WPS1_DESCRIBE_PROCESS_XML],
+    ])
+    def test_get_provider_process_description_html_with_provider_query_old_schema(self):
+        self.register_provider()
+
+        path = f"/processes/{resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID}"
+        query = {"provider": self.remote_provider_name, "f": OutputFormat.HTML, "schema": ProcessSchema.OLD}
+        resp = self.app.get(path, params=query)
+        assert resp.status_code == 200
+        assert resp.content_type == ContentType.TEXT_HTML
+        assert resources.TEST_REMOTE_SERVER_WPS1_PROCESS_ID in resp.text
+        assert self.remote_provider_name in resp.text
+        assert resources.TEST_REMOTE_SERVER_URL in resp.text
+
     @mocked_remote_server_requests_wps1([
         resources.TEST_REMOTE_SERVER_URL,
         resources.TEST_REMOTE_SERVER_WPS1_GETCAP_XML,

weaver/datatype.py

@@ -533,7 +533,8 @@ def keywords(self, container=None):
         Obtains the keywords relevant to the service provider.
         """
         wps = self.wps(container=container)
-        return wps.identification.keywords
+        keywords = [kw.strip() for kw in wps.identification.keywords]
+        return keywords
 
     def summary(self, container, fetch=True, ignore=False):
         # type: (AnySettingsContainer, bool, bool) -> Optional[JSON]
@@ -2306,6 +2307,7 @@ def _set_desc(self, description):
     def keywords(self):
         # type: () -> List[str]
         keywords = self.setdefault("keywords", [])
+        keywords = list({key.strip(): None for key in keywords})
         if self.type not in keywords:
             keywords.append(self.type)
             self["keywords"] = keywords
@@ -2314,7 +2316,8 @@ def keywords(self):
     @keywords.setter
     def keywords(self, keywords):
         # type: (List[str]) -> None
-        self["keywords"] = list(set(sd.KeywordList().deserialize(keywords)))
+        keywords = {key.strip(): None for key in keywords}
+        self["keywords"] = list(sd.KeywordList().deserialize(list(keywords)))
 
     @property
     def metadata(self):
@@ -2843,15 +2846,18 @@ def href(self, container=None):
         proc_desc = base_url + sd.process_service.path.format(process_id=self.id)
         return proc_desc
 
-    def offering(self, schema=ProcessSchema.OGC, request=None):
-        # type: (ProcessSchemaType, AnyRequestType) -> JSON
+    def offering(self, schema=ProcessSchema.OGC, request=None, **additional_properties):
+        # type: (ProcessSchemaType, AnyRequestType, **JSON) -> JSON
         """
         Obtains the :term:`JSON` or :term:`XML` serializable offering/description representation of the :term:`Process`.
 
         :param request: HTTP request that can provide more details on how to describe the process.
         :param schema:
             One of values defined by :class:`sd.ProcessDescriptionSchemaQuery` to select which
             process description representation to generate (see each schema for details).
+        :param additional_properties:
+            Additional properties to include in the JSON representation.
+            These properties still need to validate against the resolved schema if explicitly defined in it.
 
         .. note::
             Property name ``offering`` is employed to differentiate from the string process ``description`` field.
@@ -2890,9 +2896,10 @@ def offering(self, schema=ProcessSchema.OGC, request=None):
             # fields nested under 'process' + I/O as lists
             for io_type in ["inputs", "outputs"]:
                 process[io_type] = normalize_ordered_io(process[io_type], io_hints[io_type])
-            process.update({"process": dict(process)})
+            process.update({"process": dict(process), **additional_properties})
             return sd.ProcessDescriptionOLD(schema_meta_include=True).deserialize(process)
         # process fields directly at root + I/O as mappings
+        process.update(additional_properties)
         return sd.ProcessDescriptionOGC(schema_meta_include=True).deserialize(process)
 
     def summary(self, revision=False, links=True, container=None):

weaver/wps_restapi/processes/processes.py

@@ -35,7 +35,8 @@
 from weaver.visibility import Visibility
 from weaver.wps_restapi import swagger_definitions as sd
 from weaver.wps_restapi.processes.utils import get_process_list_links, get_processes_filtered_by_valid_schemas
-from weaver.wps_restapi.providers.utils import get_provider_services
+from weaver.wps_restapi.providers.providers import describe_provider_process
+from weaver.wps_restapi.providers.utils import get_provider_id, get_provider_services
 
 if TYPE_CHECKING:
     from pyramid.config import Configurator
@@ -275,7 +276,13 @@ def get_local_process(request):
     Get a registered local process information (DescribeProcess).
     """
     try:
-        process = get_process(request=request)
+        provider = None
+        provider_id = get_provider_id(request)
+        if provider_id:
+            process, service = describe_provider_process(request, provider_id)
+            provider = service.summary(request, fetch=True, ignore=True)  # fetch is lazy since done during describe
+        else:
+            process = get_process(request=request)
         process["inputs"] = opensearch.replace_inputs_describe_process(process.inputs, process.payload)
         schema = request.params.get("schema")
         ctype = guess_target_format(request)
@@ -285,7 +292,7 @@ def get_local_process(request):
         ctype_xml = add_content_type_charset(ContentType.APP_XML, "UTF-8")
         proc_url = process.href(request)
         if ctype in ContentType.ANY_XML or str(schema).upper() == ProcessSchema.WPS:
-            offering = process.offering(ProcessSchema.WPS, request=request)
+            offering = process.offering(ProcessSchema.WPS, request=request, provider=provider)
             headers = [
                 ("Link", make_link_header(f"{proc_url}?f=json", rel="alternate", type=ctype_json)),
                 ("Link", make_link_header(f"{proc_url}?f=html", rel="alternate", type=ctype_html)),
@@ -294,7 +301,7 @@ def get_local_process(request):
             ]
             return Response(offering, headerlist=headers)
         elif ctype == ContentType.APP_YAML:
-            offering = process.offering(schema)
+            offering = process.offering(schema, provider=provider)
             content = OutputFormat.convert(offering, OutputFormat.YAML)
             headers = [
                 ("Link", make_link_header(f"{proc_url}?f=json", rel="alternate", type=ctype_json)),
@@ -306,7 +313,7 @@ def get_local_process(request):
             ]
             return HTTPOk(headers=headers, content_type=ctype, charset="utf-8", body=content)
         elif ctype == ContentType.APP_JSON:
-            offering = process.offering(schema)
+            offering = process.offering(schema, provider=provider)
             request.response.content_type = ctype_json
             request.response.headers.extend([
                 ("Link", make_link_header(f"{proc_url}?f=xml", rel="alternate", type=ctype_xml)),
@@ -317,7 +324,7 @@ def get_local_process(request):
             ])
             return Box(offering)
         else:  # HTML
-            offering = process.offering(schema)
+            offering = process.offering(schema, provider=provider)
             request.response.headers.extend([
                 ("Link", make_link_header(f"{proc_url}?f=json", rel="alternate", type=ctype_json)),
                 ("Link", make_link_header(f"{proc_url}?f=yaml", rel="alternate", type=ctype_yaml)),