Add Solar Orbiter SPICE datasource and ingestion support

docs/src/source/appendix/data_sources/solo.rst

@@ -3,14 +3,40 @@ Solar Orbiter
 
 .. table:: Solar Orbiter Data Sources
 
-    +-----------+-------------+----------------+------------+-------------+
-    | Source ID | Description | Observatory    | Instrument | Measurement |
-    +===========+=============+================+============+=============+
-    | 84        | EUI FSI 174 | Solar Orbiter  | EUI        | 174         |
-    +-----------+-------------+----------------+------------+-------------+
-    | 85        | EUI FSI 304 | Solar Orbiter  | EUI        | 304         |
-    +-----------+-------------+----------------+------------+-------------+
-    | 86        | EUI HRI 174 | Solar Orbiter  | EUI        | 174         |
-    +-----------+-------------+----------------+------------+-------------+
-    | 87        | EUI HRI 1216| Solar Orbiter  | EUI        | 1216        |
-    +-----------+-------------+----------------+------------+-------------+
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | Source ID | Description    | Observatory    | Instrument | Measurement| Line                |
+    +===========+================+================+============+===========+=====================+
+    | 84        | EUI FSI 174    | Solar Orbiter  | EUI        | 174       |                     |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 85        | EUI FSI 304    | Solar Orbiter  | EUI        | 304       |                     |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 86        | EUI HRI 174    | Solar Orbiter  | EUI        | 174       |                     |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 87        | EUI HRI 1216   | Solar Orbiter  | EUI        | 1216      |                     |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 601       | SPICE 77.04    | Solar Orbiter  | SPICE      | intensity | Ne VIII 77.04 nm    |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 602       | SPICE 103.19   | Solar Orbiter  | SPICE      | intensity | O VI 103.19 nm      |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 603       | SPICE 70.38    | Solar Orbiter  | SPICE      | intensity | O III 70.38 nm      |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 604       | SPICE 78.77    | Solar Orbiter  | SPICE      | intensity | O IV 78.77 nm       |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 605       | SPICE 70.6     | Solar Orbiter  | SPICE      | intensity | Mg IX 70.60 nm      |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 606       | SPICE 97.25    | Solar Orbiter  | SPICE      | intensity | H Ly gamma 97.25 nm |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 607       | SPICE 76.51    | Solar Orbiter  | SPICE      | intensity | N IV 76.51 nm       |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+    | 608       | SPICE 97.7     | Solar Orbiter  | SPICE      | intensity | C III 97.70 nm      |
+    +-----------+----------------+----------------+------------+-----------+---------------------+
+
+SPICE Import
+^^^^^^^^^^^^
+
+The SPICE downloader reads JP2 files from the directory configured by
+``SPICE_DATA_PATH``. The expected layout under that root is ``YYYY/MM/DD``.
+
+Example::
+
+    SPICE_DATA_PATH=/path/to/spice/jp2 python install/downloader.py -b local -m localmove -d spice

install/__test__/test_spice.py

@@ -0,0 +1,127 @@
+import datetime
+import os
+
+import pytest
+
+from helioviewer.hvpull.net.daemon import ImageRetrievalDaemon
+from helioviewer.hvpull.servers.spice import SpiceDataServer
+from helioviewer.jp2parser import JP2parser
+
+
+class FakeImageData:
+    observatory = "Solar Orbiter"
+    instrument = "SPICE"
+    detector = "SW"
+    nickname = "SPICE"
+    measurement = "103.19"
+    meta = {"cmpnam": "O VI 103.19 nm"}
+    date = datetime.datetime(2023, 12, 2, 6, 19, 22)
+
+
+def test_spice_data_path_required(monkeypatch):
+    monkeypatch.delenv("SPICE_DATA_PATH", raising=False)
+
+    with pytest.raises(ValueError, match="SPICE_DATA_PATH"):
+        SpiceDataServer()
+
+
+def test_spice_data_path_defined(monkeypatch):
+    monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2")
+
+    spice = SpiceDataServer()
+
+    assert spice.uri == "/data/spice/jp2"
+    assert spice.name == "SPICE"
+
+
+def test_compute_directories(monkeypatch):
+    monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2")
+    spice = SpiceDataServer()
+
+    dirs = spice.compute_directories(
+        datetime.datetime(2023, 12, 1),
+        datetime.datetime(2023, 12, 3)
+    )
+
+    assert dirs == [
+        "/data/spice/jp2/2023/12/03",
+        "/data/spice/jp2/2023/12/02",
+        "/data/spice/jp2/2023/12/01",
+    ]
+
+
+def test_get_datetime_from_file(monkeypatch):
+    monkeypatch.setenv("SPICE_DATA_PATH", "/data/spice/jp2")
+    spice = SpiceDataServer()
+
+    assert spice.get_datetime_from_file(
+        "solo_L4_spice-n-ras_20231202T061922_218104664-000_0_3.jp2"
+    ) == datetime.datetime(2023, 12, 2, 6, 19, 22)
+    assert spice.get_datetime_from_file(
+        "solo_L4_spice-n-ras_20231202T181921_218104668-000_5_0.jp2"
+    ) == datetime.datetime(2023, 12, 2, 18, 19, 21)
+
+
+def test_spice_registered_in_daemon():
+    assert ImageRetrievalDaemon.get_servers()["spice"] == "SpiceDataServer"
+
+
+def test_spice_loads_from_spice_module():
+    daemon = ImageRetrievalDaemon.__new__(ImageRetrievalDaemon)
+
+    cls = daemon._load_class(
+        "helioviewer.hvpull.servers",
+        "spice",
+        ImageRetrievalDaemon.get_servers()["spice"]
+    )
+
+    assert cls is SpiceDataServer
+
+
+def test_spice_detection_keys_ignore_detector():
+    img = {
+        "observatory": "Solar_Orbiter",
+        "instrument": "SPICE",
+        "detector": "SW",
+        "measurement": "intensity",
+        "line": "O VI 103.19 nm",
+    }
+
+    assert JP2parser.get_detection_keys(img) == [
+        "observatory",
+        "instrument",
+        "measurement",
+        "line",
+    ]
+
+
+def test_spice_metadata_extraction(monkeypatch):
+    parser = JP2parser.__new__(JP2parser)
+    parser._filepath = "solo_L4_spice-n-ras_20231202T061922_218104664-000_0_3.jp2"
+    parser._data = {}
+
+    monkeypatch.setattr(parser, "getImageMap", lambda: FakeImageData())
+    monkeypatch.setattr(parser, "getImageDimensions", lambda: {"width": 1024, "height": 1024})
+    monkeypatch.setattr(parser, "getRefPixelCoords", lambda: {"x": 512, "y": 512})
+    monkeypatch.setattr(parser, "getImagePlateScale", lambda: 1)
+    monkeypatch.setattr(parser, "getDSun", lambda: 149597870700)
+    monkeypatch.setattr(parser, "getLayeringOrder", lambda: 1)
+    monkeypatch.setattr(parser, "_get_date", lambda image_data: image_data.date)
+    monkeypatch.setattr(parser, "get_storage_suffix", lambda image: os.path.join(image["nickname"], "2023/12/02", image["measurement"]))
+
+    image = parser.getData()
+
+    assert image["observatory"] == "Solar_Orbiter"
+    assert image["instrument"] == "SPICE"
+    assert image["detector"] == "SW"
+    assert image["measurement"] == "intensity"
+    assert image["line"] == "O VI 103.19 nm"
+
+
+def test_spice_missing_cmpnam_message():
+    parser = JP2parser.__new__(JP2parser)
+    image_data = FakeImageData()
+    image_data.meta = {}
+
+    with pytest.raises(ValueError, match="SPICE JP2 missing required CMPNAM metadata"):
+        parser._get_spice_line(image_data)

install/database/2026_07_23_spice_source.sql

@@ -0,0 +1,68 @@
+INSERT INTO datasources (
+    id,
+    name,
+    description,
+    units,
+    layeringOrder,
+    enabled,
+    sourceIdGroup,
+    displayOrder
+)
+VALUES
+    (601, 'SPICE 77.04',  'Solar Orbiter SPICE 77.04',  'NM', 1, 0, '', 0),
+    (602, 'SPICE 103.19', 'Solar Orbiter SPICE 103.19', 'NM', 1, 0, '', 0),
+    (603, 'SPICE 70.38',  'Solar Orbiter SPICE 70.38',  'NM', 1, 0, '', 0),
+    (604, 'SPICE 78.77',  'Solar Orbiter SPICE 78.77',  'NM', 1, 0, '', 0),
+    (605, 'SPICE 70.6',   'Solar Orbiter SPICE 70.6',   'NM', 1, 0, '', 0),
+    (606, 'SPICE 97.25',  'Solar Orbiter SPICE 97.25',  'NM', 1, 0, '', 0),
+    (607, 'SPICE 76.51',  'Solar Orbiter SPICE 76.51',  'NM', 1, 0, '', 0),
+    (608, 'SPICE 97.7',   'Solar Orbiter SPICE 97.7',   'NM', 1, 0, '', 0);
+
+INSERT INTO datasource_property (
+    sourceId,
+    label,
+    name,
+    fitsName,
+    description,
+    uiOrder
+)
+VALUES
+    (601, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (601, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (601, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (601, 'Line',        'Ne VIII 77.04 nm',    'Ne VIII 77.04 nm',    'Ne VIII 77.04 nm',    4),
+
+    (602, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (602, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (602, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (602, 'Line',        'O VI 103.19 nm',      'O VI 103.19 nm',      'O VI 103.19 nm',      4),
+
+    (603, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (603, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (603, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (603, 'Line',        'O III 70.38 nm',      'O III 70.38 nm',      'O III 70.38 nm',      4),
+
+    (604, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (604, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (604, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (604, 'Line',        'O IV 78.77 nm',       'O IV 78.77 nm',       'O IV 78.77 nm',       4),
+
+    (605, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (605, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (605, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (605, 'Line',        'Mg IX 70.60 nm',      'Mg IX 70.60 nm',      'Mg IX 70.60 nm',      4),
+
+    (606, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (606, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (606, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (606, 'Line',        'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 4),
+
+    (607, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (607, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (607, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (607, 'Line',        'N IV 76.51 nm',       'N IV 76.51 nm',       'N IV 76.51 nm',       4),
+
+    (608, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter',                1),
+    (608, 'Instrument',  'SPICE',               'SPICE',               'SPICE',               2),
+    (608, 'Measurement', 'intensity',           'intensity',           'intensity',           3),
+    (608, 'Line',        'C III 97.70 nm',      'C III 97.70 nm',      'C III 97.70 nm',      4);

install/helioviewer/db.py

@@ -365,6 +365,14 @@ def create_datasource_table(cursor):
 (134,  'PUNCH', 'PUNCH WFI+NFI PAM Mosaic', NULL, 1, 0, '', 0, 0, 0, 0),
 (132, 'GOES', 'CCOR-1', NULL, 1, 0, '', 0, 0, 0, 0),
 (133, 'SWFO-L1', 'CCOR-2',  NULL, 1, 0, '', 0, 0, 0, 0),
+(601, 'SPICE 77.04', 'Solar Orbiter SPICE 77.04', 'NM', 1, 0, '', 0, 0, 0, 0),
+(602, 'SPICE 103.19', 'Solar Orbiter SPICE 103.19', 'NM', 1, 0, '', 0, 0, 0, 0),
+(603, 'SPICE 70.38', 'Solar Orbiter SPICE 70.38', 'NM', 1, 0, '', 0, 0, 0, 0),
+(604, 'SPICE 78.77', 'Solar Orbiter SPICE 78.77', 'NM', 1, 0, '', 0, 0, 0, 0),
+(605, 'SPICE 70.6', 'Solar Orbiter SPICE 70.6', 'NM', 1, 0, '', 0, 0, 0, 0),
+(606, 'SPICE 97.25', 'Solar Orbiter SPICE 97.25', 'NM', 1, 0, '', 0, 0, 0, 0),
+(607, 'SPICE 76.51', 'Solar Orbiter SPICE 76.51', 'NM', 1, 0, '', 0, 0, 0, 0),
+(608, 'SPICE 97.7', 'Solar Orbiter SPICE 97.7', 'NM', 1, 0, '', 0, 0, 0, 0),
 (2000, 'GOES-R SUVI 94', 'GOES-R SUVI 94',  NULL, 1, 0, '', 0, 0, 0, 0),
 (2001, 'GOES-R SUVI 131', 'GOES-R SUVI 131',  NULL, 1, 0, '', 0, 0, 0, 0),
 (2002, 'GOES-R SUVI 171', 'GOES-R SUVI 171',  NULL, 1, 0, '', 0, 0, 0, 0),
@@ -403,7 +411,7 @@ def create_datasource_property_table(cursor):
 
     """Inserts properties about datasource
         @sourceId:      unique sidentifier which matches datasource table id
-        @label:         one of the following Observatory/Instrument/Detector/Measurement
+        @label:         one of the following Observatory/Instrument/Detector/Measurement/Line
         @name:          the name that is embedded in the jp2 file
         @fitsName       the name that is embedded in the jp2 file
         @description:   verbal description of datasource
@@ -933,7 +941,39 @@ def create_datasource_property_table(cursor):
 (133, 'Observatory', 'SWFO-L1', 'SWFO-L1', 'SWFO-L1 Observatory',1),
 (133, 'Instrument', 'CCOR-2', 'CCOR2', 'CCOR-2',2),
 (133, 'Detector', 'CCOR-2', '1', 'CCOR-2',2),
-(133, 'Measurement', 'white-light', 'white-light', 'White Light',3);""")
+(133, 'Measurement', 'white-light', 'white-light', 'White Light',3),
+(601, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(601, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(601, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(601, 'Line', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 'Ne VIII 77.04 nm', 4),
+(602, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(602, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(602, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(602, 'Line', 'O VI 103.19 nm', 'O VI 103.19 nm', 'O VI 103.19 nm', 4),
+(603, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(603, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(603, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(603, 'Line', 'O III 70.38 nm', 'O III 70.38 nm', 'O III 70.38 nm', 4),
+(604, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(604, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(604, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(604, 'Line', 'O IV 78.77 nm', 'O IV 78.77 nm', 'O IV 78.77 nm', 4),
+(605, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(605, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(605, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(605, 'Line', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 'Mg IX 70.60 nm', 4),
+(606, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(606, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(606, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(606, 'Line', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 'H Ly gamma 97.25 nm', 4),
+(607, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(607, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(607, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(607, 'Line', 'N IV 76.51 nm', 'N IV 76.51 nm', 'N IV 76.51 nm', 4),
+(608, 'Observatory', 'SOLO', 'Solar_Orbiter', 'Solar Orbiter', 1),
+(608, 'Instrument', 'SPICE', 'SPICE', 'SPICE', 2),
+(608, 'Measurement', 'intensity', 'intensity', 'intensity', 3),
+(608, 'Line', 'C III 97.70 nm', 'C III 97.70 nm', 'C III 97.70 nm', 4);""")
 
 def create_movies_table(cursor):
     """Creates movie table

install/helioviewer/hvpull/browser/basebrowser.py

@@ -1,6 +1,5 @@
 """Base data browser definition"""
-from typing import Callable, Optional
-
+from collections.abc import Callable
 
 class BaseDataBrowser:
     """BaseDataBrowser"""
@@ -10,8 +9,8 @@ def __init__(self, server):
     def get_directories(self, start_time, end_time):
         """Gets a list of directories to be queried for the given time range"""
         return None
-
-    def get_files(self, uri, extension, filter_func: Optional[Callable] = None):
+    
+    def get_files(self, uri, extension, filter_func: Callable | None = None):
         """Get all the files that end with specified extension at the uri"""
         return None
 

install/helioviewer/hvpull/browser/httpbrowser.py

@@ -2,7 +2,7 @@
 import sys
 import os
 import socket
-from typing import Callable, Optional
+from collections.abc import Callable
 from helioviewer.hvpull.browser.basebrowser import BaseDataBrowser, NetworkError
 
 if (sys.version_info >= (3, 0)):
@@ -82,7 +82,7 @@ def get_directories(self, start_date, end_date):
         # filter(lambda url: url.endswith("/"), self._query(location))
         return self.server.compute_directories(start_date, end_date)
 
-    def get_files(self, location, extension, filter_func: Optional[Callable] = None):
+    def get_files(self, location, extension, filter_func: Callable | None = None):
         """Get all the files that end with specified extension at the uri"""
         files = None
         num_retries = 0
@@ -126,4 +126,3 @@ def _query(self, location):
         urls = filter(lambda url: url[0] != "/" and url[0] != "?", result)
 
         return [os.path.join(location, url) for url in urls]
-

install/helioviewer/hvpull/browser/localbrowser.py

@@ -1,6 +1,6 @@
 """Local data browser"""
 import os
-from typing import Callable, Optional
+from collections.abc import Callable
 from helioviewer.hvpull.browser.basebrowser import BaseDataBrowser
 
 
@@ -15,7 +15,7 @@ def get_directories(self, start_date, end_date):
         """Get a list of directories at the passed uri"""
         return self.server.compute_directories(start_date, end_date)
 
-    def get_files(self, location, extension, filter_func: Optional[Callable] = None):
+    def get_files(self, location, extension, filter_func: Callable | None = None):
         """Get all the files that end with specified extension at the uri"""
 
         # ensure the location exists

install/helioviewer/hvpull/net/daemon.py

@@ -876,6 +876,7 @@ def get_servers(cls):
             "kcor": "KCORDataServer",
             "hv_kcor": "HVKCORDataServer",
             "solar_orbiter": "SolarOrbiterDataServer",
+            "spice": "SpiceDataServer",
             "suvi": "SUVIDataServer",
             "iris": "IRISDataServer",
             "hv_iris": "HvIRISDataServer",

install/helioviewer/hvpull/servers/spice.py

@@ -0,0 +1,16 @@
+import os
+from helioviewer.hvpull.servers import DataServer
+
+class SpiceDataServer(DataServer):
+    """Solar Orbiter/SPICE Datasource definition"""
+    def __init__(self):
+        """Defines the root directory where SPICE JP2 files are kept."""
+        spice_data_path = os.environ.get('SPICE_DATA_PATH')
+        if not spice_data_path:
+            raise ValueError("SPICE_DATA_PATH must be set to the SPICE JP2 root directory")
+
+        DataServer.__init__(self, spice_data_path, "SPICE")
+
+    def compute_directories(self, start_date, end_date):
+        """Computes a list of remote directories expected to contain files"""
+        return [os.path.join(self.uri, date) for date in self.get_dates(start_date, end_date)]