Layer and Site comments

scripts/upload/add_snow_depths.py

@@ -18,7 +18,7 @@ def main():
     # Site name
     start = time.time()
     site_name = 'Grand Mesa'
-    timezone = 'MST'
+    timezone = 'US/Mountain'
 
     # Read in the Grand Mesa Snow Depths Data
     base = abspath(join('../download/data/SNOWEX/SNEX20_SD.001/'))

snowex_db/metadata.py

@@ -103,13 +103,13 @@ class SnowExProfileMetadata(ProfileMetaData):
     """
     air_temp: Union[float, None] = None
     aspect: Union[float, None] = None
+    comments: Union[str, None] = None
     ground_condition: Union[str, None] = None
     ground_roughness: Union[str, None] = None
     ground_vegetation: Union[str, None] = None
     instrument: Union[str, None] = None
     instrument_model: Union[str, None] = None
     precip: Union[str, None] = None
-    site_notes: Union[str, None] = None
     sky_cover: Union[str, None] = None
     slope: Union[float, None] = None
     total_depth: Union[float, None] = None
@@ -144,6 +144,7 @@ def parse(self):
             air_temp=self.parse_air_temp(),
             aspect=self.parse_aspect(),
             campaign_name=self.parse_campaign_name(),
+            comments=self.parse_header('COMMENTS'),
             date_time=self.parse_date_time(),
             flags=self.parse_flags(),
             ground_condition=self.parse_header('GROUND_CONDITION'),
@@ -156,7 +157,6 @@ def parse(self):
             observers=self.parse_observers(),
             precip=self.parse_header('PRECIP'),
             site_name=self.parse_id(),
-            site_notes=self.parse_site_notes(),
             sky_cover=self.parse_header('SKY_COVER'),
             slope=self.parse_slope(),
             total_depth=self.parse_header('TOTAL_DEPTH'),
@@ -214,9 +214,6 @@ def parse_header(self, name):
             self.metadata_variables.entries[name].code
         )
 
-    def parse_site_notes(self):
-        return None
-
 
 # TODO: delete this?
 class DataHeader(object):

snowex_db/upload/layers.py

@@ -79,7 +79,7 @@ def __init__(
         self._instrument = kwargs.get("instrument")
         self._instrument_model = kwargs.get("instrument_model")
 
-        self._comments = kwargs.get("comments")
+        self._comments = kwargs.get("comments", '')
 
         # Read in data
         self.data = self._read()
@@ -139,18 +139,8 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame:
         columns = df.columns.values
         # Clean up comments a bit
         if 'comments' in columns:
-            df['comments'] = df['comments'].apply(
+            df['value'] = df['value'].apply(
                 lambda x: x.strip(' ') if isinstance(x, str) else x)
-            # Add pit comments
-            if profile.metadata.comments:
-                df["comments"] += profile.metadata.comments
-        else:
-            # Make comments to pit comments
-            df["comments"] = [profile.metadata.comments] * len(df)
-
-        # In case of SMP, pass comments in
-        if self._comments is not None:
-            df["comments"] = [self._comments] * len(df)
 
         # Add flags to the comments.
         flag_string = metadata.flags
@@ -185,7 +175,7 @@ def submit(self):
                     instrument = self._add_instrument(profile.metadata)
 
                 for row in df.to_dict(orient="records"):
-                    if row.get('value') is 'None':
+                    if row.get('value') == 'None':
                         continue
 
                     d = self._add_entry(
@@ -242,6 +232,14 @@ def _add_metadata(self, metadata: SnowExProfileMetadata):
             f"Point ({metadata.longitude} {metadata.latitude})",
             srid=4326
         )
+        # Combine found comments and passed in comments to this class
+        comments = '; '.join(
+            [
+                comment for comment in [metadata.comments,  self._comments]
+                if comment is not None
+
+            ]
+        )
         # Site record
         site_id = metadata.site_name
 
@@ -250,26 +248,26 @@ def _add_metadata(self, metadata: SnowExProfileMetadata):
             Site,
             dict(name=site_id),
             object_kwargs=dict(
-                name=site_id,
+                air_temp=metadata.air_temp,
+                aspect=metadata.aspect,
                 campaign=campaign,
+                comments=comments,
                 datetime=dt,
-                geom=geom,
                 doi=doi,
+                geom=geom,
+                ground_condition=metadata.ground_condition,
+                ground_roughness=metadata.ground_roughness,
+                ground_vegetation=metadata.ground_vegetation,
+                name=site_id,
                 observers=observer_list,
-                aspect=metadata.aspect,
+                precip=metadata.precip,
+                sky_cover=metadata.sky_cover,
                 slope_angle=metadata.slope,
-                air_temp=metadata.air_temp,
                 total_depth=metadata.total_depth,
+                tree_canopy=metadata.tree_canopy,
+                vegetation_height=metadata.vegetation_height,
                 weather_description=metadata.weather_description,
-                precip=metadata.precip,
-                sky_cover=metadata.sky_cover,
                 wind=metadata.wind,
-                ground_condition=metadata.ground_condition,
-                ground_roughness=metadata.ground_roughness,
-                ground_vegetation=metadata.ground_vegetation,
-                vegetation_height=metadata.vegetation_height,
-                tree_canopy=metadata.tree_canopy,
-                site_notes=metadata.site_notes,
             ))
         return campaign, observer_list, site
 
@@ -294,7 +292,6 @@ def _add_instrument(self, metadata: SnowExProfileMetadata):
             dict(name=instrumen_name, model=instrument_model)
         )
 
-
     def _add_entry(
         self, row: dict, campaign: Campaign,
         observer_list: List[Observer], site: Site, instrument: Instrument,
@@ -311,7 +308,7 @@ def _add_entry(
         Returns:
 
         """
-        # An instrument associated with a row has presedence over the
+        # An instrument associated with a row has precedence over the
         # given via arguments
         if row.get('instrument') is not None:
             instrument = self._check_or_add_object(
@@ -340,7 +337,6 @@ def _add_entry(
             depth=row["depth"],
             bottom_depth=row.get("bottom_depth"),
             value=row["value"],
-            comments=row["comments"],
             # Linked tables
             instrument=instrument,
             measurement_type=measurement_obj,

tests/data/site_details_2020.csv

@@ -0,0 +1,25 @@
+# Location,Grand Mesa
+# Site,1N20
+# PitID,COGM1N20_20200205
+# Date/Local Time,2020-02-05T13:30
+# UTM Zone,12N
+# Easting (m),743281
+# Northing (m),4324005
+# Slope (deg),5
+# Aspect (deg),S
+# Air Temp (deg C),NaN
+# Total Depth (cm),35
+# Surveyors,"C. Hiemstra, H. Lievens"
+# WISe Serial No,WIS018A
+# Weather, Sunny, cold, gusts
+# Precip,None
+# Sky,Few (< 1/4 of sky)
+# Wind,Moderate
+# Ground Condition,Frozen
+# Ground Roughness,Rough
+# Ground Vegetation,Grass
+# Vegetation Height,5
+# Tree Canopy,No Trees
+# Comments:,"Start temperature measurements (top): 13:48  End temperature
+measurements (bottom): 13:53  LWC sampler broke, no measurements were
+possible"

tests/data/stratigraphy.csv

@@ -5,6 +5,8 @@
 # UTM Zone,12N
 # Easting,743281
 # Northing,4324005
+# Pit Comments, "No additional"
+# Flags, AD
 # Top [cm],Bottom [cm],Grain Size [mm],Grain Type,Hand Hardness,Manual Wetness,Comments
 35.0,33.0,< 1 mm,DF,F,D,NaN
 33.0,30.0,< 1 mm,DF,4F,D,NaN

tests/helpers.py

@@ -17,10 +17,24 @@ def upload_file(self, session, filename):
         )
         u.submit()
 
-    def get_records(self, table, attribute, value):
-        with db_session_with_credentials() as (engine, session):
-            attribute = getattr(table, attribute)
-            return session.query(table).filter(attribute == value).all()
+    def get_records(self, session, table, attribute, value):
+        """
+        Fetches records that match criteria.
+
+        Using the session object from the test class allows for lazy loading
+        associated records.
+
+        Arguments:
+            session: The database session from the test class
+            table: Table to query
+            attribute: The name of the attribute in the table to use as a filter.
+            value: The attribute value to filter by.
+
+        Returns:
+            A list of records
+        """
+        attribute = getattr(table, attribute)
+        return session.query(table).filter(attribute == value).all()
 
     def get_value(self, table, attribute):
         with db_session_with_credentials() as (engine, session):

tests/layers/test_density_profile.py

@@ -57,7 +57,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
     @pytest.mark.parametrize(
         "data_name, attribute_to_check, filter_attribute, filter_value, expected",
         [
-            ('density', 'value', 'depth', 35, [190, 245, 'None']),
+            ('density', 'value', 'depth', 35, [190, 245]),
         ]
     )
     def test_value(
@@ -71,7 +71,7 @@ def test_value(
 
     @pytest.mark.parametrize(
         "data_name, expected", [
-            ("density", 12),
+            ("density", 8),
         ]
     )
     def test_count(self, data_name, expected, uploaded_file):
@@ -136,7 +136,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
 
     @pytest.mark.parametrize(
         "data_name, expected", [
-            ("density", 15),
+            ("density", 10),
         ]
     )
     def test_count(self, data_name, expected, uploaded_file):

tests/layers/test_layer_batch.py

@@ -22,7 +22,7 @@ class TestUploadProfileBatch(TableTestBase, WithUploadBatchFiles):
 
     @pytest.fixture(scope="class")
     def uploaded_file(self, session, data_dir):
-        fnames = ['site_details.csv', 'stratigraphy.csv', 'temperature.csv']
+        fnames = ['site_details_2020.csv', 'stratigraphy.csv', 'temperature.csv']
         fpaths = [str(data_dir.joinpath(f)) for f in fnames]
         self.upload_file(fpaths, session)
 
@@ -37,18 +37,19 @@ def test_count(self, data_name, expected):
         n = self.check_count(data_name)
         assert n == expected
 
-    def test_only_one_site(self):
+    @pytest.mark.usefixtures("uploaded_file")
+    def test_only_one_site(self, session):
         """
         The three CSVs are for the same site. Verify we only create one
         Site record and properly associate the layer information with it.
         """
-        records = self.get_records(Site, 'name', 'COGM1N20_20200205')
+        records = self.get_records(session, Site, 'name', 'COGM1N20_20200205')
         assert len(records) == 1
 
         site = records[0]
-        # The sratigraphy has 5 layers with 5 data points,
-        # plus 5 LWC measurements
-        assert len(site.layer_data) == 30
+        # The stratigraphy has 5 layers with 4 data points, plus one comment
+        # in a layer, plus 5 LWC measurements
+        assert len(site.layer_data) == 26
 
 
 class TestUploadProfileBatchErrors(TableTestBase):