Layer and Site comments

snowex_db/upload/layers.py

@@ -79,7 +79,7 @@ def __init__(
         self._instrument = kwargs.get("instrument")
         self._instrument_model = kwargs.get("instrument_model")
 
-        self._comments = kwargs.get("comments")
+        self._comments = kwargs.get("comments", '')
 
         # Read in data
         self.data = self._read()
@@ -139,18 +139,8 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame:
         columns = df.columns.values
         # Clean up comments a bit
         if 'comments' in columns:
-            df['comments'] = df['comments'].apply(
+            df['value'] = df['value'].apply(
                 lambda x: x.strip(' ') if isinstance(x, str) else x)
-            # Add pit comments
-            if profile.metadata.comments:
-                df["comments"] += profile.metadata.comments
-        else:
-            # Make comments to pit comments
-            df["comments"] = [profile.metadata.comments] * len(df)
-
-        # In case of SMP, pass comments in
-        if self._comments is not None:
-            df["comments"] = [self._comments] * len(df)
 
         # Add flags to the comments.
         flag_string = metadata.flags
@@ -185,7 +175,7 @@ def submit(self):
                     instrument = self._add_instrument(profile.metadata)
 
                 for row in df.to_dict(orient="records"):
-                    if row.get('value') is 'None':
+                    if row.get('value') == 'None':
                         continue
 
                     d = self._add_entry(
@@ -242,6 +232,14 @@ def _add_metadata(self, metadata: SnowExProfileMetadata):
             f"Point ({metadata.longitude} {metadata.latitude})",
             srid=4326
         )
+        # Combine found comments and passed in comments to this class
+        comments = '; '.join(
+            [
+                comment for comment in [metadata.comments,  self._comments]
+                if comment is not None
+
+            ]
+        )
         # Site record
         site_id = metadata.site_name
 
@@ -250,26 +248,27 @@ def _add_metadata(self, metadata: SnowExProfileMetadata):
             Site,
             dict(name=site_id),
             object_kwargs=dict(
-                name=site_id,
+                air_temp=metadata.air_temp,
+                aspect=metadata.aspect,
                 campaign=campaign,
+                comments=comments,
                 datetime=dt,
-                geom=geom,
                 doi=doi,
+                geom=geom,
+                ground_condition=metadata.ground_condition,
+                ground_roughness=metadata.ground_roughness,
+                ground_vegetation=metadata.ground_vegetation,
+                name=site_id,
                 observers=observer_list,
-                aspect=metadata.aspect,
+                precip=metadata.precip,
+                site_notes=metadata.site_notes,
+                sky_cover=metadata.sky_cover,
                 slope_angle=metadata.slope,
-                air_temp=metadata.air_temp,
                 total_depth=metadata.total_depth,
+                tree_canopy=metadata.tree_canopy,
+                vegetation_height=metadata.vegetation_height,
                 weather_description=metadata.weather_description,
-                precip=metadata.precip,
-                sky_cover=metadata.sky_cover,
                 wind=metadata.wind,
-                ground_condition=metadata.ground_condition,
-                ground_roughness=metadata.ground_roughness,
-                ground_vegetation=metadata.ground_vegetation,
-                vegetation_height=metadata.vegetation_height,
-                tree_canopy=metadata.tree_canopy,
-                site_notes=metadata.site_notes,
             ))
         return campaign, observer_list, site
 
@@ -294,7 +293,6 @@ def _add_instrument(self, metadata: SnowExProfileMetadata):
             dict(name=instrumen_name, model=instrument_model)
         )
 
-
     def _add_entry(
         self, row: dict, campaign: Campaign,
         observer_list: List[Observer], site: Site, instrument: Instrument,
@@ -311,7 +309,7 @@ def _add_entry(
         Returns:
 
         """
-        # An instrument associated with a row has presedence over the
+        # An instrument associated with a row has precedence over the
         # given via arguments
         if row.get('instrument') is not None:
             instrument = self._check_or_add_object(
@@ -340,7 +338,6 @@ def _add_entry(
             depth=row["depth"],
             bottom_depth=row.get("bottom_depth"),
             value=row["value"],
-            comments=row["comments"],
             # Linked tables
             instrument=instrument,
             measurement_type=measurement_obj,

tests/data/stratigraphy.csv

@@ -5,6 +5,7 @@
 # UTM Zone,12N
 # Easting,743281
 # Northing,4324005
+# Pit Comments, "No additional"
 # Top [cm],Bottom [cm],Grain Size [mm],Grain Type,Hand Hardness,Manual Wetness,Comments
 35.0,33.0,< 1 mm,DF,F,D,NaN
 33.0,30.0,< 1 mm,DF,4F,D,NaN

tests/layers/test_density_profile.py

@@ -57,7 +57,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
     @pytest.mark.parametrize(
         "data_name, attribute_to_check, filter_attribute, filter_value, expected",
         [
-            ('density', 'value', 'depth', 35, [190, 245, 'None']),
+            ('density', 'value', 'depth', 35, [190, 245]),
         ]
     )
     def test_value(
@@ -71,7 +71,7 @@ def test_value(
 
     @pytest.mark.parametrize(
         "data_name, expected", [
-            ("density", 12),
+            ("density", 8),
         ]
     )
     def test_count(self, data_name, expected, uploaded_file):
@@ -136,7 +136,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
 
     @pytest.mark.parametrize(
         "data_name, expected", [
-            ("density", 15),
+            ("density", 10),
         ]
     )
     def test_count(self, data_name, expected, uploaded_file):

tests/layers/test_layer_batch.py

@@ -46,9 +46,9 @@ def test_only_one_site(self):
         assert len(records) == 1
 
         site = records[0]
-        # The sratigraphy has 5 layers with 5 data points,
-        # plus 5 LWC measurements
-        assert len(site.layer_data) == 30
+        # The sratigraphy has 5 layers with 4 data points, plus one comment
+        # in a layer, plus 5 LWC measurements
+        assert len(site.layer_data) == 26
 
 
 class TestUploadProfileBatchErrors(TableTestBase):

tests/layers/test_smp_profile.py

@@ -36,6 +36,7 @@ def uploaded_file(self, session, data_dir):
             session=session,
         )
 
+    @pytest.mark.usefixtures("uploaded_file")
     @pytest.mark.parametrize(
         "table, attribute, expected_value", [
             (Site, "name", "COGM_Fakepitid123"),
@@ -51,6 +52,7 @@ def uploaded_file(self, session, data_dir):
                     'POINT (-108.16268920898438 39.03013229370117)', srid=4326
                 ),
             ),
+            (Site, "comments", "Filename: S06M0874_2N12_20200131.CSV"),
             (Campaign, "name", "Grand Mesa"),
             (Instrument, "name", "snowmicropen"),
             (Instrument, "model", "6"),
@@ -59,48 +61,46 @@ def uploaded_file(self, session, data_dir):
             (MeasurementType, "derived", [True]),
         ]
     )
-    def test_metadata(self, table, attribute, expected_value, uploaded_file):
+    def test_metadata(self, table, attribute, expected_value):
         # need:
         #  * comments = "Filename: filename"
         self._check_metadata(table, attribute, expected_value)
 
+    @pytest.mark.usefixtures("uploaded_file")
     @pytest.mark.parametrize(
         "data_name, attribute_to_check, filter_attribute, "
         "filter_value, expected",
         [
             ('force', 'value', 'depth', -53.17, [0.331]),
-            (
-                'force', 'comments', 'depth', -53.17,
-                ['Filename: S06M0874_2N12_20200131.CSV'],
-            ),
         ]
     )
     def test_value(
         self, data_name, attribute_to_check,
-        filter_attribute, filter_value, expected, uploaded_file
+        filter_attribute, filter_value, expected
     ):
         self.check_value(
             data_name, attribute_to_check,
             filter_attribute, filter_value, expected,
         )
 
+    @pytest.mark.usefixtures("uploaded_file")
     @pytest.mark.parametrize(
         "data_name, expected", [
             ("force", 154),
         ]
     )
-    def test_count(self, data_name, expected, uploaded_file):
+    def test_count(self, data_name, expected):
         n = self.check_count(data_name)
         assert n == expected
 
+    @pytest.mark.usefixtures("uploaded_file")
     @pytest.mark.parametrize(
         "data_name, attribute_to_count, expected", [
             ("force", "site_id", 1),
         ]
     )
     def test_unique_count(
-        self, data_name, attribute_to_count, expected,
-        uploaded_file
+        self, data_name, attribute_to_count, expected
     ):
         self.check_unique_count(
             data_name, attribute_to_count, expected

tests/layers/test_ssa_profile.py

@@ -47,13 +47,15 @@ def uploaded_file(self, session, data_dir):
             (
                     MeasurementType, "name",
                     [
-                        'sample_signal', 'reflectance',
+                        'sample_signal',
+                        'reflectance',
                         'specific_surface_area',
                         'equivalent_diameter',
+                        'comments',
                     ],
             ),
-            (MeasurementType, "units", ['mv', '%', 'm^2/kg', 'mm']),
-            (MeasurementType, "derived", [False] * 4),
+            (MeasurementType, "units", ['mv', '%', 'm^2/kg', 'mm', None]),
+            (MeasurementType, "derived", [False] * 5),
         ]
     )
     def test_metadata(self, table, attribute, expected_value, uploaded_file):
@@ -67,7 +69,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
             ('specific_surface_area', 'value', 'depth', 35, [11.2]),
             ('equivalent_diameter', 'value', 'depth', 80, [0.1054]),
             ('sample_signal', 'value', 'depth', 10, [186.9]),
-            ('sample_signal', 'comments', 'depth', 5, ["brush"]),
+            ('comments', 'value', 'depth', 5, ["brush"]),
         ]
     )
     def test_value(

tests/layers/test_stratigraphy_profile.py

@@ -58,7 +58,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file):
             ('grain_size', 'value', 'depth', 35, ["< 1 mm"]),
             ('grain_type', 'value', 'depth', 17, ["FC"]),
             ('manual_wetness', 'value', 'depth', 17, ["D"]),
-            ('hand_hardness', 'comments', 'depth', 17, ["Cups"]),
+            ('comments', 'value', 'depth', 17, ["Cups"]),
         ]
     )
     def test_value(