diff --git a/scripts/upload/add_snow_depths.py b/scripts/upload/add_snow_depths.py index 7a41ac9..b6ecb68 100644 --- a/scripts/upload/add_snow_depths.py +++ b/scripts/upload/add_snow_depths.py @@ -18,7 +18,7 @@ def main(): # Site name start = time.time() site_name = 'Grand Mesa' - timezone = 'MST' + timezone = 'US/Mountain' # Read in the Grand Mesa Snow Depths Data base = abspath(join('../download/data/SNOWEX/SNEX20_SD.001/')) diff --git a/snowex_db/metadata.py b/snowex_db/metadata.py index a939d30..ef38f56 100644 --- a/snowex_db/metadata.py +++ b/snowex_db/metadata.py @@ -103,13 +103,13 @@ class SnowExProfileMetadata(ProfileMetaData): """ air_temp: Union[float, None] = None aspect: Union[float, None] = None + comments: Union[str, None] = None ground_condition: Union[str, None] = None ground_roughness: Union[str, None] = None ground_vegetation: Union[str, None] = None instrument: Union[str, None] = None instrument_model: Union[str, None] = None precip: Union[str, None] = None - site_notes: Union[str, None] = None sky_cover: Union[str, None] = None slope: Union[float, None] = None total_depth: Union[float, None] = None @@ -144,6 +144,7 @@ def parse(self): air_temp=self.parse_air_temp(), aspect=self.parse_aspect(), campaign_name=self.parse_campaign_name(), + comments=self.parse_header('COMMENTS'), date_time=self.parse_date_time(), flags=self.parse_flags(), ground_condition=self.parse_header('GROUND_CONDITION'), @@ -156,7 +157,6 @@ def parse(self): observers=self.parse_observers(), precip=self.parse_header('PRECIP'), site_name=self.parse_id(), - site_notes=self.parse_site_notes(), sky_cover=self.parse_header('SKY_COVER'), slope=self.parse_slope(), total_depth=self.parse_header('TOTAL_DEPTH'), @@ -214,9 +214,6 @@ def parse_header(self, name): self.metadata_variables.entries[name].code ) - def parse_site_notes(self): - return None - # TODO: delete this? class DataHeader(object): diff --git a/snowex_db/upload/layers.py b/snowex_db/upload/layers.py index 4d9aec3..0a37c40 100644 --- a/snowex_db/upload/layers.py +++ b/snowex_db/upload/layers.py @@ -79,7 +79,7 @@ def __init__( self._instrument = kwargs.get("instrument") self._instrument_model = kwargs.get("instrument_model") - self._comments = kwargs.get("comments") + self._comments = kwargs.get("comments", '') # Read in data self.data = self._read() @@ -139,18 +139,8 @@ def build_data(self, profile: SnowExProfileData) -> gpd.GeoDataFrame: columns = df.columns.values # Clean up comments a bit if 'comments' in columns: - df['comments'] = df['comments'].apply( + df['value'] = df['value'].apply( lambda x: x.strip(' ') if isinstance(x, str) else x) - # Add pit comments - if profile.metadata.comments: - df["comments"] += profile.metadata.comments - else: - # Make comments to pit comments - df["comments"] = [profile.metadata.comments] * len(df) - - # In case of SMP, pass comments in - if self._comments is not None: - df["comments"] = [self._comments] * len(df) # Add flags to the comments. flag_string = metadata.flags @@ -185,7 +175,7 @@ def submit(self): instrument = self._add_instrument(profile.metadata) for row in df.to_dict(orient="records"): - if row.get('value') is 'None': + if row.get('value') == 'None': continue d = self._add_entry( @@ -242,6 +232,14 @@ def _add_metadata(self, metadata: SnowExProfileMetadata): f"Point ({metadata.longitude} {metadata.latitude})", srid=4326 ) + # Combine found comments and passed in comments to this class + comments = '; '.join( + [ + comment for comment in [metadata.comments, self._comments] + if comment is not None + + ] + ) # Site record site_id = metadata.site_name @@ -250,26 +248,26 @@ def _add_metadata(self, metadata: SnowExProfileMetadata): Site, dict(name=site_id), object_kwargs=dict( - name=site_id, + air_temp=metadata.air_temp, + aspect=metadata.aspect, campaign=campaign, + comments=comments, datetime=dt, - geom=geom, doi=doi, + geom=geom, + ground_condition=metadata.ground_condition, + ground_roughness=metadata.ground_roughness, + ground_vegetation=metadata.ground_vegetation, + name=site_id, observers=observer_list, - aspect=metadata.aspect, + precip=metadata.precip, + sky_cover=metadata.sky_cover, slope_angle=metadata.slope, - air_temp=metadata.air_temp, total_depth=metadata.total_depth, + tree_canopy=metadata.tree_canopy, + vegetation_height=metadata.vegetation_height, weather_description=metadata.weather_description, - precip=metadata.precip, - sky_cover=metadata.sky_cover, wind=metadata.wind, - ground_condition=metadata.ground_condition, - ground_roughness=metadata.ground_roughness, - ground_vegetation=metadata.ground_vegetation, - vegetation_height=metadata.vegetation_height, - tree_canopy=metadata.tree_canopy, - site_notes=metadata.site_notes, )) return campaign, observer_list, site @@ -294,7 +292,6 @@ def _add_instrument(self, metadata: SnowExProfileMetadata): dict(name=instrumen_name, model=instrument_model) ) - def _add_entry( self, row: dict, campaign: Campaign, observer_list: List[Observer], site: Site, instrument: Instrument, @@ -311,7 +308,7 @@ def _add_entry( Returns: """ - # An instrument associated with a row has presedence over the + # An instrument associated with a row has precedence over the # given via arguments if row.get('instrument') is not None: instrument = self._check_or_add_object( @@ -340,7 +337,6 @@ def _add_entry( depth=row["depth"], bottom_depth=row.get("bottom_depth"), value=row["value"], - comments=row["comments"], # Linked tables instrument=instrument, measurement_type=measurement_obj, diff --git a/tests/data/site_details.csv b/tests/data/site_details.csv deleted file mode 100644 index b9ac9ab..0000000 --- a/tests/data/site_details.csv +++ /dev/null @@ -1,25 +0,0 @@ -# Location,Grand Mesa -# Site,1N20 -# PitID,COGM1N20_20200205 -# Date/Time,2020-02-05-13:30 -# UTM Zone,12N -# Easting [m],743281 -# Northing [m],4324005 -# Slope [deg],5° -# Aspect [deg],S -# Air Temp [deg C],NaN -# Total Depth [cm],35 -# Surveyors,"Chris Hiemstra, Hans Lievens" -# WISe Serial No,10 -# Weather,"Sunny, cold, gusts -# Ground roughness, rough, rocks in places" -# Precip,None -# Sky,Few (< 1/4 of sky) -# Wind,Moderate -# Ground Condition,Frozen -# Ground Vegetation,['Grass'] -# Vegetation Height,"5,nan" -# Tree Canopy,No Trees -# Comments:,"Start temperature measurements (top): 13:48 End temperature -measurements (bottom): 13:53 LWC sampler broke, no measurements were -possible" diff --git a/tests/data/site_details_2020.csv b/tests/data/site_details_2020.csv new file mode 100644 index 0000000..0ba0ee5 --- /dev/null +++ b/tests/data/site_details_2020.csv @@ -0,0 +1,25 @@ +# Location,Grand Mesa +# Site,1N20 +# PitID,COGM1N20_20200205 +# Date/Local Time,2020-02-05T13:30 +# UTM Zone,12N +# Easting (m),743281 +# Northing (m),4324005 +# Slope (deg),5 +# Aspect (deg),S +# Air Temp (deg C),NaN +# Total Depth (cm),35 +# Surveyors,"C. Hiemstra, H. Lievens" +# WISe Serial No,WIS018A +# Weather, Sunny, cold, gusts +# Precip,None +# Sky,Few (< 1/4 of sky) +# Wind,Moderate +# Ground Condition,Frozen +# Ground Roughness,Rough +# Ground Vegetation,Grass +# Vegetation Height,5 +# Tree Canopy,No Trees +# Comments:,"Start temperature measurements (top): 13:48 End temperature +measurements (bottom): 13:53 LWC sampler broke, no measurements were +possible" diff --git a/tests/data/stratigraphy.csv b/tests/data/stratigraphy.csv index 65985e8..3c39351 100644 --- a/tests/data/stratigraphy.csv +++ b/tests/data/stratigraphy.csv @@ -5,6 +5,8 @@ # UTM Zone,12N # Easting,743281 # Northing,4324005 +# Pit Comments, "No additional" +# Flags, AD # Top [cm],Bottom [cm],Grain Size [mm],Grain Type,Hand Hardness,Manual Wetness,Comments 35.0,33.0,< 1 mm,DF,F,D,NaN 33.0,30.0,< 1 mm,DF,4F,D,NaN diff --git a/tests/helpers.py b/tests/helpers.py index bb342ba..028d9cb 100644 --- a/tests/helpers.py +++ b/tests/helpers.py @@ -17,10 +17,24 @@ def upload_file(self, session, filename): ) u.submit() - def get_records(self, table, attribute, value): - with db_session_with_credentials() as (engine, session): - attribute = getattr(table, attribute) - return session.query(table).filter(attribute == value).all() + def get_records(self, session, table, attribute, value): + """ + Fetches records that match criteria. + + Using the session object from the test class allows for lazy loading + associated records. + + Arguments: + session: The database session from the test class + table: Table to query + attribute: The name of the attribute in the table to use as a filter. + value: The attribute value to filter by. + + Returns: + A list of records + """ + attribute = getattr(table, attribute) + return session.query(table).filter(attribute == value).all() def get_value(self, table, attribute): with db_session_with_credentials() as (engine, session): diff --git a/tests/layers/test_density_profile.py b/tests/layers/test_density_profile.py index bdd32a9..8af44d5 100644 --- a/tests/layers/test_density_profile.py +++ b/tests/layers/test_density_profile.py @@ -57,7 +57,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): @pytest.mark.parametrize( "data_name, attribute_to_check, filter_attribute, filter_value, expected", [ - ('density', 'value', 'depth', 35, [190, 245, 'None']), + ('density', 'value', 'depth', 35, [190, 245]), ] ) def test_value( @@ -71,7 +71,7 @@ def test_value( @pytest.mark.parametrize( "data_name, expected", [ - ("density", 12), + ("density", 8), ] ) def test_count(self, data_name, expected, uploaded_file): @@ -136,7 +136,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): @pytest.mark.parametrize( "data_name, expected", [ - ("density", 15), + ("density", 10), ] ) def test_count(self, data_name, expected, uploaded_file): diff --git a/tests/layers/test_layer_batch.py b/tests/layers/test_layer_batch.py index 3a48d69..86c147b 100644 --- a/tests/layers/test_layer_batch.py +++ b/tests/layers/test_layer_batch.py @@ -22,7 +22,7 @@ class TestUploadProfileBatch(TableTestBase, WithUploadBatchFiles): @pytest.fixture(scope="class") def uploaded_file(self, session, data_dir): - fnames = ['site_details.csv', 'stratigraphy.csv', 'temperature.csv'] + fnames = ['site_details_2020.csv', 'stratigraphy.csv', 'temperature.csv'] fpaths = [str(data_dir.joinpath(f)) for f in fnames] self.upload_file(fpaths, session) @@ -37,18 +37,19 @@ def test_count(self, data_name, expected): n = self.check_count(data_name) assert n == expected - def test_only_one_site(self): + @pytest.mark.usefixtures("uploaded_file") + def test_only_one_site(self, session): """ The three CSVs are for the same site. Verify we only create one Site record and properly associate the layer information with it. """ - records = self.get_records(Site, 'name', 'COGM1N20_20200205') + records = self.get_records(session, Site, 'name', 'COGM1N20_20200205') assert len(records) == 1 site = records[0] - # The sratigraphy has 5 layers with 5 data points, - # plus 5 LWC measurements - assert len(site.layer_data) == 30 + # The stratigraphy has 5 layers with 4 data points, plus one comment + # in a layer, plus 5 LWC measurements + assert len(site.layer_data) == 26 class TestUploadProfileBatchErrors(TableTestBase): diff --git a/tests/layers/test_nondata_file.py b/tests/layers/test_nondata_file.py index 43d9a6d..e8cfcfe 100644 --- a/tests/layers/test_nondata_file.py +++ b/tests/layers/test_nondata_file.py @@ -40,10 +40,9 @@ def test_count(self, data_name, expected, uploaded_file): assert n == expected -class TestMetadata(TableTestBase, WithUploadedFile): +class TestMetadata2020(TableTestBase, WithUploadedFile): """ - Test the large amount of metadata we get from the - site details file + Test the site details file from the 2020 campaign """ kwargs = { 'timezone': 'MST', @@ -51,45 +50,65 @@ class TestMetadata(TableTestBase, WithUploadedFile): } UploaderClass = UploadProfileData - @pytest.fixture + @pytest.fixture(scope="class") def uploaded_site_details_file(self, session, data_dir): self.upload_file( - filename=str(data_dir.joinpath("site_details.csv")), session=session + filename=str(data_dir.joinpath("site_details_2020.csv")), + session=session ) + @pytest.fixture + def site_records(self, uploaded_site_details_file, session): + return self.get_records(session, Site, "name", "COGM1N20_20200205") + @pytest.mark.parametrize( - "table, attribute, expected_value", [ - (Site, "name", "COGM1N20_20200205"), - ( - Site, "datetime", - datetime(2020, 2, 5, 20, 30, tzinfo=timezone.utc), - ), + "attribute, expected_value", [ + ("datetime", datetime(2020, 2, 5, 20, 30, tzinfo=timezone.utc)), + ("aspect", 180.0), + ("slope_angle", 5.0), + ("air_temp", np.nan), + ("total_depth", 35.0), + ("weather_description", "Sunny, cold, gusts"), + ("precip", "None"), + ("sky_cover", "Few (< 1/4 of sky)"), + ("wind", "Moderate"), + ("ground_condition", "Frozen"), + ("ground_roughness", "Rough"), + ("ground_vegetation", "Grass"), + ("vegetation_height", "5"), + ("tree_canopy", "No Trees"), ( - Site, "geom", - WKTElement( - 'POINT (-108.1894813320662 39.031261970372725)', srid=4326 - ), + "comments", + "Start temperature measurements (top) 13:48 End temperature " + "measurements (bottom) 13:53 LWC sampler broke, no " + "measurements were possible; " ), - (Campaign, "name", "Grand Mesa"), - (Site, "aspect", 180.0), - (Site, "slope_angle", 5.0), - (Site, "air_temp", np.nan), - (Site, "total_depth", 35.0), - (Site, "weather_description", "Sunny, cold, gusts"), - (Site, "precip", "None"), - (Site, "sky_cover", "Few (< 1/4 of sky)"), - (Site, "wind", "Moderate"), - (Site, "ground_condition", "Frozen"), - (Site, "ground_roughness", "rough, rocks in places"), - (Site, "ground_vegetation", "[Grass]"), - (Site, "vegetation_height", "5, nan"), - (Site, "tree_canopy", "No Trees"), - (Site, "site_notes", None), - (Observer, "name", ["Chris Hiemstra", "Hans Lievens"]), ] ) - def test_metadata( - self, uploaded_site_details_file, table, attribute, - expected_value - ): - self._check_metadata(table, attribute, expected_value) + def test_site_attributes(self, attribute, expected_value, site_records): + assert len(site_records) == 1 + + site = site_records[0] + if attribute == "air_temp": + assert np.isnan(getattr(site, attribute)) + else: + assert getattr(site, attribute) == expected_value + + def test_query_by_site_geom(self, site_records, session): + """ + Test that we can find the site by its coordinates. + """ + site_coordinate = WKTElement( + 'POINT (-108.1894813320662 39.031261970372725)', srid=4326 + ) + site = self.get_records(session, Site, "geom", site_coordinate) + + assert site[0].name == site_records[0].name + assert site[0].geom == site_records[0].geom + + def test_site_campaign(self, site_records): + assert site_records[0].campaign.name == "Grand Mesa" + + def test_site_observer(self, site_records): + observers = [observer.name for observer in site_records[0].observers] + assert "Chris Hiemstra", "Hans Lievens" in observers diff --git a/tests/layers/test_smp_profile.py b/tests/layers/test_smp_profile.py index 0e59c47..1c30846 100644 --- a/tests/layers/test_smp_profile.py +++ b/tests/layers/test_smp_profile.py @@ -36,6 +36,7 @@ def uploaded_file(self, session, data_dir): session=session, ) + @pytest.mark.usefixtures("uploaded_file") @pytest.mark.parametrize( "table, attribute, expected_value", [ (Site, "name", "COGM_Fakepitid123"), @@ -51,6 +52,7 @@ def uploaded_file(self, session, data_dir): 'POINT (-108.16268920898438 39.03013229370117)', srid=4326 ), ), + (Site, "comments", "Filename: S06M0874_2N12_20200131.CSV"), (Campaign, "name", "Grand Mesa"), (Instrument, "name", "snowmicropen"), (Instrument, "model", "6"), @@ -59,48 +61,46 @@ def uploaded_file(self, session, data_dir): (MeasurementType, "derived", [True]), ] ) - def test_metadata(self, table, attribute, expected_value, uploaded_file): + def test_metadata(self, table, attribute, expected_value): # need: # * comments = "Filename: filename" self._check_metadata(table, attribute, expected_value) + @pytest.mark.usefixtures("uploaded_file") @pytest.mark.parametrize( "data_name, attribute_to_check, filter_attribute, " "filter_value, expected", [ ('force', 'value', 'depth', -53.17, [0.331]), - ( - 'force', 'comments', 'depth', -53.17, - ['Filename: S06M0874_2N12_20200131.CSV'], - ), ] ) def test_value( self, data_name, attribute_to_check, - filter_attribute, filter_value, expected, uploaded_file + filter_attribute, filter_value, expected ): self.check_value( data_name, attribute_to_check, filter_attribute, filter_value, expected, ) + @pytest.mark.usefixtures("uploaded_file") @pytest.mark.parametrize( "data_name, expected", [ ("force", 154), ] ) - def test_count(self, data_name, expected, uploaded_file): + def test_count(self, data_name, expected): n = self.check_count(data_name) assert n == expected + @pytest.mark.usefixtures("uploaded_file") @pytest.mark.parametrize( "data_name, attribute_to_count, expected", [ ("force", "site_id", 1), ] ) def test_unique_count( - self, data_name, attribute_to_count, expected, - uploaded_file + self, data_name, attribute_to_count, expected ): self.check_unique_count( data_name, attribute_to_count, expected diff --git a/tests/layers/test_ssa_profile.py b/tests/layers/test_ssa_profile.py index 7dda42a..e402c02 100644 --- a/tests/layers/test_ssa_profile.py +++ b/tests/layers/test_ssa_profile.py @@ -47,13 +47,15 @@ def uploaded_file(self, session, data_dir): ( MeasurementType, "name", [ - 'sample_signal', 'reflectance', + 'sample_signal', + 'reflectance', 'specific_surface_area', 'equivalent_diameter', + 'comments', ], ), - (MeasurementType, "units", ['mv', '%', 'm^2/kg', 'mm']), - (MeasurementType, "derived", [False] * 4), + (MeasurementType, "units", ['mv', '%', 'm^2/kg', 'mm', None]), + (MeasurementType, "derived", [False] * 5), ] ) def test_metadata(self, table, attribute, expected_value, uploaded_file): @@ -67,7 +69,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): ('specific_surface_area', 'value', 'depth', 35, [11.2]), ('equivalent_diameter', 'value', 'depth', 80, [0.1054]), ('sample_signal', 'value', 'depth', 10, [186.9]), - ('sample_signal', 'comments', 'depth', 5, ["brush"]), + ('comments', 'value', 'depth', 5, ["brush"]), ] ) def test_value( diff --git a/tests/layers/test_stratigraphy_profile.py b/tests/layers/test_stratigraphy_profile.py index ecf85b1..b0d7437 100644 --- a/tests/layers/test_stratigraphy_profile.py +++ b/tests/layers/test_stratigraphy_profile.py @@ -58,7 +58,7 @@ def test_metadata(self, table, attribute, expected_value, uploaded_file): ('grain_size', 'value', 'depth', 35, ["< 1 mm"]), ('grain_type', 'value', 'depth', 17, ["FC"]), ('manual_wetness', 'value', 'depth', 17, ["D"]), - ('hand_hardness', 'comments', 'depth', 17, ["Cups"]), + ('comments', 'value', 'depth', 17, ["Cups"]), ] ) def test_value(