Add service to publish past detections to BirdWeather

scripts/birdnet_analysis.py

@@ -3,7 +3,6 @@
 import os.path
 import re
 import signal
-import sys
 import threading
 from queue import Queue
 from subprocess import CalledProcessError
@@ -12,13 +11,13 @@
 from inotify.constants import IN_CLOSE_WRITE
 
 from server import load_global_model, run_analysis
-from utils.helpers import get_settings, ParseFileName, get_wav_files, ANALYZING_NOW
-from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, bird_weather, heartbeat, \
-    update_json_file
+from utils.helpers import get_settings, ParseFileName, get_wav_files, ANALYZING_NOW, setup_logging
+from utils.reporting import extract_detection, summary, write_to_file, write_to_db, apprise, \
+    post_current_detections_to_birdweather, heartbeat, update_json_file
 
 shutdown = False
 
-log = logging.getLogger(__name__)
+log = logging.getLogger(os.path.splitext(os.path.basename(os.path.realpath(__file__)))[0])
 
 
 def sig_handler(sig_num, curr_stack_frame):
@@ -115,7 +114,7 @@ def handle_reporting_queue(queue):
                 write_to_file(file, detection)
                 write_to_db(file, detection)
             apprise(file, detections)
-            bird_weather(file, detections)
+            post_current_detections_to_birdweather(file, detections)
             heartbeat()
             os.remove(file.file_name)
         except BaseException as e:
@@ -129,17 +128,6 @@ def handle_reporting_queue(queue):
     log.info('handle_reporting_queue done')
 
 
-def setup_logging():
-    logger = logging.getLogger()
-    formatter = logging.Formatter("[%(name)s][%(levelname)s] %(message)s")
-    handler = logging.StreamHandler(stream=sys.stdout)
-    handler.setFormatter(formatter)
-    logger.addHandler(handler)
-    logger.setLevel(logging.INFO)
-    global log
-    log = logging.getLogger('birdnet_analysis')
-
-
 if __name__ == '__main__':
     signal.signal(signal.SIGINT, sig_handler)
     signal.signal(signal.SIGTERM, sig_handler)

scripts/birdweather_past_publication.py

@@ -0,0 +1,157 @@
+"""Publish past detections to BirdWeather."""
+
+import datetime
+import logging
+import os
+import sqlite3
+from typing import Optional
+import warnings
+
+import librosa
+import pandas as pd
+from tzlocal import get_localzone
+from utils.helpers import DB_PATH, get_settings, setup_logging, Detection
+from utils.birdweather import get_birdweather_species_id, query_birdweather_detections, \
+    post_soundscape_to_birdweather, post_detection_to_birdweather
+
+log = logging.getLogger(os.path.splitext(os.path.basename(os.path.realpath(__file__)))[0])
+
+
+def get_last_run_time(script_name: str) -> Optional[datetime.datetime]:
+    """Fetch the last run time for the given script from the database."""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+
+    cursor.execute(
+        "SELECT last_run FROM scripts_metadata WHERE script_name = ?", (script_name,)
+    )
+    result = cursor.fetchone()
+
+    conn.close()
+
+    if result:
+        return datetime.datetime.fromisoformat(result[0])
+    return None
+
+
+def update_last_run_time(script_name: str):
+    """Update the last run time for the given script to the current time in the database."""
+    conn = sqlite3.connect(DB_PATH)
+    cursor = conn.cursor()
+
+    current_time = datetime.datetime.now().isoformat()
+
+    cursor.execute(
+        """
+        INSERT INTO scripts_metadata (script_name, last_run) VALUES (?, ?)
+        ON CONFLICT(script_name) DO UPDATE SET last_run = excluded.last_run;
+        """,
+        (script_name, current_time),
+    )
+
+    conn.commit()
+    conn.close()
+
+
+def get_detections_since(start_datetime: datetime.datetime) -> pd.DataFrame:
+    """Get detections from the database that occurred after the specified date."""
+    conn = sqlite3.connect(DB_PATH)
+    query = (
+        "SELECT * FROM detections "
+        "WHERE DATETIME(Date || ' ' || Time) > "
+        f"DATETIME('{start_datetime.strftime('%Y-%m-%d %H:%M:%S')}')"
+    )
+    df = pd.read_sql_query(query, conn)
+    conn.close()
+    return df
+
+
+def main():
+
+    conf = get_settings()
+    if conf["BIRDWEATHER_ID"] == "":
+        return
+
+    # Get detections since last run (defaults to 7 days if last run time is not found)
+    last_run_time = get_last_run_time(script_name=os.path.basename(os.path.realpath(__file__)))
+    if last_run_time is None:
+        last_run_time = datetime.datetime.now() - datetime.timedelta(days=7)
+    df = get_detections_since(last_run_time)
+
+    # Loop through recent detections
+    log.info(
+        f"Checking if recent detections are present in BirdWeather since {last_run_time}"
+    )
+    for detection_entry in df.itertuples():
+
+        detection_datetime = datetime.datetime.strptime(
+            f"{detection_entry.Date} {detection_entry.Time}", "%Y-%m-%d %H:%M:%S"
+        ).astimezone(get_localzone())
+
+        try:
+            # Lookup detections present in BirdWeather at the time of this detection
+            species_id = get_birdweather_species_id(
+                detection_entry.Sci_Name, detection_entry.Com_Name
+            )
+            birdweather_detections = query_birdweather_detections(
+                conf["BIRDWEATHER_ID"],
+                species_id,
+                detection_datetime,
+            )
+        except Exception as e:
+            log.error(
+                f"Script {os.path.basename(os.path.realpath(__file__))} stopped due to error: {e}"
+            )
+            return
+
+        # This detection is not present in BirdWeather
+        if birdweather_detections == []:
+
+            log.info(f"Detection not in BirdWeather: {detection_entry.File_Name}")
+
+            # Post extracted audio to BirdWeather as soundscape
+            extracted_audio_file = os.path.join(
+                conf["EXTRACTED"],
+                "By_Date",
+                detection_datetime.strftime("%Y-%m-%d"),
+                detection_entry.Com_Name.replace(" ", "_").replace("'", ""),
+                detection_entry.File_Name,
+            )
+            soundscape_id = post_soundscape_to_birdweather(
+                conf["BIRDWEATHER_ID"],
+                detection_datetime,
+                extracted_audio_file,
+            )
+
+            # Get length of extracted audio file, will be useful to post detection to BirdWeather
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore", category=FutureWarning)
+                soundscape_duration = librosa.get_duration(path=extracted_audio_file)
+
+            # Create an instance of Detection and post it to BirdWeather
+            # This Detection start and end times are equal to soundscape start and end times,
+            # because we're using an "extracted" audio file as soundscape
+            detection = Detection(
+                detection_datetime,
+                detection_datetime + datetime.timedelta(seconds=soundscape_duration),
+                f"{detection_entry.Sci_Name}_{detection_entry.Com_Name}",
+                detection_entry.Confidence,
+            )
+            post_detection_to_birdweather(
+                detection,
+                soundscape_id,
+                detection_datetime,
+                conf["BIRDWEATHER_ID"],
+                conf['LATITUDE'],
+                conf['LONGITUDE'],
+                conf['MODEL'],
+            )
+
+    update_last_run_time(script_name=os.path.basename(os.path.realpath(__file__)))
+
+
+if __name__ == "__main__":
+
+    setup_logging()
+
+    main()

scripts/createdb.sh

@@ -17,6 +17,10 @@ CREATE TABLE IF NOT EXISTS detections (
   File_Name VARCHAR(100) NOT NULL);
 CREATE INDEX "detections_Com_Name" ON "detections" ("Com_Name");
 CREATE INDEX "detections_Date_Time" ON "detections" ("Date" DESC, "Time" DESC);
+DROP TABLE IF EXISTS scripts_metadata;
+CREATE TABLE IF NOT EXISTS scripts_metadata (
+    script_name TEXT PRIMARY KEY,
+    last_run DATETIME);
 EOF
 chown $USER:$USER $HOME/BirdNET-Pi/scripts/birds.db
 chmod g+w $HOME/BirdNET-Pi/scripts/birds.db

scripts/install_helpers.sh

@@ -70,3 +70,33 @@ install_tmp_mount() {
     echo "tmp.mount is $STATE, skipping"
   fi
 }
+
+install_birdweather_past_publication() {
+  cat << EOF > $HOME/BirdNET-Pi/templates/birdweather_past_publication@.service
+[Unit]
+Description=BirdWeather Publication for %i interface
+After=network-online.target
+Wants=network-online.target
+[Service]
+Type=oneshot
+User=${USER}
+ExecStartPre= /bin/sh -c 'n=0; until curl --silent --head --fail https://app.birdweather.com >/dev/null || [ \$n -ge 30 ]; do n=\$((n+1)); sleep 5; done;'
+ExecStart=$PYTHON_VIRTUAL_ENV /usr/local/bin/birdweather_past_publication.py
+EOF
+  cat << EOF > $HOME/BirdNET-Pi/templates/50-birdweather-past-publication
+#!/bin/bash
+UNIT_NAME="birdweather_past_publication@\$IFACE.service"
+# Check if the service is active and then start it
+if systemctl is-active --quiet "\$UNIT_NAME"; then
+    echo "\$UNIT_NAME is already running."
+else
+    echo "Starting \$UNIT_NAME..."
+    systemctl start "\$UNIT_NAME"
+fi
+EOF
+  chmod +x $HOME/BirdNET-Pi/templates/50-birdweather-past-publication
+  chown root:root $HOME/BirdNET-Pi/templates/50-birdweather-past-publication
+  ln -sf $HOME/BirdNET-Pi/templates/50-birdweather-past-publication /etc/networkd-dispatcher/routable.d
+  ln -sf $HOME/BirdNET-Pi/templates/birdweather_past_publication@.service /usr/lib/systemd/system
+  systemctl enable systemd-networkd
+}

scripts/install_services.sh

@@ -20,7 +20,7 @@ install_depends() {
   apt install -qqy caddy sqlite3 php-sqlite3 php-fpm php-curl php-xml php-zip php icecast2 \
     pulseaudio avahi-utils sox libsox-fmt-mp3 alsa-utils ffmpeg \
     wget curl unzip bc \
-    python3-pip python3-venv lsof net-tools inotify-tools
+    python3-pip python3-venv lsof net-tools inotify-tools networkd-dispatcher
 }
 
 set_hostname() {
@@ -387,6 +387,13 @@ install_weekly_cron() {
 
 chown_things() {
   chown -R $USER:$USER $HOME/Bird*
+
+  # Set ownership to root for the birdweather publication networkd-dispatcher script
+  BIRDWEATHER_PAST_DISPATCHER_SCRIPT="$HOME/BirdNET-Pi/templates/50-birdweather-past-publication"
+  if [ -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ]; then
+    sudo chown root:root "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT"
+    sudo chmod 755 "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT"
+  fi
 }
 
 increase_caddy_timeout() {
@@ -409,6 +416,7 @@ install_services() {
   install_Caddyfile
   install_avahi_aliases
   install_birdnet_analysis
+  install_birdweather_past_publication
   install_birdnet_stats_service
   install_recording_service
   install_custom_recording_service # But does not enable

scripts/server.py

@@ -339,9 +339,8 @@ def run_analysis(file):
                     log.warning("Excluded as below Species Occurrence Frequency Threshold: %s", entry[0])
                 else:
                     d = Detection(
-                        file.file_date,
-                        time_slot.split(';')[0],
-                        time_slot.split(';')[1],
+                        file.file_date + datetime.timedelta(seconds=float(time_slot.split(';')[0])),
+                        file.file_date + datetime.timedelta(seconds=float(time_slot.split(';')[1])),
                         entry[0],
                         entry[1],
                     )

scripts/update_birdnet_snippets.sh

@@ -26,6 +26,9 @@ chmod g+r $HOME
 # remove world-writable perms
 chmod -R o-w ~/BirdNET-Pi/templates/*
 
+# update database schema
+$my_dir/update_db.sh
+
 APT_UPDATED=0
 PIP_UPDATED=0
 
@@ -147,6 +150,29 @@ if grep -q 'birdnet_server.service' "$HOME/BirdNET-Pi/templates/birdnet_analysis
     systemctl daemon-reload && restart_services.sh
 fi
 
+
+# Ensure networkd-dispatcher is installed
+if ! dpkg -s networkd-dispatcher >/dev/null 2>&1; then
+    echo "networkd-dispatcher is not installed. Installing it now..."
+    sudo apt update -qq
+    sudo apt install -qqy networkd-dispatcher
+fi
+
+# Add BirdWeather past publication service if not already installed
+export PYTHON_VIRTUAL_ENV="$HOME/BirdNET-Pi/birdnet/bin/python3"
+BIRDWEATHER_PAST_DISPATCHER_SCRIPT="$HOME/BirdNET-Pi/templates/50-birdweather-past-publication"
+BIRDWEATHER_PAST_SERVICE_FILE="/usr/lib/systemd/system/birdweather_past_publication@.service"
+if [ ! -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ] || [ ! -f "$BIRDWEATHER_PAST_SERVICE_FILE" ]; then
+  echo "Installing BirdWeather past publication service..."
+  install_birdweather_past_publication
+fi
+# Set ownership to root for the birdweather publication networkd-dispatcher script
+if [ -f "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT" ]; then
+  sudo chown root:root "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT"
+  sudo chmod 755 "$BIRDWEATHER_PAST_DISPATCHER_SCRIPT"
+fi
+
+
 TMP_MOUNT=$(systemd-escape -p --suffix=mount "$RECS_DIR/StreamData")
 if ! [ -f "$HOME/BirdNET-Pi/templates/$TMP_MOUNT" ]; then
    install_birdnet_mount

scripts/update_db.sh

@@ -0,0 +1,44 @@
+#!/usr/bin/env bash
+
+DB_PATH="$HOME/BirdNET-Pi/scripts/birds.db"
+
+echo "Checking database schema for updates"
+
+# Check if the tables exist
+DETECTIONS_TABLE_EXISTS=$(sqlite3 "$DB_PATH" "SELECT name FROM sqlite_master WHERE type='table' AND name='detections';")
+SCRIPTS_MTD_TABLE_EXISTS=$(sqlite3 "$DB_PATH" "SELECT name FROM sqlite_master WHERE type='table' AND name='scripts_metadata';")
+
+if [ -z "$DETECTIONS_TABLE_EXISTS" ]; then
+    echo "Table 'detections' does not exist. Creating table..."
+    sqlite3 "$DB_PATH" << EOF
+    CREATE TABLE IF NOT EXISTS detections (
+        Date DATE,
+        Time TIME,
+        Sci_Name VARCHAR(100) NOT NULL,
+        Com_Name VARCHAR(100) NOT NULL,
+        Confidence FLOAT,
+        Lat FLOAT,
+        Lon FLOAT,
+        Cutoff FLOAT,
+        Week INT,
+        Sens FLOAT,
+        Overlap FLOAT,
+        File_Name VARCHAR(100) NOT NULL);
+    CREATE INDEX "detections_Com_Name" ON "detections" ("Com_Name");
+    CREATE INDEX "detections_Date_Time" ON "detections" ("Date" DESC, "Time" DESC);
+EOF
+    echo "Table 'detections' created successfully."
+elif [ -z "$SCRIPTS_MTD_TABLE_EXISTS" ]; then
+    echo "Table 'scripts_metadata' does not exist. Creating table..."
+    sqlite3 "$DB_PATH" << EOF
+    CREATE TABLE IF NOT EXISTS scripts_metadata (
+        script_name TEXT PRIMARY KEY,
+        last_run DATETIME
+    );
+EOF
+    echo "Table 'scripts_metadata' created successfully."
+else
+    echo "Tables 'detections' and 'scripts_metadata' already exist. No changes made."
+fi
+
+echo "Database schema update complete."