do not trim off recording file extension when naming generated files

Author: bjarthur

app/post-process.py

@@ -1,3 +1,5 @@
+#!/usr/bin/env python3
+
 import sys
 import ast
 import os
@@ -25,9 +27,7 @@
 
 M.init(None, os.path.join(__dir__, "configuration.py"), False)
 
-wavfile_noext = M.trim_ext(wavfile)
-
-with open(wavfile_noext+"-classify.log",'r') as fid:
+with open(wavfile+"-classify.log",'r') as fid:
     for line in fid:
         if "labels: " in line:
             m=re.search('labels: (.+)',line)
@@ -40,13 +40,13 @@
 
 if delete_wavs:
     for label in labels:
-        fullpath = wavfile_noext+'-'+label+'.wav'
+        fullpath = wavfile+'-'+label+'.wav'
         os.remove(fullpath)
         print("deleting ", fullpath)
 
 durations = {}
 counts = {}
-with open(wavfile_noext+'-predicted-1.0pr.csv') as fid:
+with open(wavfile+'-predicted-1.0pr.csv') as fid:
     csvreader = csv.reader(fid)
     for row in csvreader:
         if row[4] not in durations:
@@ -55,7 +55,7 @@
         durations[row[4]] += (int(row[2]) - int(row[1]))
         counts[row[4]] += 1
 
-with open(wavfile_noext+"-post-process.csv",'w') as fid:
+with open(wavfile+"-post-process.csv",'w') as fid:
     fid.write("wavfile,label,duration ("+M.context_time_units+"),num events\n")
     for label in durations.keys():
         fid.write(os.path.basename(wavfile)+','+
@@ -64,8 +64,8 @@
                   str(counts[label])+'\n')
 
 if delete_logs:
-    os.remove(wavfile_noext+'-classify.log')
-    os.remove(wavfile_noext+'-ethogram.log')
-    os.remove(wavfile_noext+'-post-process.log')
+    os.remove(wavfile+'-classify.log')
+    os.remove(wavfile+'-ethogram.log')
+    os.remove(wavfile+'-post-process.log')
 
 print(str(datetime.now())+": finish time")

src/classify

@@ -84,7 +84,7 @@ def main():
 
   load_audio_read_plugin(FLAGS.audio_read_plugin, FLAGS.audio_read_plugin_kwargs)
   load_video_read_plugin(FLAGS.video_read_plugin, FLAGS.video_read_plugin_kwargs)
-  from lib import audio_read, video_read, trim_ext
+  from lib import audio_read, video_read
 
   with open(FLAGS.model_labels, 'r') as fid:
     model_labels = fid.read().splitlines()
@@ -255,13 +255,11 @@ def main():
         else:
           adjusted_probability = probability_matrix[:,ch]
         waveform = adjusted_probability*np.iinfo(np.int16).max
-        withoutext = trim_ext(FLAGS.wav)
-        filename = withoutext+'-'+labels[ch]+'.wav'
+        filename = FLAGS.wav+'-'+labels[ch]+'.wav'
         wavfile.write(filename, int(sample_rate), waveform.astype('int16'))
   else:
         waveform = probability_matrix * np.iinfo(np.int16).max
-        withoutext = trim_ext(FLAGS.wav)
-        filename = withoutext+'-.wav'
+        filename = FLAGS.wav+'-.wav'
         wavfile.write(filename, int(sample_rate), waveform.astype('int16'))
 
 if __name__ == '__main__':

src/congruence

@@ -166,15 +166,8 @@ def main():
   thresholds=set()
   for wavdir in wavdirs:
     for wavfile in wavdirs[wavdir]:
-      if has_rec[wavfile] == "False":
-        wavfile_noext  = os.path.splitext(wavfile)[0]
-      else:
-        tmp = wavfile.split('-')
-        withext, rec = '-'.join(tmp[:-1]), tmp[-1]
-        wavfile_noext  = os.path.splitext(withext)[0]+'-'+rec
-
       if not labels:
-        with open(os.path.join(FLAGS.basepath,wavdir,wavfile_noext+'-ethogram.log'), 'r') as fid:
+        with open(os.path.join(FLAGS.basepath,wavdir,wavfile+'-ethogram.log'), 'r') as fid:
           for line in fid:
             if line.startswith('logdir: '):
               m=re.search('logdir: (.*)',line)
@@ -194,7 +187,7 @@ def main():
                                                                       thresholds_file)
 
       audio_tic_rate_probabilities, half_stride_sec, probability_matrix = \
-            read_probabilities(os.path.join(FLAGS.basepath, wavdir, wavfile_noext), labels)
+            read_probabilities(os.path.join(FLAGS.basepath, wavdir, wavfile), labels)
       for threshold in np.linspace(0, 1, num=FLAGS.nprobabilities+2)[1:-1]:
         features, start_tics, stop_tics = discretize_probabilities(probability_matrix,
                                                                    threshold,
@@ -205,7 +198,7 @@ def main():
         if len(features)>0:
           thresholds.add(str(threshold)+'th')
         filename = os.path.join(FLAGS.basepath, wavdir,
-                                wavfile_noext+'-predicted-'+str(threshold)+'th.csv')
+                                wavfile+'-predicted-'+str(threshold)+'th.csv')
         temp_files.append(filename)
         isort = np.argsort(start_tics)
         with open(filename,'w') as fid:
@@ -506,7 +499,7 @@ def main():
     pool.close()
 
   def to_csv(intervals, csvbase, whichset):
-    filename = os.path.splitext(csvbase)[0]+'-disjoint-'+whichset+'.csv'
+    filename = csvbase+'-disjoint-'+whichset+'.csv'
     with open(os.path.join(FLAGS.topath,filename), 'w') as fid:
       csvwriter = csv.writer(fid, lineterminator='\n')
       for ilabel,label in enumerate(timestamps.keys()):

src/detect-plugins/template.py

@@ -64,7 +64,7 @@ def main():
     print('%s = %s' % (key, flags[key]))
 
   load_audio_read_plugin(FLAGS.audio_read_plugin, FLAGS.audio_read_plugin_kwargs)
-  from lib import audio_read, trim_ext
+  from lib import audio_read
 
   hyperparameter1 = int(FLAGS.parameters["my-simple-textbox"])
 
@@ -75,7 +75,7 @@ def main():
   amplitude = scipy.signal.medfilt(song)
 
   basename = os.path.basename(FLAGS.filename)
-  with open(trim_ext(FLAGS.filename)+'-detected.csv', 'w') as fid:
+  with open(FLAGS.filename+'-detected.csv', 'w') as fid:
       csvwriter = csv.writer(fid, lineterminator='\n')
       for i in range(1,len(amplitude)-1):
           if amplitude[i] > hyperparameter1:

src/detect-plugins/time-freq-threshold.py

@@ -100,7 +100,7 @@ def main():
     add_plugins_to_path(srcdir)
 
     load_audio_read_plugin(FLAGS.audio_read_plugin, FLAGS.audio_read_plugin_kwargs)
-    from lib import audio_read, trim_ext
+    from lib import audio_read
 
     time_sigma_signal, time_sigma_noise = [int(x) for x in FLAGS.parameters['time_sigma'].split(',')]
 
@@ -229,7 +229,7 @@ def bool2stamp(song_morphed, scale):
 
 
     basename = os.path.basename(FLAGS.filename)
-    with open(trim_ext(FLAGS.filename)+'-detected.csv', 'w') as fid:
+    with open(FLAGS.filename+'-detected.csv', 'w') as fid:
       csvwriter = csv.writer(fid, lineterminator='\n')
       for i in intervals_time_signal:
         csvwriter.writerow([basename,i[1],i[2],'detected','time'+i[3]])

src/ethogram

@@ -39,26 +39,17 @@ try:
   audio_tic_rate=float(audio_tic_rate)
 
   wavpath, wavname = os.path.split(wav_file)
-  if has_rec == "False":
-    if not os.path.isfile(wav_file):
-      print('cannot find WAV file')
-      exit()
-    wavname_noext  = os.path.splitext(wavname)[0]
-  else:
-    tmp = wavname.split('-')
-    withext, rec = '-'.join(tmp[:-1]), tmp[-1]
-    wavname_noext  = os.path.splitext(withext)[0]+'-'+rec
-    if not os.path.isfile(os.path.join(wavpath, withext)):
-      print('cannot find WAV file')
-      exit()
+  if not os.path.isfile(wav_file):
+    print('cannot find WAV file')
+    exit()
 
   precision_recall_ratios, thresholds = read_thresholds(logdir, model, thresholds_file)
 
   labels = [x[0] for x in thresholds]
   thresholds = np.array([x[1:] for x in thresholds], dtype=np.float64)
 
   audio_tic_rate_probabilities, half_stride_sec, probability_matrix = \
-        read_probabilities(os.path.join(wavpath, wavname_noext), labels)
+        read_probabilities(os.path.join(wavpath, wavname), labels)
 
   for ithreshold in range(len(precision_recall_ratios)):
     features, start_tics, stop_tics = discretize_probabilities(probability_matrix,
@@ -68,7 +59,7 @@ try:
                                                                half_stride_sec,
                                                                audio_tic_rate)
     filename = os.path.join(wavpath,
-                            wavname_noext+'-predicted-'+precision_recall_ratios[ithreshold]+'pr.csv')
+                            wavname+'-predicted-'+precision_recall_ratios[ithreshold]+'pr.csv')
     isort = np.argsort(start_tics)
     with open(filename,'w') as fid:
       csvwriter = csv.writer(fid, lineterminator='\n')

src/gui/controller.py

@@ -937,10 +937,10 @@ def pbfile_succeeded(pbfile, reftime):
     return recent_file_exists(pbfile, reftime, True)
 
 def detect_succeeded(wavfile, reftime):
-    logfile = M.trim_ext(wavfile)+'-detect.log'
+    logfile = wavfile+'-detect.log'
     if not logfile_succeeded(logfile, reftime):
         return False
-    csvfile = M.trim_ext(wavfile)+'-detected.csv'
+    csvfile = wavfile+'-detected.csv'
     if not csvfile_succeeded(csvfile, reftime):
         return False
     return True
@@ -955,7 +955,7 @@ async def detect_actuate():
 async def _detect_actuate(i, wavfiles, threads, results):
     wavfile = wavfiles.pop(0)
     currtime = time.time()
-    logfile = M.trim_ext(wavfile)+'-detect.log'
+    logfile = wavfile+'-detect.log'
     jobid = generic_actuate(os.path.join("detect-plugins", M.detect_plugin+".py"),
                             logfile,
                             M.detect_where,
@@ -1010,8 +1010,7 @@ async def misses_actuate():
         csvreader = csv.reader(fid)
         row1 = next(csvreader)
     wavfile = row1[0]
-    noext = os.path.join(basepath, os.path.splitext(wavfile)[0])
-    logfile = noext+'-misses.log'
+    logfile = os.path.join(basepath, wavfile)+'-misses.log'
     jobid = generic_actuate("misses", logfile, \
                             M.misses_where,
                             M.misses_ncpu_cores,
@@ -1855,7 +1854,7 @@ def classify_succeeded(modeldir, wavfile, reftime):
     with open(os.path.join(modeldir, 'labels.txt'), 'r') as fid:
         labels = fid.read().splitlines()
     for x in labels:
-        if not recent_file_exists(M.trim_ext(wavfile)+'-'+x+'.wav', reftime, True):
+        if not recent_file_exists(wavfile+'-'+x+'.wav', reftime, True):
             return False
     return True
 
@@ -1867,7 +1866,7 @@ async def _classify_actuate(wavfiles):
     wavfile = wavfiles.pop(0)
     currtime = time.time()
     logdir, model, _, check_point = M.parse_model_file(V.model_file.value)
-    logfile = M.trim_ext(wavfile)+'-classify.log'
+    logfile = wavfile+'-classify.log'
     args = ["--context="+V.context.value,
             "--shiftby="+V.shiftby.value,
             "--loss="+V.loss.value,
@@ -1928,7 +1927,7 @@ def ethogram_succeeded(modeldir, ckpt, wavfile, reftime):
         row1 = next(csvreader)
     precision_recalls = row1[1:]
     for x in precision_recalls:
-        if not recent_file_exists(M.trim_ext(wavfile)+'-predicted-'+x+'pr.csv', reftime, True):
+        if not recent_file_exists(wavfile+'-predicted-'+x+'pr.csv', reftime, True):
             return False
     return True
 
@@ -1946,7 +1945,7 @@ async def _ethogram_actuate(i, wavfiles, threads, results):
         thresholds_file = os.path.basename(V.model_file.value)
     else:
         thresholds_file = 'thresholds.ckpt-'+check_point+'.csv'
-    logfile = M.trim_ext(wavfile)+'-ethogram.log'
+    logfile = wavfile+'-ethogram.log'
     jobid = generic_actuate("ethogram", logfile, M.ethogram_where,
                             M.ethogram_ncpu_cores,
                             M.ethogram_ngpu_cards,

src/gui/model.py

@@ -89,7 +89,7 @@ def save_annotations():
         wavfiles = set()
         for sound in annotated_sounds:
             if not sound["label"]:  continue
-            wavfiles |= set([trim_ext(os.path.join(*sound["file"]))])
+            wavfiles |= set([os.path.join(*sound["file"])])
         for wavfile in wavfiles:
             csvfile = wavfile+"-annotated-"+songexplorer_starttime+".csv"
             annotated_csvfiles_all.add(csvfile)
@@ -104,8 +104,7 @@ def save_annotations():
         corrected_sounds=[]
         for annotation in annotated_sounds:
             if annotation['label']!="" and not annotation['label'].isspace():
-                wavfile_noext = trim_ext(os.path.join(*annotation['file']))
-                csvwriters[wavfile_noext].writerow(
+                csvwriters[os.path.join(*annotation['file'])].writerow(
                         [annotation['file'][1],
                          annotation['ticks'][0], annotation['ticks'][1],
                          'annotated', annotation['label']])
@@ -119,8 +118,7 @@ def save_annotations():
                                         columns=['file','start','stop','kind','label'])
             wavfiles = set()
             for sound in corrected_sounds:
-                wavfile_noext = trim_ext(os.path.join(*sound["file"]))
-                wavfiles |= set([wavfile_noext])
+                wavfiles |= set([os.path.join(*sound["file"])])
             for wavfile in wavfiles:
                 wavdir, wavbase = os.path.split(wavfile)
                 wavpath = os.path.join(V.groundtruth_folder.value, wavdir)
@@ -248,7 +246,7 @@ def init(_bokeh_document, _configuration_file, _use_aitch):
     global context_width_sec0, context_offset_sec0
     global xcluster, ycluster, zcluster, ndcluster, tic2pix_max, snippet_width_pix, ilayer, ispecies, iword, inohyphen, ikind, nlayers, layers, species, words, nohyphens, kinds, used_labels, snippets_gap_sec, snippets_tic, snippets_gap_tic, snippets_decimate_by, snippets_pix, snippets_gap_pix, context_decimate_by, context_width_tic, context_offset_tic, context_sound, isnippet, xsnippet, ysnippet, file_nframes, context_midpoint_tic, ilabel, used_sounds, used_starts_sorted, used_stops, iused_stops_sorted, annotated_sounds, annotated_starts_sorted, annotated_stops, iannotated_stops_sorted, annotated_csvfiles_all, nrecent_annotations, clustered_sounds, clustered_activations, used_recording2firstsound, clustered_starts_sorted, clustered_stops, iclustered_stops_sorted, songexplorer_starttime, history_stack, history_idx, wizard, action, function, statepath, state, file_dialog_root, file_dialog_filter, nearest_sounds, status_ticker_queue, waitfor_job, dfs, remaining_isounds
     global user_changed_recording, user_copied_parameters
-    global audio_read, audio_read_exts, audio_read_rec2ch, audio_read_strip_rec, trim_ext
+    global audio_read, audio_read_exts, audio_read_rec2ch, audio_read_strip_rec
     global video_read, detect_labels, doubleclick_annotation, context_data, context_data_istart, model, video_findfile
     global detect_parameters, doubleclick_parameters, model_parameters, cluster_parameters, augmentation_parameters
 
@@ -270,7 +268,7 @@ def init(_bokeh_document, _configuration_file, _use_aitch):
 
     load_audio_read_plugin(audio_read_plugin, audio_read_plugin_kwargs)
     load_video_read_plugin(video_read_plugin, video_read_plugin_kwargs)
-    from lib import audio_read, audio_read_exts, audio_read_rec2ch, audio_read_strip_rec, video_read, trim_ext
+    from lib import audio_read, audio_read_exts, audio_read_rec2ch, audio_read_strip_rec, video_read
 
     sys.path.insert(0,os.path.dirname(detect_plugin))
     tmp = importlib.import_module(os.path.basename(detect_plugin))

src/gui/view.py

@@ -1146,8 +1146,7 @@ def context_update():
         M.file_nframes = np.shape(context_cache_data)[0]
         probs = [None]*len(M.used_labels)
         for ilabel,label in enumerate(M.used_labels):
-            prob_wavfile = M.trim_ext(os.path.join(groundtruth_folder.value,
-                                                   *M.context_sound['file']))+'-'+label+'.wav'
+            prob_wavfile = os.path.join(groundtruth_folder.value, *M.context_sound['file'])+'-'+label+'.wav'
             if os.path.isfile(prob_wavfile):
                 prob_tic_rate, probs[ilabel] = spiowav.read(prob_wavfile, mmap=True)
         if istart+M.context_width_tic>0 and istart<M.file_nframes:

src/lib.py

@@ -51,15 +51,6 @@ def video_read(fullpath, start_frame=None, stop_frame=None):
         return video_read_module.video_read(fullpath, start_frame, stop_frame,
                                             **video_read_plugin_kwargs)
 
-def trim_ext(wavfile):
-    if len(audio_read_rec2ch(audio_read_strip_rec(wavfile))) > 1:
-        tmp = wavfile.split('-')
-        withext, rec = '-'.join(tmp[:-1]), tmp[-1]
-        withoutext = os.path.splitext(withext)[0]+'-'+rec
-    else:
-        withoutext = os.path.splitext(wavfile)[0]
-    return withoutext
-
 def check_config(configuration_file):
     exec(open(configuration_file).read())