feat: add --move option to ltfs_ordered_copy

configure.ac

@@ -336,12 +336,16 @@ fi
 
 dnl
 dnl Check for ICU
+dnl Prefer icu-config for legacy distros; fall back to pkg-config icu-uc/icu-i18n
+dnl (icu-config was removed in ICU 63+ and is absent on modern Debian/Ubuntu).
 dnl
 ICU_MODULE_CFLAGS="`icu-config --cppflags 2> /dev/null`";
 ICU_MODULE_LIBS="`icu-config --ldflags 2> /dev/null`";
 if test -z "$ICU_MODULE_LIBS"
 then
-    PKG_CHECK_MODULES([ICU_MODULE], [icu >= 0.21])
+    PKG_CHECK_MODULES([ICU_MODULE], [icu-uc >= 0.21 icu-i18n >= 0.21],
+        [],
+        [PKG_CHECK_MODULES([ICU_MODULE], [icu >= 0.21])])
 fi
 
 AC_MSG_CHECKING([use latest ICU])
@@ -356,6 +360,10 @@ if test "x${icu_6x}" = "xyes"
 then
     AC_MSG_CHECKING(for ICU version)
     ICU_MODULE_VERSION="`icu-config --version 2> /dev/null`";
+    if test -z "$ICU_MODULE_VERSION"
+    then
+        ICU_MODULE_VERSION="`$PKG_CONFIG --modversion icu-uc 2> /dev/null`";
+    fi
     if test "${ICU_MODULE_VERSION%%.*}" -ge "60"
     then
         AM_EXTRA_CPPFLAGS="${AM_EXTRA_CPPFLAGS} -D ICU6x"

man/ltfs_ordered_copy.1

@@ -40,6 +40,9 @@ Configure verbosity of logger. VERBOSE shall be 0-6. (Default: 4)
 .TP
 \fB-q, --quiet\fR
 No message outout
+.TP
+\fB--move\fR
+Move files instead of copying them. Each source file is removed only after it has been successfully copied to the destination, so the LTFS tape read order is preserved and no data is lost if a copy fails. This option implies \fB-p\fR so that file attributes are preserved like a normal move. After a recursive move the emptied source directories are pruned; source directories that still contain files (for example because a copy failed) are left untouched.
 .SH "COMMAND EXAMPLES"
 .PP
 This section shows various command examples.

man/sgml/ltfs_ordered_copy.sgml

@@ -104,6 +104,12 @@
             <para>No message outout</para>
           </listitem>
         </varlistentry>
+        <varlistentry>
+          <term><option>--move</option></term>
+          <listitem>
+            <para>Move files instead of copying them. Each source file is removed only after it has been successfully copied to the destination, so the LTFS tape read order is preserved and no data is lost if a copy fails. This option implies <option>-p</option> so that file attributes are preserved like a normal move. After a recursive move the emptied source directories are pruned; source directories that still contain files (for example because a copy failed) are left untouched.</para>
+          </listitem>
+        </varlistentry>
       </variablelist>
     </refsect1>
 

src/utils/ltfs_ordered_copy

@@ -46,12 +46,13 @@ from collections import deque
 
 class CopyItem:
     """"""
-    def __init__(self, src, dst, vea_pre, cp_attr, cp_xattr, logger): #initialization
+    def __init__(self, src, dst, vea_pre, cp_attr, cp_xattr, logger, move=False): #initialization
         self.src     = src
         self.dst     = dst
         self.vea_pre = vea_pre
         self.cp_attr = cp_attr
         self.cp_xattr = cp_xattr
+        self.move    = move
         self.vuuid   = ''
         self.part    = ''
         self.start   = -1
@@ -78,6 +79,7 @@ class CopyItem:
         return (self.vuuid, self.part, self.start)
 
     def run(self):
+        action = 'move' if self.move else 'copy'
         try:
             if len(self.vuuid):
                 logger.debug('"{0}" ({2}) -> "{1}"'.format(self.src, self.dst, str(self.start)))
@@ -98,9 +100,19 @@ class CopyItem:
             else: #Only copy data
                 shutil.copy(self.src, self.dst)
         except Exception as e:
-            self.logger.error('Failed to copy "{0}" to "{1}": {2}'.format(self.src, self.dst, str(str(e))))
+            self.logger.error('Failed to {3} "{0}" to "{1}": {2}'.format(self.src, self.dst, str(str(e)), action))
             return False
 
+        if self.move:
+            # The data is safely on the destination now; remove the source to
+            # complete the move. Deletion happens only after a successful copy so
+            # that nothing is lost if the copy above failed.
+            try:
+                os.remove(self.src)
+            except Exception as e:
+                self.logger.error('Copied "{0}" to "{1}" but failed to remove source: {2}'.format(self.src, self.dst, str(e)))
+                return False
+
         return True
 
     def __repr__(self):
@@ -151,7 +163,7 @@ class CopyQueue:
 
             self.items = self.items + 1
 
-    def walk_dir(self, source, dest, cp_attr, cp_xattr=False):
+    def walk_dir(self, source, dest, cp_attr, cp_xattr=False, move=False):
         (source_root, t) = os.path.split(source)
         prefix_len = len(source_root)
         dst = dest + "/" + t
@@ -171,7 +183,7 @@ class CopyQueue:
             for f in sorted(files) if self.sort_files else files:
                 self.logger.log(NOTSET + 1, 'Creating a copy item for file {}'.format(f))
                 c = CopyItem(os.path.join(root, f), os.path.join(dst, f), VEA_PREFIX,
-                             cp_attr, cp_xattr, logger)
+                             cp_attr, cp_xattr, logger, move)
                 self.add_copy_item(c)
 
         for d in walk_dirs:
@@ -280,13 +292,18 @@ parser.add_argument('-v', help='Verbose output. Set VERBOSE level 5', action='st
 parser.add_argument('--verbose', help='Configure verbosity of logger. VERBOSE shall be 0-6. default is 4', default = str(logger_info))
 parser.add_argument('-q','--quiet', help='No message output', action='store_true')
 parser.add_argument('--sort-files', help='Sort the file list before copying', action='store_true')
+parser.add_argument('--move', help='Move files instead of copying. Each source file is removed only after it has been successfully copied to the destination, preserving tape read order. Implies -p so file attributes are preserved like a normal move. Empty source directories are pruned after a recursive move.', action='store_true')
 
 args=parser.parse_args()
 
 if args.all:
     args.p = True
     args.recursive = True
 
+if args.move:
+    # A move should preserve file metadata like a regular "mv".
+    args.p = True
+
 logger = getLogger(__name__)
 basicConfig(format = '')
 if args.quiet:
@@ -336,6 +353,9 @@ if args.DEST == None:
     logger.error('No destination is specified')
     exit(2)
 
+if args.keep_tree is None:
+    args.keep_tree = ''
+
 # Special case:
 #  Copy source is only one file
 if args.recursive == False and len(args.SOURCE) == 1:
@@ -348,7 +368,12 @@ if args.recursive == False and len(args.SOURCE) == 1:
                 (new_d, t) = os.path.split(dst)
                 if not os.path.exists(new_d):
                     os.makedirs(new_d)
-            shutil.copy(args.SOURCE[0], args.DEST)
+            if args.p:
+                shutil.copy2(args.SOURCE[0], args.DEST)
+            else:
+                shutil.copy(args.SOURCE[0], args.DEST)
+            if args.move:
+                os.remove(args.SOURCE[0])
         except Exception as e:
             logger.error(str(e))
             exit(1)
@@ -387,9 +412,6 @@ if len(args.SOURCE) == 0:
         args.SOURCE.append(line.rstrip('\r\n'))
     logger.log(NOTSET + 1, 'Source: {}'.format(args.SOURCE))
 
-if args.keep_tree is None:
-    args.keep_tree = ''
-
 # Create the list of copy item
 copyq = CopyQueue(logger, args.sort_files)
 for s in args.SOURCE:
@@ -402,7 +424,7 @@ for s in args.SOURCE:
             (new_d, t) = os.path.split(dst)
             if not os.path.exists(new_d):
                 os.makedirs(new_d)
-        c = CopyItem(s, dst, VEA_PREFIX, args.p, args.all, logger)
+        c = CopyItem(s, dst, VEA_PREFIX, args.p, args.all, logger, args.move)
         copyq.add_copy_item(c)
     else:
         logger.log(NOTSET + 1, 'Creating copy item for directory {}'.format(s))
@@ -414,7 +436,7 @@ for s in args.SOURCE:
                 if not os.path.exists(new_d):
                    os.makedirs(new_d)
                 dst = new_d
-            copyq.walk_dir(s, dst, args.p, args.all)
+            copyq.walk_dir(s, dst, args.p, args.all, args.move)
         else:
             logger.warning("omitting directory '{0}'".format(s))
 
@@ -471,10 +493,24 @@ while tape != None:
     (tape_key, tape) = copyq.pop_tape()
     prog_tape.finish()
 
+# In move mode the source files have been removed as they were copied. Prune the
+# now-empty source directory trees bottom-up. Directories that still hold files
+# (e.g. because a copy failed, or a non-recursive directory was skipped) are left
+# untouched, so no data is ever discarded.
+if args.move:
+    for s in args.SOURCE:
+        if os.path.isdir(s):
+            for dirpath, dirnames, filenames in os.walk(s, topdown=False):
+                try:
+                    os.rmdir(dirpath)
+                except OSError:
+                    pass # Not empty or already removed
+
 # Return code
+done = 'Moved' if args.move else 'Copied'
 if fail:
-    logger.info("Copied {} files, Failed {} files.".format(success, fail))
+    logger.info("{} {} files, Failed {} files.".format(done, success, fail))
     exit(1)
 else:
-    logger.info("Copied {} files.".format(success, fail))
+    logger.info("{} {} files.".format(done, success))
     exit(0)