enable removing old confirmed_put files (objects) when skip_on_zero_len and replace_existing

pymods/irods_utils/irods_utils.py

@@ -350,12 +350,13 @@ def streaming_transfer_file_to_object( args ):
     return rc, hash_digest256, hash_digest512, transfer_size
 
 # Returns None or the data_object irods ref.
-def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None, datatype=None, block_size=(2**28), replace_existing=None ):
+def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None, datatype=None, block_size=(2**28), replace_existing=False, skip_on_zero_len=False ):
     logging.debug( "file_pathname: "  + file_pathname
                  + " new_obj_ipath: " + new_obj_ipath
                  + " datatype: "      + datatype
                  + " block_size: "    + str( block_size )
-                 + " replace_existing " + str( replace_existing) )
+                 + " replace_existing " + str( replace_existing),
+                 + " skip_on_zero_len " + str( skip_on_zero_len ) )
 
     # NOTE: this routine uploades the file as a temporary iRODS object.
     # Common sense suggests checking if the objct already exsists, a rare condition first.
@@ -368,6 +369,34 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
         logging.error( "irods_sesh == None" )
         return None
 
+    # Figure out if this path in the archive is occupied with pre-existing data.
+    existing_obj = None
+    try:
+        options = {kw.VERIFY_CHKSUM_KW: ''}
+        existing_obj = irods_sesh.data_objects.get( new_obj_ipath, **options )
+        logging.debug( "Object aleady exists at ipath: " + new_obj_ipath )
+    except irods_ex.DataObjectDoesNotExist:
+        logging.debug( "DataObjectDoesNotExist " + new_obj_ipath )
+    except irods_ex.OBJ_PATH_DOES_NOT_EXIST:
+        logging.debug( "OBJ_PATH_DOES_NOT_EXIST " + new_obj_ipath )
+    except Exception as ex:
+        logging.error("Failed using data_onject.get() (but not DataObjectDoesNotExist) ipath: " + new_obj_ipath + " ex: " + str(ex) + " type " + str(type(ex)) )
+        return None
+
+    try:
+        upload_file_size = os.path.getsize( file_pathname )
+    except Exception as ex:
+        logging.error( f"FATAL: Failed to get size of file to upload. file_pathname: {file_pathname} ex: {ex}" )
+        raise
+
+    if upload_file_size == 0 and existing_obj != None and replace_existing and skip_on_zero_len :
+        logging.warning( f"got zero len file with replace_existing -- removing current data object. ipath: {new_object_ipath}" )
+        try:
+            existing_obj.unlink(force=True)
+        except Exception as ex:
+            logging.warning( "Failed remove zero line file on replace_existing and skip_on_zero_len. ipath: {new_obj_ipath}  ex: {ex} " )
+            raise
+
     # Make a timestamped partial tmp file name which, if the upload somehow fails here, will be left behind
     # Hopefully this does not happen much, but we have seen at least on case so far.
     # Cleanup will need to be done elsewhere.
@@ -428,20 +457,6 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
         logging.error("metadata AVU dict >" + str( metadata_dict ) + "<" )
         return None
 
-    existing_obj = None
-
-    try:
-        options = {kw.VERIFY_CHKSUM_KW: ''}
-        existing_obj = irods_sesh.data_objects.get( new_obj_ipath, **options )
-        logging.debug( "Object aleady exists at ipath: " + new_obj_ipath )
-    except irods_ex.DataObjectDoesNotExist:
-        logging.debug( "DataObjectDoesNotExist " + new_obj_ipath )
-    except irods_ex.OBJ_PATH_DOES_NOT_EXIST:
-        logging.debug( "OBJ_PATH_DOES_NOT_EXIST " + new_obj_ipath )
-    except Exception as ex:
-        logging.error("Failed using data_onject.get() (but not DataObjectDoesNotExist) ipath: " + new_obj_ipath + " ex: " + str(ex) + " type " + str(type(ex)) )
-        return None
-
     mismatch = False
 
     if existing_obj != None:
@@ -512,7 +527,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
             dup_obj.unlink(force=True)
         except Exception as ex:
             logging.warning( "Failed remove dup when remove_existing flag set. ipath: {dup_ipath}  ex: {ex} " )
-          
+
     new_obj = None
     try:
         new_obj = irods_sesh.data_objects.get( new_obj_ipath )