From deaf75353ec5a7112fa5bab142b073c79ef87dee Mon Sep 17 00:00:00 2001
From: Blake Fitch <blake.fitch@tuebingen.mpg.de>
Date: Mon, 1 Jul 2024 16:04:04 +0200
Subject: [PATCH] enable proper handling of dcm2niix stdout and stderr when
additional DICOM files come for a series
---
pymods/irods_utils/irods_utils.py | 59 +++++++++++++++++--------------
1 file changed, 33 insertions(+), 26 deletions(-)
diff --git a/pymods/irods_utils/irods_utils.py b/pymods/irods_utils/irods_utils.py
index 2e5d91d..70d5a69 100644
--- a/pymods/irods_utils/irods_utils.py
+++ b/pymods/irods_utils/irods_utils.py
@@ -314,7 +314,8 @@ def streaming_transfer_file_to_object( args ):
irods_obj_handle = target_obj.open('w', checksum=True)
except Exception as ex:
logging.error("Failed open irods object for transfer. file: " + file_pathname + " to irods ipath: " + irods_ipath + " ex: " + str(ex) )
- return -1, hasher256, hasher512, 0
+ raise
+ #return -1, hasher256, hasher512, 0
try:
with open( file_pathname, 'rb' ) as input_file:
@@ -337,7 +338,8 @@ def streaming_transfer_file_to_object( args ):
+ " to irods_ipath: " + irods_ipath
+ " ex: " + str(ex) )
#return None
- rc = -1
+ #rc = -1
+ raise
irods_obj_handle.close()
@@ -365,9 +367,9 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
# a network file system twice due to blowing cache.
# So, we upload to temp, checkusmming along the way, and then check for a pre-existing irods object.
- if irods_sesh == None:
- logging.error( "irods_sesh == None" )
- return None
+ #if irods_sesh == None:
+ # logging.error( "irods_sesh == None" )
+ # return None
# Figure out if this path in the archive is occupied with pre-existing data.
existing_obj = None
@@ -381,7 +383,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
logging.debug( "OBJ_PATH_DOES_NOT_EXIST " + new_obj_ipath )
except Exception as ex:
logging.error("Failed using data_onject.get() (but not DataObjectDoesNotExist) ipath: " + new_obj_ipath + " ex: " + str(ex) + " type " + str(type(ex)) )
- return None
+ raise
try:
upload_file_size = os.path.getsize( file_pathname )
@@ -390,9 +392,10 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
raise
if upload_file_size == 0 and existing_obj != None and replace_existing and skip_on_zero_len :
- logging.warning( f"got zero len file with replace_existing -- removing current data object. ipath: {new_object_ipath}" )
+ logging.warning( f"got zero len file with replace_existing -- removing current data object. ipath: {new_obj_ipath}" )
try:
existing_obj.unlink(force=True)
+ return 0
except Exception as ex:
logging.warning( "Failed remove zero line file on replace_existing and skip_on_zero_len. ipath: {new_obj_ipath} ex: {ex} " )
raise
@@ -411,7 +414,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
+ " file_pathname: " + file_pathname
+ " tmp ipath: " + new_obj_ipath_tmp
+ " ex: " + str(axe) )
- return None
+ raise
run_args = [ file_pathname, irods_sesh, new_obj_ipath_tmp, block_size ]
@@ -425,11 +428,11 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
hash_digest512 = run_results[ 2 ]
transfer_size = run_results[ 3 ]
- if transfer_rc < 0:
+ if transfer_rc < 0: # Maybe not needed ... take out the rc part of the return
logging.error( "Falied transfer file to irods oobject."
+ " file_pathname: " + file_pathname
+ " tmp ipath: " + new_obj_ipath_tmp )
- return None
+ sys.exit( -1 )
# Example here: https://github.com/irods/python-irodsclient/blob/main/irods/test/helpers.py#L251
# ... in function: def compute_sha256_digest(file_path):
@@ -455,7 +458,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
except Exception as ex:
logging.error("Failed to create AVUS on iRODS object. ipath: " + new_obj_ipath + " ex: " + str(ex) )
logging.error("metadata AVU dict >" + str( metadata_dict ) + "<" )
- return None
+ raise
mismatch = False
@@ -492,7 +495,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
logging.error( "Failed to move an existing object out of the way so can not upload current version."
+ " path " + existing_obj.path
+ " ex: " + str( ex ) )
- return None
+ raise
logging.warn( f"Moved mismatching existing object (reason: {mismatch_reason}) to ipath: {dup_ipath}" )
# Fall through to move newly uploaded object into place.
else:
@@ -503,10 +506,11 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
logging.error( "Failed to inlink newly created, but duplicate object."
+ " new_obj_ipath_tmp: " + str( new_obj_ipath_tmp )
+ " ex: " + str( ex ) )
+ raise
logging.warn( "Existing object matched data and metadata at ipath (removed uploaded tmp object): "
+ " ipath: " + new_obj_ipath )
- return existing_obj
+ return transfer_size
# Move the tmp object to the proper object ipath.
try:
@@ -516,7 +520,7 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
+ " new_obj_ipath_tmp: " + new_obj_ipath_tmp
+ " new_obj_ipath: " + new_obj_ipath
+ " ex: " + str( ex ) )
- return None
+ raise
logging.debug( "file " + file_pathname + " in place at ipath " + new_obj_ipath )
@@ -527,19 +531,22 @@ def confirmed_put( irods_sesh, file_pathname, new_obj_ipath, metadata_dict=None,
dup_obj.unlink(force=True)
except Exception as ex:
logging.warning( "Failed remove dup when remove_existing flag set. ipath: {dup_ipath} ex: {ex} " )
+ raise
- new_obj = None
- try:
- new_obj = irods_sesh.data_objects.get( new_obj_ipath )
- except Exception as ex:
- logging.error("Failed data_onject.open(). ipath: " + new_obj_ipath + " ex: " + str(ex) )
- return None
-
- logging.debug( "Returning new object handle for: "
- + " file_pathname: " + file_pathname
- + " new_obj.ipath: " + new_obj.path )
-
- return new_obj
+ return transfer_size
+
+ #new_obj = None
+ #try:
+ # new_obj = irods_sesh.data_objects.get( new_obj_ipath )
+ #except Exception as ex:
+ # logging.error("Failed data_onject.open(). ipath: " + new_obj_ipath + " ex: " + str(ex) )
+ #
+ #
+ #logging.debug( "Returning new object handle for: "
+ # + " file_pathname: " + file_pathname
+ # + " new_obj.ipath: " + new_obj.path )
+ #
+ #return new_obj
# Bring in the experiment validation code
--
GitLab