src/hdf5_writer.py

@@ -191,18 +191,23 @@ def create_hdf5_file_from_filesystem_path(path_to_input_directory: str,
                     offset = 1
                 tmp_list = group_name.split('/')
                 if len(tmp_list) > offset+1:
-                    group_name = '/'.join([tmp_list[i] for i in range(offset+1)])                
-
-                # Create group called "group_name". Hierarchy of nested groups can be implicitly defined by the forward slashes
-                if not group_name in h5file.keys():                    
-                    h5file.create_group(group_name)
-                    h5file[group_name].attrs['creation_date'] = utils.created_at().encode('utf-8')
-                    #h5file[group_name].attrs.create(name='filtered_file_list',data=convert_string_to_bytes(filtered_filename_list))
-                    #h5file[group_name].attrs.create(name='file_list',data=convert_string_to_bytes(filenames_list))
-                #else:                           
-                    #print(group_name,' was already created.') 
-                    instFoldermsgStart = f'Starting data transfer from instFolder: {group_name}'
-                    print(instFoldermsgStart)
+                    group_name = '/'.join([tmp_list[i] for i in range(offset+1)])   
+
+                try:
+                    # Create group called "group_name". Hierarchy of nested groups can be implicitly defined by the forward slashes
+                    if not group_name in h5file.keys():                    
+                        h5file.create_group(group_name)
+                        h5file[group_name].attrs['creation_date'] = utils.created_at().encode('utf-8')
+                        #h5file[group_name].attrs.create(name='filtered_file_list',data=convert_string_to_bytes(filtered_filename_list))
+                        #h5file[group_name].attrs.create(name='file_list',data=convert_string_to_bytes(filenames_list))
+                    #else:                           
+                        #print(group_name,' was already created.') 
+                        instFoldermsgStart = f'Starting data transfer from instFolder: {group_name}'
+                        print(instFoldermsgStart)
+
+                except Exception as inst: 
+                    stdout = inst
+                    logging.error('Failed to create group %s into HDF5: %s', group_name, inst)
 
                 for filenumber, filename in enumerate(filtered_filenames_list):
                     

utils/g5505_utils.py

@@ -285,6 +285,11 @@ def copy_directory_with_contraints(input_dir_path, output_dir_path,
     select_file_keywords = select_file_keywords or []
     allowed_file_extensions = allowed_file_extensions or []
 
+    # Normalize paths and keywords to be consistently specified with os specific separator 
+    input_dir_path = os.path.normpath(input_dir_path)
+    output_dir_path = os.path.normpath(output_dir_path)
+    select_dir_keywords = [keyword.replace('/',os.sep) for keyword in select_dir_keywords]
+
     date = created_at('%Y_%m').replace(":", "-")
     log_dir='logs/'
     setup_logging(log_dir, f"copy_directory_with_contraints_{date}.log")
@@ -312,6 +317,10 @@ def copy_directory_with_contraints(input_dir_path, output_dir_path,
     for subpath in paths:
 
         for dirpath, _, filenames in os.walk(subpath,topdown=False):
+            
+            # Ensure composite keywords e.g., <keyword>/<keyword> are contained in the path
+            if not any([keyword in dirpath for keyword in select_dir_keywords]):
+                continue
 
             # Reduce filenames to those that are admissible
             admissible_filenames = [