You could utilize multiprocessing to help speed up process of creating 112+ rasters.
In a separate python script named WorkProcess:
"""
Description: Python script used by multithreading create rasters.
Usage: worker method
"""
# -------------------------------------------------------------------------------
# IMPORTS
import arcpy
import os
def comp_band(tifs=None):
result = {'Task': os.path.basename(tifs["outName"]), 'Error': None}
try:
arcpy.CompositeBands_management(tifs['tifPaths'], tifs['outName'])
return result
except Exception as ex:
result['Error'] = ex
return result
Save that it in the same folder that you save this main script below. Borrowing Madhi_ch parsing script above, which is great- I modified a couple lines to output the generated paths into a list of dictionaries {tifs, output name} and added the multiprocessing code:
import os
import arcpy
import multiprocessing as mp
import WorkerProcess
input_dir = r'C:\blah' # input tifs are here
output_dir = r'C:\blah\Stacked_bands' # composed tifs will be saved here
def create_tif_groups():
files = os.listdir(input_dir) # reads all files in the folder
# extracts all files that are tif and keeps the name only (drops extension)
tif_file_names = [i.split('.')[0] for i in files if i.lower().endswith('tif')]
tifGroups = []
relations = dict() # empty dictionary to save file name relations
for name in tif_file_names:
# unique_part is the unique part of the name, and we will extract that and use it
# as a key to a dictionary that holds related file names
unique_part = f"{name.split('_')[1]}_{name.split('_')[2]}" # takes 20230316165710_001, etc
if unique_part not in relations.keys(): # if the first time, creates a list and appends first file
relations[unique_part] = [name]
else: # if visited that unique part before just adds other files to the same list
relations[unique_part].append(name)
# key is the unique part of the name e.g. 20230316165710_001
# val is list of 4 file names
for key, val in relations.items():
# check point to make sure all your files have 4 files for the bands
if len(val) == 4:
pass
else:
print(f'There are only {len(val)} files for {key}') # you can also raise error and stop
# place holder variables outside of for loop scope
band_1 = None
band_2 = None
band_3 = None
band_4 = None
for file_name in val:
# I am not sure if the orders are important here
# # but just in case I manually assigned those
if file_name.endswith('_G'):
band_1 = os.path.join(input_dir, f'{file_name}.tif')
elif file_name.endswith('_NIR'):
band_2 = os.path.join(input_dir, f'{file_name}.tif')
elif file_name.endswith('_R'):
band_3 = os.path.join(input_dir, f'{file_name}.tif')
elif file_name.endswith('_RE'):
band_4 = os.path.join(input_dir, f'{file_name}.tif')
else:
print('Error! Should not land here!')
# create list of bands and its output path
tifGroups.append({'tifPaths': [band_1, band_2, band_3, band_4], 'outName': os.path.join(output_dir, f'DJI_{key}.tif')})
return tifGroups
def mp_tifs():
# get the list of tif groups and output name from create_tif_groups
tif_groups = create_tif_groups()
# set up multiprocessing, leaving one core free to do other tasks. Remove the - 1 to just let it rip on all cores
cores = mp.cpu_count() - 1
with mp.Pool(processes=cores) as pool:
jobs = []
for batchGroup in tif_groups:
# Create a job for each tif group and pass it to the worker function.
jobs.append(pool.apply_async(WorkerProcess.comp_band, (batchGroup,)))
# get results from the workers
res = [j.get() for j in jobs]
# check your results
for r in res:
if r['Error'] != None:
print(f'failed for {r["Error"]}')
if __name__ == '__main__':
mp_tifs()
