I have a script to process a bunch of polygons. The script uses nested ProcessPoolExecutors. After a certain point, I get this error:
Exception has occurred: RuntimeError (note: full exception trace is shown but execution is paused at: <module>)
The Product License has not been initialized.
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\_base.py", line 14, in <module>
import arcgisscripting
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\geoprocessing\__init__.py", line 14, in <module>
from ._base import *
File "C:\Program Files\ArcGIS\Pro\Resources\ArcPy\arcpy\__init__.py", line 77, in <module>
from arcpy.geoprocessing import gp
File "C:\Users\redacted\example.py", line 1, in <module>
import arcpy
File "<string>", line 1, in <module> (Current frame)
RuntimeError: The Product License has not been initialized.
Things I have tried yet still run into this issue:
- Using the default ArcGIS pro python environment or using a cloned environment.
- If I have ArcGIS pro currently open and logged in as well as when it is closed.
- Using my named user license or checking it out for offline use.
- Having the child process pool in a different python file and importing it.
The only "solution" I've found is to have the total number of processes (max_concurrency * max_children) stay below the mid 20's even on machines with enough cores and ram.
Here is an example script:
import arcpy
import itertools
import uuid
import time
import concurrent.futures
import random
arcpy.env.overwriteOutput = True
arcpy.env.workspace = Point towards your Geodatabase
output = "test_number"
max_concurrency = 20
max_children = 20
def process_number(number):
time.sleep(random.random() * 3)
return number
def process_numbers_multi(number, num_range):
print(f"PROCESSING STARTED ON NUMBER: {number}")
nums = list(range(number, number+num_range))
input_handler = iter(nums)
num_results = []
with concurrent.futures.ProcessPoolExecutor(max_workers=max_children) as child_executor:
futures = {
child_executor.submit(process_number, part): part
for part in itertools.islice(input_handler, max_children)
}
while futures:
done, _ = concurrent.futures.wait(
futures, return_when=concurrent.futures.FIRST_COMPLETED
)
for fut in done:
original_input = futures.pop(fut)
try:
results = []
results = fut.result()
except Exception as exc:
print(f"{original_input} generated an exception: {exc}")
else:
num_results.append(results)
if any(input_handler):
for part in itertools.islice(input_handler, len(done)):
fut = child_executor.submit(process_number, part)
futures[fut] = (part)
return num_results
def create_feature_class():
# Create transect FC. Add Fields.
trans_fc = arcpy.management.CreateFeatureclass(out_path=arcpy.env.workspace,
out_name=output)
flds = [("NUMBER_GUID", "GUID"), ("NUMBER", "DOUBLE")]
for fld_name, fld_type in flds:
arcpy.management.AddField(in_table=trans_fc, field_name=fld_name,
field_type=fld_type, field_length=1)
print("CREATED TRANSECT FC")
return trans_fc
def write_to_db(number, trans_fc):
flds = ["NUMBER_GUID", "NUMBER"]
print(f"WRITING NUMBER")
for rows in number:
with arcpy.da.InsertCursor(trans_fc, flds) as icurs:
icurs.insertRow([rows[0],rows[1]])
def main():
startTime = time.time()
print("PROCESS STARTING")
trans_fc = create_feature_class()
num_list = list(range(0,200))
input_handler = iter(num_list)
num_range = 10
full_results = []
"""tmp = process_numbers_multi(num_list[0], num_range)
for i in tmp:
full_results.append([uuid.uuid4(), i])
write_to_db(full_results, trans_fc)"""
with concurrent.futures.ProcessPoolExecutor(max_workers=max_concurrency ) as executor:
futures = {
executor.submit(process_numbers_multi, part, num_range): part
for part in itertools.islice(input_handler, max_concurrency )
}
while futures:
done, _ = concurrent.futures.wait(
futures, return_when=concurrent.futures.FIRST_COMPLETED
)
for fut in done:
original_input = futures.pop(fut)
try:
results = []
results = fut.result()
except Exception as exc:
print(f"{original_input} generated an exception: {exc}")
else:
for x in results:
full_results.append([uuid.uuid4(), x])
write_to_db(full_results, trans_fc)
full_results = []
if any(input_handler):
for part in itertools.islice(input_handler, len(done)):
fut = executor.submit(process_numbers_multi, part, num_range)
futures[fut] = (part)
endTime = time.time()
print("PROCESS COMPLETE")
print(f"Elapsed Time: {endTime - startTime}")
return True
if __name__ == '__main__':
main()
If you change the max_concurrency and max_children values it will be less likely to occur, but even when max_concurrency*max_children <= os.cpu_count() I've had this error occur. It just seems more reliable to trigger with a larger number of processes.
