Scripting batch clip in Arcgis Pro

ShayneGalloway · ‎10-30-2024

Let me say at the start that I am not a coder, so feel free to assume I'm flying completely blind here.

I've built this handy little slicer that cuts shapefiles down to region and then catchment extents, but it's creating all these extra layers and I can't suss why. The purpose is to automate batch clipping sets of shapefiles (i.e. All of DOCs available data) down to a regional level and then a subextent.

It prompts for a national set folder (a folder of shapefile folders), a regional extent (should work with .lyrx or shape), and a catchment extent (same). Lastly it asks for the target folder for where it should store the resuting clipped files.

Its meant to delete any empty layerfile and for both extents and report on which were deleted.

Then its meant to create group layer files and name them.

Then it should move the clipped layers to their respective group folders.

It does all that, but its also creating or leaving a lot of extras. Typically its a full set of the national shapes, and the regional and catchment clippled layers.

First reason is probably that I built it with ChatGPT iterations.

Any thoughts on what I need to fix and how I might go about it?

The Script:

import arcpy
import os
import tempfile
import tkinter as tk
from tkinter import filedialog

# Create a function to select a folder with a file dialog
def select_folder(title):
    root = tk.Tk()
    root.withdraw()
    folder_path = filedialog.askdirectory(title=title)
    return folder_path

# Create a function to select a layer file with a file dialog
def select_layer_file(title):
    root = tk.Tk()
    root.withdraw()
    layer_path = filedialog.askopenfilename(title=title, filetypes=[("Layer Files", "*.lyrx")])
    return layer_path

# Get user inputs for folder, regional boundary, and catchment boundary
national_set_folder = select_folder("Select the folder containing the national shapefile datasets")
regional_boundary_layer = select_layer_file("Select the regional boundary (.lyrx) file")
catchment_boundary_layer = select_layer_file("Select the catchment boundary (.lyrx) file")

# Create a new folder for the output datasets
target_parent_folder = select_folder("Select the folder where the new region and catchment folders should be created")
region_catchment_folder_name = os.path.splitext(os.path.basename(regional_boundary_layer))[0]
target_folder = os.path.join(target_parent_folder, region_catchment_folder_name)
os.makedirs(target_folder, exist_ok=True)

# Load regional and catchment boundaries as feature layers
arcpy.management.MakeFeatureLayer(regional_boundary_layer, "regional_boundary_layer")
arcpy.management.MakeFeatureLayer(catchment_boundary_layer, "catchment_boundary_layer")

# Load the ArcGIS project and get the active map
doc = arcpy.mp.ArcGISProject("CURRENT")
map_obj = doc.activeMap

# Remove any existing group layers with the same name
for layer in map_obj.listLayers():
    if layer.isGroupLayer:
        if layer.name == "Region Group Layer" or layer.name == "Catchment Group Layer":
            map_obj.removeLayer(layer)

# Create new group layers for Region and Catchment
region_group_layer = map_obj.createGroupLayer("Region Group Layer")
catchment_group_layer = map_obj.createGroupLayer("Catchment Group Layer")

# Iterate over each folder in the national dataset directory
for root, dirs, files in os.walk(national_set_folder):
    shapefiles = [file for file in files if file.endswith(".shp")]
    if shapefiles:
        # Use the first shapefile in the folder as the base name for output
        base_name = os.path.splitext(shapefiles[0])[0]

        # Create a subfolder within the target folder for storing outputs from this subfolder
        output_subfolder = os.path.join(target_folder, os.path.basename(root))
        os.makedirs(output_subfolder, exist_ok=True)

        # If there is more than one shapefile, merge them, otherwise use the single shapefile
        if len(shapefiles) > 1:
            merged_shapefile_path = os.path.join(tempfile.gettempdir(), f"{base_name}_merged_temp.shp")
            shapefile_paths = [os.path.join(root, file) for file in shapefiles]
            arcpy.management.Merge(shapefile_paths, merged_shapefile_path)
        else:
            merged_shapefile_path = os.path.join(root, shapefiles[0])

        # Repair the geometry of the merged shapefile to avoid invalid topology issues
        arcpy.management.RepairGeometry(merged_shapefile_path)

        try:
            # Clip to regional boundary and save to new folder
            region_clip_output = os.path.join(output_subfolder, f"{base_name}_Region.shp")
            arcpy.analysis.Clip(merged_shapefile_path, "regional_boundary_layer", region_clip_output)

            # Check if the clipped shapefile is empty
            if int(arcpy.management.GetCount(region_clip_output).getOutput(0)) == 0:
                arcpy.management.Delete(region_clip_output)
            else:
                # Add the region layer directly to the group layer
                layer_name = f"{base_name}_Region"
                region_layer = arcpy.management.MakeFeatureLayer(region_clip_output, layer_name)[0]
                map_obj.addLayerToGroup(region_group_layer, region_layer)

                # Clip to catchment boundary and save to new folder
                catchment_clip_output = os.path.join(output_subfolder, f"{base_name}_Catchment.shp")
                arcpy.analysis.Clip(region_clip_output, "catchment_boundary_layer", catchment_clip_output)

                # Check if the catchment clipped shapefile is empty
                if int(arcpy.management.GetCount(catchment_clip_output).getOutput(0)) == 0:
                    arcpy.management.Delete(catchment_clip_output)
                else:
                    # Add the catchment layer directly to the group layer
                    layer_name = f"{base_name}_Catchment"
                    catchment_layer = arcpy.management.MakeFeatureLayer(catchment_clip_output, layer_name)[0]
                    map_obj.addLayerToGroup(catchment_group_layer, catchment_layer)
        except arcpy.ExecuteError as e:
            print(f"Error processing merged shapefile in folder {root}: {e}")
        finally:
            # Delete the temporary merged shapefile if it was created
            if len(shapefiles) > 1 and arcpy.Exists(merged_shapefile_path):
                arcpy.management.Delete(merged_shapefile_path)

print("\nProcessing complete. Clipped shapefiles have been saved in the specified folder and loaded into the map under their respective group layers.")<div> </div>

AlfredBaldenweck · ‎10-30-2024

Do they still have all those extras if you take out the "addLayerToGroup"?

I think last time I was messing around with layers in the context of a map I had to manually remove the layer after adding it.

for fc in arcpy.ListFeatureClasses():
    fcLay = mp.addDataFromPath(os.path.join(out_ds,fc))
    mp.addLayerToGroup(outGroup, fcLay)
    mp.removeLayer(fcLay)

ShayneGalloway · ‎10-30-2024

Hi Alfred, the only place removeLayer appears is at:

# Remove any existing group layers with the same name
for layer in map_obj.listLayers():
    if layer.isGroupLayer:
        if layer.name == "Region Group Layer" or layer.name == "Catchment Group Layer":
            map_obj.removeLayer(layer)

I only use this in an otherwise empty project. I wonder if I could add code to remove any layers not included in the group layers at the end?

JoshuaSharp-Heward · ‎10-30-2024

Hi Shayne,

I think by default most geoprocessing tools add the result to the map automatically, but you should be able to prevent that by adding the following line at the top of your script:

arcpy.env.addOutputsToMap = False

which should stop those intermediary layers from being added in the first place. Hope this helps!

ShayneGalloway · ‎10-30-2024

After a couple of iterations with removeLayer and tadaaa! It worked! Thanks for the help!!

import arcpy
import os
import tempfile
import tkinter as tk
from tkinter import filedialog

# Create a function to select a folder with a file dialog
def select_folder(title):
    root = tk.Tk()
    root.withdraw()
    folder_path = filedialog.askdirectory(title=title)
    return folder_path

# Create a function to select a layer file with a file dialog
def select_layer_file(title):
    root = tk.Tk()
    root.withdraw()
    layer_path = filedialog.askopenfilename(title=title, filetypes=[("Layer Files", "*.lyrx")])
    return layer_path

# Get user inputs for folder, regional boundary, and catchment boundary
national_set_folder = select_folder("Select the folder containing the national shapefile datasets")
regional_boundary_layer = select_layer_file("Select the regional boundary (.lyrx) file")
catchment_boundary_layer = select_layer_file("Select the catchment boundary (.lyrx) file")

# Create a new folder for the output datasets
target_parent_folder = select_folder("Select the folder where the new region and catchment folders should be created")
region_catchment_folder_name = os.path.splitext(os.path.basename(regional_boundary_layer))[0]
target_folder = os.path.join(target_parent_folder, region_catchment_folder_name)
os.makedirs(target_folder, exist_ok=True)

# Load regional and catchment boundaries as feature layers
arcpy.management.MakeFeatureLayer(regional_boundary_layer, "regional_boundary_layer")
arcpy.management.MakeFeatureLayer(catchment_boundary_layer, "catchment_boundary_layer")

# Load the ArcGIS project and get the active map
doc = arcpy.mp.ArcGISProject("CURRENT")
map_obj = doc.activeMap

# Remove any existing group layers with the same name
for layer in map_obj.listLayers():
    if layer.isGroupLayer:
        if layer.name == "Region Group Layer" or layer.name == "Catchment Group Layer":
            map_obj.removeLayer(layer)

# Create new group layers for Region and Catchment
region_group_layer = map_obj.createGroupLayer("Region Group Layer")
catchment_group_layer = map_obj.createGroupLayer("Catchment Group Layer")

# Verify that the group layers were created successfully
if not region_group_layer or not catchment_group_layer:
    print("Error: Unable to create Region Group Layer or Catchment Group Layer.")
else:
    # Iterate over each folder in the national dataset directory
    for root, dirs, files in os.walk(national_set_folder):
        shapefiles = [file for file in files if file.endswith(".shp")]
        if shapefiles:
            # Use the first shapefile in the folder as the base name for output
            base_name = os.path.splitext(shapefiles[0])[0]

            # Create a subfolder within the target folder for storing outputs from this subfolder
            output_subfolder = os.path.join(target_folder, os.path.basename(root))
            os.makedirs(output_subfolder, exist_ok=True)

            # If there is more than one shapefile, merge them, otherwise use the single shapefile
            if len(shapefiles) > 1:
                merged_shapefile_path = os.path.join(tempfile.gettempdir(), f"{base_name}_merged_temp.shp")
                shapefile_paths = [os.path.join(root, file) for file in shapefiles]
                arcpy.management.Merge(shapefile_paths, merged_shapefile_path)
            else:
                merged_shapefile_path = os.path.join(root, shapefiles[0])

            # Repair the geometry of the merged shapefile to avoid invalid topology issues
            arcpy.management.RepairGeometry(merged_shapefile_path)

            try:
                # Clip to regional boundary and save to new folder
                region_clip_output = os.path.join(output_subfolder, f"{base_name}_Region.shp")
                arcpy.analysis.Clip(merged_shapefile_path, "regional_boundary_layer", region_clip_output)

                # Check if the clipped shapefile is empty
                if int(arcpy.management.GetCount(region_clip_output).getOutput(0)) == 0:
                    arcpy.management.Delete(region_clip_output)
                else:
                    # Add the region layer directly to the group layer
                    layer_name = f"{base_name}_Region"
                    region_layer = arcpy.management.MakeFeatureLayer(region_clip_output, layer_name)[0]
                    map_obj.addLayerToGroup(region_group_layer, region_layer)

                    # Clip to catchment boundary and save to new folder
                    catchment_clip_output = os.path.join(output_subfolder, f"{base_name}_Catchment.shp")
                    arcpy.analysis.Clip(region_clip_output, "catchment_boundary_layer", catchment_clip_output)

                    # Check if the catchment clipped shapefile is empty
                    if int(arcpy.management.GetCount(catchment_clip_output).getOutput(0)) == 0:
                        arcpy.management.Delete(catchment_clip_output)
                    else:
                        # Add the catchment layer directly to the group layer
                        layer_name = f"{base_name}_Catchment"
                        catchment_layer = arcpy.management.MakeFeatureLayer(catchment_clip_output, layer_name)[0]
                        map_obj.addLayerToGroup(catchment_group_layer, catchment_layer)
            except arcpy.ExecuteError as e:
                print(f"Error processing merged shapefile in folder {root}: {e}")
            finally:
                # Delete the temporary merged shapefile if it was created
                if len(shapefiles) > 1 and arcpy.Exists(merged_shapefile_path):
                    arcpy.management.Delete(merged_shapefile_path)

    # Clean up layers outside of "Region Group Layer" and "Catchment Group Layer" (retain basemap layers)
    for layer in map_obj.listLayers():
        # Skip the group layers, layers within these groups, and basemap layers
        if (layer != region_group_layer and
            layer != catchment_group_layer and
            not any(layer in group.listLayers() for group in [region_group_layer, catchment_group_layer]) and
            not layer.isBasemapLayer):
            # Remove layers that are not part of the group layers or the basemap
            map_obj.removeLayer(layer)

    print("\nProcessing complete. Clipped shapefiles have been saved in the specified folder and loaded into the map under their respective group layers.")