Select to view content in your preferred language

A python script for using a feature layer item ID for scanning the organization for which maps and apps it is used in.

746
4
05-01-2025 11:34 AM
MDB_GIS
Frequent Contributor

Not a question, but just wanted to share a script that has been helpful for us. In the past we attempted to use this incredible script created by @jcarlson to scan our organization to figure out where feature layers were being used. The script is amazing and worked for many people, but not for us. It ran, and appeared to be working, but after a few seconds, it would hang and then never complete. It took us a long time to figure out why, but we finally did. 

We had some old App Studio applications in our ArcGIS Online organization. These are classified as "Native Applications," which get lumped in with the scripts scan of all applications in the organization. Once we added a debug function to list the layer being scanned, we found that it was getting hung on those native applications every time. We had to add in a segment to skip that application type, but once we did, it worked flawlessly. Here is our full code, which will skip Native Application item types if they are causing issues for you.

"""
Script to find where a specific AGOL layer is used within an ArcGIS Online organization.
This tool connects to your AGOL org, searches specified item types for references
to the layer's URL or item ID, and summarizes the results.
"""

# ---------------------------------------------------------------------------
# Import & Warning Suppression
# ---------------------------------------------------------------------------
import warnings
from urllib3.exceptions import InsecureRequestWarning

# Suppress "Unverified HTTPS request" warnings for cleaner output
warnings.simplefilter("ignore", InsecureRequestWarning)

import os
import json
import pandas as pd
from arcgis.gis import GIS

# ---------------------------------------------------------------------------
# CONFIGURATION – set these before running
# ---------------------------------------------------------------------------
ORG_URL = "https://YOUR ORG HERE.arcgis.com/" # Your AGOL organization URL
USERNAME = "AGOL_USERNAME" # Your AGOL username
PASSWORD = "AGOL_PASSWORD" # Your AGOL password
LAYER_ID = "292q8e3ff7b94837a8f373c946bz1c5b" # Item ID of the layer to search for
ITEM_TYPES = ["Web Map", "Application"] # Which item types to scan
MAX_ITEMS = 5000 # Max items per search (-1 for no limit)


# ---------------------------------------------------------------------------

def find_usage(gis, layer_id):
"""
Search for references to a layer in specified item types.

Parameters:
gis (arcgis.gis.GIS): Authenticated GIS object.
layer_id (str): Item ID of the target layer.

Returns:
dict: Keys are item types; values are lists of items where the layer is found.
"""
# Retrieve the layer item and its service URL
layer_item = gis.content.get(layer_id)
layer_url = layer_item.url

usage = {}

for item_type in ITEM_TYPES:
# Map generic 'Application' to true web mapping apps
actual_type = (
"Web Mapping Application"
if item_type.lower() == "application"
else item_type
)

# Fetch all items of this type
items = gis.content.search(query="", item_type=actual_type, max_items=MAX_ITEMS)
print(f"[DEBUG] {actual_type}: {len(items)} items to scan")

found = []
for it in items:
# Log which item is being checked
print(f"[DEBUG] Scanning {it.id} — {it.title}")

# Skip Native Applications to avoid hangs
if it.type.startswith("Native Application"):
print(f"[DEBUG] Skipping native app {it.id}")
continue

try:
# Pull JSON payload and check for layer URL or ID
payload = json.dumps(it.get_data())
if layer_url in payload or layer_id in payload:
found.append(it)
except Exception as e:
print(f"[WARN] Error fetching {it.id}: {e}")

print(f"[DEBUG] → {len(found)} matches in {actual_type}\n")
usage[item_type] = found

return usage


def summarize(usage):
"""
Print a summary table of items that reference the layer.

Parameters:
usage (dict): Output from find_usage(), mapping item types to item lists.
"""
for itype, items in usage.items():
header = f"\n{itype}s referencing layer:\n" + "-" * (len(itype) + 12)
print(header)
if not items:
print(" None found")
continue

# Build a DataFrame for legibility
df = pd.DataFrame([{
'title': it.title,
'id': it.id,
'owner': it.owner
} for it in items])
print(df.to_string(index=False))


def main():
"""
Main execution flow: validate config, connect to AGOL, run search, and output results.
"""
# Ensure password is provided
if not PASSWORD:
print("ERROR: AGOL_PASSWORD environment variable not found. Exiting.")
return

# Inform user of connection attempt
print(f"→ Connecting to {ORG_URL} as {USERNAME}…")
try:
gis = GIS(ORG_URL, USERNAME, PASSWORD)
print(" Connected successfully.\n")
except Exception as e:
print(f" Failed to connect: {e}")
return

# Perform the usage search
print(f"→ Searching for usage of layer '{LAYER_ID}' in item types {ITEM_TYPES}…")
usage = find_usage(gis, LAYER_ID)

# Output a concise summary
summarize(usage)
print("\n→ Done.")


if __name__ == '__main__':
main()

 

 

4 Replies
MichaelCherry
Emerging Contributor

This partially worked for me! I wasn't able to get any returns for Applications but it worked for Web Maps. Completely unsure as to why. Either way the web map part was helpful

MDB_GIS
Frequent Contributor

Odd. It should output a table that looks like this in the console:

MDB_GIS_0-1759257710218.png

There are a couple of things you can try. If you change the ITEM_TYPES variable to 'ITEM_TYPES = ["*"]', it will scan ALL items in your organization. Granted, it will take a good bit longer, but it will scan everything. 

Another option is to run the script below. This will basically scan the target organization and list all items with it, as well as what their "ITEM_TYPE" is. You can then use that to fill out the "ITEM_TYPE" variable in the original script. It will also output an Excel file named "AGOL_Item_Types.csv" to your default directory. Hope this helps! Let me know if I can help in any way!

"""
AGOL Inventory with live progress: lists ITEM_TYPE for all org items,
prints each item as it is processed, and writes a CSV + summary.
"""

import os, sys, csv
from datetime import datetime
from arcgis.gis import GIS

# --- CONFIG ---
ORG_URL        = "YourORGHere.arcgis.com"
USERNAME       = "YOURUSERNAME"
PASSWORD       = "AGOL_PASSWORD"
MAX_ITEMS      = 10000
OUTPUT_CSV     = "AGOL_Item_Types_Inventory.csv"

# Progress controls
SHOW_PROGRESS  = True   # set False to silence per-item prints
PROGRESS_EVERY = 1      # print every N items (1 = every item)

CSV_FIELDS = [
    "id","title","owner","type","typeKeywords",
    "created_utc","modified_utc","url","numViews","size"
]

def utc_ms_to_iso(ms):
    try:
        return datetime.utcfromtimestamp(ms/1000.0).isoformat() + "Z"
    except Exception:
        return ""

def main():
    if not PASSWORD:
        print("ERROR: AGOL_PASSWORD environment variable not found.")
        sys.exit(1)

    print(f"→ Connecting to {ORG_URL} as {USERNAME} …")
    try:
        gis = GIS(ORG_URL, USERNAME, PASSWORD)
        print("✔ Connected.\n")
    except Exception as e:
        print(f"✖ Failed to connect: {e}")
        sys.exit(1)

    print(f"→ Searching org for ALL items (max_items={MAX_ITEMS}) …")
    items = gis.content.search(query="*", max_items=MAX_ITEMS, outside_org=False)
    total = len(items)
    print(f"✔ Retrieved {total} items.\n")
    if total == 0:
        print("No items found. Nothing to do.")
        return

    type_counts = {}

    print(f"→ Writing CSV: {OUTPUT_CSV}")
    with open(OUTPUT_CSV, "w", newline="", encoding="utf-8") as f:
        writer = csv.DictWriter(f, fieldnames=CSV_FIELDS)
        writer.writeheader()

        for idx, it in enumerate(items, start=1):
            # Safeguard per-item attributes
            t = (getattr(it, "type", "") or "").strip()
            title = getattr(it, "title", "") or ""
            it_id = getattr(it, "id", "") or ""

            # === LIVE PROGRESS PRINT ===
            if SHOW_PROGRESS and (idx % PROGRESS_EVERY == 0):
                print(f"[{idx}/{total}] {t or '(blank)'} — {title} ({it_id})", flush=True)

            # Count
            type_counts[t] = type_counts.get(t, 0) + 1

            # Row
            row = {
                "id": it_id,
                "title": title,
                "owner": getattr(it, "owner", ""),
                "type": t,
                "typeKeywords": ";".join(getattr(it, "typeKeywords", []) or []),
                "created_utc": utc_ms_to_iso(getattr(it, "created", 0) or 0),
                "modified_utc": utc_ms_to_iso(getattr(it, "modified", 0) or 0),
                "url": getattr(it, "url", ""),
                "numViews": getattr(it, "numViews", ""),
                "size": getattr(it, "size", "")
            }
            writer.writerow(row)

    # Summary
    print("\nITEM_TYPE Summary (descending count)")
    print("===================================")
    for t, c in sorted(type_counts.items(), key=lambda kv: kv[1], reverse=True):
        print(f"{t or '(blank)'}: {c}")

    print("\n✔ Done.")
    print("Tip: Set PROGRESS_EVERY=10 (or higher) if the console gets too noisy.")

if __name__ == "__main__":
    main()

 

0 Kudos
ErikLash_HiCo
Occasional Contributor

Excellent approach. Worked like a champ in my organization portal. Output exactly as you describe. Thanks!

- Erik Lash
Hawaii County GIS
0 Kudos
MDB_GIS
Frequent Contributor

Awesome! Glad it was helpful!

0 Kudos