import os
import shutil
import zipfile
import math
import time
import gc
import arcpy
import arcgis
from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection

# Set arcpy environment to overwrite existing outputs
arcpy.env.overwriteOutput = True

# Increase global ArcGIS Python API HTTP socket timeout (in seconds)
arcgis.env.client_properties = {'timeout': 3600}

# --- CONFIGURATION ---
username = os.environ.get("AGOL_USER", "xxxxxxxxx")
password = os.environ.get("AGOL_PASS", "xxxxxxxxx")
target_item_id = "xxxxxxxxxxxxxx"  # Item ID for stations_en_primary / OAG stations

source_gdb = r"C:\Users\Administrator\Documents\OAG\Stations_en.gdb"
feature_class_name = "Stations_en"  # Internal name of dataset
temp_dir = r"C:\Users\Administrator\Documents\Temp_Chunks_Stations"
chunk_size = 50000  # Smaller chunks prevent AGOL backend memory/500 errors

run_timestamp = int(time.time())  # Unique execution ID to prevent AGOL item filename collisions

# --- STEP 1: PREPARE TEMP DIRECTORY & CHUNK DATASET ---
if os.path.exists(temp_dir):
    try:
        shutil.rmtree(temp_dir)
    except PermissionError:
        print("Warning: Lock detected on temp directory. Using timestamped directory.")
        temp_dir = rf"C:\Users\Administrator\Documents\Temp_Chunks_Stations_{run_timestamp}"

os.makedirs(temp_dir, exist_ok=True)

# Auto-detect exact dataset name and path inside GDB
arcpy.env.workspace = source_gdb
datasets = arcpy.ListFeatureClasses() + arcpy.ListTables()
matching = [d for d in datasets if d.lower() == feature_class_name.lower()]

if matching:
    source_path = os.path.join(source_gdb, matching[0])
    actual_name = matching[0]
else:
    raise FileNotFoundError(f"Could not find '{feature_class_name}' inside {source_gdb}. Found datasets: {datasets}")

# Describe dataset type to dynamically choose Export tool
desc = arcpy.Describe(source_path)
dataset_type = desc.datasetType  # 'FeatureClass' or 'Table'
print(f"Detected dataset type: {dataset_type} ({actual_name})")

total_count = int(arcpy.management.GetCount(source_path)[0])
num_chunks = math.ceil(total_count / chunk_size)
print(f"Total local features: {total_count}. Splitting into {num_chunks} chunk(s)...")

zip_files = []

for i in range(num_chunks):
    offset = i * chunk_size
    chunk_gdb_name = f"Chunk_{i+1}_{run_timestamp}.gdb"
    chunk_gdb_path = os.path.join(temp_dir, chunk_gdb_name)
    chunk_zip_path = os.path.join(temp_dir, f"Chunk_{i+1}_{run_timestamp}.gdb.zip")

    print(f"\nCreating Chunk {i+1}/{num_chunks}...")

    # 1. Create chunk File GDB
    arcpy.management.CreateFileGDB(temp_dir, chunk_gdb_name)

    # 2. Export subset of features/rows using OBJECTID offset
    where_clause = f"OBJECTID > {offset} AND OBJECTID <= {offset + chunk_size}"
    out_path = os.path.join(chunk_gdb_path, actual_name)

    if dataset_type == "FeatureClass":
        arcpy.conversion.ExportFeatures(source_path, out_path, where_clause=where_clause)
    else:
        arcpy.conversion.ExportTable(source_path, out_path, where_clause=where_clause)

    # 3. Compact GDB to commit changes and release schema locks
    arcpy.management.Compact(chunk_gdb_path)

    # 4. Zip the chunk File GDB
    with zipfile.ZipFile(chunk_zip_path, 'w', zipfile.ZIP_DEFLATED) as zipf:
        for root, _, files in os.walk(chunk_gdb_path):
            for file in files:
                full_path = os.path.join(root, file)
                arcname = os.path.relpath(full_path, start=os.path.dirname(chunk_gdb_path))
                zipf.write(full_path, arcname)

    zip_files.append(chunk_zip_path)

    # 5. Explicitly delete temp GDB to free up file handles
    try:
        arcpy.management.Delete(chunk_gdb_path)
    except Exception as e:
        print(f"Note: Could not immediately delete {chunk_gdb_name}: {e}")

    # Force Python garbage collection to release C++ handles
    gc.collect()

print(f"\nAll {len(zip_files)} chunks successfully created and compressed.")

# --- STEP 2: CONNECT TO AGOL, TRUNCATE, APPEND, AND CLEAN UP ---
print("\nConnecting to ArcGIS Online...")
gis = GIS("https://ised-isde-scp.maps.arcgis.com/", username, password)

target_item = gis.content.get(target_item_id)

# Access layer or table dynamically
if dataset_type == "FeatureClass" and target_item.layers:
    target_layer = target_item.layers[0]
elif target_item.tables:
    target_layer = target_item.tables[0]
else:
    target_layer = target_item.layers[0]

flc = FeatureLayerCollection.fromitem(target_item)

sync_was_enabled = False

try:
    # Disable sync temporarily if enabled to allow truncation
    if flc.properties.get('syncEnabled', False):
        print("Disabling Sync capability on layer temporarily...")
        sync_was_enabled = True
        flc.manager.update_definition({'syncEnabled': False})

    # Single truncate call before processing chunks
    print("\nTruncating target hosted table/layer...")
    target_layer.manager.truncate()

    root_folder = gis.content.folders.get()

    # Append each chunk file sequentially
    for idx, zip_path in enumerate(zip_files, start=1):
        print(f"\n[Chunk {idx}/{len(zip_files)}] Uploading {os.path.basename(zip_path)} to AGOL...")
        fgd_properties = {
            'title': f'Temp_Append_Chunk_{idx}_{run_timestamp}',
            'type': 'File Geodatabase',
            'tags': 'temporary, automation'
        }

        fgdb_job = root_folder.add(item_properties=fgd_properties, file=zip_path)
        fgdb_item = fgdb_job.result()

        # Retry loop for server-side processing errors (e.g. transient 500 errors)
        max_retries = 3
        for attempt in range(1, max_retries + 1):
            try:
                print(f"[Chunk {idx}/{len(zip_files)}] Initiating background append (Attempt {attempt}/{max_retries})...")
                
                # Run asynchronously using future=True to prevent client HTTP socket timeouts
                append_job = target_layer.append(
                    item_id=fgdb_item.id,
                    upload_format="filegdb",
                    upsert=False,
                    field_mappings=[],
                    future=True
                )

                # Poll status safely in background
                while not append_job.done():
                    print(f"   -> AGOL processing chunk {idx}/{len(zip_files)} in background... waiting 15s")
                    time.sleep(15)

                result = append_job.result()
                print(f"[Chunk {idx}/{len(zip_files)}] Append successful: {result}")
                break  # Exit retry loop on success

            except Exception as err:
                print(f"   [WARNING] Attempt {attempt} failed for Chunk {idx}: {err}")
                if attempt == max_retries:
                    raise err
                print("   Retrying in 20 seconds...")
                time.sleep(20)

        # Delete temporary GDB content item
        fgdb_item.delete()

    print("\nSUCCESS: All chunks uploaded and appended successfully!")

    # Safely check for recycle_bin attribute
    try:
        recycle_bin = getattr(gis.content, 'recycle_bin', None)
        if recycle_bin and hasattr(recycle_bin, 'empty'):
            print("Emptying AGOL Recycle Bin...")
            recycle_bin.empty()
            print("Recycle Bin emptied successfully!")
        else:
            print("Recycle Bin auto-cleanup not required/supported in this API version.")
    except Exception as rb_err:
        print(f"Note: Could not empty Recycle Bin: {rb_err}")

except Exception as e:
    print(f"\nERROR: An issue occurred during processing: {str(e)}")

finally:
    # Re-enable Sync if it was previously enabled
    if sync_was_enabled:
        print("\nRe-enabling Sync capability on layer...")
        flc.manager.update_definition({'syncEnabled': True})

    # Clean up local temp directory and zip files
    print("Cleaning up local temporary files...")
    shutil.rmtree(temp_dir, ignore_errors=True)