Exporting AGOL data to CVS Daily?

JordanMiller4

What's the best way to automate exporting AGOL feature data down to a local CVS file daily? 

JordanMiller4

 

from time import strftime
print(strftime('%c'))
import zipfile, time, os
# importing the pandas library
import pandas as pd
from arcgis.gis import GIS

layer_Dict = ["v411 Routes"]
itemid_Dict =['ab15af29418f49*************']

from datetime import date
todayDate = str(date.today())


def current_milli_time():
    return round(time.time() * 1000)


def my_agol_export_function():
    milliTime = current_milli_time()
    unzipExport = r'C:\Users\jnmiller\Downloads\v411_Routes%s_%s.zip' %(todayDate, milliTime)
    outputAGOL = r'C:\Users\jnmiller\Downloads'
    outNameAGOL = "%s_%s" %(todayDate, milliTime)

    for i, fc in enumerate(layer_Dict):
        itemid = itemid_Dict[i]
        fc_lyr = fc
        print(itemid)
        #print(FC_ID)
        outName = '{}{}'.format(fc_lyr,outNameAGOL)
        gis = GIS('https://****.maps.arcgis.com/', 'username', 'password')
        dataitem = gis.content.get(itemid)
        dataitem.export(outName, 'CSV', parameters=None, wait=True)
        myexport = gis.content.search(outName, item_type='CSV')
        fgdb = gis.content.get(myexport[0].itemid)
        fgdb.download(save_path=outputAGOL)
        fgdb.delete()
        zip = zipfile.ZipFile(unzipExport)
        zip.extractall(r'C:\Users\jnmiller\Downloads')
        #time.sleep(2)
        f=pd.read_csv(r'C:\Users\jnmiller\Downloads\Join_Features_to_Monitoring_Routes_v411_0.csv')
        keep_col = ['Station Name','Inspected Date','Survey123','System Number','Questionnaire_Electronic','Inlet_Pressure','Average_Metered_Pressure','Avg','Intermediate_Pressure','Avg_Metered_Temp','Avg_Metered_Ft','Outlet_Pressure','Supercompressibility_Fpv2','Index_Reading','Last_Reading','MSCF_Current_Period','MSCF_Per_Day_Rate','MSCF_MTD','Tank_Guage','Fluid','Filter_Differential','Meter_Oil','Flow_Rate','RTU_Index','Notes','x','y']
        new_f = f[keep_col]
        new_f.to_csv(r'C:\Users\jnmiller\Downloads\Join_Features_to_Monitoring_Routes_v411_0.csv', index=False)

my_agol_export_function()


def my_date_conv_function():
  # reading the csv file
    df = pd.read_csv(r'C:\Users\jnmiller\Downloads\Join_Features_to_Monitoring_Routes_v411_0.csv')
  
# updating the column value/data
    df['Inspected Date'] = pd.to_datetime(df['Inspected Date']).dt.date
  
# writing into the file
    df.to_csv(r'C:\Users\jnmiller\Downloads\Join_Features_to_Monitoring_Routes_v411_0.csv', index=False)
    print(df)
    df.to_csv(r'C:\Users\jnmiller\ArcGIS\My Survey Designs\bb0fb0a89d***************\media\Join_Features_to_Monitoring_Routes_v411_0.csv', index=False)
    print(df)
    df.to_csv(r'C:\Users\jnmiller\ArcGIS\My Survey Designs\bd29657bdabc43***************\media\Join_Features_to_Monitoring_Routes_v411_0.csv', index=False)
    print(df)
my_date_conv_function()

print('Script Complete at {}'.format(strftime('%c')))