topic Re: join data after unzipping a service replica in Python Questions

join data after unzipping a service replica

Anonymous User — Sat, 11 Dec 2021 13:06:17 GMT

Hi all,

I am trying to create a script that would allow me to join data after programmatically after a replica is created.

I am not sure how to select the feature class and table located in the unzipped gdb.

Here is the script:

import urllib, urllib2, json, time, datetime, os,fnmatch, zipfile, arcpy, sys, logging
pattern = '*.zip'
gdbextension =  '*.gdb'

logging.basicConfig(filename=r"\\arcgisserver\cronjobs\stormwater\backup.log",format='%(asctime)s %(levelname)s: %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.INFO)

username = ""                                             #CHANGE
password = ""                                             #CHANGE
replicaURL = ""    #CHANGE
replicaLayers = []                                               #CHANGE
replicaName = ""                                       #CHANGE

def sendRequest(request):
    response = urllib2.urlopen(request)
    readResponse = response.read()
    jsonResponse = json.loads(readResponse)
    return jsonResponse

logging.info('..Starting process..')
print ("Generating token")
url = "https://arcgis.com/sharing/rest/generateToken"
data = {'username': username,
        'password': password,
        'referer': "https://www.arcgis.com",
        'f': 'json'}
request = urllib2.Request(url, urllib.urlencode(data))
jsonResponse = sendRequest(request)
token = jsonResponse['token']

print("Creating the replica")
data = {'f' : 'json',
    'replicaName' : replicaName,
    'layers' : replicaLayers,
    'returnAttachments' : 'true',
    'returnAttachmentsDatabyURL' : 'false',
    'syncModel' : 'none',
    'dataFormat' : 'filegdb',
    'async' : 'true',
    'token': token}
request = urllib2.Request(replicaURL, urllib.urlencode(data))
jsonResponse = sendRequest(request)

print(jsonResponse)

print("Pinging the server")
responseUrl = jsonResponse['statusUrl']
url = "{}?f=json&token={}".format(responseUrl, token)
request = urllib2.Request(url)
jsonResponse = sendRequest(request)
while not jsonResponse.get("status") == "Completed":
    time.sleep(5)
    request = urllib2.Request(url)
    jsonResponse = sendRequest(request)
    
# Create string date for today's date
dt = str(datetime.date.today())
# Create new folder with today's date
newfolder = 'C:\\replicas'+ "\\" + dt
if not os.path.exists(newfolder): os.makedirs(newfolder)

print("Downloading the replica. In case this fails note that the replica URL is: \n")
jres = jsonResponse['resultUrl']
url = "{0}?token={1}".format(jres, token)
print(url)
logging.info("Replica URL: ")
logging.info(url)
f = urllib2.urlopen(url)
with open(newfolder + "\\" + os.path.basename(jres), "wb") as local_file:
    local_file.write(f.read())
print("\n Finished!")
logging.info("finished creating replica")

for root, dirs, files in os.walk(newfolder):
    for filename in fnmatch.filter(files, pattern):
        print(os.path.join(root, filename))
        zipfile.ZipFile(os.path.join(root, filename)).extractall(os.path.join(root, os.path.splitext(filename)[0]))


    
        









‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

Re: join data after unzipping a service replica

AlexHuygen — Mon, 26 Feb 2018 09:47:10 GMT

Hi Alex, have you had any luck with the above problem?

Im using the same code and could use more functionality.

cheers

Re: join data after unzipping a service replica

Anonymous User — Sat, 11 Dec 2021 13:06:20 GMT

Yeah, I got something working here:

import urllib, urllib2, json, time, datetime, os,fnmatch, zipfile, arcpy, sys, logging, tempfile, shutil, glob, re
pattern = '*.zip'
gdbextension =  '*.gdb'

logging.basicConfig(filename=r"\\arcgisserver\cronjobs\stormwater\backup.log",format='%(asctime)s %(levelname)s: %(message)s',datefmt='%m/%d/%Y %I:%M:%S %p',level=logging.INFO)

username =                                       #CHANGE
password =                                          #CHANGE
replicaURL = "https://services.arcgis.com/UHg8l1wC48WQyDSO/ArcGIS/rest/services/swexporttest/FeatureServer/createReplica"    #CHANGE
replicaLayers = [0, 1, 2, 3, 4, 5]                                               #CHANGE
replicaName = "replicaStormWater"                                       #CHANGE

def sendRequest(request):
    response = urllib2.urlopen(request)
    readResponse = response.read()
    jsonResponse = json.loads(readResponse)
    return jsonResponse

logging.info('..Starting process..')
print ("Generating token")
url = "https://arcgis.com/sharing/rest/generateToken"
data = {'username': username,
        'password': password,
        'referer': "https://www.arcgis.com",
        'f': 'json'}
request = urllib2.Request(url, urllib.urlencode(data))
jsonResponse = sendRequest(request)
token = jsonResponse['token']

print("Creating the replica")
data = {'f' : 'json',
    'replicaName' : replicaName,
    'layers' : replicaLayers,
    'returnAttachments' : 'true',
    'returnAttachmentsDatabyURL' : 'false',
    'syncModel' : 'none',
    'dataFormat' : 'filegdb',
    'async' : 'true',
    'token': token}
request = urllib2.Request(replicaURL, urllib.urlencode(data))
jsonResponse = sendRequest(request)

print(jsonResponse)

print("Pinging the server")
responseUrl = jsonResponse['statusUrl']
url = "{}?f=json&token={}".format(responseUrl, token)
request = urllib2.Request(url)
jsonResponse = sendRequest(request)
while not jsonResponse.get("status") == "Completed":
    time.sleep(5)
    request = urllib2.Request(url)
    jsonResponse = sendRequest(request)
    
# Create string date for today's date
dt = str(datetime.date.today())
# Create new folder with today's date
newfolder = 'C:\\replicas'+ "\\" + "Alex"
if not os.path.exists(newfolder): os.makedirs(newfolder)

print("Downloading the replica. In case this fails note that the replica URL is: \n")
jres = jsonResponse['resultUrl']
url = "{0}?token={1}".format(jres, token)
print(url)
logging.info("Replica URL: ")
logging.info(url)
f = urllib2.urlopen(url)
with open(newfolder + "\\" + os.path.basename(jres), "wb") as local_file:
    local_file.write(f.read())
print("\n Finished!")
logging.info("finished creating replica")

for root, dirs, files in os.walk(newfolder):
    for filename in fnmatch.filter(files, pattern):
        print(os.path.join(root, filename))
        zipfile.ZipFile(os.path.join(root, filename)).extractall(os.path.join(root, os.path.splitext(filename)[0]))

for root, dirs, files in os.walk(newfolder):
    for filegdb in fnmatch.filter(dirs, gdbextension):
        print(os.path.join(root, filegdb))
        pathgdb = os.path.join(root, filegdb)

        #Set up Workspace
        print("Setting workspace...")
        arcpy.env.workspace = pathgdb
        arcpy.env.overwriteOutput = True

        #Add GUID to feature class
        print("Adding GUID field...")
        arcpy.AddField_management("SW_CONVEY", "GUID", "GUID", "", "", "", "", "NULLABLE", "REQUIRED", "")
        print("Done Adding field...")

        #Calculate field GUID
        print("Calculating GUID field...")
        arcpy.CalculateField_management("SW_CONVEY", "GUID", '!GlobalID!', "PYTHON_9.3", "")
        print("Done Calculating field...")

        # Use FeatureToPoint function to find a point inside each park
        print("Converting line to points...")
        arcpy.FeatureToPoint_management("SW_CONVEY", "SW_CONVEY_PTS", "INSIDE")
        print "Line converted to points."

        # Execute AddXY
        print("Adding XY  field to point layer...")
        arcpy.AddXY_management("SW_CONVEY_PTS")
        print "XY added."

        # Create a feature layer from FC
        print("Create a feature layer from FC...")
        arcpy.MakeFeatureLayer_management ("SW_CONVEY_PTS",  "SW_CONVEY_PTS_FL")
        print "Feature Layer created."
        
        # Create view from table
        print("Create view from table...")
        arcpy.MakeTableView_management("SW_CONVEY_ASSET_MAINT", "SW_CONVEY_ASSET_MAINT_VIEW", "", "", "")
        print "Table View created."

        #Add join
        print("Add Join...")
        arcpy.AddJoin_management( "SW_CONVEY_ASSET_MAINT_VIEW", "Convey_guid", "SW_CONVEY_PTS_FL", "GUID")
        print "Join created."

        # Copy the layer to a new table
        print("Copying Joined results to new table...")
        arcpy.CopyRows_management("SW_CONVEY_ASSET_MAINT_VIEW", "SW_CONVEY_ASSET_MAINT_VIEW_TABLE")
        print "Table created."

        # Make the XY event layer...
        sr = arcpy.SpatialReference(102642)
        print("Making a point from xy table...")
        arcpy.MakeXYEventLayer_management("SW_CONVEY_ASSET_MAINT_VIEW_TABLE", "SW_CONVEY_PTS_POINT_X", "SW_CONVEY_PTS_POINT_Y", "SW_CONVEY_ASSET_MAINT_PTS", sr, "")
        print "XY layer Created."

        # Execute CreateFileGDB
        print("Creating a fgdb ...")
        arcpy.CreateFileGDB_management(r"Z:\Agole", "Assign.gdb")
        
        #Copy features
        print("Making feature class...")
        arcpy.FeatureClassToFeatureClass_conversion("SW_CONVEY_ASSET_MAINT_PTS", r"Z:\Agole\Assign.gdb", "Assignments", "")
        print "Features are successfully copied."
        

        


‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

Re: join data after unzipping a service replica

AlexHuygen — Wed, 28 Feb 2018 07:11:45 GMT

sweet.

You're helping a lot of people with this i guess.

thank you