Data Scrape and Add Features (need attachments)

kapalczynski

I have some code that I am using to Scrape a Service and based on a where clause copy specific records to another dataset.  It scrapes the service and creates json files for every 1000 records... It then reads the JSON files and uses the below to add them to the Service ..... This is working great.. but I need to modify it to move attachments as well... this is where I am confused ... 

Because I am writing the feature to JSON and then using that to add features I am not sure how to include the attachments for each of those features in the JSON file.

Any thoughts very appreciated... 

I am doing to add at the service level with 'edit_features' : 

add_result = ports_layer.edit_features(adds = featureAddingAdd)

 

# SNIP 

portal_item = gis.content.get('73xxxxxxxxxxxxxxxxxxxxxxxxx5')
ports_layer = portal_item.tables[0]

class DataScraper():
    def __init__(self):
        # URL to map service you want to extract data from
        self.service_url = s123URL
    def getServiceProperties(self, url):
        URL = url
        PARAMS = {'f' : 'json'}
        r = requests.get(url = URL, params = PARAMS)
        service_props = r.json()
        return service_props
    def getLayerIds(self, url, query=None):
        URL = url + '/query'
        print(URL)
        PARAMS = {'f':'json', 'returnIdsOnly': True, 'where' : "Imported = 'No'"}

        if query:
            PARAMS['where'] = "ST = '{}'".format(query)
        r = requests.get(url = URL, params = PARAMS)
        data = r.json()
        
        return data['objectIds']
    def getLayerDataByIds(self, url, ids):
        # ids parameter should be a list of object ids
        URL = url + '/query'
        field = 'OBJECTID'
        value = ', '.join([str(i) for i in ids])
        PARAMS = {'f': 'json', 'where': '{} IN ({})'.format(field, value), 'returnIdsOnly': False, 'returnCountOnly': False,
                  'returnGeometry': True, 'outFields': '*'}
        r = requests.post(url=URL, data=PARAMS)
        layer_data = r.json()
        return layer_data
    def chunks(self, lst, n):
        # Yield successive n-sized chunks from list
        for i in range(0, len(lst), n):
            yield lst[i:i + n]
            
def scrapeData():
    try:
        service_props = ds.getServiceProperties(ds.service_url)
        max_record_count = service_props['maxRecordCount']
        layer_ids = ds.getLayerIds(ds.service_url)
        
        id_groups = list(ds.chunks(layer_ids, max_record_count))
        
        for i, id_group in enumerate(id_groups):
            print('  group {} of {}'.format(i+1, len(id_groups)))
            layer_data = ds.getLayerDataByIds(ds.service_url, id_group)
            level = str(i)
            outjsonpath = outputVariable + level + ".json"

            layer_data_final = layer_data
            print('Writing JSON file...')
            with open(outjsonpath, 'w') as out_json_file:
                json.dump(layer_data_final, out_json_file)
                
    except Exception:
        # Handle errors accordingly...this is generic
        tb = sys.exc_info()[2]
        tb_info = traceback.format_tb(tb)[0]
        pymsg = 'PYTHON ERRORS:\n\tTraceback info:\t{tb_info}\n\tError Info:\t{str(sys.exc_info()[1])}\n'
        msgs = 'ArcPy ERRORS:\t{arcpy.GetMessages(2)}\n'
        print(pymsg)
        print(msgs)

def addAAHData():
    try:
        for x in os.listdir(path):
            if x.startswith("output"):
                filetoImport = path + x
                print("Appending: " + x)
                f = open(filetoImport)
                data = json.load(f)
                featureAddingAdd = data['features']

                add_result = ports_layer.edit_features(adds = featureAddingAdd)
                
    except Exception:
        # Handle errors accordingly...this is generic
        tb = sys.exc_info()[2]
        tb_info = traceback.format_tb(tb)[0]
        pymsg = 'PYTHON ERRORS:\n\tTraceback info:\t{tb_info}\n\tError Info:\t{str(sys.exc_info()[1])}\n'
        msgs = 'ArcPy ERRORS:\t{arcpy.GetMessages(2)}\n'
        print(pymsg)
        print(msgs)

 

kapalczynski

Thanks... Still working through this... 

As of now this is the only syntax that seems to work

{"1":{"where": "IMPORTED = 'No'"}}