Error when cloning Portal file based items

843
2
12-23-2021 07:48 AM
OlivierLefevre
New Contributor III

Hi,

I'm trying to set up some Python code (in ArcGIS Pro Notebook) which aims at cloning Portal users, groups and items like in https://developers.arcgis.com/python/sample-notebooks/clone-portal-users-groups-and-content/

Cloning of users and groups is fine.

But I get errors when cloning Shapefile and File Geodatabase which are file based items :

Shapefile
[WinError 32] The process cannot access the file because it is being used by another process: 'C:\\Users\\O7E12~1.LEF\\AppData\\Local\\Temp\\ArcGISProTemp19440\\tmpagr42c_r\\Extrairelesdonnesseptembre2220202.57.15PM.zip'
File Geodatabase
[WinError 32] The process cannot access the file because it is being used by another process: 'C:\\Users\\O7E12~1.LEF\\AppData\\Local\\Temp\\ArcGISProTemp19440\\tmp7xrfzcrk\\diagpelousecalci.zip'

 Here is my code.

 

from arcgis.gis import GIS
import tempfile

target_username = "xxxxxxx"
target_password = "zzzzzzz"
source = GIS("pro") # Connected Portal in ArcGIS Pro
target = GIS("https://xoxoxo.xoxoxoxo.fr/portal", target_username, target_password)

source_users = source.users.search('!esri_*')
target_users = target.users.search('!esri_*')

source_groups = source.groups.search("!owner:esri_* AND !title:\"Fond de carte\"")
target_groups = target.groups.search("!owner:esri_* AND !title:\"Fond de carte\"")

# For each user create a mapping of itemId to the Item
source_items_by_id = {}
for user in source_users:
        num_items = 0
        num_folders = 0
        print("Collecting item ids for {}".format(user.username), end="\t\t")
        user_content = user.items()

        # Get item ids from root folder first
        for item in user_content:
            num_items += 1
            source_items_by_id[item.itemid] = item 

        # Get item ids from each of the folders next
        folders = user.folders
        for folder in folders:
            num_folders += 1
            folder_items = user.items(folder=folder['title'])
            for item in folder_items:
                num_items += 1
                source_items_by_id[item.itemid] = item
        
        print("Number of folders {} # Number of items {}".format(str(num_folders), str(num_items)))

# Prepare sharing information for each item
for group in source_groups:
    #iterate through each item shared to the source group
    for group_item in group.content():
        try:
            #get the item
            item = source_items_by_id[group_item.itemid]
            if item is not None:
                if not 'groups' in item:
                    item['groups'] = []
                
                #assign the target portal's corresponding group's name
                item['groups'].append(group['title'])
        except:
            print("Cannot find item : " + group_item.itemid)

# Copy Items
TEXT_BASED_ITEM_TYPES = frozenset(['Web Map', 'Feature Service', 'Map Service','Web Scene',
                                   'Image Service', 'Feature Collection', 
                                   'Feature Collection Template',
                                   'Web Mapping Application', 'Mobile Application', 
                                   'Symbol Set', 'Color Set',
                                   'Windows Viewer Configuration'])

FILE_BASED_ITEM_TYPES = frozenset(['File Geodatabase','CSV', 'Image', 'KML', 'Locator Package',
                                  'Map Document', 'Shapefile', 'Microsoft Word', 'PDF',
                                  'Microsoft Powerpoint', 'Microsoft Excel', 'Layer Package',
                                  'Mobile Map Package', 'Geoprocessing Package', 'Scene Package',
                                  'Tile Package', 'Vector Tile Package'])

ITEM_COPY_PROPERTIES = ['title', 'type', 'typeKeywords', 'description', 'tags',
                        'snippet', 'extent', 'spatialReference', 'name',
                        'accessInformation', 'licenseInfo', 'culture', 'url']

def copy_item(target, source_item):
    try:
        with tempfile.TemporaryDirectory() as temp_dir:
            item_properties = {}
            for property_name in ITEM_COPY_PROPERTIES:
                item_properties[property_name] = source_item[property_name]

            data_file = None
            
            if source_item.type in TEXT_BASED_ITEM_TYPES:
                # If its a text-based item, then read the text and add it to the request.
                text = source_item.get_data(False)
                item_properties['text'] = text
            
            elif source_item.type in FILE_BASED_ITEM_TYPES:
                # download data and add to the request as a file
                data_file = source_item.download(temp_dir)
                
            thumbnail_file = source_item.download_thumbnail(temp_dir)
            metadata_file = source_item.download_metadata(temp_dir)

            #find item's owner
            source_item_owner = source.users.search(source_item.owner)[0]
            
            #find item's folder
            item_folder_titles = [f['title'] for f in source_item_owner.folders 
                                  if f['id'] == source_item.ownerFolder]
            folder_name = None
            if len(item_folder_titles) > 0:
                folder_name = item_folder_titles[0]

            #if folder does not exist for target user, create it
            if folder_name:
                target_user = target.users.search(source_item.owner)[0]
                target_user_folders = [f['title'] for f in target_user.folders
                                       if f['title'] == folder_name]
                if len(target_user_folders) == 0:
                    #create the folder
                    target.content.create_folder(folder_name, source_item.owner)
            
            #if item has a non-federated server source, add username and password in properties
            if source_item.type in ('Feature Service', 'Map Service') and 'Service Proxy' in source_item.typeKeywords:
                source_url = source_item.sourceUrl
                item_properties['url'] = source_url
                if source_url.startswith('https://non-federated.arcgis-server.fr'):
                    #token = target._con.token
                    item_properties['serviceUsername'] = "????????"
                    item_properties['servicePassword'] = "!!!!!!!!"
                    #item_properties['token'] = token            
            
            #if item is a hosted layer
            if source_item.type in ('Feature Service', 'Map Service') and 'Hosted Service' in source_item.typeKeywords:
                #Use 'clone_items' method so that Feature/Map Service is also created 
                target_item = target.content.clone_items(items=[source_item], copy_data=True, owner=source_item.owner)[0]
            #for other item types
            else:
                #Use 'add' method like in esri example @ https://developers.arcgis.com/python/sample-notebooks/clone-portal-users-groups-and-content/
                target_item = target.content.add(item_properties, data_file, thumbnail_file, 
                                             metadata_file, 
                                             source_item.owner, folder_name)

            #Set sharing (privacy) information
            share_everyone = source_item.access == 'public'
            share_org = source_item.access in ['org', 'public']
            share_groups = []
            #if source_item.access == 'shared':
                #share_groups = source_item.groups
            if 'groups' in source_item:
                share_groups = [group.id for group in target_groups if group.title in source_item.groups]
                
            target_item.share(share_everyone, share_org, share_groups)
            
            return target_item
        
    except Exception as copy_ex:
        print("\tError copying " + source_item.title)
        print("\t" + str(copy_ex))
        return None

source_target_itemId_map = {}
for key, source_item in source_items_by_id.items():
    print("Copying {} ({}) \tfor\t {}".format(source_item.title, source_item.type, source_item.owner))
    target_item = copy_item(target, source_item)
    if target_item:
        source_target_itemId_map[key] = target_item.itemid
    else:
        source_target_itemId_map[key] = None

 

I guess the download method does not finish before item is added. But I don't know how to debug this.

Would you have some advice please ?

Olivier

0 Kudos
2 Replies
JakeSkinner
Esri Esteemed Contributor

@OlivierLefevre it's been awhile since I've used the following tool, but the below should help you copy File Based items:

https://community.esri.com/t5/arcgis-enterprise-documents/copy-content-between-portals/ta-p/920460

0 Kudos
OlivierLefevre
New Contributor III

Thank you @JakeSkinner for your quick answer.

I already had a look at this code and it looks pretty similar to the one from https://developers.arcgis.com/python/sample-notebooks/clone-portal-users-groups-and-content/

0 Kudos