Clone "complex" Survey123

Question

Hi,

Our goal is to clone pre-defined Survey123 templates. Our environment is ArcGIS Enterprise. Based on this tutorial: https://developers.arcgis.com/survey123/guide/clone-surveys-from-one-organization-to-another/ it was pretty straightforward to clone a single survey.

Things get more tricky if the SurveyForm references a WebMap item. Although we are able to clone the WebMap item, based on the tutorial from the link, the Survey123 Form still references the WebMap Item from the template and not the cloned one.

As a workaround we found out that we can download the Survey123 Form as a zip file, open the *.info json and update the displayInfo -> map -> defaultType -> name reference and reupload the zip (like described here: https://developers.arcgis.com/survey123/guide/update-contents-of-the-media-folder-for-an-arcgis-survey123-form-item/#Download-survey) - but this feels like a very, very hacky way of doing something that sounds like it should be done via some python API commands rather than doing a zip download/upload.

Is there any better way of doing this? Updating the reference to the WebMap is just one usecase - in the future our templates will have more complex things like additional JavaScript files, Media Content etc. It would be very frustrating to update our GP Tool / Script every time a new template is defined to check if we don't miss any edge-case for cloning a Survey123 template.

Bernd_Loigge · Answer

If someone is interested - that's what we came up with so far. Guess it still will need a lot of tweaking. But so far it:

* clones a Form and all its relations
* additional dashboards which belong to our "template user" with the same tag as the form
* references (no copy) all layers in all Web Map Items from the dashboards
* references (no copy) all layers in the Survey Form - Web Map
* Updates the linked Web Map in the cloned Form

from arcpy import GetParameterAsText, AddMessage, AddError
from arcgis.gis import GIS, Item, User
import tempfile
import zipfile
import shutil
import os
import json

# Helper Functions
def updateItemMappingDictForWebMap(gis, mappingDict, webMap, template_tag):
    for layer_description in webMap.get_data()['operationalLayers']:
        layer_item_id = layer_description["itemId"]        
        layer_item = Item(gis, layer_item_id)
        if template_tag not in layer_item.tags:
            mappingDict[layer_item_id] = layer_item_id

return mappingDict

def extractZIP(filename, folder):
    zfile = zipfile.ZipFile(filename)
    zfile.extractall(folder)

# Main Function
def ScriptTool(form_item_id, project_id, template_tag, form_owner_id, portal_url, portal_user_id, portal_password):
    """
    --------------------------------------------------
    Initial creation of the gis object
    Check if item already exists in the target folder.
    --------------------------------------------------
    """
    AddMessage("Init and checks ...")

# Get the GIS Object which is the root connector to the portal
    gis = GIS(url=portal_url, username=portal_user_id, password=portal_password)

# Get the form_item
    form_item = Item(gis, form_item_id)
    form_title = form_item.title

# Get the form_owner_user
    form_owner_user = User(gis, form_owner_id)

# Get the portal_user
    portal_user = User(gis, portal_user_id)

# Check if the portal_user already has a cloned form_item - if so exit
    new_form_title = f"{project_id}_{form_title}"
    for folder in portal_user.folders:
        if folder["title"] == project_id:
            for item in portal_user.items(project_id):
                if item["title"] == new_form_title:
                    AddError("A cloned version of this item in the target folder of the portal user already exists!")
                    return

"""
    ------------------------------------------
    Initial mapping dict and obtain relations.
    ------------------------------------------
    """
    AddMessage("Obtaining relations ...")

# Init item_mapping_dict. This dict will map elements from items we clone which should be referenced rather than get cloned (e.g.: feture layer of Web Map Items)
    item_mapping_dict = {}

# Obtain the related_items of the form_item 
    related_items = form_item.related_items("Survey2Service", "forward")

# Obtain the additional_items of the form_item 
    additional_items = form_item.related_items("Survey2Data", "forward")

# Obtain additional applications which should be cloned - need to have the same template_tag and belong to the form_owner_id
    dashboards = gis.content.search(query=f'owner:"{form_owner_id}" AND type:"Dashboard" AND tags:"{template_tag}"')

"""
    ----------------------------------------------------------------------------
    Update the item_mapping_dict. Only survey feature layers need to be copied.
    All others should be referenced.
    ----------------------------------------------------------------------------
    """
    AddMessage("Updating mapping dict ...")

# For dashboards
    for dashboard in dashboards:
        # Obtain all widgets inside the dashboard
        dashboard_data_dict = dashboard.get_data()
        dashboard_data_dict_widgets = dashboard_data_dict["widgets"]
        if dashboard_data_dict_widgets:
            # Iterate dashboard widgets
            for widget in dashboard_data_dict_widgets:
                # If a widget is from type mapWidget we update the item_mapping_dict
                if widget["type"] == "mapWidget":
                    dashboard_map = Item(gis, widget["itemId"]) 
                    updateItemMappingDictForWebMap(gis, item_mapping_dict, dashboard_map, template_tag)

# For webmaps inside the form
    for item in additional_items:
        if item.type == "Web Map":
            updateItemMappingDictForWebMap(gis, item_mapping_dict, item, template_tag)

"""
    ------------------------------------
    Clone Items and rename cloned items.
    Reset references.
    ------------------------------------
    """
    AddMessage("Cloning items ...")

# Merging all items we want to clone
    all_items = [form_item] + related_items + additional_items + dashboards

# Clone Items
    cloned_items = gis.content.clone_items(
      items = all_items, 
      folder = project_id, 
      search_existing_items = False, 
      copy_data = False,
      item_mapping = item_mapping_dict
    )

cloned_form = None
    cloned_web_map = None
    cloned_addtional_items = []

# Iterate cloned_items to get access to certain items and rename
    for cloned_item in cloned_items:
        # Find the cloned form
        if cloned_item.title == form_title:
            cloned_form = cloned_item

# Find cloned_additional_items
        for additional_item in additional_items:
            if cloned_item.title == additional_item.title:
                cloned_addtional_items.append(cloned_item)

cloned_item_title = cloned_item.title
        new_title = f"{project_id}_{cloned_item_title}" 
        cloned_item.update(item_properties={"title": new_title})

# Set relationship for cloned_additional_items
    for cloned_additional_item in cloned_addtional_items:
        # Find any cloned WebMap
        if (cloned_additional_item.type == "Web Map"):
            cloned_web_map = cloned_additional_item

# Set relationship
        cloned_form.add_relationship(cloned_additional_item, "Survey2Data")

"""
    ---------------------------------------------------------
    Download the form item and update additional information:
        1) Update Web Map in Survey
    ---------------------------------------------------------
    """
    AddMessage("Additional updates ...")

# Update cloned_form
    tmpdir = tempfile.TemporaryDirectory()
    download_folder = tmpdir.name
    savedZip = cloned_form.download(save_path=download_folder)

# Extract Zip
    extracted_folder_path = os.path.join(download_folder + "/_extracted/")
    extractZIP(savedZip, extracted_folder_path)

# Read form *.info JSON
    cloned_form_info_json_path = os.path.join(extracted_folder_path + "/esriinfo" + "/" + form_title + ".info")
    cloned_form_info_json = open(cloned_form_info_json_path, "r")
    cloned_form_info_json_data = json.load(cloned_form_info_json)
    cloned_form_info_json.close()

# Update form *.info
    # Change path of displayInfo -> map -> defaultType
    cloned_form_info_json_data["displayInfo"]["map"]["defaultType"]["name"] = f"id:{cloned_web_map.id}||{cloned_web_map.title}"

# Write to *.info JSON
    cloned_form_info_json = open(cloned_form_info_json_path, "w+")
    cloned_form_info_json.write(json.dumps(cloned_form_info_json_data))
    cloned_form_info_json.close()

# Upload ZIP File
    updateZip = shutil.make_archive(cloned_form.title, 'zip', extracted_folder_path + "/")
    cloned_form.update({}, updateZip)

AddMessage("Done!")
    return

# This is used to execute code if the file was run but not imported
if __name__ == '__main__':

# Parameter 1 is the id of the form we want to clone
    form_item_id = GetParameterAsText(0)

# Parameter 2 is the name of the project id where we want the form to be cloned to
    project_id = GetParameterAsText(1)

# Parameter 3 is the name of the project id where we want the form to be cloned to
    template_tag = GetParameterAsText(2)

# Parameter 4 is the id of the form_owner
    form_owner_id = GetParameterAsText(3)

# Parameter 5 is the portal_url
    portal_url = GetParameterAsText(4)

# Parameter 6 is the id of the portal_user
    portal_user_id = GetParameterAsText(5)

# Parameter 7 is the portal_password of the portal_user
    portal_password = GetParameterAsText(6)

ScriptTool(form_item_id, project_id, template_tag, form_owner_id, portal_url, portal_user_id, portal_password)

Bernd_Loigge · Answer

Thank you Don,

Unfortunately your answer confirms my assumption that there is no way to modify the cloned items of a survey using the python API. The whole concept of the Portal needs to be reworked IMHO. Why is it using plain xlm and json files and not a database?

DonMorrison1 · Answer

Hi - I created a process for cloning a Survey 123 'template', but it is probably even more hacky - but it does the job for us so far. I don't know if it could handle your complexity. Our survey is used by a large number of organizations that we have a relationship with. Each organization gets it's own survey and the survey is customized for that org (underlying web map, thumbnail, title, etc), based on the template. There are a couple of steps involved. I'll include the code as-is so it might take a bit to decipher (variables prefixed with 'dp_' contain data from a local configuration database that describes the participating organizations). 1. Whenever the template changes, we run an export to the local file system import os from row.deploy.util.app_info import AppInfo import row.db.organization import row.deploy.util.common from zipfile import ZipFile import shutil import row.consolidation.logger logger = row.consolidation.logger.get('row_log') def export(template_folder, template_title, dp_form_pid): logger.info ("Logging to %s" % (row.consolidation.logger.LOG_FILE)) logger.info("Exporting survey 123 form %s from %s to Git repository" % (template_title, template_folder)) # Find the form's AGOL item gis_item = __get_gis_item (template_folder, template_title, 'Form') # Get the zip file attached to the item and unzip it into the GIT repository logger.info("Downloading spec files") zip_file_name = gis_item.get_data(False) git_dir = os.path.join(AppInfo.path_code_base, 'config', 'templates', dp_form_pid) if os.path.exists(git_dir): shutil.rmtree (git_dir, ignore_errors=True) if os.path.exists(git_dir): os.rmdir(git_dir) os.makedirs(git_dir) logger.info("Survey spec files unzipping to %s" % (git_dir)) with ZipFile(zip_file_name, 'r') as zip_obj: # Extract all the contents of zip file in different directory zip_obj.extractall(git_dir) # Delete the original zip file os.remove(zip_file_name) def __get_gis_item (folder_id, title, type_): for gis_item in AppInfo.gis_agol.users.me.items(folder=folder_id): if gis_item.title == title and gis_item.type == type_: logger.info ("%s: Found survey 123 %s '%s'. GIS ID=%s" % (folder_id, type_, title, gis_item.id)) return gis_item raise Exception ("%s: Can not find survey 123 %s %s" % (folder_id, type_, title)) 2. When we on-board a new organization we created an ArcGIS Online folder for the org, create the web map/data source in that folder, run the 'create' function 3. When we want to propagate template change to the org-specific versions we run 'update' for each org import os import arcpy from row.deploy.util.app_info import AppInfo import row.deploy.util.common import row.deploy.util.deployment_plan from zipfile import ZipFile import zipfile import shutil import re import json import row.consolidation.logger logger = row.consolidation.logger.get('row_log') def create (dp, dp_form): # Clean up any orphans for gis_item in AppInfo.gis_agol.users.me.items(folder=dp['org_id']): if (gis_item.title == dp_form['title'] or __get_short_title(gis_item.title) == dp_form['title']) and gis_item.type == 'Form': logger.info ("%s: Form '%s' already exists. Deleting it now" % (dp['org_id'], dp_form['title'])) delete (gis_item) # Create the new form with no content information logger.info ("%s: Creating form: '%s'" % (dp['org_id'], __get_full_title(dp_form))) item_properties={ 'title': __get_full_title(dp_form), 'type': 'Form', 'typeKeywords': ["Form", "Survey123", "Survey123 Connect", "xForm"] } # Create the Survey 123 form gis_form = AppInfo.gis_agol.content.add(item_properties=item_properties, folder=dp['org_id']) row.db.persistent_id_map.add_entry (gis_form.id, dp_form['persistent_id'], dp['org_id']) update (dp, dp_form, gis_form) return gis_form def update (dp, dp_form, gis_form): logger.info ("%s: Updating form: '%s'" % (dp['org_id'], __get_full_title(dp_form))) # Make a copy of the template files located in the git directory git_dir = os.path.join(AppInfo.path_code_base, 'config', 'templates', 'SRVY_001') temp_dir = os.path.join(arcpy.env.scratchFolder, dp['org_id'] + '_s123') if os.path.exists(temp_dir): shutil.rmtree (temp_dir, ignore_errors=True) shutil.copytree (git_dir, temp_dir) # Get the item ID of this org's feature class dp_fs = row.deploy.util.deployment_plan.get_pfs (dp, dp_form['xref_pfs_pid']) gis_pfs = AppInfo.gis_agol.content.get(row.db.persistent_id_map.lookup_gis_item_id (dp['org_id'], dp_fs['persistent_id'])) # Get the names of the spec files from the forminfo.json file fn = os.path.join(temp_dir, 'esriinfo', 'forminfo.json') json_ = __read_json_file (dp['org_id'], fn) template_title = json_['name'] # Edit the submission URL in the webform fn = os.path.join(temp_dir, 'esriinfo', template_title + '.webform') text = __read_text_file (dp['org_id'], fn) text = re.sub(r'action=\"https://www.arcgis.com/sharing/rest/content/items/[a-z0-9]{32}\"', 'action=\"https://www.arcgis.com/sharing/rest/content/items/%s\"' % (gis_pfs.id), text) __write_text_file (dp['org_id'], fn, text) # Edit the submission URL and header label in the xml form fn = os.path.join(temp_dir, 'esriinfo', template_title + '.xml') text = __read_text_file (dp['org_id'], fn) text = text.replace('', '%s ' % dp['org_id'], 1) text = re.sub(r'action="https://www.arcgis.com/sharing/rest/content/items/[a-z0-9]{32}"', 'action="https://www.arcgis.com/sharing/rest/content/items/%s"' % (gis_pfs.id), text) __write_text_file (dp['org_id'], fn, text) # Edit the lat,long in the .info file fn = os.path.join(temp_dir, 'esriinfo', template_title + '.info') lat, lng = __calculate_midpoint (dp, dp_fs) json_ = __read_json_file (dp['org_id'], fn) json_['displayInfo']['map']['home']['latitude'] = lat json_['displayInfo']['map']['home']['longitude'] = lng __write_json_file (dp['org_id'], fn, json_) # Zip up the files and upload into the item zip_file_name = os.path.join(temp_dir, 'ROW.zip') logger.debug("%s: Zipping survey spec files to %s" % (dp['org_id'], zip_file_name)) owd = os.getcwd() os.chdir(temp_dir) with ZipFile(zip_file_name, 'w', zipfile.ZIP_DEFLATED) as zip_obj: zip_obj.write('esriinfo') for root, dirs, files in os.walk('esriinfo'): for file in files: zip_obj.write(os.path.join(root, file)) os.chdir(owd) # Upload the modified zip file gis_form.update(None, data=zip_file_name) # Update the item metadata item_spec = {} item_spec['title'] = __get_full_title(dp_form) item_spec['description'] = dp_form['description'] item_spec['tags'] = dp['tags'] item_spec['snippet'] = dp['summary'] item_spec['ownerFolder'] = gis_form['ownerFolder'] url = AppInfo.agol_connect_host_name + '/sharing/rest/content/users/' + AppInfo.agol_connect_user_name + '/' + gis_form['ownerFolder'] + '/items/' + gis_form.id + '/update' resp = row.deploy.util.common.submit_http_req (url, item_spec, 'AGOL') AppInfo.gis_agol.content.get(resp['id']) # Reset the relationship with the underlying feature layer for related_item in gis_form.related_items('Survey2Service', 'forward'): gis_form.delete_relationship(related_item, 'Survey2Service') gis_form.add_relationship(gis_pfs, 'Survey2Service') # Upload thumbnail row.deploy.util.common.upload_thumbnail (gis_form, dp_form['thumbnail']) def delete (gis_form): try: if gis_form is not None: logger.info ("Deleting form: '" + gis_form.title + "'") gis_form_id = gis_form.id gis_form.delete() row.db.persistent_id_map.remove_entry (gis_form_id) else: logger.info ("Could not find form: '" + str(gis_form) + "'") except Exception: logger.info ("Could not delete form: '" + str(gis_form) + "'") def __calculate_midpoint (dp, dp_fs): dp_ags_rest = row.deploy.util.deployment_plan.get_ags_rest (dp, dp_fs['xref_ags_rest_id']) ags_rest_layer_url = dp_ags_rest['url'] + '/' + str(0) url = ags_rest_layer_url.replace(AppInfo.ags_service_rest_base, AppInfo.ags_service_rest_base_internal) + '/query' params = {'returnExtentOnly' : 'true', 'returnCountOnly': 'true', 'where': '1=1', 'returnGeometry': 'false', 'spatialRel': 'esriSpatialRelIntersects', 'outSR': '4326'} data = row.deploy.util.common.submit_http_req (url, params, 'AGS_CONSUMER', False) x_min, y_min, x_max, y_max = [data['extent']['xmin'], data['extent']['ymin'], data['extent']['xmax'], data['extent']['ymax']] if x_min != 'NaN' and y_min != 'NaN' and x_max != 'NaN' and y_max != 'NaN': lng = (x_min + x_max) / 2 lat = (y_min + y_max) / 2 else: lat = 38.8627768334364 lng = -96.6298169140754 return lat, lng def __read_text_file (org_id, fn): logger.debug ("%s: Opening text file for edit: '%s'" % (org_id, fn)) with open(fn) as text_file: return text_file.read() def __write_text_file (org_id, fn, text): logger.debug ("%s: Updating text: '%s'" % (org_id, fn)) with open(fn, 'w') as text_file: text_file.write(text) def __read_json_file (org_id, fn): logger.debug ("%s: Opening text file for edit: '%s'" % (org_id, fn)) with open(fn) as json_file: return json.load(json_file) def __write_json_file (org_id, fn, json_): logger.debug ("%s: Updating json: '%s'" % (org_id, fn)) with open(fn, 'w') as json_file: json.dump(json_, json_file, indent=4) def __get_full_title (dp_form): return dp_form['title'] + ' ' + dp_form['version'] def __get_short_title (full_title): return full_title.rsplit(' ', 1) [0]