We are trying to setup a scheduled task to update a vector tile cache in ArcGIS Enterprise making use of python and the requests library.
We can run the REST API from a browser (Chrome)
<<server url>>/rest/services/System/CachingControllers/GPServer/Manage Vector Tile Cache/ submitJob
when logged in as portaladmin and get the jobId back we can then monitor the progress of the job using
<<server url>>/rest/services/System/CachingControllers/GPServer/Manage Vector Tile Cache/jobs/<<jobId>>
This runs to completion with esriJobSucceeded status.
When I try to do similar calls from python with the requests library the job fails with esriJobFailed and error messages
{
'jobId': 'jf4d02b8cfcc74481b8ce6777d1c79174',
'jobStatus': 'esriJobFailed',
'messages': [{
'type': 'esriJobMessageTypeInformative',
'description': 'Submitted.'
}, {
'type': 'esriJobMessageTypeInformative',
'description': 'Executing...'
}, {
'type': 'esriJobMessageTypeInformative',
'description': 'Start Time: Friday, 05 November 2021 15:52:19'
}, {
'type': 'esriJobMessageTypeError',
'description': 'ERROR 001359: Failed to connect to the server.'
}, {
'type': 'esriJobMessageTypeError',
'description': 'Failed to execute (Manage Vector Tile Cache).'
}, {
'type': 'esriJobMessageTypeInformative',
'description': 'Failed at Friday, 05 November 2021 15:52:19 (Elapsed Time: 0,00 seconds)'
}, {
'type': 'esriJobMessageTypeError',
'description': 'Failed.'
}
]
}
The token that we generate uses a referer client with the <<server url>> as referrer.
This token is part of the request that I send to the geoprocessing task to authenticate with.
When I use this token in the browser the job also fails.
Has anyone encountered similar problems and / or found a way around them.
Here is the code that we used
import os
import requests
import logging
from logging.handlers import RotatingFileHandler
import time
class CacheUpdater:
"""
Class to update the tile cache
"""
def __init__(self):
self.logger = setup_logger(os.environ['LOG_FOLDER'])
self.portal_url = os.environ['PORTAL_URL']
self.token = None
def update_cache(self):
self.logger.info(f'Updating cache on {self.portal_url}...')
try:
self.token = self.get_gis_portal_token()
self.update_vector_tile_cache()
# self.logger.info(f'Geopoint count {self.query_geopoint_count()}')
self.logger.info('Updated cache')
except Exception as ex:
self.logger.error(f'Updating Vector Tile Cache failed with an exception')
self.logger.exception(ex)
def get_gis_portal_token(self):
self.logger.info('Getting portal token...')
token_url = self.portal_url + '/portal/sharing/rest/generateToken'
# token_url = self.portal_url + '/hosting/tokens/generateToken'
# 'client': 'referer',
# 'referer': self.portal_url + '/hosting',
query_parameters = {
'username': os.environ['PORTAL_USER'],
'password': os.environ['PORTAL_PASS'],
'client': 'requestip',
'f': 'json'
}
result = requests.post(token_url, data=query_parameters)
result.raise_for_status()
response_json = result.json()
if 'token' in response_json:
return response_json['token']
elif 'error' in response_json:
raise RuntimeError(f'generateToken failed with {response_json["error"]}')
self.logger.info('Got portal token')
def update_vector_tile_cache(self):
"""
Update the vector tile cache
:return:
"""
self.logger.info('Updating vector tile cache...')
url = self.portal_url + '/hosting/rest/services/System/CachingControllers/GPServer/Manage Vector Tile ' \
'Cache/submitJob'
params = {
'serviceName': 'EA_Vector_Tile',
'serviceFolder': 'Hosted',
'minScale': 36111.909643,
'maxScale': 564.248588,
'tilingFormat': 'INDEXED',
'token': self.token,
'f': 'json'
}
result = requests.post(url, params)
result.raise_for_status()
response_json = result.json()
if 'error' in response_json:
raise RuntimeError(f'submitJob failed with {response_json["error"]}')
if ("jobStatus" in response_json) and (response_json["jobStatus"] != 'esriJobSubmitted'):
raise RuntimeError(f'submitJob failed with {response_json}')
self.logger.info(f'Submitted job {response_json["jobId"]} with status {response_json["jobStatus"]}')
url = self.portal_url + f'/hosting/rest/services/System/CachingControllers/GPServer/Manage Vector Tile ' \
f'Cache/jobs/{response_json["jobId"]}'
params = {
'token': self.token,
'f': 'json'
}
completed = False
while not completed:
time.sleep(5)
result = requests.post(url, params)
result.raise_for_status()
response_json = result.json()
if 'error' in response_json:
raise RuntimeError(f'job failed with {response_json["error"]}')
if ("jobStatus" in response_json) and (response_json["jobStatus"] == 'esriJobFailed'):
raise RuntimeError(f'submitJob failed with {response_json}')
completed = response_json["jobStatus"] == 'esriJobSucceeded'
self.logger.info(f' Job status {response_json["jobStatus"]}')
self.logger.info('Updated vector tile cache')
def query_geopoint_count(self):
"""
Query the geo-point layer
:return:
"""
url = self.portal_url + '/hosting/rest/services/EnumerationSet/StatsSAEnumerationSet_DV/FeatureServer/0/query'
params = {
'where': '1=1',
'returnCountOnly': 'true',
'token': self.token,
'f': 'json'
}
result = requests.get(url, params)
result.raise_for_status()
response_json = result.json()
if 'error' in response_json:
raise RuntimeError(f'submitJob failed with {response_json["error"]}')
return response_json['count']
def setup_logger(log_folder: str, logging_level: int = logging.INFO) -> logging.Logger:
""" Setup the rolling logger
Args:
log_folder ([type]): FOlder to store logs
logging_level ([logging], optional): Logging level. Defaults to logging.INFO.
Returns:
logging.Logger: Logger to use
"""
new_logger = logging.getLogger('tilecache')
new_logger.setLevel(logging_level)
if len(new_logger.handlers) > 0:
return new_logger
logging_format = logging.Formatter(fmt='%(asctime)s: %(levelname)s: %(message)s',
datefmt='%Y-%m-%d %H:%M:%S')
stream_handler = logging.StreamHandler()
stream_handler.setFormatter(logging_format)
new_logger.addHandler(stream_handler)
if not os.path.exists(log_folder):
os.makedirs(log_folder)
logfile_name: str = os.path.join(log_folder, r'tile_cache.log')
file_handler = RotatingFileHandler(logfile_name, maxBytes=2 ** 22, backupCount=10, mode='a')
file_handler.setFormatter(logging_format)
new_logger.addHandler(file_handler)
return new_logger
if __name__ == '__main__':
c = CacheUpdater()
c.update_cache()
