I recently had to create a script to enable downloading of map packages from a SLIP connection here in Western Australia. There was code available from the providers but a few tweaks were needed to allow it to work in python 3.
Object is to download a series of map packages, then expand them to be accessed as file geodatabases.
First: the secrets file in json format
{
"username":"someone@somewere"
"password":"xxxxxxxxxx"
}
which are acquired from the SLIP provider
Second: config.py
download_directory = "<a drive path to where you want the map package to go.. they can be large"
packages_directory = "<where you want the packages extracted to>"
Lastly: program.py
import json
import requests
import shutil
from urllib.parse import urlparse
import config
import arcpy
import os
def getSecrets():
# Secrets from file stored outside of revison control
with open(r"./secrets.json") as f:
secrets = json.load(f)
return secrets
def fetchDownloadSnapshot(URL):
response = s.get(URL, allow_redirects=False, stream=True)
if response.status_code == 302:
parsed_uri = urlparse(response.headers['Location'])
domain = "{uri.scheme}://{uri.netloc}/".format(uri=parsed_uri)
if parsed_uri.netloc.startswith("sso.slip.wa.gov.au") or parsed_uri.netloc.startswith("maps.slip.wa.gov.au"):
response = fetchDownloadSnapshot(response.headers['Location'])
else:
raise Exception("Receieved a redirect to an unknown domain '%s' for %s" % (parsed_uri.netloc, response.headers['Location']))
if response.status_code == 200:
if response.headers["Content-Type"] == "application/zip" or response.headers["Content-Type"] == "application/octet-stream":
return response
raise Exception("Received an invalid Content-Type response - should be 'application/zip' or 'application/octet-stream', but was '{}'".format(response.headers["Content-Type"]))
else:
raise Exception("Received a '%s' response for the URL %s" % (response.status_code, URL))
def extractPackage(filetoextract,directory_name):
arcpy.env.overwriteOutput = True
arcpy.ExtractPackage_management(filetoextract,config.packages_directory)
# Secrets
secrets = getSecrets()
s = requests.Session()
s.auth = (secrets["username"], secrets["password"])
s.headers.update({"User-Agent": "SLIPAppUser"})
def process_files():
for urllocation in config.URLlist:
#response = fetchDownloadSnapshot(urllocation)
filename = urllocation.split('/')[-1]
with open(config.download_directory+'\\'+filename, mode="wb") as localfile:
shutil.copyfileobj(response.raw, localfile)
response.close()
os.mkdir(config.packages_directory+'\\'+filename.split('.')[0])
arcpy.ExtractPackage_management(config.download_directory+'\\'+filename, config.packages_directory+'\\'+filename.split('.')[0])
def main():
process_files()
if __name__ == '__main__':
main()
and that's it. Give it a crack, you may need to pip in some pieces though.
This can also be automated via a scheduler
Cheers
