Solution in Python 3 for SLIP connections

[MichaelPemberton1]

I recently had to create a script to enable downloading of map packages from a SLIP connection here in Western Australia.  There was code available from the providers but a few tweaks were needed to allow it to work in python 3.

Object is to download a series of map packages, then expand them to be accessed as file geodatabases.

First:  the secrets file in json format

{

   "username":"someone@somewere"

   "password":"xxxxxxxxxx"

}

which are acquired from the SLIP provider

Second:  config.py

download_directory = "<a drive path to where you want the map package to go.. they can be large"

packages_directory = "<where you want the packages extracted to>"

Lastly: program.py

import json

import requests

import shutil

from urllib.parse import urlparse

import config

import arcpy

import os

def getSecrets():

    # Secrets from file stored outside of revison control

    with open(r"./secrets.json") as f:

        secrets = json.load(f)

    return secrets

def fetchDownloadSnapshot(URL):

    response = s.get(URL, allow_redirects=False, stream=True)

    if response.status_code == 302:

        parsed_uri = urlparse(response.headers['Location'])

        domain = "{uri.scheme}://{uri.netloc}/".format(uri=parsed_uri)

        if parsed_uri.netloc.startswith("sso.slip.wa.gov.au") or parsed_uri.netloc.startswith("maps.slip.wa.gov.au"):

            response = fetchDownloadSnapshot(response.headers['Location'])

        else:

            raise Exception("Receieved a redirect to an unknown domain '%s' for %s" % (parsed_uri.netloc, response.headers['Location']))

    if response.status_code == 200:

        if response.headers["Content-Type"] == "application/zip" or response.headers["Content-Type"] == "application/octet-stream":

            return response

        raise Exception("Received an invalid Content-Type response - should be 'application/zip' or 'application/octet-stream', but was '{}'".format(response.headers["Content-Type"]))

    else:

        raise Exception("Received a '%s' response for the URL %s" % (response.status_code, URL))

def extractPackage(filetoextract,directory_name):

    arcpy.env.overwriteOutput = True

    arcpy.ExtractPackage_management(filetoextract,config.packages_directory)

# Secrets

secrets = getSecrets()

s = requests.Session()

s.auth = (secrets["username"], secrets["password"])

s.headers.update({"User-Agent": "SLIPAppUser"})

def process_files():

    for urllocation in config.URLlist:

        #response = fetchDownloadSnapshot(urllocation)

        filename = urllocation.split('/')[-1]

        with open(config.download_directory+'\\'+filename, mode="wb") as localfile:

           shutil.copyfileobj(response.raw, localfile)

        response.close()

        os.mkdir(config.packages_directory+'\\'+filename.split('.')[0])

        arcpy.ExtractPackage_management(config.download_directory+'\\'+filename, config.packages_directory+'\\'+filename.split('.')[0])

def main():

    process_files()

if __name__ == '__main__':

    main()

and that's it.  Give it a crack, you may need to pip in some pieces though.

This can also be automated via a scheduler 

Cheers