Using the ArcGIS API for Python with a REST service

JustinBridwell2 · ‎08-15-2018

Hey All- I have a public REST service that has several layers and feature classes that I want to access. According to the documentation. Once I have the 'arcgis' package imported, I can inject my outside REST service as such:

*Note: I am using the Jupyter Notebook to test this code.

from arcgis.gis import GIS
gis = GIS("https://hazards.fema.gov/gis/nfhl/rest/services/CSLF/Prelim_CSLF/MapServer")

Since this is a public service, I didn't have to include any username or password parameters. The service has several layers (item_type="Feature Layer) that I want to access; namely Special Flood Hazard Area Change layer, but perhaps others:

Using the example from the API docs, I tried to do a search in the service just to see all the layers and any info I might need in future calls. I am using the '.search' method from the content manager.

# search and list all feature layers in my contents
search_result = gis.content.search(query="", item_type="Feature Layer") 
search_result

Every time I try this, I get an either a KeyError='num' or a TypeError: must be str, not int (if I try to add a query). What am I doing wrong here and what is the proper way to search within this REST service?

darrenng123123 · ‎05-17-2021

This is the code i am running. Its the exact same code except for the url change. Unfortunately i cannot give you the exact URL, not that it matters because its within our organisation's intranet.

from arcgis.mapping import MapImageLayermil = MapImageLayer(r"https://xxx.xxx.xx/arcgis/rest/services/MapServices/xxxxxx/MapServer",gis=None)for layer in mil.layers:
    print(layer.properties["name"], layer.properties["type"])

This is the error code i got. To add, i verified that i am able to resolve the url from my environment

---------------------------------------------------------------------------
gaierror                                  Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
    156             conn = connection.create_connection(
--> 157                 (self._dns_host, self.port), self.timeout, **extra_kw
    158             )
/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
     60 
---> 61     for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
     62         af, socktype, proto, canonname, sa = res
/opt/conda/lib/python3.7/socket.py in getaddrinfo(host, port, family, type, proto, flags)
    747     addrlist = []
--> 748     for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
    749         af, socktype, proto, canonname, sa = res
gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
NewConnectionError                        Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    671                 headers=headers,
--> 672                 chunked=chunked,
    673             )
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    375         try:
--> 376             self._validate_conn(conn)
    377         except (SocketTimeout, BaseSSLError) as e:
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    993         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 994             conn.connect()
    995 
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in connect(self)
    333         # Add certificate verification
--> 334         conn = self._new_conn()
    335         hostname = self.host
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
    168             raise NewConnectionError(
--> 169                 self, "Failed to establish a new connection: %s" % e
    170             )
NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
MaxRetryError                             Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    719             retries = retries.increment(
--> 720                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    721             )
/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    435         if new_retry.is_exhausted():
--> 436             raise MaxRetryError(_pool, url, error or ResponseError(cause))
    437 
MaxRetryError: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
ConnectionError                           Traceback (most recent call last)
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    686                                           cert=cert,
--> 687                                           files=files)
    688         except requests.exceptions.SSLError as err:
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in post(self, url, data, json, **kwargs)
    580 
--> 581         return self.request('POST', url, data=data, json=json, **kwargs)
    582 
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    532         send_kwargs.update(settings)
--> 533         resp = self.send(prep, **send_kwargs)
    534 
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in send(self, request, **kwargs)
    645         # Send the request
--> 646         r = adapter.send(request, **kwargs)
    647 
/opt/conda/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    515 
--> 516             raise ConnectionError(e, request=request)
    517 
ConnectionError: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
ConnectionError                           Traceback (most recent call last)
<ipython-input-42-293792572c7c> in <module>
      1 from arcgis.mapping import MapImageLayer
      2 
----> 3 mil = MapImageLayer(r"https://xxxxx.xxx.xxxx/arcgis/rest/services/MapServices/xxxxxxxx/MapServer",gis=None)
      4 
      5 for layer in mil.layers:
~/.local/lib/python3.7/site-packages/arcgis/mapping/_types.py in __init__(self, url, gis)
   3312         :param gis: the GIS to which this layer belongs
   3313         """
-> 3314         super(MapImageLayer, self).__init__(url, gis)
   3315 
   3316         self._populate_layers()
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, gis)
  11561 
  11562     def __init__(self, url, gis=None):
> 11563         super(Layer, self).__init__(url, gis)
  11564         self.filter = None
  11565         self._time_filter = None
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, gis)
  11412 
  11413         if gis is None:
> 11414             gis = GIS(set_active=False)
  11415             self._gis = gis
  11416             self._con = gis._con
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, username, password, key_file, cert_file, verify_cert, set_active, client_id, profile, **kwargs)
    357                                    "argument when connecting to the GIS.")
    358             else:
--> 359                 raise e
    360         try:
    361             if url.lower().find("arcgis.com") > -1 and \
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, username, password, key_file, cert_file, verify_cert, set_active, client_id, profile, **kwargs)
    341                                            custom_auth=custom_auth, #token=self._utoken,
    342                                            client_secret=client_secret,
--> 343                                            trust_env=kwargs.get("trust_env", None))
    344             if self._is_hosted_nb_home:
    345                 # For GIS("home") objects, force no referer passed in
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in __init__(self, url, username, password, key_file, cert_file, expiration, referer, proxy_host, proxy_port, connection, workdir, tokenurl, verify_cert, client_id, custom_auth, token, **kwargs)
    171                                       trust_env=trust_env)
    172         #self.get_version(True)
--> 173         self.get_properties(True)
    174 
    175 
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in get_properties(self, force)
   1136                     resp = self.con.get(path, ssl=True) # issue seen with key, cert auth
   1137                 if not resp:
-> 1138                     raise e
   1139 
   1140             if resp:
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in get_properties(self, force)
   1124             resp = None
   1125             try:
-> 1126                 resp = self.con.post(path, self._postdata(), ssl=True)
   1127             except Exception as e:
   1128                 if not self.con._verify_cert and \
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    694         except requests.exceptions.ConnectionError as errCE:
    695             raise requests.exceptions.ConnectionError(
--> 696                 "A connection error has occurred: %s" % errCE)
    697         except requests.exceptions.InvalidHeader as errIH:
    698             raise requests.exceptions.InvalidHeader(
ConnectionError: A connection error has occurred: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))

darrenng123123 · ‎05-17-2021

Here is the code that i am using. Its the exact same one as suggested. Unfortunately i cannot share the url that i am connecting to, not that it matters because it is hosted in our organisation's intranet.

from arcgis.mapping import MapImageLayer
mil = MapImageLayer(r"https://xxx.xxx.xx/arcgis/rest/services/MapServices/xxxxxx/MapServer",gis=None)
for layer in mil.layers:
    print(layer.properties["name"], layer.properties["type"])

and here is the error code: also note that the mapserver url used is resolvable in my environment :

---------------------------------------------------------------------------
gaierror                                  Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
    156             conn = connection.create_connection(
--> 157                 (self._dns_host, self.port), self.timeout, **extra_kw
    158             )
/opt/conda/lib/python3.7/site-packages/urllib3/util/connection.py in create_connection(address, timeout, source_address, socket_options)
     60 
---> 61     for res in socket.getaddrinfo(host, port, family, socket.SOCK_STREAM):
     62         af, socktype, proto, canonname, sa = res
/opt/conda/lib/python3.7/socket.py in getaddrinfo(host, port, family, type, proto, flags)
    747     addrlist = []
--> 748     for res in _socket.getaddrinfo(host, port, family, type, proto, flags):
    749         af, socktype, proto, canonname, sa = res
gaierror: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
NewConnectionError                        Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    671                 headers=headers,
--> 672                 chunked=chunked,
    673             )
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in _make_request(self, conn, method, url, timeout, chunked, **httplib_request_kw)
    375         try:
--> 376             self._validate_conn(conn)
    377         except (SocketTimeout, BaseSSLError) as e:
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in _validate_conn(self, conn)
    993         if not getattr(conn, "sock", None):  # AppEngine might not have  `.sock`
--> 994             conn.connect()
    995 
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in connect(self)
    333         # Add certificate verification
--> 334         conn = self._new_conn()
    335         hostname = self.host
/opt/conda/lib/python3.7/site-packages/urllib3/connection.py in _new_conn(self)
    168             raise NewConnectionError(
--> 169                 self, "Failed to establish a new connection: %s" % e
    170             )
NewConnectionError: <urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known
During handling of the above exception, another exception occurred:
MaxRetryError                             Traceback (most recent call last)
/opt/conda/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    448                     retries=self.max_retries,
--> 449                     timeout=timeout
    450                 )
/opt/conda/lib/python3.7/site-packages/urllib3/connectionpool.py in urlopen(self, method, url, body, headers, retries, redirect, assert_same_host, timeout, pool_timeout, release_conn, chunked, body_pos, **response_kw)
    719             retries = retries.increment(
--> 720                 method, url, error=e, _pool=self, _stacktrace=sys.exc_info()[2]
    721             )
/opt/conda/lib/python3.7/site-packages/urllib3/util/retry.py in increment(self, method, url, response, error, _pool, _stacktrace)
    435         if new_retry.is_exhausted():
--> 436             raise MaxRetryError(_pool, url, error or ResponseError(cause))
    437 
MaxRetryError: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
ConnectionError                           Traceback (most recent call last)
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    686                                           cert=cert,
--> 687                                           files=files)
    688         except requests.exceptions.SSLError as err:
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in post(self, url, data, json, **kwargs)
    580 
--> 581         return self.request('POST', url, data=data, json=json, **kwargs)
    582 
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in request(self, method, url, params, data, headers, cookies, files, auth, timeout, allow_redirects, proxies, hooks, stream, verify, cert, json)
    532         send_kwargs.update(settings)
--> 533         resp = self.send(prep, **send_kwargs)
    534 
/opt/conda/lib/python3.7/site-packages/requests/sessions.py in send(self, request, **kwargs)
    645         # Send the request
--> 646         r = adapter.send(request, **kwargs)
    647 
/opt/conda/lib/python3.7/site-packages/requests/adapters.py in send(self, request, stream, timeout, verify, cert, proxies)
    515 
--> 516             raise ConnectionError(e, request=request)
    517 
ConnectionError: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))
During handling of the above exception, another exception occurred:
ConnectionError                           Traceback (most recent call last)
<ipython-input-42-293792572c7c> in <module>
      1 from arcgis.mapping import MapImageLayer
      2 
----> 3 mil = MapImageLayer(r"https://xxxxx.xxx.xxxx/arcgis/rest/services/MapServices/xxxxxxxx/MapServer",gis=None)
      4 
      5 for layer in mil.layers:
~/.local/lib/python3.7/site-packages/arcgis/mapping/_types.py in __init__(self, url, gis)
   3312         :param gis: the GIS to which this layer belongs
   3313         """
-> 3314         super(MapImageLayer, self).__init__(url, gis)
   3315 
   3316         self._populate_layers()
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, gis)
  11561 
  11562     def __init__(self, url, gis=None):
> 11563         super(Layer, self).__init__(url, gis)
  11564         self.filter = None
  11565         self._time_filter = None
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, gis)
  11412 
  11413         if gis is None:
> 11414             gis = GIS(set_active=False)
  11415             self._gis = gis
  11416             self._con = gis._con
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, username, password, key_file, cert_file, verify_cert, set_active, client_id, profile, **kwargs)
    357                                    "argument when connecting to the GIS.")
    358             else:
--> 359                 raise e
    360         try:
    361             if url.lower().find("arcgis.com") > -1 and \
~/.local/lib/python3.7/site-packages/arcgis/gis/__init__.py in __init__(self, url, username, password, key_file, cert_file, verify_cert, set_active, client_id, profile, **kwargs)
    341                                            custom_auth=custom_auth, #token=self._utoken,
    342                                            client_secret=client_secret,
--> 343                                            trust_env=kwargs.get("trust_env", None))
    344             if self._is_hosted_nb_home:
    345                 # For GIS("home") objects, force no referer passed in
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in __init__(self, url, username, password, key_file, cert_file, expiration, referer, proxy_host, proxy_port, connection, workdir, tokenurl, verify_cert, client_id, custom_auth, token, **kwargs)
    171                                       trust_env=trust_env)
    172         #self.get_version(True)
--> 173         self.get_properties(True)
    174 
    175 
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in get_properties(self, force)
   1136                     resp = self.con.get(path, ssl=True) # issue seen with key, cert auth
   1137                 if not resp:
-> 1138                     raise e
   1139 
   1140             if resp:
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_portalpy.py in get_properties(self, force)
   1124             resp = None
   1125             try:
-> 1126                 resp = self.con.post(path, self._postdata(), ssl=True)
   1127             except Exception as e:
   1128                 if not self.con._verify_cert and \
~/.local/lib/python3.7/site-packages/arcgis/gis/_impl/_con/_connection.py in post(self, path, params, files, **kwargs)
    694         except requests.exceptions.ConnectionError as errCE:
    695             raise requests.exceptions.ConnectionError(
--> 696                 "A connection error has occurred: %s" % errCE)
    697         except requests.exceptions.InvalidHeader as errIH:
    698             raise requests.exceptions.InvalidHeader(
ConnectionError: A connection error has occurred: HTTPSConnectionPool(host='www.arcgis.com', port=443): Max retries exceeded with url: /sharing/rest/portals/self (Caused by NewConnectionError('<urllib3.connection.VerifiedHTTPSConnection object at 0x7ff1811f1a90>: Failed to establish a new connection: [Errno -2] Name or service not known'))