Select to view content in your preferred language

Dataframe works in IDE but not in AGO notebook

915
3
09-15-2022 03:02 AM
HaydenWilson3
Emerging Contributor

Good day.

I am experiencing some strange behaviour from a Notebook in ArcGIS Online.

I want to convert a related table into a pandas dataframe. when I run this script locally on my machine it runs fine:

from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection
import tempfile
import pandas as pd
from datetime import datetime, timedelta
import numpy as np

# ArcGIS Online Login
gis = GIS(url='https://XXXXXXX.maps.arcgis.com/', username='XXXX', password='XXXXXXX')

#create a temp scratch workspace
tmpdir = tempfile.TemporaryDirectory()
download_folder = tmpdir.name

fl = gis.content.get('8fbdac5a21ae4fa8a72b7fb6bd96e6ca')
items = fl.tables[0]
df = pd.DataFrame.spatial.from_layer(items)
print(df)

 

Yet when I run this slightly amended version in ArcGIS Online from a notebook:

from arcgis.gis import GIS
from arcgis.features import FeatureLayerCollection
import tempfile
import pandas as pd
from datetime import datetime, timedelta
import numpy as np
gis = GIS("home")

#create a temp scratch workspace
tmpdir = tempfile.TemporaryDirectory()
download_folder = tmpdir.name


fl = gis.content.get('8fbdac5a21ae4fa8a72b7fb6bd96e6ca')
items = fl.tables[0]

df = pd.DataFrame.spatial.from_layer(items)
print(df)

I get the following error:

---------------------------------------------------------------------------
IntCastingNaNError                        Traceback (most recent call last)
File /opt/conda/lib/python3.9/site-packages/arcgis/features/geo/_accessor.py:2688, in GeoAccessor.from_layer(layer)
   2686     from arcgis.features.geo._io.serviceops import from_layer
-> 2688     return from_layer(layer=layer)
   2689 except ImportError:

File /opt/conda/lib/python3.9/site-packages/arcgis/features/geo/_io/serviceops.py:187, in from_layer(layer, query)
    186     raise ValueError("Invalid inputs: must be FeatureLayer or Table")
--> 187 sdf = layer.query(where=query, as_df=True)
    188 sdf.spatial._meta.source = layer

File /opt/conda/lib/python3.9/site-packages/arcgis/features/layer.py:3720, in Table.query(self, where, out_fields, time_filter, return_count_only, return_ids_only, return_distinct_values, group_by_fields_for_statistics, statistic_filter, result_offset, result_record_count, object_ids, gdb_version, order_by_fields, out_statistics, return_all_records, historic_moment, sql_format, return_exceeded_limit_features, as_df, having, **kwargs)
   3718 import pandas as pd
-> 3720 df = self._query_df(url, params)
   3721 dt_fields = [
   3722     fld["name"]
   3723     for fld in self.properties.fields
   3724     if fld["type"] == "esriFieldTypeDate"
   3725 ]

File /opt/conda/lib/python3.9/site-packages/arcgis/features/layer.py:3308, in FeatureLayer._query_df(self, url, params, **kwargs)
   3307 if dtypes:
-> 3308     df = df.astype(dtypes)
   3310 if "SHAPE" in featureset_dict:

File /opt/conda/lib/python3.9/site-packages/pandas/core/generic.py:5799, in NDFrame.astype(self, dtype, copy, errors)
   5797 if col_name in dtype:
   5798     results.append(
-> 5799         col.astype(dtype=dtype[col_name], copy=copy, errors=errors)
   5800     )
   5801 else:

File /opt/conda/lib/python3.9/site-packages/pandas/core/generic.py:5815, in NDFrame.astype(self, dtype, copy, errors)
   5813 else:
   5814     # else, only a single dtype is given
-> 5815     new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors)
   5816     return self._constructor(new_data).__finalize__(self, method="astype")

File /opt/conda/lib/python3.9/site-packages/pandas/core/internals/managers.py:418, in BaseBlockManager.astype(self, dtype, copy, errors)
    417 def astype(self: T, dtype, copy: bool = False, errors: str = "raise") -> T:
--> 418     return self.apply("astype", dtype=dtype, copy=copy, errors=errors)

File /opt/conda/lib/python3.9/site-packages/pandas/core/internals/managers.py:327, in BaseBlockManager.apply(self, f, align_keys, ignore_failures, **kwargs)
    326     else:
--> 327         applied = getattr(b, f)(**kwargs)
    328 except (TypeError, NotImplementedError):

File /opt/conda/lib/python3.9/site-packages/pandas/core/internals/blocks.py:591, in Block.astype(self, dtype, copy, errors)
    589 values = self.values
--> 591 new_values = astype_array_safe(values, dtype, copy=copy, errors=errors)
    593 new_values = maybe_coerce_values(new_values)

File /opt/conda/lib/python3.9/site-packages/pandas/core/dtypes/cast.py:1309, in astype_array_safe(values, dtype, copy, errors)
   1308 try:
-> 1309     new_values = astype_array(values, dtype, copy=copy)
   1310 except (ValueError, TypeError):
   1311     # e.g. astype_nansafe can fail on object-dtype of strings
   1312     #  trying to convert to float

File /opt/conda/lib/python3.9/site-packages/pandas/core/dtypes/cast.py:1257, in astype_array(values, dtype, copy)
   1256 else:
-> 1257     values = astype_nansafe(values, dtype, copy=copy)
   1259 # in pandas we don't store numpy str dtypes, so convert to object

File /opt/conda/lib/python3.9/site-packages/pandas/core/dtypes/cast.py:1168, in astype_nansafe(arr, dtype, copy, skipna)
   1167 elif np.issubdtype(arr.dtype, np.floating) and np.issubdtype(dtype, np.integer):
-> 1168     return astype_float_to_int_nansafe(arr, dtype, copy)
   1170 elif is_object_dtype(arr):
   1171 
   1172     # work around NumPy brokenness, #1987

File /opt/conda/lib/python3.9/site-packages/pandas/core/dtypes/cast.py:1213, in astype_float_to_int_nansafe(values, dtype, copy)
   1212 if not np.isfinite(values).all():
-> 1213     raise IntCastingNaNError(
   1214         "Cannot convert non-finite values (NA or inf) to integer"
   1215     )
   1216 return values.astype(dtype, copy=copy)

IntCastingNaNError: Cannot convert non-finite values (NA or inf) to integer

During handling of the above exception, another exception occurred:

Exception                                 Traceback (most recent call last)
Input In [5], in <cell line: 17>()
     15 utr_items = predeploymnet_master_fl.tables[0]
     16 print(utr_items)
---> 17 utr_df = pd.DataFrame.spatial.from_layer(utr_items)
     18 print(utr_df)

File /opt/conda/lib/python3.9/site-packages/arcgis/features/geo/_accessor.py:2697, in GeoAccessor.from_layer(layer)
   2692     raise Exception(
   2693         "Malformed response from server, could not load the dataset: %s"
   2694         % str(je)
   2695     )
   2696 except Exception as e:
-> 2697     raise Exception("Could not load the dataset: %s" % str(e))

Exception: Could not load the dataset: Cannot convert non-finite values (NA or inf) to integer

 

Can anyone please assist? 

0 Kudos
3 Replies
HaydenWilson3
Emerging Contributor

I have tried using:

df = items.query(as_df=True, out_fields = ['XXX','XXX','XXX'])

 

but sadly some of the fields contain (NA or inf) values

0 Kudos
EarlMedina
Esri Regular Contributor

Are there any obvious difference between the two environments? Are the arcgis package versions the same? What about the python versions?

0 Kudos
HaydenWilson3
Emerging Contributor

Okay, Figured it out!

In ArcGIS Online, Rather than using :

pd.DataFrame.spatial.from_layer(items)

 

Use:

gis.content.get('8fbdac5a21ae4fa8a72b7fb6bd96e6ca').tables[0].query().sdf
0 Kudos