I often use pd.DataFrame.spatial.from_layer(***) in my scripts to convert the layer to a pandas dataframe for running reports or otherwise manipulating the data. My scripts have worked fine for the past year or more, but I'm getting an error, now.
Last week, I updated ArcPro to version 3.0.2, and it currently says my ArcGIS Pro version is up to date. I did discover that the Python versions have changed, and since I often run my scripts from Anaconda, I re-cloned the python environment so I'm using the same versions of the arcpy libraries.
However, I'm noticing some funny behaviors with the DataFrame.spatial.from_layer(***) command. I am getting an error due to having nulls in the layers. For one layer, I was able to fix this by setting all nulls to 0 or 1, since these are realistic default codes. However, in other layers, this is not possible. The data represents information being collected in the field, and assuming a particular value when nothing is entered is a bad data management practice. These fields allow nulls and that is required.
Up until last week, the conversion to a dataframe worked fine with nulls. However, I am now getting the error that it is trying to convert null values to an integer, even if I don't have any integer-type classes in the layer. I'm not sure what changed or how to fix this. Please help as I need to run these scripts ASAP (regulatory reporting for the end of the quarter). I am desperate for any help you can provide!
Here is the full error:
---------------------------------------------------------------------------
IntCastingNaNError Traceback (most recent call last)
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\arcgis\features\geo\_accessor.py in from_layer(layer)
2687
-> 2688 return from_layer(layer=layer)
2689 except ImportError:
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\arcgis\features\geo\_io\serviceops.py in from_layer(layer, query)
186 raise ValueError("Invalid inputs: must be FeatureLayer or Table")
--> 187 sdf = layer.query(where=query, as_df=True)
188 sdf.spatial._meta.source = layer
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\arcgis\features\layer.py in query(self, where, out_fields, time_filter, return_count_only, return_ids_only, return_distinct_values, group_by_fields_for_statistics, statistic_filter, result_offset, result_record_count, object_ids, gdb_version, order_by_fields, out_statistics, return_all_records, historic_moment, sql_format, return_exceeded_limit_features, as_df, having, **kwargs)
3793
-> 3794 df = self._query_df(url, params)
3795 count += len(df)
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\arcgis\features\layer.py in _query_df(self, url, params, **kwargs)
3307 if dtypes:
-> 3308 df = df.astype(dtypes)
3309
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\generic.py in astype(self, dtype, copy, errors)
5798 results.append(
-> 5799 col.astype(dtype=dtype[col_name], copy=copy, errors=errors)
5800 )
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\generic.py in astype(self, dtype, copy, errors)
5814 # else, only a single dtype is given
-> 5815 new_data = self._mgr.astype(dtype=dtype, copy=copy, errors=errors)
5816 return self._constructor(new_data).__finalize__(self, method="astype")
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\internals\managers.py in astype(self, dtype, copy, errors)
417 def astype(self: T, dtype, copy: bool = False, errors: str = "raise") -> T:
--> 418 return self.apply("astype", dtype=dtype, copy=copy, errors=errors)
419
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\internals\managers.py in apply(self, f, align_keys, ignore_failures, **kwargs)
326 else:
--> 327 applied = getattr(b, f)(**kwargs)
328 except (TypeError, NotImplementedError):
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\internals\blocks.py in astype(self, dtype, copy, errors)
590
--> 591 new_values = astype_array_safe(values, dtype, copy=copy, errors=errors)
592
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\dtypes\cast.py in astype_array_safe(values, dtype, copy, errors)
1308 try:
-> 1309 new_values = astype_array(values, dtype, copy=copy)
1310 except (ValueError, TypeError):
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\dtypes\cast.py in astype_array(values, dtype, copy)
1256 else:
-> 1257 values = astype_nansafe(values, dtype, copy=copy)
1258
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\dtypes\cast.py in astype_nansafe(arr, dtype, copy, skipna)
1167 elif np.issubdtype(arr.dtype, np.floating) and np.issubdtype(dtype, np.integer):
-> 1168 return astype_float_to_int_nansafe(arr, dtype, copy)
1169
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\pandas\core\dtypes\cast.py in astype_float_to_int_nansafe(values, dtype, copy)
1212 if not np.isfinite(values).all():
-> 1213 raise IntCastingNaNError( 1214 "Cannot convert non-finite values (NA or inf) to integer"
IntCastingNaNError: Cannot convert non-finite values (NA or inf) to integer
During handling of the above exception, another exception occurred:
Exception Traceback (most recent call last)
<ipython-input-15-1882b171487b> in <module>
8
9 assets_df = pd.DataFrame.spatial.from_layer(nf1)
---> 10 poles_df = pd.DataFrame.spatial.from_layer(nt1)
11 #circ_df = pd.DataFrame.spatial.from_layer(nt2)
C:\ProgramData\Anaconda3\envs\arcgispy3\lib\site-packages\arcgis\features\geo\_accessor.py in from_layer(layer)
2695 )
2696 except Exception as e:
-> 2697 raise Exception("Could not load the dataset: %s" % str(e))
2698
2699 # ----------------------------------------------------------------------
Exception: Could not load the dataset: Cannot convert non-finite values (NA or inf) to integer
And here is the analysis of my fields:
I do have some fields that are type Long, but they also do allow nulls.
Any suggestions?
