POST
|
Maybe a firewall issue on my side? Can I download it and install locally? I try at this location Tryin to install urllib3 I can get their via web browser
... View more
08-05-2022
10:08 AM
|
0
|
0
|
2600
|
POST
|
Hello... I am trying to grab some JSON text that is on a webpage via a WordPress Api output. There is nothing on this page except the JSON string. If I go to CMD prompt and navigate to my Python folder I can confirm I have python and PIP C:\Python310>py --version
Python 3.10.6
C:\Python310>py -m pip --version
pip 22.2.1 from C:\Python310\lib\site-packages\pip (python 3.10) import arcpy
import requests
import json
import uuid
from urllib.request import urlopen
import urllib
import requests
url = "https://somewebsite/"
response = urlopen(url)
data_json = json.loads(response.read)
print (data_json) I am trying to above code but keep getting this error: Traceback (most recent call last): File "C:\Users\t\Desktop\JSONSamples\readJson.py", line 7, in <module> from urllib.request import urlopen ImportError: No module named request Any thoughts? Do I need to install something additional to Python? Any help would be much appreciated.
... View more
08-05-2022
09:34 AM
|
0
|
10
|
2833
|
POST
|
@Anonymous User man this is it... THANK YOU very much for your help...I learned a ton here and will pay it forward.....I didnt know which one to mark as the ANSWERED so I picked to one that directly addressed my initial question. Here is my Final Script if anyone has any questions or comments please feel free to ask. import arcpy
import json
import uuid
import os
import re
arcpy.env.workspace = 'C:/Users/t/Desktop/JSONSamples/'
dataInput = r"C:/Users/t/Desktop/JSONSamples/JSON_New3.json"
dataOutput = r"C:/Users/t/Desktop/JSONSamples/JSON_New4.json"
f = open(dataInput)
data = json.load(f)
# Added this So I could use it in the For loop
data1 = {"BirdTrailLocations": data }
print data1
print ""
converted_list = []
for i in data1['BirdTrailLocations']:
feat_dict = {}
feat_dict["geometry"] = {"x": i["site_longitude"], "y": i["site_latitude"]}
# the list created a value type is incompatible with the field type. [site_facilities] so we nee dto convert it to a string
i["site_facilities"] = str(i["site_facilities"])
feat_dict["attributes"] = i
converted_list.append(feat_dict)
print converted_list
print ""
# Closing file
f.close()
new_dict = {"geometryType" : "esriGeometryPoint",
"spatialReference" : {"wkid" : 4326},
'fields': [{"name": str(k), "type": "esriFieldTypeString", "alias": str(k), "length": 255} for k, v in data[0].items()],
'features': converted_list}
print(new_dict)
with open(dataOutput, 'w') as f:
json.dump(new_dict, f)
output_fc = r'C:\\Users\\t\\Desktop\\JSONSamples\\outgdb.gdb\test'
arcpy.env.workspace = r"C:\\Users\\t\\Desktop\\JSONSamples\\outgdb.gdb"
cws = arcpy.env.workspace
fc_Delete = ["test"]
for fc in fc_Delete:
fc_path = os.path.join(cws, fc)
if arcpy.Exists(fc_path):
arcpy.Delete_management(fc_path)
arcpy.JSONToFeatures_conversion(dataOutput, output_fc)
# old field is used for the key for the new field details.
fld_dict = {"site_longitude": {'field_name': 'longitude', 'field_type': "LONG"},
"site_latitude": {'field_name': 'latitude', 'field_type': "DOUBLE"},
"new_field": {'field_name': 'new_field', 'field_type': 'SHORT', 'field_precision': None, 'field_scale': None, 'field_length': None,
'field_alias': 'new alias', 'field_is_nullable': True, 'field_is_required': False, 'field_domain': None},
"new_fieldText": {'field_name': 'new_fieldText', 'field_type': 'TEXT', 'field_precision': None, 'field_scale': None, 'field_length': 50,
'field_alias': 'new Text', 'field_is_nullable': True, 'field_is_required': False, 'field_domain': None}}
for fld, val in fld_dict.items():
value = str({val.get("field_name")})
print "adding Field: " + value
arcpy.AddField_management(output_fc, val.get('field_name'), val.get('field_type'), val.get('field_precision'), val.get('field_scale'), val.get('field_length'),
val.get('field_alias'), val.get('field_is_nullable'), val.get('field_is_required'), val.get('field_domain'))
##### Clean up the fields.
flds = [f.name for f in arcpy.ListFields(output_fc)]
p_tag = re.compile(r'.*?>(.*)<')
print ""
print "--- Fixing Data ---"
print ""
def extract_p_tag_descriptions(invalue):
result = p_tag.search(invalue)
if result:
newVal = result.group(1)
else:
newVal = invalue
print 'Fixing p Tag ' + newVal + ' to ' + newVal
print ""
return newVal
def extract_p_tag_directions(invalue):
result = p_tag.search(invalue)
if result:
newVal = result.group(1)
else:
newVal = invalue
print 'Fixing p Tag Directions' + newVal + ' to ' + newVal
print ""
return newVal
def fix_website(invalue):
newVal = re.sub(r'/', r'\\', invalue)
newVal = newVal.replace('\\\\', '\\')
#arcpy.AddMessage(f'Fixing: {invalue} to {newVal}')
print 'Fixing website ' + invalue + ' to ' + newVal
print ""
return str(newVal)
def fix_facilities(invalue):
newVal = invalue.replace("u'", "'").replace('[', '').replace(']', '').replace(r"'", r'')
#arcpy.AddMessage(f'Fixing: {invalue} to {newVal}')
print 'Fixing facilities ' + invalue + ' to ' + newVal
print ""
return newVal
def fix_na(invalue):
newVal = invalue.replace('\/', '\\')
#arcpy.AddMessage(f'Fixing: {invalue} to {newVal}')
print 'Fixing NA' + invalue + ' to ' + newVal
print ""
return str(newVal)
with arcpy.da.UpdateCursor(output_fc, flds) as uCur:
int_example = 0
text_example = "new text"
for row in uCur:
# new field- migrate the previous info into the new field
row[flds.index('longitude')] = float(extract_p_tag_descriptions(row[flds.index('site_longitude')]))
row[flds.index('latitude')] = float(extract_p_tag_descriptions(row[flds.index('site_latitude')]))
row[flds.index('new_field')] = int_example
row[flds.index('new_fieldText')] = text_example
row[flds.index('site_description')] = extract_p_tag_descriptions(row[flds.index('site_description')])
row[flds.index('site_website')] = fix_website(row[flds.index('site_website')])
row[flds.index('site_ebird_url')] = fix_website(row[flds.index('site_ebird_url')])
row[flds.index('site_facilities')] = fix_facilities(row[flds.index('site_facilities')])
row[flds.index('seabirds')] = fix_na(row[flds.index('seabirds')])
uCur.updateRow(row)
int_example =+ 1
for fld, val in fld_dict.items():
# use the fld dict again to delete the 'old' fields no longer needed, but save the new fields by putting the name in the list:
if fld not in ['new_field', 'new_fieldText']:
arcpy.DeleteField_management(output_fc, fld)
... View more
08-05-2022
07:43 AM
|
1
|
0
|
155
|
POST
|
@Anonymous User thanks again.,,,Last question as I think this is now working great... You mentioned I can modify the field type and lengths etc. Is that done in the def as seen below for each Field? What would the syntax be to set a text field with length vs Int vs double etc def extract_p_tag(invalue):
result = p_tag.search(invalue)
newVal = result.group(1)
print 'Fixing p Tag ' + result + ' to ' + newVal
return newVal
... View more
08-05-2022
05:00 AM
|
0
|
2
|
170
|
POST
|
OK so I am trying to clean up the data and all the reformats work fine except this one... thoughts? ERROR: Traceback (most recent call last): File "C:\Users\t\Desktop\JSONSamples\AddPoints.py", line 104, in <module> row[flds.index('site_description')] = extract_p_tag(row[flds.index('site_description')]) File "C:\Users\tjv36463\Desktop\JSONSamples\AddPoints.py", line 77, in extract_p_tag newVal = result.group(1) AttributeError: 'NoneType' object has no attribute 'group' ##### Clean up the fields.
flds = [f.name for f in arcpy.ListFields(output_fc)]
p_tag = re.compile(r'.*?>(.*)<')
def extract_p_tag(invalue):
result = p_tag.search(invalue)
newVal = result.group(1)
#arcpy.AddMessage(f'Fixing: {invalue} to {newVal}')
print 'Fixing p Tag ' + result + ' to ' + newVal
print ""
return newVal
... View more
08-04-2022
10:45 AM
|
0
|
4
|
681
|
POST
|
@Anonymous User wow thanks man... very appreciated... Can you elaborate on: You'll need to import re for some regex to fix the values Not sure what this is.... a simple Import re or do I need to install something
... View more
08-04-2022
06:29 AM
|
0
|
6
|
692
|
POST
|
I just do not think I am getting to the outgdb.gdb correctly.... I think this is a pathing issue? Not sure c:/Users/t/Desktop/JSONSamples/outgdb.gdb
... View more
08-04-2022
06:16 AM
|
0
|
0
|
697
|
POST
|
OK you can see my result in the other post... the resulting JSON is out of order as to what is specified in the python script....I am using this code to try and create the Feature Class but getting an error... I get this error: Traceback (most recent call last): File "C:/Users/t/Desktop/JSONSamples/AddPoints.py", line 47, in <module> arcpy.JSONToFeatures_conversion(json_object, os.path.join("outgdb.gdb", "myfeatures")) File "C:\Program Files (x86)\ArcGIS\Desktop10.8\ArcPy\arcpy\conversion.py", line 602, in JSONToFeatures raise e ExecuteError: Failed to execute. Parameters are not valid. ERROR 000814: Invalid file type Failed to execute (JSONToFeatures). import arcpy
import requests
import json
import uuid
import os
arcpy.env.workspace = "C:/Users/t/Desktop/JSONSamples/"
dataInput = r"C:/Users/t/Desktop/JSONSamples/JSON4.json"
f = open(dataInput)
data = json.load(f)
data1 = {"BirdTrailLocations": data }
converted_list = []
for i in data1['BirdTrailLocations']:
feat_dict = {}
feat_dict["geometry"] = {"x": i["site_longitude"], "y": i["site_latitude"]}
# the list created a value type is incompatible with the field type. [site_facilities] so we nee dto convert it to a string
i["site_facilities"] = str(i["site_facilities"])
feat_dict["attributes"] = i
converted_list.append(feat_dict)
# Closing file
f.close()
esri_json = {"geometryType" : "esriGeometryPoint",
"spatialReference" : {"wkid" : 4326},
'fields': [{"name": str(k), "type": "esriFieldTypeString", "alias": str(k), "length": 255} for k, v in data[0].items()],
'features': converted_list}
json_object = json.dumps(esri_json)
print json_object
print ""
arcpy.JSONToFeatures_conversion(json_object, os.path.join("outgdb.gdb", "myfeatures"))
... View more
08-04-2022
05:42 AM
|
0
|
9
|
705
|
POST
|
OK so great making progress here... So I added/modified the code and its reading ok but not in the order you specified I was wondering if that was OK???? You can see it writes Fields - geometry type - features - spatial reference But in the code I specified esri_json = {"geometryType" : "esriGeometryPoint",
"spatialReference" : {"wkid" : 4326},
'fields': [{"name": str(k), "type": "esriFieldTypeString", "alias": str(k), "length": 255} for k, v in data[0].items()],
'features': converted_list} {'fields': [{'alias': 'site_name', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_name'}, {'alias': 'insects', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'insects'}, {'alias': 'site_number', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_number'}, {'alias': 'site_id', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_id'}, {'alias': 'seabirds', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'seabirds'}, {'alias': 'black_bear', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'black_bear'}, {'alias': 'site_ebird_url', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_ebird_url'}, {'alias': 'shorebirds', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'shorebirds'}, {'alias': 'loop_name', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'loop_name'}, {'alias': 'last_updated', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'last_updated'}, {'alias': 'lizards', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'lizards'}, {'alias': 'hawks', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'hawks'}, {'alias': 'frogs', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'frogs'}, {'alias': 'site_ebird_site_id', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_ebird_site_id'}, {'alias': 'site_access', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_access'}, {'alias': 'site_loop', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_loop'}, {'alias': 'site_season', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_season'}, {'alias': 'carto_id', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'carto_id'}, {'alias': 'cms_id', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'cms_id'}, {'alias': 'general_birding', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'general_birding'}, {'alias': 'phase', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'phase'}, {'alias': 'site_contact', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_contact'}, {'alias': 'site_latitude', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_latitude'}, {'alias': 'marine_mammals', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'marine_mammals'}, {'alias': 'bald_eagle', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'bald_eagle'}, {'alias': 'site_longitude', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_longitude'}, {'alias': 'site_website', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_website'}, {'alias': 'site_facilities', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_facilities'}, {'alias': 'site_description', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'site_description'}, {'alias': 'waterfowl', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'waterfowl'}, {'alias': 'grassland_birds', 'length': 255, 'type': 'esriFieldTypeString', 'name': 'grassland_birds'}], 'geometryType': 'esriGeometryPoint', 'features': [{'geometry': {'y': 37.44901, 'x': -77.581618}, 'attributes': {u'site_name': u'Rockwood Park', u'insects': u'Unknown', u'site_number': u'1', u'site_id': u'CAP01', u'seabirds': u'Na', u'black_bear': u'Unknown', u'site_ebird_url': u'https:ebird.orgbarchart?r=L617671&yr=all&m=', u'shorebirds': u'Na', u'loop_name': u'Appomattox Loop', u'last_updated': u'May 20, 2022', u'lizards': u'Unknown', u'hawks': u'Na', u'frogs': u'Unknown', u'site_ebird_site_id': u'L617671', u'site_access': u'Free, Daily', u'site_loop': u'CAP', u'site_season': u'All', u'carto_id': u'01', u'cms_id': u'4799', u'general_birding': u'Available', u'phase': u'Coastal Phase', u'site_contact': u'Chesterfield', u'site_latitude': 37.44901, u'marine_mammals': u'Na', u'bald_eagle': u'Unknown', u'site_longitude': -77.581618, u'site_website': u'www.chesterfield.gov3824Rockwood-Park', u'site_facilities': "[u'environmental_study_area', u'handicap_accessible', u'hiking_trails', u'information', u'interpretive_nature_program', u'parking', u'phone', u'picnic', u'restrooms']", u'site_description': u'This 160-acre', u'waterfowl': u'Na', u'grassland_birds': u'Unknown'}}], 'spatialReference': {'wkid': 4326}}
... View more
08-04-2022
05:35 AM
|
0
|
0
|
706
|
POST
|
OK this is what I have and getting another error.....sorry for my ignorance...tryin to learn here. Its rewriting to the X and Y field names so thats good... But getting error Traceback (most recent call last): File "C:/Users/t/Desktop/JSONSamples/AddPoints.py", line 44, in <module> arcpy.JSONToFeatures_conversion(json_object, os.path.join("outgdb.gdb", "myfeatures")) File "C:\Program Files (x86)\ArcGIS\Desktop10.8\ArcPy\arcpy\conversion.py", line 602, in JSONToFeatures raise e ExecuteError: Failed to execute. Parameters are not valid. ERROR 000814: Invalid file type Failed to execute (JSONToFeatures). Do I have to convert the JSON Object to something before I use the JSONToFeatures_conversion Maybe its my output location and gdb. I have a file gdb called outgdb.gdb at the same location as where the python file is located. import arcpy
import requests
import json
import uuid
import os
arcpy.env.workspace = "C:/Users/t/Desktop/JSONSamples/"
dataInput = r"C:/Users/t/Desktop/JSONSamples/JSON3.json"
f = open(dataInput)
data = json.load(f)
converted_list = []
for i in data['BirdTrailLocations']:
i['x'] = i.pop('site_longitude')
i['y'] = i.pop('site_latitude')
converted_list.append(i)
varsearchSiteID = i["site_id"]
varsearchSiteName = i["site_name"]
print ("site_id: " + varsearchSiteID)
print ("site_name: " + varsearchSiteName)
print ""
print(i)
print ""
# Closing file
f.close()
json_object = json.dumps(converted_list)
print json_object
print ""
print converted_list
outfeatureLocation = "C:/Users/t/Desktop/JSONSamples/OutputGeodatabase/"
arcpy.JSONToFeatures_conversion(json_object, os.path.join("outgdb.gdb", "myfeatures"))
... View more
08-03-2022
01:21 PM
|
0
|
12
|
726
|
POST
|
If I do this I get this error import arcpy
import os
arcpy.env.workspace = "C:/Users/t/Desktop/JSONSamples/"
outfeatureLocation = "C:/Users/t/Desktop/JSONSamples/OutputGeodatabase"
arcpy.JSONToFeatures_conversion("JSON.json", outfeatureLocation) Traceback (most recent call last): File "C:/Users/t/Desktop/JSONSamples/AddPoints.py", line 7, in <module> arcpy.JSONToFeatures_conversion("JSON.json", outfeatureLocation) File "C:\Program Files (x86)\ArcGIS\Desktop10.8\ArcPy\arcpy\conversion.py", line 602, in JSONToFeatures raise e ExecuteError: ERROR 001558: Error parsing json file 'JSON.json'. Failed to execute (JSONToFeatures). If I do this: import arcpy
import os
arcpy.env.workspace = "C:/Users/t/Desktop/JSONSamples/"
outfeatureLocation = "C:/Users/t/Desktop/JSONSamples/OutputGeodatabase"
arcpy.JSONToFeatures_conversion("JSON.json", os.path.join("outgdb.gdb"), 'POINT')
Traceback (most recent call last): File "C:/Users/tjv36463/Desktop/JSONSamples/AddPoints.py", line 7, in <module> arcpy.JSONToFeatures_conversion("JSON.json", outfeatureLocation, 'POINT') TypeError: JSONToFeatures() takes at most 2 arguments (3 given)
... View more
08-03-2022
12:09 PM
|
0
|
1
|
689
|
POST
|
Tried this import arcpy
import os
arcpy.env.workspace = "C:/Users/t/Desktop/JSONSamples/"
arcpy.JSONToFeatures_conversion("JSON.json", os.path.join("outgdb.gdb", "myfeatures"), {POINT}) Error: Traceback (most recent call last): File "C:/Users/tjv36463/Desktop/JSONSamples/AddPoints.py", line 4, in <module> arcpy.JSONToFeatures_conversion("JSON.json", os.path.join("outgdb.gdb", "myfeatures"), {POINT}) NameError: name 'POINT' is not defined
... View more
08-03-2022
11:59 AM
|
0
|
3
|
699
|
POST
|
What is "myfeatures" referring to in the example you gave me? I thought that it took 3 parameters and the third would be POINT, LINE, POLYGON etc. arcpy.conversion.JSONToFeatures(in_json_file, out_features, {geometry_type}) EXAMPLE arcpy.JSONToFeatures_conversion("myjsonfeatures.json", os.path.join("outgdb.gdb", "myfeatures"))
... View more
08-03-2022
11:52 AM
|
0
|
0
|
701
|
POST
|
@Anonymous User did you look at my attachment... I am not sure how to specify the X Y field to get this to work... there is no geometry in the json file.
... View more
08-03-2022
11:03 AM
|
0
|
6
|
714
|
POST
|
I have a json string that is being read from a word press API URL.... it looks like the attached. I need to get this to a feature class. In addition to all the fields, it only has an X and Y field which I have to convert to point geometry. Any thoughts on how I can achieve this via Python? So I need to read the URL for the json string then create a FC from it. Python to Table and then Table to FC? Any help would be appreciated.
... View more
08-03-2022
10:11 AM
|
0
|
22
|
1633
|
Title | Kudos | Posted |
---|---|---|
1 | 09-15-2022 11:02 AM | |
1 | 05-21-2021 07:35 AM | |
1 | 08-09-2022 12:39 PM | |
1 | 11-10-2016 11:20 AM | |
2 | 11-15-2016 03:33 PM |
Online Status |
Offline
|
Date Last Visited |
09-19-2022
09:23 PM
|