We found that although the stated limit is 10MB, we received that same generic error when we attempted to upload chunks as small as 4MB, and that we only were able to reliably and consistently succeed with multipart uploads when the chunks were around 3MB. Our guess is that mutipart-form encoding is simply far less compact, so when a file or part of a file is converted to form-encoded UTF-8 it's much bigger and hits the 10MB cutoff sooner. That means that although you can get right up to 10MB for a standard upload, once you cross that 10MB threshold, you have to upload in much smaller chunks because of the multipart encoding. That said, it's not the end of the world once you've got the plumbing for multipart uploads coded, just set the chunk limit smaller. Our simplified code block is below, hope it helps.
def uploadMultipartFeatures(token, feat_service, file):
import time, requests, os
""" Add the item to the portal in chunks."""
def read_in_chunks(file_object, chunk_size=3000000):
"""Generate file chunks of 3MB"""
while True:
data = file_object.read(chunk_size)
if not data:
break
yield data
try:
title = os.path.basename(file)
url = feat_service + "/uploads/register"
params = {"description": "Multipart Upload",
"f": "json",
"token": token,
'itemName': title}
response = requests.post(url, data=params)
json_response = response.json()
print("response was",json_response)
if 'success' in json_response.keys():
success = json_response['success']
if not success:
return json_response
itemID = json_response["item"]["itemID"]
url = feat_service + "/uploads/{}/uploadPart".format(itemID)
with open(file, 'rb') as f:
for part_num, piece in enumerate(read_in_chunks(f), start=1):
params = {"f": "json", "partId":str(part_num)}
fileObj = {"file": piece}
print("posting",url,params,file)
response = requests.post(url, data=params, files=fileObj)
json_response = response.json()
print("response was",json_response)
if 'success' in json_response.keys():
success = json_response['success']
if not success: break
if success:
url = feat_service + "/uploads/{}/commit".format(itemID)
params = {"f": "json", "token": token}
print("posting",url,params)
response = requests.post(url, data=params)
json_response = response.json()
print("response was",json_response)
return json_response
except Exception as e:
raise ValueError("Problem with uploadFeatures. Url: {} Params: {} Response: {} Exception: {}".format(url, params, response.text, e))
