nav[aria-label="Primary Navigation"] { padding: 0; & ul { list-style: none; width: 100%; display: flex; flex-direction: row; justify-content: start; align-items: start; gap: 30px; padding: 0; & li { margin: 0; } & ul li { list-style: none; } } }

How to solve the error:out of memory

HuaShang

I want to use the clip function in python, the code is as follow:

import os

from multiprocessing import Pool

import arcpy

from arcpy import env

def data_clip(shp):

env.workspace = r"F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ShpFile/f"

clip_features = "D:/Research/boundry.shp"

pre, suf = os.path.splitext(shp)

new = pre + "_china" + suf

in_mem = os.path.join("in_memory", pre)

out_feature_class = os.path.join("F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ChinaShpFile", new)

arcpy.Clip_analysis(shp, clip_features, in_mem)

arcpy.CopyFeatures_management(in_mem, out_feature_class)

arcpy.DeleteFeatures_management(in_mem)

if __name__ == "__main__":

env.overwriteOutput = True

env.workspace = r"F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ShpFile/f"

p = Pool(7)

fcs = arcpy.ListFeatureClasses()

p.map(data_clip, fcs)

print('Clip finish!')

But I got the "out of memory" error.

If anyone knows how to solve this problem?

Thank you so much!!!

Find more posts tagged with

Accepted answers

All comments

DanPatterson_Retired

when it comes to multiprocessing, have you had a look at the links Curtis Price posted here... it is in response to another thread https://community.esri.com/docs/DOC-3824#comment-11549

HuaShang

Thanks! I'll check that. But when I didn't use multiprocessing, it comes out the same error.

JeffKaplan_Boston_0-1744902404235.png

DanPatterson_Retired

p = Pool(7) this line confuses me then. Perhaps the files are just too big then. ... and I did notice that you tried it in-memory

RebeccaStrauch__GISP

You may want to try to move your data to a simpler path. Slight change that it doesn't like the _ and - ( ) etc in the path name. Even though the error doesn't point to this, it doesn't hurt to eliminate that as an issue.

Also, it would be easier to debug if you tried Posting Code blocks in the new GeoNet

ChrisSmith7

Have you thought about smoothing/generalizing your polygons, if that's an option for you? Reducing the vertices will result in a smaller file, but at the expense of losing some spatial resolution. What about splitting-up some of your features? Have you considered increasing your memory? How much do you have installed?

XanderBakker

... and what if you take out the multiprocessing and the in_memory workspace. Does it work? If I run into memory problems it is often related to the in_memory workspace.

ChrisSnyder

Before you run the Clip tool, run the Dice tool (Dicing Godzillas (features with too many vertices) | ArcGIS Blog) on the layer you are trying to clip. The Dice tool is specifically for the case when the input geometry is "too big to process" (one of the symptoms being the 'out of memory' error).

DanPatterson_Retired

with the caveat that it requires an Advanced license Dice—Help | ArcGIS for Desktop

XanderBakker

... but if you want to Dice a polyline featureclass with a basic license, this script may be helpful:

#-------------------------------------------------------------------------------
# Name:        Dice.py
#
# Author:      xbakker
#
# Created:     07/01/2015
#-------------------------------------------------------------------------------

import arcpy

def main():
    import sys
    import traceback
    import os
    arcpy.env.overwriteOutput = True

    try:
        fc_in = arcpy.GetParameterAsText(0)
        fc_out = arcpy.GetParameterAsText(1)
        max_pnts = arcpy.GetParameter(2)

        sr = arcpy.Describe(fc_in).spatialReference
        fld_shpin = arcpy.Describe(fc_in).shapeFieldName

        # create output featureclass
        out_ws, out_name = os.path.split(fc_out)
        arcpy.AddMessage("Crear output featureclass '{0}'".format(out_name))
        arcpy.CreateFeatureclass_management(out_ws, out_name, "POLYLINE", fc_in, "DISABLED", "DISABLED", sr)

        # make fields list
        flds_use = correctFieldList(arcpy.ListFields(fc_in))

        # create output cursor
        cnt_out = 0
        with arcpy.da.InsertCursor(fc_out, flds_use) as curs_out:

            # loop through input features
            cnt_in = 0
            with arcpy.da.SearchCursor(fc_in, flds_use) as curs_in:
                for row_in in curs_in:
                    cnt_in += 1
                    if cnt_in % 50 == 0:
                        arcpy.AddMessage("Input Feature: {0}, feature de salida: {1}".format(cnt_in, cnt_out))

                    polyline_in = row_in[0]
                    pnt_cnt = polyline_in.pointCount
                    if pnt_cnt > max_pnts:
                        for part in polyline_in:
                            if len(part) > max_pnts:
                                lst_part = list(part)
                                lst_chunks = chunks(lst_part, max_pnts)
                                for p in lst_chunks:
                                    arr_part = arcpy.Array(p)
                                    polyline_out = arcpy.Polyline(arr_part)
                                    if polyline_out.length != 0:
                                        cnt_out += 1
                                        row_out = buildRow(row_in, flds_use, polyline_out)
                                        curs_out.insertRow(row_out)

                            else:
                                # write part
                                arr_part = arcpy.Array(part)
                                polyline_out = arcpy.Polyline(arr_part)
                                cnt_out += 1
                                row_out = buildRow(row_in, flds_use, polyline_out)
                                curs_out.insertRow(row_out)

                    else:
                        # polyline is OK, write polyline
                        cnt_out += 1
                        row_out = buildRow(row_in, flds_use, polyline_in)
                        curs_out.insertRow(row_out)

    except:
        tb = sys.exc_info()[2]
        tbinfo = traceback.format_tb(tb)[0]
        pymsg = "Errores de Python:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info()[1])
        msgs = "Errores de ArcPy:\n" + arcpy.GetMessages(2) + "\n"
        arcpy.AddError(pymsg)
        arcpy.AddError(msgs)


def correctFieldList(flds):
    flds_use = ['Shape@']
    fldtypes_not = ['Geometry', 'Guid', 'OID']
    for fld in flds:
        if not fld.type in fldtypes_not:
            flds_use.append(fld.name)
    return flds_use

def chunks(l, n):
    """ Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l

def buildRow(row_in, flds_use, polyline_out):
    try:
        lst_in = list(row_in)
        lst_in[0] = polyline_out
        return tuple(lst_in)

    except Exception, e:
        arcpy.AddWarning("buildRow exception: {0}".format(e))
        return None

    finally:
        pass

if __name__ == '__main__':
    main()

DanPatterson_Retired

Problem resolved? or still ongoing?

HuaShang

Thanks for help. But still got the problem. I'm wondering if it is not the multiprocess problem. I described my question more detail here,Out of memory error I'm really appreciate if I could get your help.

HuaShang

Thank you for help! Do I need to add this code before my code directly? I am a beginner, so it looks so complex for me, but I will try this! Thanks!

HuaShang

Thank you !

Yes , I tried the simpler path, it still got that error.

Join the Conversation

Signed in members can post, follow updates, and more. New here? Register a free account.

Quick Links

Community Help
Find useful guides, FAQs, and documents to help you navigate and make the most of Esri Community.