How to solve the error:out of memory

HuaShang · ‎01-10-2016

I want to use the clip function in python, the code is as follow:

import os

from multiprocessing import Pool

import arcpy

from arcpy import env

def data_clip(shp):

env.workspace = r"F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ShpFile/f"

clip_features = "D:/Research/boundry.shp"

pre, suf = os.path.splitext(shp)

new = pre + "_china" + suf

in_mem = os.path.join("in_memory", pre)

out_feature_class = os.path.join("F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ChinaShpFile", new)

arcpy.Clip_analysis(shp, clip_features, in_mem)

arcpy.CopyFeatures_management(in_mem, out_feature_class)

arcpy.DeleteFeatures_management(in_mem)

if __name__ == "__main__":

env.overwriteOutput = True

env.workspace = r"F:/GCM_CMIP5/FGOALS-g2(LASG-CESS)/mrro/ShpFile/f"

p = Pool(7)

fcs = arcpy.ListFeatureClasses()

p.map(data_clip, fcs)

print('Clip finish!')

But I got the "out of memory" error.

If anyone knows how to solve this problem?

Thank you so much!!!

XanderBakker · ‎01-12-2016

... but if you want to Dice a polyline featureclass with a basic license, this script may be helpful:

#-------------------------------------------------------------------------------
# Name:        Dice.py
#
# Author:      xbakker
#
# Created:     07/01/2015
#-------------------------------------------------------------------------------

import arcpy

def main():
    import sys
    import traceback
    import os
    arcpy.env.overwriteOutput = True

    try:
        fc_in = arcpy.GetParameterAsText(0)
        fc_out = arcpy.GetParameterAsText(1)
        max_pnts = arcpy.GetParameter(2)

        sr = arcpy.Describe(fc_in).spatialReference
        fld_shpin = arcpy.Describe(fc_in).shapeFieldName

        # create output featureclass
        out_ws, out_name = os.path.split(fc_out)
        arcpy.AddMessage("Crear output featureclass '{0}'".format(out_name))
        arcpy.CreateFeatureclass_management(out_ws, out_name, "POLYLINE", fc_in, "DISABLED", "DISABLED", sr)

        # make fields list
        flds_use = correctFieldList(arcpy.ListFields(fc_in))

        # create output cursor
        cnt_out = 0
        with arcpy.da.InsertCursor(fc_out, flds_use) as curs_out:

            # loop through input features
            cnt_in = 0
            with arcpy.da.SearchCursor(fc_in, flds_use) as curs_in:
                for row_in in curs_in:
                    cnt_in += 1
                    if cnt_in % 50 == 0:
                        arcpy.AddMessage("Input Feature: {0}, feature de salida: {1}".format(cnt_in, cnt_out))

                    polyline_in = row_in[0]
                    pnt_cnt = polyline_in.pointCount
                    if pnt_cnt > max_pnts:
                        for part in polyline_in:
                            if len(part) > max_pnts:
                                lst_part = list(part)
                                lst_chunks = chunks(lst_part, max_pnts)
                                for p in lst_chunks:
                                    arr_part = arcpy.Array(p)
                                    polyline_out = arcpy.Polyline(arr_part)
                                    if polyline_out.length != 0:
                                        cnt_out += 1
                                        row_out = buildRow(row_in, flds_use, polyline_out)
                                        curs_out.insertRow(row_out)

                            else:
                                # write part
                                arr_part = arcpy.Array(part)
                                polyline_out = arcpy.Polyline(arr_part)
                                cnt_out += 1
                                row_out = buildRow(row_in, flds_use, polyline_out)
                                curs_out.insertRow(row_out)

                    else:
                        # polyline is OK, write polyline
                        cnt_out += 1
                        row_out = buildRow(row_in, flds_use, polyline_in)
                        curs_out.insertRow(row_out)

    except:
        tb = sys.exc_info()[2]
        tbinfo = traceback.format_tb(tb)[0]
        pymsg = "Errores de Python:\nTraceback info:\n" + tbinfo + "\nError Info:\n" + str(sys.exc_info()[1])
        msgs = "Errores de ArcPy:\n" + arcpy.GetMessages(2) + "\n"
        arcpy.AddError(pymsg)
        arcpy.AddError(msgs)


def correctFieldList(flds):
    flds_use = ['Shape@']
    fldtypes_not = ['Geometry', 'Guid', 'OID']
    for fld in flds:
        if not fld.type in fldtypes_not:
            flds_use.append(fld.name)
    return flds_use

def chunks(l, n):
    """ Yield successive n-sized chunks from l."""
    for i in xrange(0, len(l), n):
        yield l

def buildRow(row_in, flds_use, polyline_out):
    try:
        lst_in = list(row_in)
        lst_in[0] = polyline_out
        return tuple(lst_in)

    except Exception, e:
        arcpy.AddWarning("buildRow exception: {0}".format(e))
        return None

    finally:
        pass

if __name__ == '__main__':
    main()

HuaShang · ‎01-20-2016

Thank you for help! Do I need to add this code before my code directly? I am a beginner, so it looks so complex for me, but I will try this! Thanks!

XanderBakker · ‎01-20-2016

Hi Hua Shang , I think that my code is not the first option you should consider. I would first remove the multi processing and the in_memory workspace. The Dice tool is used to subdivide features with a lot of vertices, which may produce problems when you clip the features.

The code I attached to this thread has been used for a toolbox and reads the input featureclass, name and workspace for the output featureclass and the maximum number of points a feature may have (sorry for the Spanish interface):

It will divide polylines in parts (chunks) if a feature has more the maximum given number of vertices.

DanPatterson_Retired · ‎01-19-2016

Problem resolved? or still ongoing?

HuaShang · ‎01-20-2016

Thanks for help. But still got the problem. I'm wondering if it is not the multiprocess problem. I described my question more detail here,Out of memory error I'm really appreciate if I could get your help.