How do I loop a folder of mxds

14728
43
05-10-2016 08:23 AM
DevinUnderwood2
Occasional Contributor

I want to loop many mxds within a folder to retrieve layer,connection,etc. information using python.

I just need the syntax for looping through the mxds.

0 Kudos
43 Replies
PeterWilson
Occasional Contributor III

Hi Devin

I wrote the following python script to copy the datasets (feature classes & rasters) for each mxd within a folder into a new File Geodatabase. You could use the following as a starting point and instead of copy it out write the location of the layers (feature classes and rasters) into a summary table.

'''
Created on Jan 20, 2016

Copy all feature classes and

rasters from each mxd in a folder

into a new File Geodatabase.

@author: PeterW
'''
import os
import time
import arcpy

# set arguments
folder = arcpy.GetParameterAsText(0)
out_gdb = arcpy.GetParameterAsText(1)

# folder = arcpy.GetParameterAsText(0)
# out_gdb = arcpy.GetParameterAsText(1)

# Processing time
def hms_string(sec_elapsed):
    h = int(sec_elapsed / (60 * 60))
    m = int(sec_elapsed % (60 * 60) / 60)
    s = sec_elapsed % 60
    return "{}h:{:>02}m:{:>05.2f}s".format(h, m, s)

start_time1 = time.time()


# copy layers function
def copy_features():
    try:
        if arcpy.Exists(os.path.join(out_gdb, layer.datasetName)):
            arcpy.AddMessage("Feature class already exists, it will be skipped")
        else:
            arcpy.FeatureClassToGeodatabase_conversion(lyr_source, out_gdb)
    except:
        arcpy.AddMessage("Error copying: " + layer.name)
        arcpy.AddError(arcpy.GetMessages())


def copy_rasters():
    try:
        if arcpy.Exists(os.path.join(out_gdb, layer.datasetName)):
            arcpy.AddMessage("Raster already exists, it will be skipped")
        else:
            out_raster = os.path.join(out_gdb, layer.datasetName)
            arcpy.CopyRaster_management(lyr_source, out_raster)
    except:
        arcpy.AddMessage("Error copying: " + layer.name)
        arcpy.AddError(arcpy.GetMessages())


# Loop through each data frame, layer and copy to new file geodatabase
for filename in os.listdir(folder):
    fullpath = os.path.join(folder, filename)
    if os.path.isfile(fullpath):
        basename, extension = os.path.splitext(fullpath)
        if extension.lower() == ".mxd":
            arcpy.AddMessage("Processing: " + basename)
            mxd = arcpy.mapping.MapDocument(fullpath)
            dfs = arcpy.mapping.ListDataFrames(mxd)
            for df in dfs:
                arcpy.AddMessage("DataFrame: " + df.name)
                layers = arcpy.mapping.ListLayers(mxd, "", df)
                for layer in layers:
                    if layer.isFeatureLayer:
                        lyr_source = layer.dataSource
                        lyr_name = layer.name.encode("utf8", "replace")
                        arcpy.AddMessage("Copying: {}".format(lyr_name))
                        copy_features()
                    if layer.isRasterLayer:
                        lyr_source = layer.dataSource
                        lyr_name = layer.name.encode("utf8", "replace")
                        arcpy.AddMessage("Copying: {}".format(lyr_name))
                        copy_rasters()

# Determine the time take to copy features
end_time1 = time.time()
print ("It took {} to copy all layers to file geodatabase".format(hms_string(end_time1 - start_time1)))

Let me know if you need help amending the following to meet you needs.

0 Kudos
DevinUnderwood2
Occasional Contributor

This is what worked for me. I used only what I needed and added openpyxl so I can write to excel. Yet I am still working on it successfully writing to excel

#Import Modules
import arcpy,os
from openpyxl import Workbook

#Set folder space
folder = xxxxx

#Set variables
# create excel worksheets
wb = Workbook()
ws1 = wb.create_sheet("yyy")

for filename in os.listdir(folder):
    fullpath = os.path.join(folder, filename)
    if os.path.isfile(fullpath):
        basename, extension = os.path.splitext(fullpath)
        if extension.lower() == ".mxd":
            arcpy.AddMessage("Processing: " + basename)
            mxd = arcpy.mapping.MapDocument(fullpath)
            dfs = arcpy.mapping.ListDataFrames(mxd)
            for df in dfs:
                arcpy.AddMessage("DataFrame: " + df.name)
                layers = arcpy.mapping.ListLayers(mxd, "", df)
                for layer in layers:
                    if layer.isFeatureLayer:
                        lyr_source = layer.dataSource
                        lyr_name = layer.name.encode("utf8", "replace")
                        arcpy.AddMessage("Copying: {}".format(lyr_name))
                        print  fullpath + lyr_source

wb.save('aaaaaaa.xlsx')

Thank you for your help. My next step is the openpyxl writing to excel.

GerardHavik
New Contributor III

Devin, have fun figuring out openpyyxl, sounds interesting. My choice sofar was writing to a csv file using open and write. Reading the question titel I think with this te discussion is complete.

0 Kudos
DevinUnderwood2
Occasional Contributor

I have tried write to csv also, but it seems that openpyxl is more versatile, e.g. write to a native xlsx .

I will see what works out for me and let you know.

0 Kudos
DevinUnderwood2
Occasional Contributor

I am going with csv instead, so I can share the script not worrying whether a person may or may not have openpyxl. Csv should suffice, but I am having a little trouble.

I have the following  and can see the csv file in windows explorer processing/looping the files, but when I open the csv it has one file just repeated in several rows. You know what may be the cause?

with open('CSVLISTLAYERS.csv', 'wb') as outputcsv:

                            writer = csv.writer(outputcsv, dialect = 'excel')

                            for filename in (fullpath + lyr_source):

                                writer.writerow ([fullpath + lyr_source])

0 Kudos
GerardHavik
New Contributor III

I found a script.

import arcpy, os
#Read input parameters from GP dialog
folderPath = arcpy.GetParameterAsText(0)
if folderPath=="":
  folderPath = r"D:\TESTFOLDER"
#Loop through ech MXD file
for filename in os.listdir(folderPath):
  fullpath = os.path.join(folderPath, filename)
  if os.path.isfile(fullpath):
    if filename.lower().endswith(".mxd"):
      #open rapportfile for MXD
      outFilename=fullpath[:fullpath.rfind(".")]+".csv"
      mes= '\nMXD: %s' % (fullpath)
      print mes
      arcpy.AddMessage(mes)
      rapportfile=open(outFilename,"w")
      header='MXD;WORKSPACE;FEATURECLASS'
      rapportfile.write(header+'\n')
      mxd = arcpy.mapping.MapDocument(fullpath)
      for df in arcpy.mapping.ListDataFrames(mxd):
        layerList = arcpy.mapping.ListLayers(mxd, "", df)
        mes='MXD %s bevat %s layers' % (filename, len(layerList))
        arcpy.AddMessage(mes)
        print mes
        for lyr in layerList:
          if lyr.supports("dataSource"):
            workspace=lyr.workspacePath
            fc=lyr.datasetName
            print 'WorkspacePath: %s' % workspace
            print 'FeatureClass:  %s' % fc
            reg='%s;%s;%s' % (filename,workspace,fc)
            #arcpy.AddMessage(reg)
            rapportfile.write(reg+'\n')
      mes='Papportbestand: %s\n' % (outFilename)
      print mes
      arcpy.AddMessage(mes)
      rapportfile.close()
      del mxd
0 Kudos
DevinUnderwood2
Occasional Contributor

I am still working on trying to get the csv to write. You know about readlines and writelines? I just leaned of these, yet I haven't noticed anyone mentioning this option, ever.

0 Kudos
DanPatterson_Retired
MVP Emeritus

the csv module is well documented 13.1. csv — CSV File Reading and Writing — Python 2.7.11 documentation  and there are thousands of examples online

0 Kudos
DevinUnderwood2
Occasional Contributor

I have looked at many regarding csv writer, yet I am not successful yet.

0 Kudos
RebeccaStrauch__GISP
MVP Emeritus

This is part of my Python addin for data inventory and “broken-link” repair.​ tool box, but I'll include the first script here.  It inventories and lists the fgdb to out put csv, and it also coverts it to an .xsl  (not .xlsx).  Maybe you can pick out what you need from this.

EDIT: bTW replace all the myMsgs with print or arcpy add message .

'''
---------------------------------------------------------------------------
Tool:    FCInventoryReport 
Toolbox: CheckAndFixLinks.tbx
Script:  2_InventoryFCs.py
Purpose: 
   This script will walk thru all the feature classes within a folder 
   and create a text report, comma-delimted and an Excel .xls file.
   Include shapes, coverages, rasters, tables, connections, and FGDB

   Column names:  FType    FCname    FullPath
-------------------------------------------------------------------------------
 Author:  Rebecca Strauch - ADFG-DWC-GIS
 Created on: 4/10/2013 
   last modification: August 10, 2015

 Description: To create list of the features classes within a folder, including
   coverages (pts, poly, arc, anno), shapes, grids and FGDB data. Outputs list
   to a text report file (not much formatting) in the folder being scanned named
     FCInventoryYYYYMMDD_HHMM.txt
   with the date and time as part of the name. This should always create
   a new file, unless you run it twice within same minute.  

   Must have write permissions on the output folder in question.

   Known issue (with built-in workaround): for some reason some, but not  
     all (ArcInfo) grids want to duplicate the folder name before the 
     describe. I'm now checking to make sure it exists, if not, I am removing
     the duplicate portion, and letting it run.  Seems to work.
------------------------------------------------------------------------------
 Arguments:  
   [0] theWorkspace: Folder/directory to search (walk thru, includes subfolders)
   [1] outFile: output base filename, default GDBList, script appends YYYYMMDD_HHMM

 Updates:          
---------------------------------------------------------------------------
'''
# Import modules
import arcpy
import os
from _miscUtils import *
from _gpdecorators import *

# catch_errors decorator must preceed a function using the @ notation.
@catch_errors
def main():
  """
  Main function to create text file report of all Feature Classes in folder
  """
  #setup environment
  arcpy.env.overwriteOutput = True

  # Script arguments...    
  """ If running as standalone, hardcode theWorkspace and outFile  """
  theWorkspace = arcpy.GetParameterAsText(0)
  if not theWorkspace:
    theWorkspace = r"C:\__Data1\_TalkeetnaBU" # r"D:\_dataTest"    

  outFile = arcpy.GetParameterAsText(1)
  if not outFile:
    outFile = "FCInventory" 
  # Create new output name name tagged with YYYYMMDD_HHMM
  fileDateTime = curFileDateTime()
  currentDate = curDate()

  # Create new output name tagged with YYYYMMDD_HHMM
  outfileTXT = os.path.join(theWorkspace, outFile) + fileDateTime + ".txt" #theWorkspace + "\FCInventory" + fileDateTime + ".txt"
  outFileCSV = os.path.join(theWorkspace, outFile) + fileDateTime + ".csv"  #theWorkspace + "\FCInventory" + fileDateTime + ".csv"
  outFileXLS = os.path.join(theWorkspace, outFile) + fileDateTime + ".xls"
  arcpy.AddMessage(theWorkspace + ", " + outfileTXT)
  reportFile = open(outfileTXT, 'w')
  csvFile = open(outFileCSV, 'w')
  arcpy.AddMessage(  "File {0} is open? {1}".format(outfileTXT, str(not reportFile.closed)))
  arcpy.AddMessage(  "File {0} is open? {1}".format(str(outFileCSV), str(not csvFile.closed)))
  #arcpy.AddMessage(  "File " + str(csvFile) + " is closed?  " + str(csvFile.closed))     
  arcpy.AddMessage("Writing the report to: " + outfileTXT + " and " + outFileCSV)

  outText = "List of all GIS data in " + theWorkspace + " on " + currentDate + '\n'
  outText += "  Includes coverages (pts, poly, arc, anno), shapes, and FGDB data." + '\n'
  outText += "-----------------------------------------------------" + '\n'

  reportFile.write(outText)
  csvFile.write("FType, FCname, FullPath\n")

  def inventory_data(workspace, datatypes):
    for path, path_names, data_names in arcpy.da.Walk(
      workspace, datatype=datatypes):
      if "tic" in data_names:
        data_names.remove('tic')
      for data_name in data_names:
        fcName = os.path.join(path, data_name)
        #arcpy.AddMessage("Show for debug: " + fcName)
        if not arcpy.Exists(fcName):
          # workaround for raster folder name duplicating
          fcName = os.path.dirname(fcName)
        desc = arcpy.Describe(fcName)
        #arcpy.AddMessage("debug, desc it to me: " + desc.dataType)
        yield [path, data_name, desc.dataType] #, desc]

  i = 0
  for feature_class in inventory_data(theWorkspace, "FeatureClass"):
    """ last modified data not working for gdb or FC in fgdb ..."""
    #lastMod = time.strftime('%m/%d/%Y %H:%M', time.localtime(os.path.getmtime(feature_class[0])))
    if i == 0:
      #arcpy.AddMessage("{0}  modified: {1}".format(feature_class[0], lastMod)) 
      arcpy.AddMessage("{0}".format(feature_class[0]))
      outText = ' ' + feature_class[0] + '\n'
      reportFile.write(outText)
      path0 = feature_class[0]
      i =+ 1
    elif not path0 == feature_class[0]:
      #arcpy.AddMessage("{0}  modified: {1}".format(feature_class[0], lastMod)) 
      arcpy.AddMessage("{0}".format(feature_class[0]))
      outText = ' ' + feature_class[0] + '\n'
      reportFile.write(outText)
      i = 0
    if feature_class[2] == "ShapeFile":
      shpfile = arcpy.os.path.join(feature_class[0], feature_class[1])
      lastMod = time.strftime('%m/%d/%Y %H:%M', time.localtime(os.path.getmtime(shpfile)))
      arcpy.AddMessage("       {0}: {1} modified: {2}".format(feature_class[2], feature_class[1], lastMod))
      outText = ("       {0}: {1} modified: {2}\n".format(feature_class[2], feature_class[1], lastMod))
    else:
      arcpy.AddMessage("       {0}: {1}".format(feature_class[2], feature_class[1]))
      outText = ("       {0}: {1}\n".format(feature_class[2], feature_class[1]))
    reportFile.write(outText)
    csvFile.write("{},{}, {}\n".format(feature_class[2], feature_class[1], feature_class[0]))
    

  reportFile.close()
  csvFile.close()
  arcpy.AddMessage(  "File {0} is closed? {1}".format(outfileTXT, str(reportFile.closed)))
  arcpy.AddMessage(  "File {0} is closed? {1}".format(outFileCSV, str(csvFile.closed)))

  # Creates Excel .xls file from the .csv ....easier to edit (ver 1)
  arcpy.TableToExcel_conversion(outFileCSV, outFileXLS) 

  arcpy.AddMessage('!!! Success !!!  ')

# End main function

if __name__ == '__main__':
  main()