AnsweredAssumed Answered

How do I generate logs from my rec and post script

Question asked by EgbuleI on Sep 28, 2017
Latest reply on Sep 29, 2017 by rastrauch

I created a python script to reconcile all edits on the users versions, then  post the reconciled version to a QAQC and then to the sde or dbo default version. After this process the script will proceed to compress analyze and rebuild indexes for the tables in the sde database. This script will be scheduled on windows scheduler to run silently at 2:00am on Thursday and  Saturday morning when users are offline.

If there should be any error encountered, the script is to print the errors and also generate a log file containing the errors and store the logfile with date and time in the specified drive location. Since the script was scheduled to run behind scene on windows scheduler, it was difficult for me to see errors when it happens. This was a big issue because users edits weren't reflected on our default database after sometime due to the error encountered by the script, his actually necessitated the need for a functional logfile that will enable me identify the errors.

The initial problem I had was that  the script executed very well but could only print the errors on the screen but it generates an empty logfile.

This was corrected by putting in 3 lines at the end of the script to get it to write the printed errors intothe logfile, save it with time and date.

 

The Initial Script - executes but generates an empty log file

# This script will reconcile and post all edits made in child versions of Natural Resources Geodatabase to QA\QC and then to SDE default database,it will go further to compress the database, analyze and rebuild the indexes on the database tables.
# Script developed by Irene F. Egbulefu - GIS Analyst ( TNR Dept)
import arcpy
import sys
import os
import datetime
import traceback

# Database Connection
editDB ="Database Connections\\Frogmouth_Natural_ResourcesTC.sde"
# Current Day
Day = time.strftime("%m-%d-%Y", time.localtime())
# Current Time
Time = time.strftime("%I:%M:%S %p", time.localtime())


# Set workspace
workspace = editDB

# Set the workspace environment
arcpy.env.workspace = workspace
arcpy.env.overwriteOutput = True

try:
    # Start Time
    print 'Process Started at ' + str(Day) + " " + str(Time)

# block new connections to the PublicWorks Geodatabase on Frogmouth Server.
    print "Blocking Connections..."
    arcpy.AcceptConnections(editDB, False)
   
# disconnect all users from the PublicWorks Geodatabase on Frogmouth Server.
    print "Disconnecting Users..."
    arcpy.DisconnectUser(editDB, "ALL")

# Get a list of all child versions besides QAQC and DEFAULT to pass into the ReconcileVersions tool.
    ver1List = [ver1.name for ver1 in arcpy.da.ListVersions(editDB) if ver1.name != 'TC_USER.QA/QC' and ver1.name != 'sde.DEFAULT']

# Execute the ReconcileVersions tool with QAQC Target Version and do not delete child versions
    print "Reconcile/post versions to QAQC...."
    arcpy.ReconcileVersions_management(editDB, "ALL_VERSIONS", "TC_USER.QA/QC", ver1List, "LOCK_ACQUIRED", "ABORT_CONFLICTS", "BY_OBJECT", "FAVOR_TARGET_VERSION", "POST", "KEEP_VERSION")

# Extract QAQC version from the list of versions to pass to ReconcileVersions tool.
    ver2List = [ver2.name for ver2 in arcpy.da.ListVersions(editDB) if ver2.name == 'TC_USER.QA/QC']

# Execute the ReconcileVersions tool with DEFAULT Target Version and do not delete QAQC version
    print "Reconcile/post QAQC to DEFAULT..."
    arcpy.ReconcileVersions_management(editDB, "ALL_VERSIONS", "sde.DEFAULT", ver2List, "LOCK_ACQUIRED", "ABORT_CONFLICTS", "BY_OBJECT", "FAVOR_TARGET_VERSION", "POST", "KEEP_VERSION")

# Run the compress tool.
    print "Compressing database..."
    arcpy.Compress_management(editDB)
   
# /////////////////////////////////// ANALYZE DATASETS AND CALC STATISTICS /////////////////////////////////////

    # NOTE: Rebuild indexes can accept a Python list of datasets.

    # Get a list of all the datasets the user has access to.
    # First, get all the stand alone tables, feature classes and rasters.
    dataList = arcpy.ListTables() + arcpy.ListFeatureClasses() + arcpy.ListRasters()
   
# Next, for feature datasets get all of the datasets and featureclasses
    # from the list and add them to the master list.
    for dataset in arcpy.ListDatasets("*", "Feature"):
        arcpy.env.workspace = os.path.join(workspace, dataset)
        dataList += arcpy.ListFeatureClasses() + arcpy.ListDatasets()

      # reset the workspace
    arcpy.env.workspace = workspace

    # Concatenate all datasets into a list
    datasetList = [ds for ds in dataList]
   
    print "rebuilding indexes"
    # Execute rebuild indexes
    # Note: to use the "SYSTEM" option the workspace user must be an administrator.
    arcpy.RebuildIndexes_management(workspace, "NO_SYSTEM", datasetList, "ALL")
    print('Rebuild Complete')
   
    print "analyzing datasets"
    arcpy.AnalyzeDatasets_management(workspace, "NO_SYSTEM", datasetList, "ANALYZE_BASE", "ANALYZE_DELTA", "ANALYZE_ARCHIVE")
    print "analysis complete"

    #Allow the database to begin accepting connections again
    print "Set databases to allow connections..."
    arcpy.AcceptConnections(editDB, True)

# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ script initiation, Rec/Post process error handling \\\\\\\\\\\\\\\\\\\\\\\\\\\\\

except:
    print 'An error occured'
    failMsg = '\nSCRIPT FAILURE IN SCRIPT INITIATION OR RECONCILE-POST PROCESS, \n'
    failMsg += 'Most recent GP messages below.\n'
    failMsg += arcpy.GetMessages() +'\n'
    failMsg += '\nTraceback messages below.\n'
    failMsg += traceback.format_exc().splitlines()[-1]
    print failMsg
   
# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ write error log info\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
# move to working directory
    os.chdir (u'Y:\\TOOLS\\Logs\\')
## define function with variable filename and the format of the timestamp
    def timeStamped(filename, fmt='%m-%d-%y-%H.%M.%S-{filename}'):
       return datetime.datetime.now().strftime(fmt).format(filename=filename)
## assign local variable filename and use whatever file name and extension you need
    filename = timeStamped('Natural_ResourcesTC_toQC_Log.txt')
## Create the logfile and assign write permission
    Open(filename, "w")

 

Final corrected script - executes and generates logfile, prints errors on the screen and also writes error information  into the logfile it

# This script will reconcile and post all edits made in child versions of Natural Resources Geodatabase to QA\QC and then to SDE default database,it will go further to compress the database, analyze and rebuild the indexes on the database tables
# Script developed by Irene F. Egbulefu - GIS Analyst ( TNR Dept)
import arcpy
import sys
import os
import datetime
import traceback

# Database Connection
editDB ="Database Connections\\Frogmouth_Natural_ResourcesTC.sde"
# Current Day
Day = time.strftime("%m-%d-%Y", time.localtime())
# Current Time
Time = time.strftime("%I:%M:%S %p", time.localtime())


# Set workspace
workspace = editDB

# Set the workspace environment
arcpy.env.workspace = workspace
arcpy.env.overwriteOutput = True

try:
    # Start Time
    print 'Process Started at ' + str(Day) + " " + str(Time)

# block new connections to the PublicWorks Geodatabase on Frogmouth Server.
    print "Blocking Connections..."
    arcpy.AcceptConnections(editDB, False)
   
# disconnect all users from the PublicWorks Geodatabase on Frogmouth Server.
    print "Disconnecting Users..."
    arcpy.DisconnectUser(editDB, "ALL")

# Get a list of all child versions besides QAQC and DEFAULT to pass into the ReconcileVersions tool.
    ver1List = [ver1.name for ver1 in arcpy.da.ListVersions(editDB) if ver1.name != 'TC_USER.QA/QC' and ver1.name != 'sde.DEFAULT']

# Execute the ReconcileVersions tool with QAQC Target Version and do not delete child versions
    print "Reconcile/post versions to QAQC...."
    arcpy.ReconcileVersions_management(editDB, "ALL_VERSIONS", "TC_USER.QA/QC", ver1List, "LOCK_ACQUIRED", "ABORT_CONFLICTS", "BY_OBJECT", "FAVOR_TARGET_VERSION", "POST", "KEEP_VERSION")

# Extract QAQC version from the list of versions to pass to ReconcileVersions tool.
    ver2List = [ver2.name for ver2 in arcpy.da.ListVersions(editDB) if ver2.name == 'TC_USER.QA/QC']

# Execute the ReconcileVersions tool with DEFAULT Target Version and do not delete QAQC version
    print "Reconcile/post QAQC to DEFAULT..."
    arcpy.ReconcileVersions_management(editDB, "ALL_VERSIONS", "sde.DEFAULT", ver2List, "LOCK_ACQUIRED", "ABORT_CONFLICTS", "BY_OBJECT", "FAVOR_TARGET_VERSION", "POST", "KEEP_VERSION")

# Run the compress tool.
    print "Compressing database..."
    arcpy.Compress_management(editDB)
   
# /////////////////////////////////// ANALYZE DATASETS AND CALC STATISTICS /////////////////////////////////////

    # NOTE: Rebuild indexes can accept a Python list of datasets.

    # Get a list of all the datasets the user has access to.
    # First, get all the stand alone tables, feature classes and rasters.
    dataList = arcpy.ListTables() + arcpy.ListFeatureClasses() + arcpy.ListRasters()
   
# Next, for feature datasets get all of the datasets and featureclasses
    # from the list and add them to the master list.
    for dataset in arcpy.ListDatasets("*", "Feature"):
        arcpy.env.workspace = os.path.join(workspace, dataset)
        dataList += arcpy.ListFeatureClasses() + arcpy.ListDatasets()

      # reset the workspace
    arcpy.env.workspace = workspace

    # Concatenate all datasets into a list
    datasetList = [ds for ds in dataList]
   
    print "rebuilding indexes"
    # Execute rebuild indexes
    # Note: to use the "SYSTEM" option the workspace user must be an administrator.
    arcpy.RebuildIndexes_management(workspace, "NO_SYSTEM", datasetList, "ALL")
    print('Rebuild Complete')
   
    print "analyzing datasets"
    arcpy.AnalyzeDatasets_management(workspace, "NO_SYSTEM", datasetList, "ANALYZE_BASE", "ANALYZE_DELTA", "ANALYZE_ARCHIVE")
    print "analysis complete"

    #Allow the database to begin accepting connections again
    print "Set databases to allow connections..."
    arcpy.AcceptConnections(editDB, True)

# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ script initiation, Rec/Post process error handling \\\\\\\\\\\\\\\\\\\\\\\\\\\\\

except:
    print 'An error occured'
    failMsg = '\nSCRIPT FAILURE IN SCRIPT INITIATION OR RECONCILE-POST PROCESS, \n'
    failMsg += 'Most recent GP messages below.\n'
    failMsg += arcpy.GetMessages() +'\n'
    failMsg += '\nTraceback messages below.\n'
    failMsg += traceback.format_exc().splitlines()[-1]
    print failMsg
   
# \\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\\ write error log info\\\\\\\\\\\\\\\\\\\\\\\\\\\\\
# move to working directory
    os.chdir (u'Y:\\TOOLS\\Logs\\')
## define function with variable filename and the format of the timestamp
    def timeStamped(filename, fmt='%m-%d-%y-%H.%M.%S-{filename}'):
       return datetime.datetime.now().strftime(fmt).format(filename=filename)
## assign local variable filename and use whatever file name and extension you need
    filename = timeStamped('Natural_ResourcesTC_toQC_Log.txt')
## Create the logfile and assign write permission
    f = Open(filename, "w")
    f.write(failmsg)
    f.close()

Outcomes