Need major, major help converting Tifs into a Mosaic with Python

JohnPavek · ‎02-26-2020

import os, cx_Oracle, string, datetime, time
import arcgisscripting,arcpy
import shutil


class PlateMapGeneration:
pass

user = ''
sdeusr = ''
fldr = ''
path = ''
asset_path = ''
sde = ''
projection = ''
connect_string = ''

timestamp_d = ''
timestamp_t = ''

filelist = []
maplist = []

local_path = ''
final_path = ''
temp_table_conn = ''
sde_conn = ''
plates_gis = ''

def start_process(self, username_, ************_, ************_, sde_, db_):
global user
global *************
global **********
global path
global asset_path
global sde
global connect_string
global timestamp_d
global timestamp_t
global projection
global local_path
global final_path
global sde_conn
global plates_gis

user = username_
*********** = ***************_
********* = *************
path = '\\' + '\\' + '**************\\pythonadmin\\data\\'+*************+'\\plates\\tif\\'

sde = sde_
connect_string = user + '/' + user + '***************' + db_

#local_path = 'C:\\Users\\pythonadmin\\AppData\\Roaming\\ESRI\\Desktop10.3\\ArcCatalog\\' 
#local_path = 'C:\\Python_Cloud_Scripts\\conn\\'
local_path = 'E:\\GIS_MXD\\**************\\PLATES_CAD_GIS1.gdb\\'
final_path = 'E:\\GIS_MXD\\**************\\PLATES_CAD_GIS.gdb\\'
sde_conn = local_path + string.upper(sdeusr) + '.sde\\' + sde
plates_gis = '*******************'

asset_path = '\\' + '\\' + '****************\\pythonadmin\\data\\'+************+'\\assets\\' + plates_gis + '\\'

#print start time
starttime = datetime.datetime.today()
print 'Start Time: ' + str(starttime)

timestamp_d = str(starttime.date())
timestamp_t = str(starttime.time()).replace(':','_')

if string.upper(user) == '*************':
projection = "PROJCS['NAD_1983_StatePlane_New_Jersey_FIPS_2900_Feet',GEOGCS['GCS_North_American_1983',DATUM['D_North_American_1983',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',492125.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-74.5],PARAMETER['Scale_Factor',0.9999],PARAMETER['Latitude_Of_Origin',38.83333333333334],UNIT['Foot_US',0.3048006096012192]];IsHighPrecision"
elif string.upper(user) == 'CUC_DELMARVA':
projection = "PROJCS['NAD_1983_StatePlane_Delaware_FIPS_0700_Feet',GEOGCS['GCS_North_American_1983',DATUM['D_North_American_1983',SPHEROID['GRS_1980',6378137.0,298.257222101]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Transverse_Mercator'],PARAMETER['False_Easting',656166.6666666665],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',-75.41666666666667],PARAMETER['Scale_Factor',0.999995],PARAMETER['Latitude_Of_Origin',38.0],UNIT['Foot_US',0.3048006096012192]];IsHighPrecision"
else:
projection = "PROJCS['WGS_1984_Web_Mercator',GEOGCS['GCS_WGS_1984_Major_Auxiliary_Sphere',DATUM['D_WGS_1984_Major_Auxiliary_Sphere',SPHEROID['WGS_1984_Major_Auxiliary_Sphere',6378137.0,0.0]],PRIMEM['Greenwich',0.0],UNIT['Degree',0.0174532925199433]],PROJECTION['Mercator'],PARAMETER['False_Easting',0.0],PARAMETER['False_Northing',0.0],PARAMETER['Central_Meridian',0.0],PARAMETER['Standard_Parallel_1',0.0],UNIT['Meter',1.0]];IsHighPrecision"

def end_process(self):
endtime = datetime.datetime.today()
print 'End Time: ' + str(endtime)

def MakeFolder(self,**********,gis):
gp = arcgisscripting.create()

folderpath = '\\' + '\\' + '****************\\pythonadmin\\data\\' + ************ + '\\assets\\' + gis + '\\'
if os.path.exists(folderpath) == False:
print 'Folder does not exist'
os.mkdir(folderpath)
os.chdir(folderpath)
os.mkdir(folderpath + 'spatial\\')
gp.CreateFileGDB(folderpath, str(gis)+'.gdb')
else:
print 'Folder already exists'
filelist = []
gp.Delete_management(folderpath+gis+'.gdb')
filelist = os.listdir(folderpath + 'spatial\\')
a=0
while a < len(filelist):
os.remove(folderpath + 'spatial\\'+ filelist[a])
a=a+1
gp.CreateFileGDB(folderpath, str(gis)+'.gdb') 
a=0
print 'Folder creation complete'

def FormatTupleNum(self,num):
numstr = ""
numstr = num
return numstr.lstrip('(').rstrip('),')

def get_filelist(self):
global filelist
global path

filelist = os.listdir(path)

print "Files:"
print filelist

def get_map(self):
global connect_string
global maplist

x=0

con = cx_Oracle.connect(connect_string)
cur = con.cursor()
#maps = 'CC3D'
#cur.execute('select b.name from PLATE_MAPS_DETAIL a, MAP_VIEWS b where A.MAP_NAME = B.NAME and a.map_name = '+'\''+maps+'\'')
cur.execute('select b.name from PLATE_MAPS_DETAIL a, MAP_VIEWS b where A.MAP_NAME = B.NAME order by b.name')
#cur.execute('select map_name from asset_plates_gis order by creation_date')

maplist = cur.fetchall()

while x < len(maplist):
#mapstr = ''
mapstr = self.FormatTupleNum(str(maplist[x]))
maplist[x] = mapstr.replace('\'','')
x=x+1
print 'Maps:'
print maplist 
con.commit()
#con.close()

def create_tfw(self):
global path
global maplist

x=0
y=0

self.get_map()

con = cx_Oracle.connect(connect_string)
cur = con.cursor()

coordnumber = '1.3369988'


cur.execute('delete from ASSET_PLATES')

con.commit()

while x < len(maplist):
mapstr = ''
mapstr = maplist[x]
print mapstr
try:
print path + mapstr +'.tif'
modified_time = time.asctime(time.localtime(os.path.getmtime(path + mapstr +'.tif')))


cur.callproc('pkg_assets.populate_asset_plates',(mapstr,modified_time))

cur.execute('SELECT b.LLX FROM PLATE_MAPS_DETAIL a, MAP_VIEWS b WHERE A.MAP_NAME = B.NAME and a.map_name = '+'\''+mapstr+'\'')
coordllxlist = cur.fetchone()
coord_llx = ''
coord_llx = self.FormatTupleNum(str(coordllxlist[0]))

cur.execute('SELECT B.URY FROM PLATE_MAPS_DETAIL a, MAP_VIEWS b WHERE A.MAP_NAME = B.NAME and a.map_name = '+'\''+mapstr+'\'')
coordurylist = cur.fetchone()
coord_ury = ''
coord_ury = self.FormatTupleNum(str(coordurylist[0]))

f = open(path + maplist[x] + '.tfw','w')
f.write(coordnumber + '\n')
f.write('0' + '\n')
f.write('0' + '\n')
f.write('-'+coordnumber + '\n')
f.write(coord_llx + '\n')
f.write(coord_ury)
print maplist[x] + '.tfw created' 

f.close()
con.commit()
except:
print"Cannot find image " + mapstr
x=x+1
con.commit()
#con.close()

def generate_plate_gis(self):
global user
global maplist
global plates_gis
global local_path
global temp_table_conn
global sde_conn
global connect_string

con = cx_Oracle.connect(connect_string)
cur = con.cursor()

x=0
counter=0

#populate new plates into ASSET_PLATES_GIS
cur.callproc('pkg_assets.get_new_asset_plates')
con.commit()

#self.MakeFolder(user, plates_gis)
print local_path+plates_gis

if arcpy.Exists(local_path+plates_gis) == True:
while x < len(maplist):
mapstr = ''
mapstr = maplist[x]

counter = cur.callfunc('pkg_assets.update_plates',cx_Oracle.NUMBER,[mapstr])
if counter > 0:
print mapstr + ' will be updated.'
try:
sqlexpression = "\"NAME\" = \'" + mapstr + ".tif\' OR " + "\"NAME\" = \'" + mapstr + ".TIF\' OR " +"\"NAME\" = \'" + mapstr + "\'"
print sqlexpression
sttime = datetime.datetime.today()
print 'Start Time: ' + str(sttime)
arcpy.MakeRasterCatalogLayer_management (local_path+plates_gis, mapstr+"_view", sqlexpression, "", "") 
sttime = datetime.datetime.today()
#print 'Make Raster: ' + str(sttime)
#print 'Raster Created'
arcpy.DeleteRasterCatalogItems_management(mapstr+"_view") 
sttime = datetime.datetime.today()
#print 'Deleted Raster: ' + str(sttime)
#print 'Deleted existing Raster Catalog record.'
arcpy.RasterToGeodatabase_conversion(path+mapstr+'.tif',local_path+plates_gis)
#arcpy.RasterToGeodatabase_conversion('C:\Python_Cloud_Scripts\*********\BC1Q.tif',sde_conn+plates_gis) 
sttime = datetime.datetime.today()
#print 'RasterTo Geo: ' + str(sttime)
#print 'RasterToGeodatabase_conversion'
cur.callproc('pkg_assets.asset_plates_gis_createtime',[mapstr])
con.commit()
print mapstr +' added to SDE Raster Catalog.'
except Exception, e:
print e.message
x=x+1

#arcpy.WorkspaceToRasterCatalog_management(asset_path+plates_gis+'.gdb',sde_conn+plates_gis,"NONE","NONE")
else:
print plates_gis + ' does not exist'

con.commit()
#con.close()
filelist = []
files = os.listdir (local_path)
for filename in files:
if filename.endswith(".lock"):
print 'lock'
else:
filelist.append(filename)
fullpath = local_path + filename
shutil.copy(fullpath, final_path)
#shutil.copy(local_path, final_path)

self.end_process()‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍‍

If needed we could just add to the current code and move it into the mosaic dataset from the GDB. Any...any...help would be massively appreciated as we are all at our programming limits.

JohnPavek · ‎02-26-2020

Our main goal though is to go from the TIFFS straight to the Mosaic dataset if possible

DavidPike · ‎02-26-2020

Does it have to be from python? Creation of a mosaic dataset and importing rasters is an easy process and you can just provide a workspace. I'm sure it can be done in arcpy if needs be though.

JohnPavek · ‎02-27-2020

Thanks for your reply David!

The way the process works ibased on the code I provided is this...the TIFFs are modified Then thru a process that runs in Task Scheduler every morning at 7 AM any changed TIFFs are found and added to the raster GDB raster dataset overwriting what is there. Then the GDB and its associated TIFFs are served out to a webservice. So we have to automate the process. But like I said we could just keep the process as is but add a step to take the raster GDB into a mosaic.

JohnPavek · ‎02-28-2020

To this community: we are really stuck here and searching for help anywhere we can find it. But I figured that this community would have the best experience and ability to help. Please...please...if you can offer something let me know. I'm going to look into Modelbuilder and see if it can help. But I'm really a blind person now swinging in the dark with 1 hand tied behind my back.

DavidPike · ‎02-28-2020

Can you explain the steps of the code and what its doing/think it's doing. Does that code block work? I can see raster catalogues mentioned in the code. Full explanations of what the data is, looks like and what you want to achieve will help people to help you.

JohnPavek · ‎02-28-2020

Hi David.

Ok...I'm a;ways trying to find the right balance of info in GeoNet vs. too much...

The workflow is this...people make changes in the TIFFs using CAD. Each morning at 7 AM a job in Task Scheduler kicks off the code. This code will go and find ones that have been changed and replace them into the raster dataset. An earlier version just recopied all the TIFFs each time but I think this version just dealt with the changed ones. Then this raster dataset is used in a few mapservices. This code works fine and I have learned that if it works let it be as much as possible as I have found not my type of thing.

Now there is a request to go to the Mosaic dataset instead of the raster dataset. I figured since the raster dataset is there and working why not take the raster dataset and just change that to a Mosaic dataset. That way I just add a new codeblock and work that in instead of rewriting the whole code. I've included a few images of the raster dataset below.

DavidPike · ‎02-28-2020

We need to get terminology right.

.tif is a tiff file outside a geodatabase.

A raster dataset is a raster inside a geodatabase in FGDBR format.

A Raster catalog is either unmanaged or managed (references raster file paths or contains the rasters respectively) and sits inside a geodatabase. This is also very outdated.

A mosaic dataset sits inside a geodatabase and references rasters and is the recommended container.

JohnPavek · ‎02-28-2020

Ok...I think it is a raster catalog if I remember my ArcCatalog symbols right. Plus I tried something and an error i got mentioned that it is "not unmanaged" which I would assume means managed (yes...that is what the message said). If you look at the images shown on the last message it includes the images themselves. I definitely need to get it to a mosaic dataset. The person that created things left a long time ago and the documentation is very weak. So putting together what I can. I'll try to find out more. Is there something I can do that would definitely show what it is?

DavidPike · ‎02-28-2020

If you right click, properties, fields tab then at the bottom it may have a managed yes/no box.

It looks like line 239 is deleting your old tiff and 243 is inserting the new one.

I would copy all your tiffs into a new mosaic dataset, then set line 243 to copy new tiffs into the mosaic dataset. If this works be aware that line 239 will have to change to remove old tiffs from the mosaic dataset, or simply set 243 up to replace existing.

I'd get someone to rewrite that whole script also.