#!/usr/bin/python #========================================================================= # SCOPE: SCOPE = 'retrieveRadarFiles.py looks for new or updated data files on '+\ ' a remote server and copies them to the local machine using FTP. ' # # # Written by: Dan Robinson, Remote Sensing Solutions # Date: June 2012 # # Version Number versionNumber = '1.0' # # Modifications # (list version number, author, date and modifications) # #========================================================================= # Import libraries # - Standard modules that should come with standard Python install import ftplib, sys, os, string, time, gzip #--------------------------------------------------------------------------- # Build file list function # - Callback function. Called by FTP object upon returning list of files. # This function is called iteratively, as file names are returned one # at a time. # # Parameters: # - fileName: data returned from call to retrlines # def buildFileList(fileName): serverFiles.append(fileName) #--------------------------------------------------------------------------- # Write status File function # - Writes status to a log file # # Parameters: # - statusFilePath: name of the status file that is being created # - statusString: string describing status that has occurred # def writeStatusFile( statusFilePath, statusString ): if not (os.path.exists( statusFilePath ) ): statusLog = open( statusFilePath, 'w' ) else: statusLog = open( statusFilePath, 'a' ) statusString = time.strftime('%Y%m%d %H:%M:%S') + ' ' + statusString print statusString statusLog.write( statusString ) statusLog.close( ) #--------------------------------------------------------------------------- # Setup varibles and file system #------------------------------- # Local Variables sleepTime = 60. # time to wait before checking address again # Command Line Argument Syntax syntax = '\n' syntax += 'SYNTAX: \n' syntax += '\n' syntax += '-> retrieveRadarFiles.py <-h or --help> \n' syntax += ' <-v or --version>\n' syntax += ' <-d or --debug>\n' syntax += ' <-x all|new>\n' syntax += ' <-m or --mission>\n' syntax += ' <-f or --file filename>\n' syntax += '\n\n\n OPTIONS:\n\n' syntax += ' -h or --help: prints syntax to stdout and exits\n' syntax += ' -v or --version: print version number exits\n' syntax += ' -d or --debug: print extra status information during execution\n' syntax += ' -m [mission ID]: Mission ID. This is used to define file directories. REQUIRED.\n' syntax += ' -x or --xfer [all|new]: file transfer instructions, where:\n' syntax += ' all = transfer all files\n' syntax += ' new = transfer all files not already transferred (default)\n' syntax += ' -f or --file [configfile]: Name of the config file to use. Default = decomp.cfg\n' print 'Running %s...', sys.argv[0] exitNow = 0 # Check for arguments fileCompare = '' bRetrieve = False bDebug = False missionIDFound = False missionID = '' skip = 0 configName = 'ftpRetr' for i in range(len(sys.argv)-1): if skip: skip = 0 continue if sys.argv[i+1] == '-h' or sys.argv[i+1] == '--help': print syntax exitNow = 1 elif sys.argv[i+1] == '-v' or sys.argv[i+1] == '--version': print '\n retrieveRadarFiles.py written by Remote Sensing Solutions' print '\tVersion: '+version+' \n' exitNow = 1 elif sys.argv[i+1] == '-m' or sys.argv[i+1] == '--mission': missionID = sys.argv[i+2] missionIDFound = True skip = 1 elif sys.argv[i+1] == '-f' or sys.argv[i+1] == '--file': configName = sys.argv[i+2] skip = 1 elif sys.argv[i+1] == '-d' or sys.argv[i+1] == '--debug': bDebug = True elif sys.argv[i+1] == '-r' or sys.argv[i+1] == '--retr': fileCompare = sys.argv[i+2] fileCompare = fileCompare.lower() bRetrieve = True skip = 1 else: print '\n Incorrect Syntax! \n'+syntax exitNow = 1 if not missionIDFound: print 'Mission ID is a required argument.' exitNow = 1 if exitNow == 1: exitTime = 15. print 'Script will exit in %d seconds.' % exitTime time.sleep( exitTime ) sys.exit() # Get FTP info # Open Underlying File zippedFile = open(configName, mode='rb') # Create the uncompressing wrapper for the zipped file # This allow us to treat the file as if it was not zipped ftpInfoFile = gzip.GzipFile(fileobj=zippedFile, mode='rt') ftpConfigLines = ftpInfoFile.readlines() ftpInfoFile.close() zippedFile.close() addressFound = False portFound = False serverFilePathFound = False localFilePathFound = False usernameFound = False passwordFound = False serverFilePath = '' localFilePath = './' for ftpConfig in ftpConfigLines: #skip comments if not ftpConfig.startswith('\\'): continue ftpConfig = ftpConfig.strip('\n') if bDebug: print ftpConfig ftpConfig = ftpConfig.split() if (len(ftpConfig) < 2): continue elif ftpConfig[0] == '\\address': address = ftpConfig[1] addressFound = True elif ftpConfig[0] == '\\port': port = ftpConfig[1] portFound = True elif ftpConfig[0] == '\\username': username = ftpConfig[1] usernameFound = True elif ftpConfig[0] == '\\password': password = ftpConfig[1] passwordFound = True elif ftpConfig[0] == '\\serverpath': serverFilePath = os.path.expanduser(ftpConfig[1]) serverFilePathFound = True elif ftpConfig[0] == '\\localpath': localFilePath = os.path.expanduser(ftpConfig[1]) localFilePathFound = True if not serverFilePathFound: print 'No Server File Path provided.' if not addressFound: print 'No Web Site provided.' if not portFound: print 'No Port Number provided.' if not usernameFound: print 'No Username provided.' if not passwordFound: print 'No Password provided.' if not serverFilePathFound or not addressFound or not portFound or not usernameFound or not passwordFound: print 'Please correct the configuration file and try again. Exiting.' sys.exit() if not localFilePathFound: print 'No Local File Path provided, using default: %s' % localFilePath # Create full server file path with mission ID serverFilePath = os.path.join(serverFilePath, missionID) # Print to standard out socket settings print '-> Running with the following parameters:' print ' - Remote Server: %s:%s'%(address, port) print ' - Server File Path: %s'%(serverFilePath) print ' - Local File Path: %s'%(localFilePath) # Assign directories if not(os.path.exists(localFilePath)): print '-> Creating path: ' + localFilePath os.makedirs( localFilePath ) logDirectory = os.path.join( localFilePath, 'log' ) if not(os.path.exists(logDirectory)): print '-> Creating path: ' + logDirectory os.makedirs( logDirectory ) looping = True fileCount = 0 timeStamp = time.strftime('%Y%m%d_%H%M%S') statusFileName = 'ftp_status_' + timeStamp + '.log' statusFilePath = os.path.join(logDirectory, statusFileName) iterationCount = 0 serverListFileName = '%s_server_files_%s.list' % (missionID, timeStamp) serverListFile = open( os.path.join(logDirectory, serverListFileName), 'w') while looping: try: iterationCount += 1 # Retrieve current file list currentDirList = os.listdir( localFilePath ) currentFileList = [] for thisEntry in currentDirList: if ( os.path.isfile(os.path.join(localFilePath, thisEntry)) ): currentFileList.append( thisEntry ) currentFileList.sort() # Determine what files need to be retrieved writtenFiles = [] serverFiles = [] fileName = os.path.join(logDirectory, 'filesRetrievedByFTP.list') if bRetrieve: if ( fileCompare == 'all' ): # Overwrite the entire list of files written writtenFileList = open( fileName, 'w+' ) else: # Read in sent file list writtenFileList = open( fileName, 'a+' ) writtenFiles = writtenFileList.readlines() # File names have carriage return appended i = 0 while i < len( writtenFiles ): writtenFiles[i] = writtenFiles[i].rstrip() i += 1 ftpConnected = False # connect and log in if not ftpConnected: # Create and open FTP session try: f = ftplib.FTP( ) if bDebug: print 'Connecting...' f.connect( address, port ) except ftplib.all_errors, error: statusString = 'Cannot connect. Error: %s\n' % error writeStatusFile( statusFilePath, statusString ) else: try: if bDebug: print 'Logging in...' f.login( username, password ) except ftplib.all_errors, error: statusString = 'Login failed. Error: %s\n' % error writeStatusFile( statusFilePath, statusString ) else: # Change to directory from which files will be transferred try: if bDebug: print 'CWD %s...' % serverFilePath f.cwd( serverFilePath ) except ftplib.all_errors, error: statusString = 'Directory ' + serverFilePath + ' Error: %s\n' % error writeStatusFile( statusFilePath, statusString ) else: ftpConnected = True # Retrieve a list of the files on the server try: serverFiles = [] if bDebug: print 'NLST...' #f.retrlines( 'NLST', buildFileList ) serverFiles = f.nlst() except ftplib.all_errors, error: statusString = 'Unable to retrieve list of files on remote server. Error: %s\n' % error writeStatusFile( statusFilePath, statusString ) if ftpConnected: # Loop through local files, write them out if appropriate if bDebug: print 'File count from server: %d' % len(serverFiles) for dataFileName in serverFiles: if dataFileName not in writtenFiles: # Retrieve the file from the remote server f.sendcmd("TYPE i") fileInfoStr = '%s\t%d\n' % (dataFileName, f.size(dataFileName)) serverListFile.write( fileInfoStr ) if bDebug: print 'Found file %s with size %d' % (dataFileName, f.size(dataFileName)) if bRetrieve: command = 'RETR ' + dataFileName localfile = open(os.path.join(localFilePath, dataFileName), 'wb') try: f.retrbinary( command, lambda data: localfile.write(data) ) statusString = 'Retrieved: ' + dataFileName writeStatusFile( statusFilePath, statusString+'\n' ) writtenFileList.write( dataFileName+'\n' ) fileCount += 1 except ftplib.all_errors, error: statusString = 'File ' + dataFileName + ' Error: %s\n' % error writeStatusFile( statusFilePath, statusString ) if ftpConnected: f.quit() ftpConnected = False if bRetrieve: # close written file list and continue looping writtenFileList.close( ) print 'Checking for new files in', sleepTime, 'seconds.' time.sleep( sleepTime ) else: # break loop looping = 0 except KeyboardInterrupt: print 'Keyboard interrupt. Shutting down.' looping = 0 serverListFile.close() print 'Done.'