""" mahali_trimble_interface.py $Id: mahali_trimble_interface.py 480 2015-08-13 23:01:52Z flind $ This program connects to the trimble receiver collects metadata, downloads a list of files, compares to files in local storage, and then ftp transfers files which have not been copied. """ # import sys import os import serial # http://pyserial.sourceforge.net/shortintro.html import string import time import struct import datetime from pytz import timezone import daemon # in preparation for running as a daemon import optparse # command line parsing import fnmatch import urllib2 import ftplib import redis from mahali_common import * # #----------------------------------------------------------------------------------- # # GLOBALS # #----------------------------------------------------------------------------------- # #----------------------------------------------------------------------------------- # class TrimbleGPSInterface: def __init__(self,redis_db, config, options, logQ): self.redis_db = redis_db self.config = config self.options = options self.logQ = logQ self.start_time = time.time() self.update_time = -1 self.ipaddr = self.config['mahali-instrument-trimble']['ipaddr'] self.gps_type = self.config['mahali-instrument-trimble']['gps_type'] self.local_path = fixup_path(self.config['mahali-data-service']['rinex_data_path']) # GPS derived metadata self.gps_time_valid = False self.gps_longsecond_time = -1 # unix longsecond time self.satellite_list = [] self.position = {} self.trimble_storage = None # GPS file lists self.gps_files = [] self.local_files = [] self.pending_file_list = [] # http timeout limit self.timeout = 5 def __getGPSFileList(self): # get year list try: cmd = "http://%s/prog/show?Directory&path=/Internal" % (self.ipaddr) if self.options.simulate: self.logQ.logDebug('simulate getGPSFileList with cmd %s' % (cmd)) response = '\nsize 4294967296\navailable 4249906348\ndirectory name=201508\ndirectory name=RTP_SI\ndirectory\nname=lost+found\n' else: r = urllib2.urlopen(cmd,timeout = self.timeout) response = r.read() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with getGPSFileList directory command %s to trimble in mahali_trimble_interface." % (exp_str, cmd) self.logQ.logError(emsg) return sdir = string.split(response,'\n') # opportunistically pull the current remaining storage on the trimble self.trimble_storage = int(string.split(sdir[2],' ')[1]) gps_dir = [] # GPS directory format is set on the receiver. We require the YYYYMM format # this limits the number of directories we need to traverse. for sp in sdir[3:-4]: gps_dir.append(string.split(sp,'=')[1]) self.gps_files = [] for gd in gps_dir: try: if self.options.simulate: self.logQ.logDebug('simulate getGPSFileList directory get %s' % (gd)) response = '\nsize 4294967296\navailable 4249840900\nfile name=5311K51736201508110000M.T02 size=615513 ctime=1123286400 attr=00008180\nfile name=5311K51736201508110100M.T02 size=714920 ctime=1123290000 attr=00008180\n' else: cmd = "http://%s/prog/show?Directory&path=/Internal/%s" % (self.ipaddr,gd) r = urllib2.urlopen(cmd,timeout = self.timeout) response = r.read() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with getGPSFileList directory command %s to trimble in mahali_trimble_interface." % (exp_str, cmd) self.logQ.logError(emsg) return fdir = string.split(response,'\n') for fg in fdir[3:-2]: fval = string.split(fg,' ') fnm = string.split(fval[1],'=') ftag = string.split(fnm[1],'.') if ftag[-1] == 'T02': # fixup the file name for FTP generation of rinex output # doesn't seem to work over the web interface f = '/Internal/%s' % (gd) + '/' + ftag[0] + '.RINEX.2.11.zip' self.gps_files.append(f) #if self.options.debug and self.options.verbose: # self.logQ.logDebug(self.gps_files) def __getLocalFileList(self): matches = [] for root, dirnames, filenames in os.walk(self.local_path): for filename in fnmatch.filter(filenames, '*.RINEX.2.11.zip'): matches.append(os.path.join(root, filename)) self.local_files = matches def checkNewData(self): try: self.__getGPSFileList() self.__getLocalFileList() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with checkNewData in mahali_trimble_interface." % (exp_str) self.logQ.logError(emsg) return False self.pending_file_list = [] # get rid of paths and just keep file names gpsfl = [] for gf in self.gps_files: try: gpsfl.append(string.split(gf,'/')[-1]) except: continue locfl = [] for lf in self.local_files: try: locfl.append(string.split(lf,'/')[-1]) except: continue # use set difference sgps = set(gpsfl) slocal = set(locfl) pending = sgps - slocal if self.options.debug and self.options.verbose: self.logQ.logDebug(list(pending)) # now go through the GPS list and match the pending list for gf in self.gps_files: if string.split(gf,'/')[-1] in pending: self.pending_file_list.append(gf) if self.pending_file_list == []: return False else: return True def __buildPath(self, fpath): if not os.path.exists(os.path.dirname(fpath)): os.makedirs(os.path.dirname(fpath)) if self.options.debug and self.options.verbose: self.logQ.logDebug('build_path: created %s' % (fpath)) def __buildLocalFilePath(self, fname): # chop out time stamp from RINEX file name year = fname[-28:-24] month = fname[-24:-22] day = fname[-22:-20] sdate = '%s-%s-%s' % (year,month,day) fpath = self.local_path + '%s/' % (sdate) self.__buildPath(fpath) if self.options.debug and self.options.verbose: self.logQ.logDebug('file_info: fpath %s' % (fpath)) return fpath + fname def collectData(self): """ collect data from the Trimble GPS. We use FTP here due to the availability of rinex translation as part of the retreival. It isn't clear if this is in the http interface or not. """ # open ftp connection if not self.options.simulate: ftp = ftplib.FTP(self.ipaddr,timeout = self.timeout) ftp.login() for fgps in self.pending_file_list: if self.options.simulate: self.logQ.logDebug('simulate collection of GPS data %s ' % (fgps)) continue try: # build output filename fname = string.split(fgps,'/')[-1] lfname = self.__buildLocalFilePath(fname) gdname = string.split(fgps,fname)[0] if os.path.isfile(lfname): self.logQ.logDebug('warning : file %s exists, will not overwrite.' % (lfname)) continue if self.options.debug and self.options.verbose: self.logQ.logDebug('ftp retreival of file %s on path %s to %s.' % (fname,gdname,lfname)) # open output file fid = open(lfname,'wb') if self.options.debug and self.options.verbose: self.logQ.logDebug('ftp cwd to %s' % (gdname)) ftp.cwd(gdname) if self.options.debug and self.options.verbose: self.logQ.logDebug('ftp binary RETR from %s' % (fname)) result = ftp.retrbinary("RETR " + fname, fid.write) fid.close() self.logQ.logInfo('ftp retreival of file %s to %s from GPS complete with result %s.' % (fgps,lfname,result)) # close output file except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with getCollectData directory ftp command for file %s to trimble in mahali_trimble_interface." % (exp_str, f) self.logQ.logError(emsg) continue if not self.options.simulate: ftp.quit() def mirrorState(self): active_time = time.time() - self.start_time self.update_time = time.time() state_info = {'start_time':self.start_time, 'active_time':active_time, 'update_time':self.update_time, 'gps_time_valid':self.gps_time_valid, 'gps_longsecond_time':self.gps_longsecond_time, 'satellite_list':self.satellite_list, 'position':self.position, 'trimble_storage':self.trimble_storage} # watch out for recursion on including the "data" field mahali_set_object(self.redis_db,self.logQ,'mahali-gps-interface-state', state_info) def timeCommand(self): """ This method commands the GPS to return the GPS derived time. Simplifies setting the system clock if needed because we can mirror this to redis for use there. However, the trimble normally acts as an NTP server. """ # write command try: cmd = 'http://%s/prog/show?UtcTime' % (self.ipaddr) if self.options.simulate: self.logQ.logDebug('simulate getTimeCommand with cmd %s' % (cmd)) response = 'UtcTime year=2015 month=8 day=11 hour=13 minute=18 second=15 julianDay=223' else: r = urllib2.urlopen(cmd,timeout = self.timeout) response = r.read() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with gpstime command %s to trimble in mahali_trimble_interface." % (exp_str, cmd) self.logQ.logError(emsg) return try: if self.options.debug and self.options.verbose: self.logQ.logDebug('gpstime got back %s' % (response)) # format is UtcTime year=2015 month=8 day=10 hour=15 minute=26 second=40 julianDay=222 sval = string.split(response,' ') year = int(string.split(sval[1],'=')[1]) month = int(string.split(sval[2],'=')[1]) day = int(string.split(sval[3],'=')[1]) hour = int(string.split(sval[4],'=')[1]) minute = int(string.split(sval[5],'=')[1]) second = int(string.split(sval[6],'=')[1]) dtime = datetime.datetime(year,month,day,hour,minute,second,0,timezone('UTC')) self.gps_longsecond_time = (dtime-datetime.datetime(1970,1,1,0,0,0,0,timezone('UTC'))).total_seconds() self.gps_time_valid = True tms = {'long second' : self.gps_longsecond_time, 'gps utc' : response} self.logQ.logInfo(tms) except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem getting GPS time response from trimble. Got value %s in mahali_trimble_interface." % (exp_str, response) self.logQ.logError(emsg) self.gps_time_valid = False self.gps_longsecond_time = -1 return def positionCommand(self): """ This method commands the GPS to return the GPS derived position. """ # write command try: cmd = 'http://%s/prog/show?Position' % (self.ipaddr) if self.options.simulate: self.logQ.logDebug('simulate getPositionCommand with cmd %s' % (cmd)) response = '\nGpsWeek 1857\nWeekSeconds 220782.6\nLatitude 42.6231663939 deg\nLongitude -71.4890326324 deg\nAltitude 93.376 meters\nQualifiers WGS84,3D,SBAS-DGPS\nSatellites 4,7,9,11,16,19,23,27,30,138\nClockOffset 0.000006 msec\nClockDrift 0.000021 ppm\nVelNorth 0.02 m/sec\nVelEast 0.00 m/sec\nVelUp 0.00 m/sec\nPDOP 1.5\nHDOP 0.9\nVDOP 1.2\nTDOP 0.8\n' else: r = urllib2.urlopen(cmd,timeout = self.timeout) response = r.read() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with gpstime command %s to trimble in mahali_trimble_interface." % (exp_str, cmd) self.logQ.logError(emsg) return try: if self.options.debug and self.options.verbose: self.logQ.logDebug('gps position got back %s' % (response)) # format is \nGpsWeek 1857\nWeekSeconds 143433.0\nLatitude 42.6231598119 deg\nLongitude -71.4890343100 deg\nAltitude 90.317 meters\nQualifiers WGS84,3D,Autonomous\nSatellites 1,4,7,11,19,28,30\nClockOffset 0.000007 msec\nClockDrift -0.000389 ppm\nVelNorth 0.01 m/sec\nVelEast -0.01 m/sec\nVelUp -0.05 m/sec\nPDOP 5.0\nHDOP 1.6\nVDOP 4.7\nTDOP 3.7\n\n sval = string.split(response,'\n') lat = string.split(sval[3],' ')[-2] lon = string.split(sval[4],' ')[-2] alt = string.split(sval[5],' ')[-2] vel_n = string.split(sval[10],' ')[-2] vel_e = string.split(sval[11],' ')[-2] self.position = {'latitude':lat, 'longitude':lon, 'altitude':alt, 'velocity north':vel_n, 'velocity east':vel_e, 'response': response} self.logQ.logInfo(self.position) except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem getting GPS position response from trimble. Got value %s in mahali_trimble_interface." % (exp_str, response) self.logQ.logError(emsg) self.gps_time_valid = False self.gps_longsecond_time = -1 return def visibleCommand(self): """ This method commands the GPS to return the visible satellite list. Very useful metadata to collect. """ # format is ['', 'Prn=8 Sys=GPS Elv=50 Azm=052 IODE=58 URA=2.8 L1snr=46 L2snr=37 L2Csnr=44 L5snr=52', ... ] try: cmd = 'http://%s/prog/show?TrackingStatus' % (self.ipaddr) if self.options.simulate: self.logQ.logDebug('simulate getVisibleCommand with cmd %s' % (cmd)) response = '\nPrn=4 Sys=GPS Elv=7 Azm=162 IODE=90 URA=2 L1snr=24\nPrn=7 Sys=GPS Elv=43 Azm=310 IODE=68 URA=2 L1snr=47 L2snr=36 L2Csnr=45\n' else: r = urllib2.urlopen(cmd,timeout = self.timeout) response = r.read() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem reading satvis response to trimble. For command %s in mahali_trimble_interface." % (exp_str, cmd) self.logQ.logError(emsg) return self.satellite_list = [] sats = string.split(response,'\n') self.logQ.logInfo('satellite visible: %s' % (sats[1:-2])) self.satellite_list = sats[1:-2] def mahali_trimble_interface(options): # create redis interface redis_db = redis.StrictRedis(host='localhost', port=6379, db=0) # register with redis in the mahali service list mahali_register_service(redis_db, 'mahali-trimble-interface') # create logging instance logQ = redisLoggerQueue('trimble-interface',redis_db, options) # load configuration mahali_cfg = mahaliConfiguration('mahali-gps-array', options.CPATH, redis_db, logQ) config = mahali_cfg.load('mahali-trimble-config') # get interval interval = config['mahali-instrument-trimble']['data_interval'] # create output handler path = config['mahali-data-service']['raw_data_path'] name = config['mahali-site']['name'] + '-' + config['mahali-site']['id'] # create GPS interface GPS = TrimbleGPSInterface(redis_db, config, options, logQ) # do initial receiver setup logQ.logInfo('startup') try: while True: # continue on errors try: GPS.visibleCommand() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with GPS satellite visibility collection in mahali_trimble_interface." % (exp_str) logQ.logError(emsg) try: GPS.timeCommand() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with GPS time retreival in mahali_trimble_interface." % (exp_str) logQ.logError(emsg) try: GPS.positionCommand() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with GPS positionretreival in mahali_trimble_interface." % (exp_str) logQ.logError(emsg) try: GPS.mirrorState() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with GPS state mirror in mahali_trimble_interface." % (exp_str) logQ.logError(emsg) # Note : each data block for trimble is a complete rinex file of a size set on the receiver. It is downloaded # whole from the receiver and written out to the appropriate path with the same file name. The data check looks # at the existing list of files and compares it to those on the receiver. Any not previously copied are retreived. if not GPS.checkNewData(): print "sleep" time.sleep(interval / 2.0) else: print "no sleep" try: GPS.collectData() except Exception as eobj: exp_str = str(ExceptionString(eobj)) emsg = "exception: %s. Problem with GPS data collection in mahali_trimble_interface." % (exp_str) logQ.logError(emsg) except Exception as eobj: exp_str = str(ExceptionString(eobj)) if options.debug: logQ.logDebug("exception: %s. Unresolved problem with mahali_trimble_interface data collection and recording." % (exp_str)) # publish shutdown logQ.logInfo('shutdown') if not mahali_unregister_service(redis_db, 'mahali-trimble-interface'): sys.exit() # end main_program # # #------------------------------------------------------------------- # def parse_command_line(): # parser = optparse.OptionParser() parser.add_option("-v", "--verbose",action="store_true", dest="verbose", default=False,help="prints debug output and additional detail.") parser.add_option("-d", "--debug",action="store_true", dest="debug", default=False,help="run in debug mode and not service context.") parser.add_option("-s", "--simulate",action="store_true", dest="simulate", default=False,help="simulate receiver commands.") parser.add_option("-c", "--config",dest="CPATH",help="Use configuration files in CPATH.") parser.add_option("-f", "--foreground",action="store_true",dest="foreground",help="Execute in foreground and not daemon context.") (options, args) = parser.parse_args() return (options, args) # end parse_command_line # #------------------------------------------------------------------- # # MAIN # #------------------------------------------------------------------- # if __name__ == '__main__': options, args = parse_command_line() # parse command line options if (options.foreground): print "Mahali Trimble Interface in DEBUG mode" mahali_trimble_interface(options) else: with daemon.DaemonContext(): mahali_trimble_interface(options)