Added old coaa scripts to subversion
authorJean-Michel Nirgal Vourgère <jmv@nirgal.com>
Thu, 12 Aug 2010 09:42:30 +0000 (09:42 +0000)
committerJean-Michel Nirgal Vourgère <jmv@nirgal.com>
Thu, 12 Aug 2010 09:42:30 +0000 (09:42 +0000)
bin/extras/planeplotter_coaa.py [new file with mode: 0755]
bin/extras/planeplotter_parse.py [new file with mode: 0755]
bin/extras/shipplotter_coaa.py [new file with mode: 0755]
bin/extras/shipplotter_parselogs.py [new file with mode: 0755]

diff --git a/bin/extras/planeplotter_coaa.py b/bin/extras/planeplotter_coaa.py
new file mode 100755 (executable)
index 0000000..5af73d6
--- /dev/null
@@ -0,0 +1,39 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import urllib2, StringIO, gzip, time, random
+from urlgrabber.keepalive import HTTPHandler
+
+from ais.ntools import read_cfg
+
+#plane plotter doen't use keep alive
+#keepalive_handler = HTTPHandler()
+#opener = urllib2.build_opener(keepalive_handler)
+#urllib2.install_opener(opener)
+
+__config_filename__ = '/etc/ais/planeplotter'
+
+request = urllib2.Request('http://www.coaa.co.uk/planeinfo9x1.php')
+request.add_header('User-Agent', 'PlanePlotter')
+#request.add_header('Accept-encoding', 'gzip, deflate')
+request.add_header('Cache-Control', 'no-cache')
+
+
+cfg = read_cfg(__config_filename__)
+reg = cfg['reg']
+extracode = cfg['extracode']
+ser = cfg['ser']
+mouset = random.randint(1,60)
+tt = int(time.time())
+
+uo = urllib2.urlopen(request, 'Lines=0&Reg=%(reg)s&Extracode=%(extracode)s&Uponly=0&PosLess=0&Mouset=%(mouset)d&LatN=144.450356&LatS=-143.075878&LonE=+237.636617&LonW=-217.642360&Fwd=0&Ver=5.3.3.7&Tt=%(tt)d&Ser=%(ser)s' % locals()) # Sometimes add '&Des=A15CB0' for selected plane
+httpdata = uo.read()
+uo.close()
+if uo.headers.get('Content-Encoding')=='gzip':
+    compressedstream = StringIO.StringIO(httpdata)
+    gzipper = gzip.GzipFile(fileobj=compressedstream)
+    httpdata = gzipper.read()
+print httpdata
+
+#time.sleep(10)
+#keepalive_handler.close_all()
diff --git a/bin/extras/planeplotter_parse.py b/bin/extras/planeplotter_parse.py
new file mode 100755 (executable)
index 0000000..7e46bda
--- /dev/null
@@ -0,0 +1,232 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+from optparse import OptionParser
+from datetime import datetime
+from pprint import pprint
+
+from ais.ntools import alarm, clean_alnum
+from ais.db import *
+
+def file_to_uset(filename):
+    '''
+    Load a file and make a set of string out of its lines
+    '''
+    s = set()
+    for iline, line in enumerate(file(filename).readlines()):
+        while len(line) and line[-1] in '\r\n':
+            line = line[:-1]
+        try:
+            tmp = unicode(line)
+        except:
+            print >> sys.stderr, 'ERROR on line', iline, ':', repr(line), ' is not a valid utf8 sequence'
+            raise
+        s.add(line)
+    return s
+
+def parse_log_line(line):
+    def _to_unicode(txt):
+        return unicode(txt.strip(), 'latin1', 'replace')
+        #return unicode(txt, 'latin1', 'replace')
+    def _clean_alnum(txt):
+        return unicode(clean_alnum(txt))
+        
+    line_struct = [
+        ('flight',       8, _to_unicode                                ),
+        ('reg',          8, _clean_alnum                               ),
+        ('ads',          8, _clean_alnum                               ),
+        ('type',         4, _to_unicode                                ),
+        ('route',        9, _to_unicode                                ),
+        ('latitude',    10, float                                      ),
+        ('longitude',   12, float                                      ),
+        ('altitude',  None, float                                      ),
+        ('course',    None, float                                      ),
+        ('speed',     None, float                                      ),
+        ('updated',   None, lambda x: datetime.utcfromtimestamp(int(x))),
+        ('userid',    None, _to_unicode                                ),
+        ('dummy',     None, _to_unicode                                ),
+        ('squawk',    None, _to_unicode                                ),
+    ]
+
+    info = {}
+    #pprint (line_struct)
+    pos = 0
+    for ik, klp in enumerate(line_struct):
+        k,l,parser=klp
+        if l is None:
+            while pos < len(line) and line[pos]==' ':
+                pos += 1
+            # compute length
+            spos = line.find(' ', pos)
+            if spos==-1:
+                l = len(line)-pos+1
+            else:
+                l = spos-pos+1
+        v = line[pos:pos+l]
+        if parser:
+            v = parser(v)
+        info[k] = v
+        pos += l
+    assert len(line)+1==pos, 'Error when parsing "%(line)s". Found extra characters at end of line' % locals()
+
+    # cleanup data
+    if info['reg'] in (u'.NOREG', u'NOREG'):
+        info['reg'] = u'        '
+    #else:
+    #    info['reg'] += ' '*(8-len(info['reg']))
+    if info['ads'] == u'?':
+        info['ads'] == u'        '
+    #else:
+    #    info['ads'] += ' '*(8-len(info['ads']))
+    if info['type'] == u'':
+        info['type'] = None
+
+
+    #if info['course']>360:
+    #    info['course'] = None
+    #if info['heading']>360:
+    #    info['heading'] = None
+    #info['infosrc'] = u'ShPl-'+info['userid']
+
+    #eta = info['eta']
+    #info['eta'] = None
+    #if len(eta)>0 and eta!='unknown' and eta[0] not in ('*.'):
+    #    M = J = m = h = None
+    #    try:
+    #        monthes='Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec'.split(',')
+    #        M = monthes.index(eta[:3])+1
+    #        J = int(eta[3:5])
+    #        h = int(eta[6:8])
+    #        m = int(eta[9:11])
+    #    except Exception, e:
+    #        pass
+    #        #print e
+    #    else:
+    #        if M is not None and J is not None:
+    #            info['eta'] = '%02d%02d' % (M, J)
+    #            if h is not None:
+    #                if m is None:
+    #                    m = 0
+    #                info['eta'] += '%02d%02d' % (h, m)
+    ##if info['eta'] is None:
+    ##    print eta, '=>', info['eta']
+
+    return info
+    
+
+def log_file_reader(filename):
+    for line in file(filename).readlines():
+        while line and line[-1] in '\r\n\0':
+            line = line[:-1]
+        if len(line)==0:
+            continue # ignore blank lines
+        if line.startswith('Alert'):
+            continue # ignore
+        yield line
+        
+
+
+class Localizator:
+    def __init__(self, user):
+        self.user = user
+        self.lat = 0.
+        self.lon = 0.
+        self.count = 0
+
+    def add_info(self, info):
+        user = info['userid'][1:3]
+        if user == self.user and info['latitude'] and info['longitude']:
+            self.lat += info['latitude']
+            self.lon += info['longitude']
+            self.count += 1
+    def add_line(self, line):
+        self.add_info(parse_log_line(line))
+    def add_info_file(self, filename):
+        for line in log_file_reader(filename):
+            self.add_line(line)
+    def getposition(self):
+        return self.lat/self.count, self.lon/self.count
+
+
+def parse_log_file(filename):
+    target_reg = file_to_uset('planeplotter/target_reg.txt')
+    target_icao24 = file_to_uset('planeplotter/target_icao24.txt')
+    target_types = file_to_uset('planeplotter/target_types.txt')
+    
+    for line in log_file_reader(filename):
+
+        info = parse_log_line(line)
+        
+        # alarm
+        is_local = info['userid'][1:3]=='dh'
+        is_target = info['reg'] in target_reg or info['ads'] in target_icao24 
+        if is_target or (info['type'] in target_types and is_local):
+            if is_local:
+                print '*'*80
+                print '* LOCAL DETECTION *'
+                print '*'*80
+                alarm()
+            print filename
+            print line
+            if info['latitude'] and info['longitude']:
+                alarm()
+
+            if is_target:
+                sqlparams = { 'flight': info['flight'], 'reg': info['reg'], 'ads':info['ads'], 'type':info['type'], 'usr':info['userid'][1:3], 'updated':info['updated'] }
+                cond = []
+                if info['reg'] and info['reg']!=u'.NOREG':
+                    cond.append(get_common_cursor().mogrify('reg=%(reg)s', sqlparams))
+                if info['ads']:
+                    cond.append(get_common_cursor().mogrify('ads=%(ads)s', sqlparams))
+                whereor = 'WHERE '+' OR '.join(cond)
+                whereand = 'WHERE '+' AND '.join(cond)
+                    
+                sqlexec(u'INSERT INTO plane (flight, reg, ads, type, usr, updated) SELECT %(flight)s, %(reg)s, %(ads)s, %(type)s, %(usr)s, %(updated)s WHERE NOT EXISTS (SELECT * FROM plane WHERE reg=%(reg)s AND ads=%(ads)s)', sqlparams)
+
+                sqlexec(u'UPDATE plane SET (usr, updated) = (%(usr)s, %(updated)s) WHERE reg=%(reg)s AND ads=%(ads)s', sqlparams)
+
+                if info['flight']:
+                    sqlexec(u'UPDATE plane SET flight = %(flight)s WHERE reg=%(reg)s AND ads=%(ads)s AND flight IS NULL', sqlparams)
+
+                if info['type']:
+                    sqlexec(u'UPDATE plane SET type = %(type)s WHERE reg=%(reg)s AND ads=%(ads)s AND type IS NULL', sqlparams)
+
+                dbcommit()
+
+
+if __name__ == "__main__":
+    parser = OptionParser()
+    parser.add_option('--user', '-u', help="localise sharing user", action='store', dest='user')
+    parser.add_option('--debug-sql', help="print all sql queries to stdout before running them", action='store_true', dest='debug_sql', default=False)
+    (options, args) = parser.parse_args()
+
+    if len(args)==0:
+        print >> sys.stderr, "Need file names"
+        sys.exit(1)
+    
+    if options.debug_sql:
+        sql_setdebug(True)
+    
+    if options.user:
+        user = options.user
+        localizator = Localizator(options.user)
+        for iarg in range(len(args)-1, -1, -1):
+            filename = args[iarg]
+            localizator.add_info_file(filename)
+            if localizator.count > 100:
+                break
+        lat, lon = localizator.getposition()
+        sqlexec('INSERT INTO ppuser (usr, lat, lon) SELECT %(user)s, %(lat)s, %(lon)s WHERE NOT EXISTS (SELECT * FROM ppuser WHERE usr=%(user)s)', locals())
+        sqlexec('UPDATE ppuser SET (usr, lat, lon) = (%(user)s, %(lat)s, %(lon)s) WHERE usr=%(user)s', locals())
+        dbcommit()
+        sys.exit(0)
+     
+    for filename in args:
+        parse_log_file(filename)
+
+    sqlexec('SELECT plane.usr FROM plane WHERE NOT EXISTS (SELECT * FROM ppuser WHERE ppuser.usr=plane.usr)')
+    
+    for (usr,) in get_common_cursor().fetchall():
+        print 'User', usr, 'location is unknown.'
+    
diff --git a/bin/extras/shipplotter_coaa.py b/bin/extras/shipplotter_coaa.py
new file mode 100755 (executable)
index 0000000..01a98ab
--- /dev/null
@@ -0,0 +1,74 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import urllib2, StringIO, gzip, time, random, os
+from urlgrabber.keepalive import HTTPHandler
+from datetime import datetime
+#from shipplotter_parselogs import parse_file
+
+from ais.ntools import read_cfg
+
+__config_filename__ = '/etc/ais/shipplotter'
+
+#Warning: keep-alive handler does mixes with User-Agent!
+#keepalive_handler = HTTPHandler()
+#opener = urllib2.build_opener(keepalive_handler)
+#urllib2.install_opener(opener)
+
+request = urllib2.Request('http://www.coaa.co.uk/shipinfo.php')
+request.add_header('User-Agent', 'ShipPlotter')
+request.add_header('Accept-encoding', 'gzip, deflate')
+request.add_header('Cache-Control', 'no-cache')
+
+
+def get_httpdata():
+    cfg = read_cfg(__config_filename__)
+    reg = cfg['reg']
+    extracode = cfg['extracode']
+
+    # If sharing 1 + 2 + 1->2, Fwd(1)=10, Fwd(2)=0
+    # If sharing 1 + 2, Fwd(1)=0, Fwd(2)=0
+    mouset = random.randint(1,60)
+    tt = int(time.time())
+    
+    # tt must be UTC
+    uo = urllib2.urlopen(request, 'Lines=0&Reg=' + reg + '&Extracode=' + extracode + '&Uponly=0&Mouset='+str(mouset)+'&LatN=161.549709&LatS=-161.548334&LonE=+266.288827&LonW=-245.287049&Fwd=0&Ver=12.4.3&Tt='+str(tt))
+    httpdata = uo.read()
+    uo.close()
+    if uo.headers.get('Content-Encoding')=='gzip':
+        compressedstream = StringIO.StringIO(httpdata)
+        gzipper = gzip.GzipFile(fileobj=compressedstream)
+        httpdata = gzipper.read()
+    return httpdata
+
+def save_httpdata(httpdata):
+    now = datetime.utcnow()
+    foldername = 'shipplotter/'+now.strftime('%Y%m%d')
+    try:
+        os.mkdir(foldername)
+    except OSError, oserr:
+        if oserr.errno != 17:
+            raise
+        # else folder allrady exists, perfect
+    filename = foldername+'/'+now.strftime('%H%M%S')
+    file = open(filename, 'wb')
+    file.write(httpdata)
+    file.close()
+    return filename
+
+
+if __name__ == '__main__':
+    from optparse import OptionParser
+    parser = OptionParser()
+    #parser.add_option('--db-path', help="set db path. default=%default", action='store', dest='dbpath', default='db')
+    options, args = parser.parse_args()
+    #for i in range(10):
+    #    data = get_httpdata()
+    #    filename = save_httpdata(data)
+    #    print filename
+    #    parse_file(options.dbpath, filename)
+    #    print 'parsed ok'
+    #    time.sleep(60)
+    data = get_httpdata()
+    print data
+    #keepalive_handler.close_all()
diff --git a/bin/extras/shipplotter_parselogs.py b/bin/extras/shipplotter_parselogs.py
new file mode 100755 (executable)
index 0000000..9822278
--- /dev/null
@@ -0,0 +1,181 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+import sys
+from optparse import OptionParser
+from pprint import pprint
+import logging
+
+from ais.common import add_nmea1, add_nmea5_partial, AIS_LATLON_SCALE, AIS_LAT_NOT_AVAILABLE, AIS_LON_NOT_AVAILABLE, AIS_SOG_SCALE, AIS_SOG_NOT_AVAILABLE, AIS_COG_SCALE, AIS_COG_NOT_AVAILABLE, AIS_NO_HEADING, AIS_ROT_NOT_AVAILABLE, DBPATH
+from ais.ntools import clean_alnum, clean_ais_charset
+
+LINE_STRUCT = (
+    ('mmsi',         9, None         ),
+    ('timestamp',   10, int          ),
+    ('status',       2, int          ),
+    ('type',        -1, int          ), # some MRCC wronglyfull send type over 99
+    ('latitude',    10, float        ),
+    ('longitude',   11, float        ),
+    ('sog',         -1, float        ),
+    ('cog',          5, float        ),
+    ('heading',      3, int          ),
+    ('draught',      4, float        ),
+    ('length',       3, int          ),
+    ('width',       -1, int          ),
+    ('name',        20, None         ),
+    ('callsign',     7, None         ),
+    ('destination', 20, None         ),
+    ('eta',         11, str.strip    ),
+    ('userid1',      1, None         ),
+    ('imo',          7, int          ),
+    ('dim_bow',      3, int          ),
+    ('dim_port',     2, int          ),
+    ('userid2',      1, None         ),
+)
+
+def parse_log_line(line):
+    info = {}
+    #pprint (LINE_STRUCT)
+    pos = 0
+    for ik, klp in enumerate(LINE_STRUCT):
+        k, l, parser = klp
+        if ik:
+            if line[pos-1] != ' ':
+                logging.error('when parsing. Key=%s. Pos=%s. Expected a space.', k, pos)
+                logging.error('%s', line)
+                logging.error(' '*(pos-1)+'^')
+                #print >> sys.stderr, 'Error when parsing. Key='+`k`+'. Pos='+`pos`+'. Expected a space.\n'+line+'\n'+' '*(pos-1)+'^'
+                return None
+        if l == -1:
+            # compute length
+            l = line.find(' ', pos)
+            if l == -1:
+                l = len(line)-pos # Not tested
+            else:
+                l -= pos
+        v = line[pos:pos+l]
+        if parser:
+            v = parser(v)
+        info[k] = v
+        pos += l+1
+    if len(line)+1 != pos:
+        # TODO: loggin
+        print >> sys.stderr, 'Error when parsing "%(line)s". Found extra characters at end of line' % locals()
+    return info
+    
+def parse_file(filename, dry_run):
+    monthes = 'Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec'.split(',')
+    f = file(filename)
+    for line in f.read().split('\n'):
+        while line and line[-1] in '\r\n\0':
+            line = line[:-1]
+        if len(line)==0:
+            continue # ignore blank lines
+        #print line
+        info = parse_log_line(line)
+        if not info:
+            continue # something went wrong, line skiped
+
+        #pprint(info)
+        mmsi = info['mmsi']
+        timestamp = info['timestamp']
+        status = info['status']
+        if status > 15:
+            continue
+        type_ = info['type']
+        if type_ > 100:
+            # FIXME
+            continue
+        latitude = int(info['latitude']*AIS_LATLON_SCALE)
+        if latitude == 0:
+            latitude = AIS_LAT_NOT_AVAILABLE
+        longitude = int(info['longitude']*AIS_LATLON_SCALE)
+        if longitude == 0:
+            longitude = AIS_LON_NOT_AVAILABLE
+        sog = int(info['sog']*AIS_SOG_SCALE)
+        if sog > AIS_SOG_NOT_AVAILABLE:
+            sog = AIS_SOG_NOT_AVAILABLE
+        cog = int(info['cog']*AIS_COG_SCALE)
+        if cog > AIS_COG_NOT_AVAILABLE:
+            cog = AIS_COG_NOT_AVAILABLE
+        heading = info['heading']
+        if heading > AIS_NO_HEADING:
+            heading = AIS_NO_HEADING
+        draught = int(info['draught']*10.0)
+        dim_bow = info['dim_bow']
+        dim_stern = info['length'] - dim_bow
+        dim_port = info['dim_port']
+        dim_starboard = info['width'] - dim_port
+        if dim_stern < 0 or dim_stern > 255 or dim_starboard < 0 or dim_starboard > 255:
+            dim_bow = dim_stern = dim_port = dim_starboard = 0
+        name = clean_ais_charset(info['name'])
+        if name.startswith(str(mmsi)):
+            #print >> sys.stderr, 'Ignoring shipname', name
+            name = '' # we don't wants rescue name, only real ones
+        callsign = info['callsign']
+        if callsign == 'unknown':
+            callsign = ''
+        else:
+            callsign = clean_alnum(callsign)
+        
+        destination = info['destination']
+        if destination in ('unknown', '....................'):
+            info['destination'] = ''
+        else:
+            destination = clean_ais_charset(destination)
+    
+        eta = info['eta']
+        eta_M = eta_D = 0
+        eta_h = 24
+        eta_m = 60
+        if len(eta) > 0 and eta != 'unknown' and eta[0] not in ('*.'):
+            try:
+                eta_M = monthes.index(eta[:3])+1
+                eta_D = int(eta[3:5])
+                eta_h = int(eta[6:8])
+                eta_m = int(eta[9:11])
+            except Exception:
+                pass
+        source = 'SP'+info['userid1']+info['userid2']
+        imo = info['imo']
+
+        if not dry_run:
+            if not add_nmea1(mmsi, timestamp, status, AIS_ROT_NOT_AVAILABLE, sog, latitude, longitude, cog, heading, source):
+                logging.debug('%s nmea1 not updated', mmsi)
+
+            #for i in (mmsi, timestamp, imo, name, callsign, type_, dim_bow, dim_stern, dim_port, dim_starboard, eta_M, eta_D, eta_h, eta_m, draught, destination, source):
+            #    print repr(i),
+            #print
+            if not add_nmea5_partial(mmsi, timestamp, imo, name, callsign, type_, dim_bow, dim_stern, dim_port, dim_starboard, eta_M, eta_D, eta_h, eta_m, draught, destination, source):
+                logging.debug('%s nmea5 not updated', mmsi)
+    f.close()
+
+
+def main():
+    global DBPATH
+    parser = OptionParser('%prog [options] filename [filename2]...')
+    parser.add_option('-d', '--debug',
+        action='store_true', dest='debug', default=False,
+        help="debug mode")
+    parser.add_option('--print-filename', help="prints each file name before it's processed", action='store_true', dest='print_filename', default=False)
+    parser.add_option('--db-path', help="set db path. default=%default", action='store', dest='dbpath', default=DBPATH)
+    parser.add_option('--dry-run',
+        action='store_true', dest='dry_run', default=False,
+        help="don't actually write anything to the db")
+    (options, args) = parser.parse_args()
+
+    DBPATH = options.dbpath
+
+    if options.debug:
+        loglevel = logging.DEBUG
+    else:
+        loglevel = logging.INFO
+    logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s')
+
+    for filename in args:
+        if options.print_filename:
+            logging.info('Processing %s', filename)
+        parse_file(filename, options.dry_run)
+
+if __name__ == '__main__':
+    main()