All code now assumes it's bases on the "ais" python module
authorJean-Michel Nirgal Vourgère <jmv@nirgal.com>
Fri, 11 Jun 2010 17:35:53 +0000 (17:35 +0000)
committerJean-Michel Nirgal Vourgère <jmv@nirgal.com>
Fri, 11 Jun 2010 17:35:53 +0000 (17:35 +0000)
21 files changed:
INSTALL
bin/__init__.py
bin/ais.py [deleted file]
bin/common.py [new file with mode: 0755]
bin/dj.py
bin/djais/basicauth.py
bin/djais/models.py
bin/djais/templatetags/ais_extras.py
bin/djais/urls.py
bin/djais/views.py
bin/gpsdecoded.py
bin/inputs/common.py
bin/inputs/peers.py
bin/inputs/serialin.py
bin/inputs/stats.py
bin/inputs/tcpout.py
bin/inputs/udp.py
bin/make-countries.py
bin/nmea.py
bin/show_targets_planes.py
bin/show_targets_ships.py

diff --git a/INSTALL b/INSTALL
index 2ce118ae16036af93e71a1fa86a620797726f5b7..4b1179fa8eee8eadb0c7994a88a28ead4f91dbd3 100644 (file)
--- a/INSTALL
+++ b/INSTALL
@@ -11,5 +11,8 @@ adduser www-data ais
 
 change apache umask in /etc/apache2/envvars from 022 to 002 so that new folders are group writables
 
+ln -s /home/nirgal/kod/ais/bin /usr/lib/python2.5/ais
+
 Required packages:
+postgresql
 python-rrdtool
index e69de29bb2d1d6434b8b29ae775ad8c2e48c5391..635d9b747b4b2babd4ab135b77381b8548ed1009 100644 (file)
@@ -0,0 +1,3 @@
+'''
+Module to handle ships Automatic Authentification System.
+'''
diff --git a/bin/ais.py b/bin/ais.py
deleted file mode 100755 (executable)
index 3bc3fcd..0000000
+++ /dev/null
@@ -1,2000 +0,0 @@
-#!/usr/bin/env python
-# -*- coding: utf-8 -*-
-__all__ = [
-    'DB_STARTDATE', 'DBPATH',
-    'COUNTRIES_MID', 'STATUS_CODES', 'SHIP_TYPES',
-    'AIS_STATUS_NOT_AVAILABLE',
-    'AIS_ROT_HARD_LEFT', 'AIS_ROT_HARD_RIGHT', 'AIS_ROT_NOT_AVAILABLE',
-    'AIS_LATLON_SCALE', 'AIS_LON_NOT_AVAILABLE', 'AIS_LAT_NOT_AVAILABLE',
-    'AIS_COG_SCALE', 'AIS_COG_NOT_AVAILABLE',
-    'AIS_NO_HEADING',
-    'AIS_SOG_SCALE', 'AIS_SOG_NOT_AVAILABLE', 'AIS_SOG_FAST_MOVER', 'AIS_SOG_MAX_SPEED',
-    #'_hash3_pathfilename',
-    'db_bydate_addrecord',
-    'db_lastinfo_setrecord_ifnewer',
-    'sql_add_nmea5',
-    #'aivdm_record123_format',
-    #'aivdm_record123_length',
-    #'aivdm_record5_format',
-    #'aivdm_record5_length',
-    'add_nmea1',
-    'add_nmea5_full',
-    'add_nmea5_partial',
-    'strmmsi_to_mmsi',
-    'mmsi_to_strmmsi',
-    'Nmea1',
-    'Nmea5',
-    'Nmea',
-    'BankNmea1',
-    'Nmea1Feeder',
-    'BankNmea5',
-    'Nmea5Feeder',
-    'NmeaFeeder',
-    'all_mmsi_generator',
-    'load_fleet_to_uset',
-    'filter_area',
-    'filter_knownposition',
-    'filter_speedcheck',
-    ]
-            
-
-import sys
-import os
-import struct
-import logging
-from datetime import datetime, timedelta, date, time
-from fcntl import lockf, LOCK_EX, LOCK_UN, LOCK_SH
-import csv
-
-from ntools import *
-from db import *
-from area import load_area_from_kml_polygon
-from earth3d import dist3_latlong_ais
-
-DB_STARTDATE = datetime(2008, 6, 1)
-
-# This is the location of the filesystem database
-DBPATH = '/var/lib/ais/db'
-
-# see make-countries.py
-COUNTRIES_MID = {
-    201: u'Albania',
-    202: u'Andorra',
-    203: u'Austria',
-    204: u'Azores',
-    205: u'Belgium',
-    206: u'Belarus',
-    207: u'Bulgaria',
-    208: u'Vatican City State',
-    209: u'Cyprus',
-    210: u'Cyprus',
-    211: u'Germany',
-    212: u'Cyprus',
-    213: u'Georgia',
-    214: u'Moldova',
-    215: u'Malta',
-    216: u'Armenia',
-    218: u'Germany',
-    219: u'Denmark',
-    220: u'Denmark',
-    224: u'Spain',
-    225: u'Spain',
-    226: u'France',
-    227: u'France',
-    228: u'France',
-    230: u'Finland',
-    231: u'Faroe Islands',
-    232: u'United Kingdom',
-    233: u'United Kingdom',
-    234: u'United Kingdom',
-    235: u'United Kingdom',
-    236: u'Gibraltar',
-    237: u'Greece',
-    238: u'Croatia',
-    239: u'Greece',
-    240: u'Greece',
-    242: u'Morocco',
-    243: u'Hungary',
-    244: u'Netherlands',
-    245: u'Netherlands',
-    246: u'Netherlands',
-    247: u'Italy',
-    248: u'Malta',
-    249: u'Malta',
-    250: u'Ireland',
-    251: u'Iceland',
-    252: u'Liechtenstein',
-    253: u'Luxembourg',
-    254: u'Monaco',
-    255: u'Madeira',
-    256: u'Malta',
-    257: u'Norway',
-    258: u'Norway',
-    259: u'Norway',
-    261: u'Poland',
-    262: u'Montenegro',
-    263: u'Portugal',
-    264: u'Romania',
-    265: u'Sweden',
-    266: u'Sweden',
-    267: u'Slovak Republic',
-    268: u'San Marino',
-    269: u'Switzerland',
-    270: u'Czech Republic',
-    271: u'Turkey',
-    272: u'Ukraine',
-    273: u'Russian Federation',
-    274: u'The Former Yugoslav Republic of Macedonia',
-    275: u'Latvia',
-    276: u'Estonia',
-    277: u'Lithuania',
-    278: u'Slovenia',
-    279: u'Serbia',
-    301: u'Anguilla',
-    303: u'Alaska',
-    304: u'Antigua and Barbuda',
-    305: u'Antigua and Barbuda',
-    306: u'Netherlands Antilles',
-    307: u'Aruba',
-    308: u'Bahamas',
-    309: u'Bahamas',
-    310: u'Bermuda',
-    311: u'Bahamas',
-    312: u'Belize',
-    314: u'Barbados',
-    316: u'Canada',
-    319: u'Cayman Islands',
-    321: u'Costa Rica',
-    323: u'Cuba',
-    325: u'Dominica',
-    327: u'Dominican Republic',
-    329: u'Guadeloupe',
-    330: u'Grenada',
-    331: u'Greenland',
-    332: u'Guatemala',
-    334: u'Honduras',
-    336: u'Haiti',
-    338: u'United States of America',
-    339: u'Jamaica',
-    341: u'Saint Kitts and Nevis',
-    343: u'Saint Lucia',
-    345: u'Mexico',
-    347: u'Martinique',
-    348: u'Montserrat',
-    350: u'Nicaragua',
-    351: u'Panama',
-    352: u'Panama',
-    353: u'Panama',
-    354: u'Panama',
-    355: u'Panama',
-    356: u'Panama',
-    357: u'Panama',
-    358: u'Puerto Rico',
-    359: u'El Salvador',
-    361: u'Saint Pierre and Miquelon',
-    362: u'Trinidad and Tobago',
-    364: u'Turks and Caicos Islands',
-    366: u'United States of America',
-    367: u'United States of America',
-    368: u'United States of America',
-    369: u'United States of America',
-    370: u'Panama',
-    371: u'Panama',
-    372: u'Panama',
-    375: u'Saint Vincent and the Grenadines',
-    376: u'Saint Vincent and the Grenadines',
-    377: u'Saint Vincent and the Grenadines',
-    378: u'British Virgin Islands',
-    379: u'United States Virgin Islands',
-    401: u'Afghanistan',
-    403: u'Saudi Arabia',
-    405: u'Bangladesh',
-    408: u'Bahrain',
-    410: u'Bhutan',
-    412: u'China',
-    413: u'China',
-    416: u'Taiwan',
-    417: u'Sri Lanka',
-    419: u'India',
-    422: u'Iran',
-    423: u'Azerbaijani Republic',
-    425: u'Iraq',
-    428: u'Israel',
-    431: u'Japan',
-    432: u'Japan',
-    434: u'Turkmenistan',
-    436: u'Kazakhstan',
-    437: u'Uzbekistan',
-    438: u'Jordan',
-    440: u'Korea',
-    441: u'Korea',
-    443: u'Palestine',
-    445: u"Democratic People's Republic of Korea",
-    447: u'Kuwait',
-    450: u'Lebanon',
-    451: u'Kyrgyz Republic',
-    453: u'Macao',
-    455: u'Maldives',
-    457: u'Mongolia',
-    459: u'Nepal',
-    461: u'Oman',
-    463: u'Pakistan',
-    466: u'Qatar',
-    468: u'Syrian Arab Republic',
-    470: u'United Arab Emirates',
-    473: u'Yemen',
-    475: u'Yemen',
-    477: u'Hong Kong',
-    478: u'Bosnia and Herzegovina',
-    501: u'Adelie Land',
-    503: u'Australia',
-    506: u'Myanmar',
-    508: u'Brunei Darussalam',
-    510: u'Micronesia',
-    511: u'Palau',
-    512: u'New Zealand',
-    514: u'Cambodia',
-    515: u'Cambodia',
-    516: u'Christmas Island',
-    518: u'Cook Islands',
-    520: u'Fiji',
-    523: u'Cocos',
-    525: u'Indonesia',
-    529: u'Kiribati',
-    531: u"Lao People's Democratic Republic",
-    533: u'Malaysia',
-    536: u'Northern Mariana Islands',
-    538: u'Marshall Islands',
-    540: u'New Caledonia',
-    542: u'Niue',
-    544: u'Nauru',
-    546: u'French Polynesia',
-    548: u'Philippines',
-    553: u'Papua New Guinea',
-    555: u'Pitcairn Island',
-    557: u'Solomon Islands',
-    559: u'American Samoa',
-    561: u'Samoa',
-    563: u'Singapore',
-    564: u'Singapore',
-    565: u'Singapore',
-    567: u'Thailand',
-    570: u'Tonga',
-    572: u'Tuvalu',
-    574: u'Viet Nam',
-    576: u'Vanuatu',
-    578: u'Wallis and Futuna Islands',
-    601: u'South Africa',
-    603: u'Angola',
-    605: u'Algeria',
-    607: u'Saint Paul and Amsterdam Islands',
-    608: u'Ascension Island',
-    609: u'Burundi',
-    610: u'Benin',
-    611: u'Botswana',
-    612: u'Central African Republic',
-    613: u'Cameroon',
-    615: u'Congo',
-    616: u'Comoros',
-    617: u'Cape Verde',
-    618: u'Crozet Archipelago',
-    619: u"C\xc3\xb4te d'Ivoire",
-    621: u'Djibouti',
-    622: u'Egypt',
-    624: u'Ethiopia',
-    625: u'Eritrea',
-    626: u'Gabonese Republic',
-    627: u'Ghana',
-    629: u'Gambia',
-    630: u'Guinea-Bissau',
-    631: u'Equatorial Guinea',
-    632: u'Guinea',
-    633: u'Burkina Faso',
-    634: u'Kenya',
-    635: u'Kerguelen Islands',
-    636: u'Liberia',
-    637: u'Liberia',
-    642: u"Socialist People's Libyan Arab Jamahiriya",
-    644: u'Lesotho',
-    645: u'Mauritius',
-    647: u'Madagascar',
-    649: u'Mali',
-    650: u'Mozambique',
-    654: u'Mauritania',
-    655: u'Malawi',
-    656: u'Niger',
-    657: u'Nigeria',
-    659: u'Namibia',
-    660: u'Reunion',
-    661: u'Rwanda',
-    662: u'Sudan',
-    663: u'Senegal',
-    664: u'Seychelles',
-    665: u'Saint Helena',
-    666: u'Somali Democratic Republic',
-    667: u'Sierra Leone',
-    668: u'Sao Tome and Principe',
-    669: u'Swaziland',
-    670: u'Chad',
-    671: u'Togolese Republic',
-    672: u'Tunisia',
-    674: u'Tanzania',
-    675: u'Uganda',
-    676: u'Democratic Republic of the Congo',
-    677: u'Tanzania',
-    678: u'Zambia',
-    679: u'Zimbabwe',
-    701: u'Argentine Republic',
-    710: u'Brazil',
-    720: u'Bolivia',
-    725: u'Chile',
-    730: u'Colombia',
-    735: u'Ecuador',
-    740: u'Falkland Islands',
-    745: u'Guiana',
-    750: u'Guyana',
-    755: u'Paraguay',
-    760: u'Peru',
-    765: u'Suriname',
-    770: u'Uruguay',
-    775: u'Venezuela',
-}
-
-STATUS_CODES = {
-     0:  'Under way using engine',
-     1:  'At anchor',
-     2:  'Not under command',
-     3:  'Restricted manoeuverability',
-     4:  'Constrained by her draught',
-     5:  'Moored',
-     6:  'Aground',
-     7:  'Engaged in Fishing',
-     8:  'Under way sailing',
-     9:  '9 - Reserved for future amendment of Navigational Status for HSC',
-    10:  '10 - Reserved for future amendment of Navigational Status for WIG',
-    11:  '11 - Reserved for future use',
-    12:  '12 - Reserved for future use',
-    13:  '13 - Reserved for future use',
-    14:  '14 - Reserved for future use', # Land stations
-    15:  'Not defined', # default
-}
-
-SHIP_TYPES = {
-     0: 'Not available (default)',
-     1: 'Reserved for future use',
-     2: 'Reserved for future use',
-     3: 'Reserved for future use',
-     4: 'Reserved for future use',
-     5: 'Reserved for future use',
-     6: 'Reserved for future use',
-     7: 'Reserved for future use',
-     8: 'Reserved for future use',
-     9: 'Reserved for future use',
-    10: 'Reserved for future use',
-    11: 'Reserved for future use',
-    12: 'Reserved for future use',
-    13: 'Reserved for future use',
-    14: 'Reserved for future use',
-    15: 'Reserved for future use',
-    16: 'Reserved for future use',
-    17: 'Reserved for future use',
-    18: 'Reserved for future use',
-    19: 'Reserved for future use',
-    20: 'Wing in ground (WIG), all ships of this type',
-    21: 'Wing in ground (WIG), Hazardous category A',
-    22: 'Wing in ground (WIG), Hazardous category B',
-    23: 'Wing in ground (WIG), Hazardous category C',
-    24: 'Wing in ground (WIG), Hazardous category D',
-    25: 'Wing in ground (WIG), Reserved for future use',
-    26: 'Wing in ground (WIG), Reserved for future use',
-    27: 'Wing in ground (WIG), Reserved for future use',
-    28: 'Wing in ground (WIG), Reserved for future use',
-    29: 'Wing in ground (WIG), Reserved for future use',
-    30: 'Fishing',
-    31: 'Towing',
-    32: 'Towing: length exceeds 200m or breadth exceeds 25m',
-    33: 'Dredging or underwater ops',
-    34: 'Diving ops',
-    35: 'Military ops',
-    36: 'Sailing',
-    37: 'Pleasure Craft',
-    38: 'Reserved',
-    39: 'Reserved',
-    40: 'High speed craft (HSC), all ships of this type',
-    41: 'High speed craft (HSC), Hazardous category A',
-    42: 'High speed craft (HSC), Hazardous category B',
-    43: 'High speed craft (HSC), Hazardous category C',
-    44: 'High speed craft (HSC), Hazardous category D',
-    45: 'High speed craft (HSC), Reserved for future use',
-    46: 'High speed craft (HSC), Reserved for future use',
-    47: 'High speed craft (HSC), Reserved for future use',
-    48: 'High speed craft (HSC), Reserved for future use',
-    49: 'High speed craft (HSC), No additional information',
-    50: 'Pilot Vessel',
-    51: 'Search and Rescue vessel',
-    52: 'Tug',
-    53: 'Port Tender',
-    54: 'Anti-pollution equipment',
-    55: 'Law Enforcement',
-    56: 'Spare - Local Vessel',
-    57: 'Spare - Local Vessel',
-    58: 'Medical Transport',
-    59: 'Ship according to RR Resolution No. 18',
-    60: 'Passenger, all ships of this type',
-    61: 'Passenger, Hazardous category A',
-    62: 'Passenger, Hazardous category B',
-    63: 'Passenger, Hazardous category C',
-    64: 'Passenger, Hazardous category D',
-    65: 'Passenger, Reserved for future use',
-    66: 'Passenger, Reserved for future use',
-    67: 'Passenger, Reserved for future use',
-    68: 'Passenger, Reserved for future use',
-    69: 'Passenger, No additional information',
-    70: 'Cargo', # 'Cargo, all ships of this type',
-    71: 'Cargo, Hazardous category A',
-    72: 'Cargo, Hazardous category B',
-    73: 'Cargo, Hazardous category C',
-    74: 'Cargo, Hazardous category D',
-    75: 'Cargo', # 'Cargo, Reserved for future use',
-    76: 'Cargo', # 'Cargo, Reserved for future use',
-    77: 'Cargo', # 'Cargo, Reserved for future use',
-    78: 'Cargo', # 'Cargo, Reserved for future use',
-    79: 'Cargo', # 'Cargo, No additional information',
-    80: 'Tanker', # 'Tanker, all ships of this type',
-    81: 'Tanker, Hazardous category A',
-    82: 'Tanker, Hazardous category B',
-    83: 'Tanker, Hazardous category C',
-    84: 'Tanker, Hazardous category D',
-    85: 'Tanker', # 'Tanker, Reserved for future use',
-    86: 'Tanker', # 'Tanker, Reserved for future use',
-    87: 'Tanker', # 'Tanker, Reserved for future use',
-    88: 'Tanker', # 'Tanker, Reserved for future use',
-    89: 'Tanker, No additional information',
-    90: 'Other Type, all ships of this type',
-    91: 'Other Type, Hazardous category A',
-    92: 'Other Type, Hazardous category B',
-    93: 'Other Type, Hazardous category C',
-    94: 'Other Type, Hazardous category D',
-    95: 'Other Type, Reserved for future use',
-    96: 'Other Type, Reserved for future use',
-    97: 'Other Type, Reserved for future use',
-    98: 'Other Type, Reserved for future use',
-    99: 'Other Type, no additional information',
-    100: 'Default Navaid',
-    101: 'Reference point',
-    102: 'RACON',
-    103: 'Offshore Structure',
-    104: 'Spare',
-    105: 'Light, without sectors',
-    106: 'Light, with sectors',
-    107: 'Leading Light Front',
-    108: 'Leading Light Rear',
-    109: 'Beacon, Cardinal N',
-    110: 'Beacon, Cardinal E',
-    111: 'Beacon, Cardinal S',
-    112: 'Beacon, Cardinal W',
-    113: 'Beacon, Port hand',
-    114: 'Beacon, Starboard hand',
-    115: 'Beacon, Preferred Channel port hand',
-    116: 'Beacon, Preferred Channel starboard hand',
-    117: 'Beacon, Isolated danger',
-    118: 'Beacon, Safe water',
-    119: 'Beacon, Special mark',
-    120: 'Cardinal Mark N',
-    121: 'Cardinal Mark E',
-    122: 'Cardinal Mark S',
-    123: 'Cardinal Mark W',
-    124: 'Port hand Mark',
-    125: 'Starboard hand Mark',
-    126: 'Preferred Channel Port hand',
-    127: 'Preferred Channel Starboard hand',
-    128: 'Isolated danger',
-    129: 'Safe Water',
-    130: 'Manned VTS / Special Mark',
-    131: 'Light Vessel / LANBY',
-}
-
-AIS_STATUS_NOT_AVAILABLE = 15
-AIS_ROT_HARD_LEFT = -127
-AIS_ROT_HARD_RIGHT = 127
-AIS_ROT_NOT_AVAILABLE = -128 # not like gpsd
-
-AIS_LATLON_SCALE = 600000.0
-AIS_LON_NOT_AVAILABLE = 0x6791AC0
-AIS_LAT_NOT_AVAILABLE = 0x3412140
-AIS_COG_SCALE = 10.0
-AIS_COG_NOT_AVAILABLE = 3600
-AIS_NO_HEADING = 511
-AIS_SOG_SCALE = 10.0
-AIS_SOG_NOT_AVAILABLE = 1023
-AIS_SOG_FAST_MOVER = 1022
-AIS_SOG_MAX_SPEED = 1021
-
-
-def _hash3_pathfilename(filename):
-    """
-    Returns a level 3 directory hashed filename on that basis:
-    123456789 -> 1/12/123/123456789
-    """
-    return os.path.join(filename[0], filename[:2], filename[:3], filename)
-
-
-def db_bydate_addrecord(basefilename, record, timestamp):
-    strdt = datetime.utcfromtimestamp(timestamp).strftime('%Y%m%d')
-    filename = os.path.join(DBPATH, 'bydate', strdt, _hash3_pathfilename(basefilename))
-    f = open_with_mkdirs(filename, 'ab')
-    lockf(f, LOCK_EX)
-    #f.seek(0,2) # go to EOF
-    assert f.tell() % len(record) == 0, 'Invalid length for %s' % filename
-    f.write(record)
-    f.close()
-
-
-def db_lastinfo_setrecord_ifnewer(basefilename, record, timestamp):
-    '''
-    Overwrite last information if date is newer
-    Input record must be complete
-    '''
-    filename = DBPATH+'/last/'+_hash3_pathfilename(basefilename)
-
-    try:
-        f = open(filename, 'r+b')
-    except IOError, ioerr:
-        if ioerr.errno != 2:
-            raise
-        # File was not found? Ok, create it. FIXME: we should lock something...
-        f = open_with_mkdirs(filename, 'wb')
-        f.write(record)
-        updated = True
-    else:
-        lockf(f, LOCK_EX)
-        assert f.tell() == 0
-        oldrecord = f.read(4)
-        assert len(oldrecord) == 4
-        oldtimestamp = struct.unpack('I', oldrecord)[0]
-        f.seek(0)
-        assert f.tell() == 0
-        if timestamp > oldtimestamp:
-            f.write(record)
-            assert f.tell() == len(record), \
-                "tell=%s size=%s" % (f.tell(), len(record))
-            updated = True
-        else:
-            updated = False
-    f.close()
-    return updated
-
-
-def sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
-    dim_bow, dim_stern, dim_port, dim_starboard, \
-    eta_M, eta_D, eta_h, eta_m, draught, destination, source):
-    ''' Don't call directly '''
-    sqlinfo = {}
-    sqlinfo['mmsi'] = strmmsi_to_mmsi(strmmsi)
-    sqlinfo['updated'] = datetime.utcfromtimestamp(timestamp)
-    sqlinfo['imo'] = imo or None
-    sqlinfo['name'] = name or None
-    sqlinfo['callsign'] = callsign or None
-    sqlinfo['type'] = type
-    sqlinfo['destination'] = None
-    if destination:
-        destination = destination.replace('\0', ' ').rstrip(' @\0')
-    sqlinfo['destination'] = destination or None
-    sqlinfo['source'] = source
-    sqlexec(u'''INSERT INTO vessel (mmsi, updated) SELECT %(mmsi)s, '1970-01-01T00:00:00' WHERE NOT EXISTS (SELECT * FROM vessel WHERE mmsi=%(mmsi)s)''', sqlinfo)
-    if sqlinfo['imo']:
-        sqlexec(u'UPDATE vessel SET imo = %(imo)s WHERE mmsi=%(mmsi)s AND (imo IS NULL OR updated<%(updated)s)', sqlinfo)
-    if sqlinfo['name']:
-        sqlexec(u'UPDATE vessel SET name = %(name)s WHERE mmsi=%(mmsi)s AND (name IS NULL OR updated<%(updated)s)', sqlinfo)
-    if sqlinfo['callsign']:
-        sqlexec(u'UPDATE vessel SET callsign = %(callsign)s WHERE mmsi=%(mmsi)s AND (callsign IS NULL OR updated<%(updated)s)', sqlinfo)
-    if sqlinfo['type']:
-        sqlexec(u'UPDATE vessel SET type = %(type)s WHERE mmsi=%(mmsi)s AND (type IS NULL OR updated<%(updated)s)', sqlinfo)
-    if sqlinfo['destination']:
-        sqlexec(u'UPDATE vessel SET destination = %(destination)s WHERE mmsi=%(mmsi)s AND (destination IS NULL OR updated<%(updated)s)', sqlinfo)
-    sqlexec(u'UPDATE vessel SET (updated, source) = (%(updated)s, %(source)s) WHERE mmsi=%(mmsi)s AND updated<%(updated)s', sqlinfo)
-    dbcommit()
-
-
-
-
-aivdm_record123_format = 'IBbhiiII4s'
-aivdm_record123_length = struct.calcsize(aivdm_record123_format)
-aivdm_record5_format = 'II20s7sBHHBBBBBBH20s4s'
-aivdm_record5_length = struct.calcsize(aivdm_record5_format)
-
-
-def add_nmea1(strmmsi, timestamp, status, rot, sog, \
-              latitude, longitude, cog, heading, source):
-    '''
-    Input is raw data, unscaled
-    '''
-    record = struct.pack(aivdm_record123_format, timestamp, status, rot, sog, latitude, longitude, cog, heading, source)
-    #print repr(record)
-    filename = strmmsi+'.nmea1'
-    db_bydate_addrecord(filename, record, timestamp)
-    # There's no need to be smart: all the information are taken, or none.
-    return db_lastinfo_setrecord_ifnewer(filename, record, timestamp)
-
-
-def add_nmea5_full(strmmsi, timestamp, imo, name, callsign, type, \
-                   dim_bow, dim_stern, dim_port, dim_starboard, \
-                   eta_M, eta_D, eta_h, eta_m, draught, destination, source):
-    '''
-    Input is raw data, unscaled
-    All fields are set, and can be upgraded if the record is newer
-    '''
-    record = struct.pack(aivdm_record5_format, timestamp, imo, name, callsign, \
-                         type, dim_bow, dim_stern, dim_port, dim_starboard, \
-                         eta_M, eta_D, eta_h, eta_m, draught, destination, source)
-    #print repr(record)
-    filename = strmmsi+'.nmea5'
-    db_bydate_addrecord(filename, record, timestamp)
-    updated = db_lastinfo_setrecord_ifnewer(filename, record, timestamp)
-    if updated:
-        sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
-                      dim_bow, dim_stern, dim_port, dim_starboard, \
-                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
-    return updated
-
-def add_nmea5_partial(strmmsi, timestamp, imo, name, callsign, type, \
-                      dim_bow, dim_stern, dim_port, dim_starboard, \
-                      eta_M, eta_D, eta_h, eta_m, draught, destination, source):
-    '''
-    Input is raw data, unscaled
-    All fields are not set. Only some of them can be upgraded, if they're newer
-    '''
-    record = struct.pack(aivdm_record5_format, \
-                         timestamp, imo, name, callsign, type, \
-                         dim_bow, dim_stern, dim_port, dim_starboard, \
-                         eta_M, eta_D, eta_h, eta_m, draught, destination, \
-                         source)
-    #print repr(record)
-    filename = strmmsi + '.nmea5'
-    db_bydate_addrecord(filename, record, timestamp)
-
-    updated = False
-    filename = os.path.join(DBPATH, 'last', _hash3_pathfilename(filename))
-    try:
-        f = open(filename, 'r+b')
-    except IOError, ioerr:
-        if ioerr.errno != 2:
-            raise
-        # File was not found? Ok, create it. FIXME: we should lock something...
-        f = open_with_mkdirs(filename, 'wb')
-        lockf(f, LOCK_EX)
-        f.write(record)
-        # keep the lock
-        updated = True
-    else:
-        lockf(f, LOCK_EX)
-        oldrecord = f.read(aivdm_record5_length)
-        oldtimestamp, oldimo, oldname, oldcallsign, oldtype, \
-        olddim_bow, olddim_stern, olddim_port, olddim_starboard, \
-        oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
-        olddraught, olddestination, oldsource \
-                  = struct.unpack(aivdm_record5_format, oldrecord)
-        if timestamp > oldtimestamp:
-            # we have incoming recent information
-            if imo == 0:
-                imo = oldimo
-            if name == '':
-                name = oldname
-            if callsign == '':
-                callsign = oldcallsign
-            if type == 0:
-                type = oldtype
-            if dim_bow == 0:
-                dim_bow = olddim_bow
-            if dim_stern == 0:
-                dim_stern = olddim_stern
-            if dim_port == 0:
-                dim_port = olddim_port
-            if dim_starboard == 0:
-                dim_starboard = olddim_starboard
-            if eta_M == 0 or eta_D == 0 or eta_h == 24 or eta_m == 60 \
-                          or destination == '':
-                eta_M = oldeta_M
-                eta_D = oldeta_D
-                eta_h = oldeta_h
-                eta_m = oldeta_m
-                destination = olddestination
-            if draught == 0:
-                draught = olddraught
-            record = struct.pack(aivdm_record5_format, \
-                                 timestamp, imo, name, callsign, type, \
-                                 dim_bow, dim_stern, dim_port, dim_starboard, \
-                                 eta_M, eta_D, eta_h, eta_m, draught, \
-                                 destination, source)
-            f.seek(0)
-            f.write(record)
-            updated = True
-        else:
-            # we received an obsolete info, but maybe there are some new things in it
-            if oldimo == 0 and imo != 0:
-                oldimo = imo
-                updated = True
-            if oldname == '' and name != '':
-                oldname = name
-                updated = True
-            if oldcallsign == '' and callsign != '':
-                oldcallsign = callsign
-                updated = True
-            if oldtype == 0 and type != 0:
-                oldtype = type
-                updated = True
-            if olddim_bow == 0 and dim_bow != 0:
-                olddim_bow = dim_bow
-                updated = True
-            if olddim_stern == 0 and dim_stern != 0:
-                olddim_stern = dim_stern
-                updated = True
-            if olddim_port == 0 and dim_port != 0:
-                olddim_port = dim_port
-                updated = True
-            if olddim_starboard == 0 and dim_starboard != 0:
-                olddim_starboard = dim_starboard
-                updated = True
-            # FIXME
-            if (oldeta_M == 0 or oldeta_D == 0 or olddestination == '') \
-                    and ((eta_M != 0 and eta_D != 0) or destination!=''):
-                oldeta_M = eta_M
-                oldeta_D = eta_D
-                oldeta_h = eta_h
-                oldeta_m = eta_m
-                olddestination = destination
-                updated = True
-            if olddraught == 0 and draught != 0:
-                olddraught = draught
-                updated = True
-            if updated:
-                oldsource = source
-                record = struct.pack(aivdm_record5_format, \
-                                     oldtimestamp, oldimo, oldname, \
-                                     oldcallsign, oldtype, \
-                                     olddim_bow, olddim_stern, \
-                                     olddim_port, olddim_starboard, \
-                                     oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
-                                     olddraught, olddestination, oldsource)
-            
-                f.seek(0)
-                f.write(record)
-    # keep the file locked during SQL updates
-    if updated:
-        sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
-                      dim_bow, dim_stern, dim_port, dim_starboard, \
-                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
-    f.close()
-    return updated
-
-
-
-def strmmsi_to_mmsi(strmmsi):
-    """
-    Convert from str mmsi to sql-int mmsi
-    Special treatment manal input
-    """
-    if strmmsi.isdigit():
-        return int(strmmsi)
-    else:
-        assert strmmsi[3:5] == 'MI'
-        strmmsi = strmmsi[:3]+'00'+strmmsi[5:]
-        return int('-'+strmmsi)
-
-
-def mmsi_to_strmmsi(mmsi):
-    """
-    Convert from sql-into mmsi to str mmsi
-    Special treatment manal input
-    """
-    if mmsi >= 0:
-        return "%08d" % mmsi
-    strmmsi = "%08d" % -mmsi
-    assert strmmsi[3:5] == '00'
-    strmmsi = strmmsi[:3]+'MI'+strmmsi[5:]
-    return strmmsi
-
-
-__misources__ = {} # cache of manual source names
-def _get_mi_sourcename(id):
-    """
-    Get the nice name for sources whose id4 starts with 'MI'
-    """
-    global __misources__
-    if not __misources__:
-        sqlexec(u'SELECT id, name FROM mi_source')
-        while True:
-            row = get_common_cursor().fetchone()
-            if row is None:
-                break
-            __misources__[row[0]] = row[1]
-    result = __misources__.get(id, None)
-    if result is None:
-        return u"Manual input #%s" % id
-    return result
-
-
-class Nmea1:
-    def __init__(self, timestamp, status=AIS_STATUS_NOT_AVAILABLE, rot=AIS_ROT_NOT_AVAILABLE, sog=AIS_SOG_NOT_AVAILABLE, latitude=AIS_LAT_NOT_AVAILABLE, longitude=AIS_LON_NOT_AVAILABLE, cog=AIS_COG_NOT_AVAILABLE, heading=AIS_NO_HEADING, source='\x00\x00\x00\x00'):
-        self.timestamp_1 = timestamp
-        self.status      = status
-        self.rot         = rot
-        self.sog         = sog
-        self.latitude    = latitude
-        self.longitude   = longitude
-        self.cog         = cog
-        self.heading     = heading
-        self.source_1    = source
-
-    from_values = __init__
-
-    def to_values(self):
-        return self.timestamp_1, self.status, self.rot, self.sog, self.latitude, self.longitude, self.cog, self.heading, self.source_1
-
-    def from_record(self, record):
-        values = struct.unpack(aivdm_record123_format, record)
-        Nmea1.__init__(self, *values)
-
-    @staticmethod
-    def new_from_record(record):
-        values = struct.unpack(aivdm_record123_format, record)
-        return Nmea1(*values)
-
-    def to_record(self):
-        return struct.pack(aivdm_record123_format, *Nmea1.to_values())
-        
-    def from_file(self, file):
-        record = file.read(aivdm_record123_length)
-        Nmea1.from_record(self, record)
-
-    @staticmethod
-    def new_from_file(file):
-        record = file.read(aivdm_record123_length)
-        return Nmea1.new_from_record(record)
-
-    def from_lastinfo(self, strmmsi):
-        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
-        try:
-            f = file(filename_nmea1, 'rb')
-        except IOError:
-            logging.debug("file %s doesn't exists" % filename_nmea1)
-            return
-        lockf(f, LOCK_SH)
-        Nmea1.from_file(self, f)
-        f.close()
-
-    @staticmethod
-    def new_from_lastinfo(strmmsi):
-        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
-        try:
-            f = file(filename_nmea1, 'rb')
-        except IOError:
-            logging.debug("file %s doesn't exists" % filename_nmea1)
-            return None
-        lockf(f, LOCK_SH)
-        record = f.read(aivdm_record123_length)
-        f.close()
-        return Nmea1.new_from_record(record)
-
-
-    def dump_to_stdout(self):
-        """
-        Prints content to stdout
-        """
-        print datetime.utcfromtimestamp(self.timestamp_1), 
-        for i in (self.status, self.rot, self.sog, self.latitude/AIS_LATLON_SCALE, self.longitude/AIS_LATLON_SCALE, self.cog, self.heading, self.source_1):
-            print repr(i),
-        print
-    @staticmethod
-    def _clean_str(txt):
-        if txt is None:
-            return ''
-        return txt.replace('\0','').replace('@', '').strip()
-
-    def get_status(self, default='Unknown'):
-        return STATUS_CODES.get(self.status, default)
-    def get_sog_str(self, default='Unknown'):
-        if self.sog == AIS_SOG_NOT_AVAILABLE:
-            return default
-        if self.sog == AIS_SOG_FAST_MOVER:
-            return 'over 102.2 kts'
-        return '%.1f kts' % (self.sog/AIS_SOG_SCALE)
-
-    def get_rot_str(self, default='Unknown'):
-        if self.rot == AIS_ROT_NOT_AVAILABLE:
-            return default
-        if self.rot == 0:
-            return 'Not turning'
-        if self.rot < 0:
-            side = 'port'
-        else:
-            side = 'starboard'
-        rot = abs(self.rot)
-        if rot == 127:
-            result = 'To '
-        else:
-            result = '%d %% to ' % rot*100./127
-        return result + side
-
-    @staticmethod
-    def _decimaldegree_to_dms(f, emispheres):
-        if f >= 0:
-            e = emispheres[0]
-        else:
-            f = -f
-            e = emispheres[1]
-        result = '%d°' % int(f)
-        f = (f%1)*60
-        result += '%02.05f\' ' % f
-        result += e
-        return result
-
-    def get_latitude_str(self, default='Unknown'):
-        if self.latitude == AIS_LAT_NOT_AVAILABLE:
-            return default
-        return Nmea1._decimaldegree_to_dms(self.latitude / AIS_LATLON_SCALE, 'NS')
-
-    def get_longitude_str(self, default='Unknown'):
-        if self.longitude == AIS_LON_NOT_AVAILABLE:
-            return default
-        return Nmea1._decimaldegree_to_dms(self.longitude / AIS_LATLON_SCALE, 'EW')
-
-    def get_cog_str(self, default='Unknown'):
-        if self.cog == AIS_COG_NOT_AVAILABLE:
-            return default
-        return '%.1f°' % (self.cog/10.)
-
-    def get_heading_str(self, default='Unknown'):
-        if self.heading == AIS_NO_HEADING:
-            return default
-        return '%s°' % self.heading
-
-    def get_source_1_str(self):
-        return Nmea.format_source(self.source_1)
-
-class Nmea5:
-    def __init__(self, timestamp, imo=0, name='', callsign='', type=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
-        self.timestamp_5   = timestamp
-        self.imo           = imo
-        self.name          = name         
-        self.callsign      = callsign
-        self.type          = type
-        self.dim_bow       = dim_bow
-        self.dim_stern     = dim_stern
-        self.dim_port      = dim_port
-        self.dim_starboard = dim_starboard
-        self.eta_M         = eta_M
-        self.eta_D         = eta_D
-        self.eta_h         = eta_h
-        self.eta_m         = eta_m
-        self.draught       = draught
-        self.destination   = destination
-        self.source_5      = source
-
-    from_values = __init__
-
-    def merge_from_values(self, timestamp, imo=0, name='', callsign='', type=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
-        updated = False
-        if self.imo == 0 or imo != 0:
-            self.imo = imo
-            updated = True
-        if self.name == '' or name != '':
-            self.name = name
-            updated = True
-        if self.callsign == '' or callsign != '':
-            self.callsign = callsign
-            updated = True
-        if self.type == 0 or type != 0:
-            self.type = type
-            updated = True
-        if self.dim_bow == 0 or dim_bow != 0:
-            self.dim_bow = dim_bow
-            updated = True
-        if self.dim_stern == 0 or dim_stern != 0:
-            self.dim_stern = dim_stern
-            updated = True
-        if self.dim_port == 0 or dim_port != 0:
-            self.dim_port = dim_port
-            updated = True
-        if self.dim_starboard == 0 or dim_starboard != 0:
-            self.dim_starboard = dim_starboard
-            updated = True
-        if (self.eta_M == 0 and self.eta_D == 0 and self.eta_h == 24 and self.eta_m == 60) or eta_M != 0 or eta_D != 0 or eta_h != 24 or eta_m != 60:
-            self.eta_M = eta_M
-            self.eta_D = eta_D
-            self.eta_h = eta_h
-            self.eta_m = eta_m
-            updated = True
-        if self.draught == 0 or draught != 0:
-            self.draught = draught
-            updated = True
-        if self.destination == '' or destination != '':
-            self.destination = destination
-            updated = True
-        if updated:
-            self.timestamp_5 = timestamp
-            self.source_5 = source
-        return updated
-
-    def to_values(self):
-        return self.timestamp_5, self.imo, self.name, self.callsign, self.type, self.dim_bow, self.dim_stern, self.dim_port, self.dim_starboard, self.eta_M, self.eta_D, self.eta_h, self.eta_m, self.draught, self.destination, self.source_5
-
-    def from_record(self, record):
-        values = struct.unpack(aivdm_record5_format, record)
-        Nmea5.__init__(self, *values)
-
-    @staticmethod
-    def new_from_record(record):
-        values = struct.unpack(aivdm_record5_format, record)
-        return Nmea5(*values)
-
-    def to_record(self):
-        return struct.pack(aivdm_record5_format, *Nmea5.to_values(self))
-        
-    def from_file(self, file):
-        record = file.read(aivdm_record5_length)
-        Nmea5.from_record(self, record)
-
-    @staticmethod
-    def new_from_file(file):
-        record = file.read(aivdm_record5_length)
-        return Nmea5.new_from_record(record)
-
-    def from_lastinfo(self, strmmsi):
-        filename_nmea5 = os.path.join(DBPATH,
-                                      'last',
-                                      _hash3_pathfilename(strmmsi+'.nmea5'))
-        try:
-            f = file(filename_nmea5, 'rb')
-        except IOError:
-            logging.debug("file %s doesn't exists" % filename_nmea5)
-            return
-        lockf(f, LOCK_SH)
-        Nmea5.from_file(self, f)
-        f.close()
-
-    @staticmethod
-    def new_from_lastinfo(strmmsi):
-        filename_nmea5 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea5')
-        try:
-            f = file(filename_nmea5, 'rb')
-        except IOError:
-            logging.debug("file %s doesn't exists" % filename_nmea5)
-            return None
-        lockf(f, LOCK_SH)
-        record = f.read(aivdm_record5_length)
-        f.close()
-        return Nmea5.new_from_record(record)
-
-    @staticmethod
-    def _clean_str(txt):
-        if txt is None:
-            return ''
-        return txt.replace('\0','').replace('@', '').strip()
-
-    def get_name(self, default='Unknown'):
-        result = self._clean_str(self.name)
-        if result:
-            return result
-        return default
-
-    def get_callsign(self, default='Unknown'):
-        return self._clean_str(self.callsign) or default
-
-    def get_shiptype(self, default='Unknown'):
-        return SHIP_TYPES.get(self.type, default)
-
-    def get_length(self):
-        return self.dim_bow + self.dim_stern
-
-    def get_width(self):
-        return self.dim_port + self.dim_starboard
-
-    _monthes = 'Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec'.split(',')
-    def get_eta_str(self, default='Unknown'):
-        if not self.eta_M and not self.eta_D:
-            return default
-        result = ''
-        if self.eta_M:
-            if self.eta_M <= len(Nmea5._monthes):
-                result += Nmea5._monthes[self.eta_M - 1]
-            else:
-                result += '%02d' % self.eta_M
-        else:
-            result += '***'
-        result += ' '
-        if self.eta_D:
-            result += '%02d' % self.eta_D
-        else:
-            result += '**'
-        if self.eta_h != 24:
-            result += ' %02d' % self.eta_h
-            if self.eta_m == 60:
-                result += 'h'
-            else:
-                result += ':%02d' % self.eta_m
-        return result
-    
-    def get_draught_str(self, default='Unknown'):
-        if not self.draught:
-            return default
-        return '%.1f meters' % (self.draught/10.)
-
-    def get_destination(self, default='Unknown'):
-        return self._clean_str(self.destination) or default
-
-    def get_source_5_str(self):
-        return Nmea.format_source(self.source_5)
-
-class Nmea(Nmea1, Nmea5):
-    """
-    This is nmea info, a merge of nmea1 and nmea5 packets
-    """
-    def __init__(self, strmmsi):
-        self.strmmsi = strmmsi
-        Nmea1.__init__(self, timestamp=0)
-        Nmea5.__init__(self, timestamp=0)
-
-    ########################
-    # Because of multiple inheritance some functions are unavailable:
-    def _nmea_not_implemented(*args, **kargs):
-        # used to avoid conflicting inherited members
-        raise NotImplementedError
-    from_values = _nmea_not_implemented
-    to_values = _nmea_not_implemented
-    from_record = _nmea_not_implemented
-    new_from_record = _nmea_not_implemented
-    to_record = _nmea_not_implemented
-    from_file = _nmea_not_implemented
-    new_from_file = _nmea_not_implemented
-    ########################
-
-    def from_lastinfo(self, strmmsi):
-        Nmea1.from_lastinfo(self, strmmsi)
-        Nmea5.from_lastinfo(self, strmmsi)
-    
-    @staticmethod
-    def new_from_lastinfo(strmmsi):
-        # better than unimplemented, but not optimal
-        nmea = Nmea(strmmsi)
-        nmea.from_lastinfo(strmmsi)
-        return nmea
-
-
-    def get_flag(self, default=u'Unknown'):
-        if self.strmmsi.startswith('00') and self.strmmsi[3:5]!='MI':
-            ref_mmsi = self.strmmsi[2:]
-        else:
-            ref_mmsi = self.strmmsi
-        country_mid = int(ref_mmsi[0:3])
-        country_name = COUNTRIES_MID.get(country_mid, default)
-        return country_name
-
-    def get_mmsi_public(self, default='Unknown'):
-        if self.strmmsi.isdigit():
-            return self.strmmsi
-        return default
-
-    def get_title(self):
-        """
-        Returns the name of the ship if available
-        Or its mmsi
-        """
-        return self.get_name(None) or self.get_mmsi_public()
-
-    def get_last_timestamp(self):
-        """
-        Returns the most recent of update from timestamp1, timestamp5
-        """
-        if self.timestamp_1 > self.timestamp_5:
-            return self.timestamp_1
-        else:
-            return self.timestamp_5
-
-    def get_last_updated_str(self):
-        """
-        Returns a pretty formated update data as a string
-        """
-        lastupdate = self.get_last_timestamp()
-        if lastupdate == 0:
-            return u'Never'
-        dt_lastupdate = datetime.utcfromtimestamp(lastupdate)
-        delta = datetime.utcnow() - dt_lastupdate
-        def nice_timedelta_str(delta):
-            strdelta = ''
-            if delta.days:
-                strdelta += str(delta.days)
-                if delta.days > 1:
-                    strdelta += ' days '
-                else:
-                    strdelta += ' day '
-            delta_s = delta.seconds
-            delta_m = delta_s / 60
-            delta_s -= delta_m * 60
-            delta_h = delta_m / 60
-            delta_m -= delta_h * 60
-
-            if delta_h:
-                strdelta += str(delta_h)
-                if delta_h > 1:
-                    strdelta += ' hours '
-                else:
-                    strdelta += ' hour '
-            if delta_m:
-                strdelta += str(delta_m)
-                if delta_m > 1:
-                    strdelta += ' minutes '
-                else:
-                    strdelta += ' minute '
-            if delta_s:
-                strdelta += str(delta_s)
-                if delta_s > 1:
-                    strdelta += ' seconds '
-                else:
-                    strdelta += ' second '
-            if not strdelta:
-                strdelta = 'less than a second '
-            strdelta += ' ago'
-            return strdelta
-        return nice_timedelta_str(delta) + ' (' + dt_lastupdate.strftime('%Y-%m-%d %H:%M:%S GMT') + ')'
-
-    @staticmethod
-    def format_source(infosrc):
-        if infosrc == '\0\0\0\0':
-            return u'(empty)'
-        elif infosrc.startswith('MI'):
-            if len(infosrc) == 4:
-                return _get_mi_sourcename(struct.unpack('<2xH', infosrc)[0])
-            else:
-                return u'Manual input'
-        elif infosrc.startswith('U'):
-            return u'User input'
-        elif infosrc.startswith('NM'):
-            return u'NMEA packets from '+xml_escape(infosrc[2:])
-        elif infosrc.startswith('SP'):
-            return u"ShipPlotter user %s" % infosrc[2:]
-        elif infosrc == u'MTWW':
-            return u'MarineTraffic.com web site'
-        elif infosrc == u'MTTR':
-            return u'MarineTraffic.com track files'
-        else:
-            return infosrc
-
-    csv_headers = [
-        'mmsi',
-        'flag',
-        'name',
-        'imo',
-        'callsign',
-        'type',
-        'length',
-        'width',
-        'datetime',
-        'status',
-        'sog',
-        'latitude',
-        'longitude',
-        'cog',
-        'heading',
-        'destination',
-        'eta',
-        'draught',
-        ]
-
-    def get_dump_row(self):
-        result = []
-        def _clean(txt):
-            if txt is None:
-                return ''
-            return txt.replace('\0','').replace('@', '').strip()
-        result.append(self.strmmsi)
-        country_mid = int(self.strmmsi[:3])
-        country_name = COUNTRIES_MID.get(country_mid, u'unknown')
-        result.append(country_name.encode('utf-8'))
-        result.append(_clean(self.name))
-        result.append(str(self.imo))
-        result.append(_clean(self.callsign))
-        result.append(str(self.type) + '-' + SHIP_TYPES.get(self.type, 'unknown'))
-        d = self.dim_bow + self.dim_stern
-        if d:
-            result.append(d)
-        else:
-            result.append(None)
-        d = self.dim_port + self.dim_starboard
-        if d:
-            result.append(d)
-        else:
-            result.append(None)
-        result.append(datetime.utcfromtimestamp(self.timestamp_1).strftime('%Y-%m-%dT%H:%M:%SZ'))
-        result.append(STATUS_CODES.get(self.status, 'unknown'))
-        if self.sog != AIS_SOG_NOT_AVAILABLE:
-            result.append(str(self.sog/AIS_SOG_SCALE))
-        else:
-            result.append(None)
-        if self.latitude != AIS_LAT_NOT_AVAILABLE:
-            result.append(str(self.latitude/AIS_LATLON_SCALE))
-        else:
-            result.append(None)
-        if self.longitude != AIS_LON_NOT_AVAILABLE:
-            result.append(str(self.longitude/AIS_LATLON_SCALE))
-        else:
-            result.append(None)
-        if self.cog != AIS_COG_NOT_AVAILABLE:
-            result.append(str(self.cog/10.))
-        else:
-            result.append(None)
-        if self.heading != AIS_NO_HEADING:
-            result.append(str(self.heading))
-        else:
-            result.append(None)
-        result.append(self.get_destination(''))
-        result.append(self.get_eta_str(''))
-        result.append(self.draught)
-        result.append(self.source_5)
-        return result
-
-
-class BankNmea1(list):
-    """
-    That class handle a .nmea1 archive file
-    """
-    def __init__(self, strmmsi, dt):
-        list.__init__(self)
-        self.strmmsi = strmmsi
-        if isinstance(dt, date):
-            dt = dt.strftime('%Y%m%d')
-        self.date = dt
-
-    def get_filename(self):
-        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea1'))
-
-    def __load_from_file(self, file):
-        '''
-        Adds all record from opened file in this bank
-        File must be locked before call
-        '''
-        while True:
-            record = file.read(aivdm_record123_length)
-            if not record:
-                break
-            self.append(Nmea1.new_from_record(record))
-
-    def _write_in_file(self, file):
-        '''
-        Write all records from that bank in opened file
-        File must be locked before call
-        File should be truncated after call
-        '''
-        for nmea1 in self:
-            file.write(nmea1.to_record())
-
-    def __load(self):
-        try:
-            file = open(self.get_filename(), 'rb')
-            lockf(file, LOCK_SH)
-        except IOError, ioerr:
-            if ioerr.errno == 2: # No file
-                return
-            raise
-        self.__load_from_file(file)
-        file.close()
-        
-    def __iter__(self):
-        """
-        Each call reload the file
-        """
-        self.__load()
-        self.sort_by_date_reverse()
-        return list.__iter__(self)
-
-    def packday(remove_manual_input=False):
-        # FIXME broken
-        #print "MMSI", strmmsi
-
-        self = BankNmea1(self.strmmsi, self.date)
-        filename = self.get_filename()
-        try:
-            file = open(filename, 'r+b') # read/write binary
-        except IOError, ioerr:
-            if ioerr.errno != 2: # No file
-                raise
-            return self # no data
-        lockf(file, LOCK_EX)
-        self.__load_from_file(file)
-        self.sort_by_date()
-
-        file_has_changed = False
-        file_must_be_unlinked = False
-
-        #print "PACKING..."
-        file_has_changed = self.remove_duplicate_timestamp() or file_has_changed
-
-        if remove_manual_input:
-            #print "REMOVING MANUAL INPUT..."
-            file_has_changed = self.remove_manual_input() or file_has_changed
-
-        if file_has_changed:
-            file.seek(0)
-            self._write_in_file(file)
-            file.truncate()
-            if file.tell() == 0:
-                file_must_be_unlinked = True
-
-        file.close()
-        
-        if file_must_be_unlinked:
-            # FIXME we release the lock before unlinking
-            # another process might encounter an empty file (not handled)
-            logging.warning('file was truncated to size 0. unlinking')
-            os.unlink(filename) # we have the lock (!)
-
-    def dump_to_stdout(self):
-        """
-        Print contents to stdout
-        """
-        for nmea1 in self:
-            nmea1.dump_to_stdout()
-
-    def sort_by_date(self):
-        self.sort(lambda n1, n2: n1.timestamp_1 - n2.timestamp_1)
-
-    def sort_by_date_reverse(self):
-        self.sort(lambda n1, n2: n2.timestamp_1 - n1.timestamp_1)
-
-    def remove_duplicate_timestamp(self):
-        file_has_changed = False
-        if len(self) <= 1:
-            return file_has_changed
-        last_timestamp = self[0].timestamp_1
-        i = 1
-        while i < len(self):
-            if self[i].timestamp_1 == last_timestamp:
-                del self[i]
-                file_has_changed = True
-            else:
-                last_timestamp = self[i].timestamp_1
-                i += 1
-        return file_has_changed
-        
-    def remove_manual_input(self):
-        file_has_changed = False
-        i = 0
-        while i < len(self):
-            if self[i].source_1[:2] == 'MI':
-                del self[i]
-                file_has_changed = True
-            else:
-                i += 1
-        return file_has_changed
-
-class Nmea1Feeder:
-    """
-    Yields all nmea1 packets between two given datetimes
-    in REVERSE order (recent information first)
-    """
-    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
-        self.strmmsi = strmmsi
-        assert datetime_end is not None
-        self.datetime_end = datetime_end
-        self.datetime_begin = datetime_begin or DB_STARTDATE
-        self.max_count = max_count
-
-    def __iter__(self):
-        dt_end = self.datetime_end
-        d_end = dt_end.date()
-        ts_end = datetime_to_timestamp(dt_end)
-        if self.datetime_begin:
-            dt_begin = self.datetime_begin
-            d_begin = dt_begin.date()
-            ts_begin = datetime_to_timestamp(dt_begin)
-        else:
-            dt_begin = None
-            d_begin = None
-            ts_begin = None
-
-        d = d_end
-        count = 0
-        while True:
-            if d_begin is not None and d < d_begin:
-                return
-            bank = BankNmea1(self.strmmsi, d)
-            for nmea1 in bank:
-                if ts_begin is not None and nmea1.timestamp_1 < ts_begin:
-                    return
-                if nmea1.timestamp_1 > ts_end:
-                    continue
-                
-                yield nmea1
-               
-                count += 1
-                if self.max_count and count >= self.max_count:
-                    return
-            d += timedelta(-1)
-
-
-class BankNmea5(list):
-    """
-    That class handle a .nmea5 archive file
-    """
-    def __init__(self, strmmsi, dt):
-        list.__init__(self)
-        self.strmmsi = strmmsi
-        if isinstance(dt, date):
-            try:
-                dt = dt.strftime('%Y%m%d')
-            except ValueError:
-                logging.critical('dt=%s', dt)
-                raise
-        self.date = dt
-
-    def get_filename(self):
-        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea5'))
-
-    def __load_from_file(self, file):
-        '''
-        Adds all record from opened file in this bank
-        File must be locked before call
-        '''
-        while True:
-            record = file.read(aivdm_record5_length)
-            if not record:
-                break
-            self.append(Nmea5.new_from_record(record))
-
-    def _write_in_file(self, file):
-        '''
-        Write all records from that bank in opened file
-        File must be locked before call
-        File should be truncated after call
-        '''
-        for nmea5 in self:
-            file.write(nmea5.to_record())
-
-    def __load(self):
-        try:
-            file = open(self.get_filename(), 'rb')
-            lockf(file, LOCK_SH)
-        except IOError, ioerr:
-            if ioerr.errno == 2: # No file
-                return
-            raise
-        self.__load_from_file(file)
-        file.close()
-        
-    def __iter__(self):
-        """
-        Each call reload the file
-        """
-        self.__load()
-        self.sort_by_date_reverse()
-        return list.__iter__(self)
-
-    def sort_by_date(self):
-        self.sort(lambda n1, n2: n1.timestamp_5 - n2.timestamp_5)
-
-    def sort_by_date_reverse(self):
-        self.sort(lambda n1, n2: n2.timestamp_5 - n1.timestamp_5)
-
-class Nmea5Feeder:
-    """
-    Yields all nmea5 packets between two given datetimes
-    in REVERSE order (recent information first)
-    """
-    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
-        self.strmmsi = strmmsi
-        assert datetime_end is not None
-        self.datetime_end = datetime_end
-        self.datetime_begin = datetime_begin or DB_STARTDATE
-        self.max_count = max_count
-
-    def __iter__(self):
-        dt_end = self.datetime_end
-        d_end = dt_end.date()
-        ts_end = datetime_to_timestamp(dt_end)
-        if self.datetime_begin:
-            dt_begin = self.datetime_begin
-            d_begin = dt_begin.date()
-            ts_begin = datetime_to_timestamp(dt_begin)
-        else:
-            dt_begin = None
-            d_begin = None
-            ts_begin = None
-
-        d = d_end
-        count = 0
-        while True:
-            if d_begin is not None and d < d_begin:
-                return
-            bank = BankNmea5(self.strmmsi, d)
-            for nmea1 in bank:
-                if ts_begin is not None and nmea1.timestamp_5 < ts_begin:
-                    return
-                if nmea1.timestamp_5 > ts_end:
-                    continue
-                
-                yield nmea1
-               
-                count += 1
-                if self.max_count and count >= self.max_count:
-                    return
-            d += timedelta(-1)
-
-
-class NmeaFeeder:
-    """
-    Yields nmea packets matching criteria.
-    """
-    def __init__(self, strmmsi, datetime_end, datetime_begin=None, filters=None, granularity=1, max_count=None):
-        if granularity <= 0:
-            logging.warning('Granularity=%d generates duplicate entries', granularity)
-        self.strmmsi = strmmsi
-        assert datetime_end is not None
-        self.datetime_end = datetime_end
-        self.datetime_begin = datetime_begin or DB_STARTDATE
-        self.filters = filters or []
-        self.granularity = granularity
-        self.max_count = max_count
-
-    def __iter__(self):
-        nmea = Nmea(self.strmmsi)
-        if self.datetime_begin:
-            nmea5_datetime_begin = self.datetime_begin - timedelta(30) # go back up to 30 days to get a good nmea5 packet
-        else:
-            nmea5_datetime_begin = None
-        nmea5_iterator = Nmea5Feeder(self.strmmsi, self.datetime_end, nmea5_datetime_begin).__iter__()
-        nmea5 = Nmea5(self.strmmsi, sys.maxint)
-
-        count = 0
-        lasttimestamp = sys.maxint
-        for nmea1 in Nmea1Feeder(self.strmmsi, self.datetime_end, self.datetime_begin):
-            Nmea1.from_values(nmea, *nmea1.to_values())
-            
-            # try to get an nmea5 paket older
-            nmea5_updated = False
-            while nmea5 is not None and nmea5.timestamp_5 > nmea1.timestamp_1:
-                try:
-                    nmea5 = nmea5_iterator.next()
-                    nmea5_updated = True
-                except StopIteration:
-                    nmea5 = None
-            
-            if nmea5_updated and nmea5 is not None:
-                Nmea5.merge_from_values(nmea, *nmea5.to_values())
-
-            filtered_out = False
-            for is_ok in self.filters:
-                if not is_ok(nmea):
-                    filtered_out = True
-                    break
-            if filtered_out:
-                continue
-
-            if nmea.timestamp_1 <= lasttimestamp - self.granularity:
-                yield nmea
-                count += 1
-                if self.max_count and count >= self.max_count:
-                    return
-                lasttimestamp = nmea.timestamp_1
-
-
-def all_mmsi_generator():
-    """
-    Returns an array of all known strmmsi.
-    """
-    for dirname, dirs, fnames in os.walk(os.path.join(DBPATH, 'last')):
-        for fname in fnames:
-            if fname[-6:] == '.nmea1':
-                yield fname[:-6]
-
-
-def load_fleet_to_uset(fleetname):
-    """
-    Loads a fleet by name-id.
-    Returns an array of strmmsi.
-    """
-    result = []
-    sqlexec(u"SELECT mmsi FROM fleet_vessel WHERE fleet=%(fleetname)s", {'fleetname': fleetname})
-    cursor = get_common_cursor()
-    while True:
-        row = cursor.fetchone()
-        if not row:
-            break
-        mmsi = row[0]
-        result.append(mmsi_to_strmmsi(mmsi))
-    logging.debug('fleet=%s', result)
-    return result
-
-
-def filter_area(nmea, area):
-    """
-    Returns false if position is out of area.
-    """
-    if nmea.latitude == AIS_LAT_NOT_AVAILABLE or nmea.longitude == AIS_LON_NOT_AVAILABLE:
-        return False
-    if not area.contains((nmea.latitude/AIS_LATLON_SCALE, nmea.longitude/AIS_LATLON_SCALE)):
-        return False
-    return True
-
-def filter_knownposition(nmea):
-    """
-    Returns false if position is not fully known
-    """
-    # we are filtering out latitude=0 and longitude=0, that is not supposed to be necessary...
-    return nmea.latitude != AIS_LAT_NOT_AVAILABLE and nmea.longitude != AIS_LON_NOT_AVAILABLE and nmea.latitude != 0 and nmea.longitude != 0
-
-
-_filter_positioncheck_last_mmsi = None
-def filter_speedcheck(nmea, max_mps):
-    """
-    mps is miles per seconds
-    """
-    global _filter_positioncheck_last_mmsi
-    global _filter_positioncheck_last_time
-    global _filter_positioncheck_last_time_failed
-    global _filter_positioncheck_last_lat
-    global _filter_positioncheck_last_lon
-    global _filter_positioncheck_error_count
-    if nmea.strmmsi != _filter_positioncheck_last_mmsi:
-        _filter_positioncheck_last_time = None
-        _filter_positioncheck_last_mmsi = nmea.strmmsi
-        _filter_positioncheck_error_count = 0
-    if _filter_positioncheck_last_time is not None:
-        seconds = _filter_positioncheck_last_time - nmea.timestamp_1
-        distance = dist3_latlong_ais((_filter_positioncheck_last_lat, _filter_positioncheck_last_lon), (nmea.latitude, nmea.longitude))
-        if seconds:
-            speed = distance/seconds
-            if speed > max_mps:
-                if _filter_positioncheck_error_count < 10:
-                    logging.debug("Ignoring point: distance = %s, time = %s, speed = %s kt, source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
-                    if _filter_positioncheck_error_count == 0 or _filter_positioncheck_last_time_failed != nmea.timestamp_1:
-                        _filter_positioncheck_error_count += 1
-                        _filter_positioncheck_last_time_failed = nmea.timestamp_1
-                    return False
-                else:
-                    logging.warning("Discontinous position accepted after too many failures: %.2f nm in %s s (%.0f kt), source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
-            _filter_positioncheck_error_count = 0
-    _filter_positioncheck_last_time = nmea.timestamp_1
-    _filter_positioncheck_last_lat = nmea.latitude
-    _filter_positioncheck_last_lon = nmea.longitude
-    return True
-
-
-def main():
-    """
-    Perform various operation on the database
-    For usage, see "ais --help"
-    """
-    from optparse import OptionParser, OptionGroup
-    global DBPATH
-
-    parser = OptionParser(usage='%prog [options] { mmsi | @fleet }+ | all')
-
-    parser.add_option('-d', '--debug',
-        action='store_true', dest='debug', default=False,
-        help="debug mode")
-
-    parser.add_option('-e', '--end',
-        action='store', dest='sdt_end', metavar="'YYYYMMDD HHMMSS'",
-        help='End data processing on that GMT date time.'
-             'Default is now.'
-             'If a date is provided without time, time defaults to 235959.')
-    parser.add_option('-s', '--start',
-        action='store', dest='sdt_start', metavar="'YYYYMMDD HHMMSS'",
-        help='Start data processing on that date.'
-             'Using that option enables multiple output of the same boat.'
-             'Disabled by default.'
-             'If a date is provided without time, time default to 000000.'
-             'If other options enable multiple output, default to 1 day before'
-             ' --end date/time.')
-    parser.add_option('-g', '--granularity',
-        action='store', type='int', dest='granularity', metavar='SECONDS',
-        help='Dump only one position every granularity seconds.'
-             'Using that option enables multiple output of the same boat.'
-             'If other options enable multiple output, defaults to 600'
-             ' (10 minutes)')
-    parser.add_option('--max',
-        action='store', type='int', dest='max_count', metavar='NUMBER',
-        help='Dump a maximum of NUMBER positions every granularity seconds.'
-             'Using that option enables multiple output of the same boat.')
-
-    parser.add_option('--filter-knownposition',
-        action='store_true', dest='filter_knownposition', default=False,
-        help="Eliminate unknown positions from results.")
-
-    parser.add_option('--filter-speedcheck',
-        action='store', type='int', dest='speedcheck', default=200, metavar='KNOTS',
-        help='Eliminate erroneaous positions from results,' 
-             ' based on impossible speed.')
-
-    parser.add_option('--filter-type',
-        action='append', type='int', dest='type_list', metavar='TYPE',
-        help="process a specific ship type.")
-    parser.add_option('--help-types',
-        action='store_true', dest='help_types', default=False,
-        help="display list of available types")
-
-    parser.add_option('--filter-area',
-        action='store', type='str', dest='area_file', metavar="FILE.KML",
-        help="only process a specific area as defined in a kml polygon file.")
-
-    parser.add_option('--filter-destination',
-        action='store', type='str', dest='filter_destination', metavar="DESTINATION",
-        help="Only print ships with that destination.")
-
-    parser.add_option('--no-headers',
-        action='store_false', dest='csv_headers', default=True,
-        help="skip CSV headers")
-    #
-
-    expert_group = OptionGroup(parser, "Expert Options",
-        "You normaly don't need any of these")
-
-    expert_group.add_option('--db',
-        action='store', dest='db', default=DBPATH,
-        help="path to filesystem database. Default=%default")
-
-    expert_group.add_option('--debug-sql',
-        action='store_true', dest='debug_sql', default=False,
-        help="print all sql queries to stdout before running them")
-
-    expert_group.add_option('--action',
-        choices=('dump', 'removemanual', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump',
-        help='Possible values are:\n'
-            'dump: dump values in csv format. This is the default.\n'
-            'removemanual: Delete Manual Input entries from the database.\n'
-            'mmsidump: Dump mmsi')
-    parser.add_option_group(expert_group)
-
-    (options, args) = parser.parse_args()
-
-
-    if options.help_types:
-        print "Known ship types:"
-        keys = SHIP_TYPES.keys()
-        keys.sort()
-        for k in keys:
-            print k, SHIP_TYPES[k]
-        sys.exit(0)
-
-    DBPATH = options.db
-
-    if options.debug:
-        loglevel = logging.DEBUG
-    else:
-        loglevel = logging.INFO
-    logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s')
-
-    if options.debug_sql:
-        sql_setdebug(True)
-
-    #
-    # Ships selections
-    #
-
-    if len(args)==0:
-        print >> sys.stderr, "No ship to process"
-        sys.exit(1)
-
-    target_mmsi_iterator = []
-    all_targets = False
-    for arg in args:
-        if arg == 'all':
-            all_targets = True
-        elif arg.startswith('@'):
-            target_mmsi_iterator += load_fleet_to_uset(arg[1:])
-        else:
-            target_mmsi_iterator.append(arg)
-    if all_targets:
-        if target_mmsi_iterator:
-            logging.warning('Selecting all ships, ignoring other arguments')
-        target_mmsi_iterator = all_mmsi_generator()
-
-    #
-    # Dates selections
-    #
-
-    if options.sdt_end:
-        # remove non digit characters
-        options.sdt_end = "".join([ c for c in options.sdt_end if c.isdigit()])
-        if len(options.sdt_end)==14:
-            dt_end = datetime.strptime(options.sdt_end, '%Y%m%d%H%M%S')
-        elif len(options.sdt_end)==8:
-            dt_end = datetime.strptime(options.sdt_end, '%Y%m%d')
-            dt_end = datetime.combine(dt_end.date(), time(23,59,59))
-        else:
-            print >> sys.stderr, "Invalid format for --end option"
-            sys.exit(1)
-    else:
-        dt_end = datetime.utcnow()
-    logging.debug('--end is %s', dt_end)
-
-    if options.sdt_start or options.granularity is not None or options.max_count:
-        # time period is enabled
-        if options.sdt_start:
-            options.sdt_start = "".join([ c for c in options.sdt_start if c.isdigit()])
-            if len(options.sdt_start)==14:
-                dt_start = datetime.strptime(options.sdt_start, '%Y%m%d%H%M%S')
-            elif len(options.sdt_start)==8:
-                dt_start = datetime.strptime(options.sdt_start, '%Y%m%d')
-            else:
-                print >> sys.stderr, "Invalid format for --start option"
-                sys.exit(1)
-        else:
-            dt_start = dt_end - timedelta(1)
-        if options.granularity is None:
-            options.granularity = 600
-    else:
-        dt_start = None
-        options.max_count = 1
-        if options.granularity is None:
-            options.granularity = 600
-    logging.debug('--start is %s', dt_start)
-
-    #
-    # Filters
-    #
-
-    filters = []
-    
-    if options.filter_knownposition:
-        filters.append(filter_knownposition)
-
-    if options.speedcheck != 0:
-        maxmps = options.speedcheck / 3600. # from knots to NM per seconds
-        filters.append(lambda nmea: filter_speedcheck(nmea, maxmps))
-
-    if options.area_file:
-        area = load_area_from_kml_polygon(options.area_file)
-        filters.append(lambda nmea: filter_area(nmea, area))
-    
-    if options.type_list:
-        def filter_type(nmea):
-            return nmea.type in options.type_list
-        filters.append(filter_type)
-
-    if options.filter_destination:
-        filters.append(lambda nmea: nmea.destination.startswith(options.filter_destination))
-
-    #
-    # Processing
-    #
-
-    if options.action == 'dump':
-        output = csv.writer(sys.stdout)
-        if options.csv_headers:
-            output.writerow(Nmea.csv_headers)
-        for mmsi in target_mmsi_iterator:
-            logging.debug('Considering %s', repr(mmsi))
-            assert dt_end is not None
-            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
-                output.writerow(nmea.get_dump_row())
-
-    elif options.action == 'removemanual':
-        if filters:
-            print >> sys.stderr, "removemanual action doesn't support filters"
-            sys.exit(1)
-
-        for dt in dates:
-            logging.info("Processing date %s", dt)
-            for mmsi in target_mmsi_iterator:
-                BankNmea1(mmsi, dt).packday(remove_manual_input=True)
-    
-    elif options.action == 'mmsidump':
-        for strmmsi in target_mmsi_iterator :
-            print strmmsi
-
-    elif options.action == 'fixdestination':
-        for mmsi in target_mmsi_iterator:
-            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
-                destination = nmea.destination.rstrip(' @\0')
-                if destination:
-                    sqlexec(u'UPDATE vessel SET destination = %(destination)s WHERE mmsi=%(mmsi)s AND destination IS NULL', {'mmsi':strmmsi_to_mmsi(mmsi), 'destination':destination})
-                    logging.info('%s -> %s', mmsi, repr(destination))
-                    dbcommit()
-                    break # go to next mmsi
-
-
-if __name__ == '__main__':
-    main()
diff --git a/bin/common.py b/bin/common.py
new file mode 100755 (executable)
index 0000000..c322f12
--- /dev/null
@@ -0,0 +1,2000 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+__all__ = [
+    'DB_STARTDATE', 'DBPATH',
+    'COUNTRIES_MID', 'STATUS_CODES', 'SHIP_TYPES',
+    'AIS_STATUS_NOT_AVAILABLE',
+    'AIS_ROT_HARD_LEFT', 'AIS_ROT_HARD_RIGHT', 'AIS_ROT_NOT_AVAILABLE',
+    'AIS_LATLON_SCALE', 'AIS_LON_NOT_AVAILABLE', 'AIS_LAT_NOT_AVAILABLE',
+    'AIS_COG_SCALE', 'AIS_COG_NOT_AVAILABLE',
+    'AIS_NO_HEADING',
+    'AIS_SOG_SCALE', 'AIS_SOG_NOT_AVAILABLE', 'AIS_SOG_FAST_MOVER', 'AIS_SOG_MAX_SPEED',
+    #'_hash3_pathfilename',
+    'db_bydate_addrecord',
+    'db_lastinfo_setrecord_ifnewer',
+    'sql_add_nmea5',
+    #'aivdm_record123_format',
+    #'aivdm_record123_length',
+    #'aivdm_record5_format',
+    #'aivdm_record5_length',
+    'add_nmea1',
+    'add_nmea5_full',
+    'add_nmea5_partial',
+    'strmmsi_to_mmsi',
+    'mmsi_to_strmmsi',
+    'Nmea1',
+    'Nmea5',
+    'Nmea',
+    'BankNmea1',
+    'Nmea1Feeder',
+    'BankNmea5',
+    'Nmea5Feeder',
+    'NmeaFeeder',
+    'all_mmsi_generator',
+    'load_fleet_to_uset',
+    'filter_area',
+    'filter_knownposition',
+    'filter_speedcheck',
+    ]
+            
+
+import sys
+import os
+import struct
+import logging
+from datetime import datetime, timedelta, date, time
+from fcntl import lockf, LOCK_EX, LOCK_UN, LOCK_SH
+import csv
+
+from ais.ntools import *
+from ais.db import *
+from ais.area import load_area_from_kml_polygon
+from ais.earth3d import dist3_latlong_ais
+
+DB_STARTDATE = datetime(2008, 6, 1)
+
+# This is the location of the filesystem database
+DBPATH = '/var/lib/ais/db'
+
+# see make-countries.py
+COUNTRIES_MID = {
+    201: u'Albania',
+    202: u'Andorra',
+    203: u'Austria',
+    204: u'Azores',
+    205: u'Belgium',
+    206: u'Belarus',
+    207: u'Bulgaria',
+    208: u'Vatican City State',
+    209: u'Cyprus',
+    210: u'Cyprus',
+    211: u'Germany',
+    212: u'Cyprus',
+    213: u'Georgia',
+    214: u'Moldova',
+    215: u'Malta',
+    216: u'Armenia',
+    218: u'Germany',
+    219: u'Denmark',
+    220: u'Denmark',
+    224: u'Spain',
+    225: u'Spain',
+    226: u'France',
+    227: u'France',
+    228: u'France',
+    230: u'Finland',
+    231: u'Faroe Islands',
+    232: u'United Kingdom',
+    233: u'United Kingdom',
+    234: u'United Kingdom',
+    235: u'United Kingdom',
+    236: u'Gibraltar',
+    237: u'Greece',
+    238: u'Croatia',
+    239: u'Greece',
+    240: u'Greece',
+    242: u'Morocco',
+    243: u'Hungary',
+    244: u'Netherlands',
+    245: u'Netherlands',
+    246: u'Netherlands',
+    247: u'Italy',
+    248: u'Malta',
+    249: u'Malta',
+    250: u'Ireland',
+    251: u'Iceland',
+    252: u'Liechtenstein',
+    253: u'Luxembourg',
+    254: u'Monaco',
+    255: u'Madeira',
+    256: u'Malta',
+    257: u'Norway',
+    258: u'Norway',
+    259: u'Norway',
+    261: u'Poland',
+    262: u'Montenegro',
+    263: u'Portugal',
+    264: u'Romania',
+    265: u'Sweden',
+    266: u'Sweden',
+    267: u'Slovak Republic',
+    268: u'San Marino',
+    269: u'Switzerland',
+    270: u'Czech Republic',
+    271: u'Turkey',
+    272: u'Ukraine',
+    273: u'Russian Federation',
+    274: u'The Former Yugoslav Republic of Macedonia',
+    275: u'Latvia',
+    276: u'Estonia',
+    277: u'Lithuania',
+    278: u'Slovenia',
+    279: u'Serbia',
+    301: u'Anguilla',
+    303: u'Alaska',
+    304: u'Antigua and Barbuda',
+    305: u'Antigua and Barbuda',
+    306: u'Netherlands Antilles',
+    307: u'Aruba',
+    308: u'Bahamas',
+    309: u'Bahamas',
+    310: u'Bermuda',
+    311: u'Bahamas',
+    312: u'Belize',
+    314: u'Barbados',
+    316: u'Canada',
+    319: u'Cayman Islands',
+    321: u'Costa Rica',
+    323: u'Cuba',
+    325: u'Dominica',
+    327: u'Dominican Republic',
+    329: u'Guadeloupe',
+    330: u'Grenada',
+    331: u'Greenland',
+    332: u'Guatemala',
+    334: u'Honduras',
+    336: u'Haiti',
+    338: u'United States of America',
+    339: u'Jamaica',
+    341: u'Saint Kitts and Nevis',
+    343: u'Saint Lucia',
+    345: u'Mexico',
+    347: u'Martinique',
+    348: u'Montserrat',
+    350: u'Nicaragua',
+    351: u'Panama',
+    352: u'Panama',
+    353: u'Panama',
+    354: u'Panama',
+    355: u'Panama',
+    356: u'Panama',
+    357: u'Panama',
+    358: u'Puerto Rico',
+    359: u'El Salvador',
+    361: u'Saint Pierre and Miquelon',
+    362: u'Trinidad and Tobago',
+    364: u'Turks and Caicos Islands',
+    366: u'United States of America',
+    367: u'United States of America',
+    368: u'United States of America',
+    369: u'United States of America',
+    370: u'Panama',
+    371: u'Panama',
+    372: u'Panama',
+    375: u'Saint Vincent and the Grenadines',
+    376: u'Saint Vincent and the Grenadines',
+    377: u'Saint Vincent and the Grenadines',
+    378: u'British Virgin Islands',
+    379: u'United States Virgin Islands',
+    401: u'Afghanistan',
+    403: u'Saudi Arabia',
+    405: u'Bangladesh',
+    408: u'Bahrain',
+    410: u'Bhutan',
+    412: u'China',
+    413: u'China',
+    416: u'Taiwan',
+    417: u'Sri Lanka',
+    419: u'India',
+    422: u'Iran',
+    423: u'Azerbaijani Republic',
+    425: u'Iraq',
+    428: u'Israel',
+    431: u'Japan',
+    432: u'Japan',
+    434: u'Turkmenistan',
+    436: u'Kazakhstan',
+    437: u'Uzbekistan',
+    438: u'Jordan',
+    440: u'Korea',
+    441: u'Korea',
+    443: u'Palestine',
+    445: u"Democratic People's Republic of Korea",
+    447: u'Kuwait',
+    450: u'Lebanon',
+    451: u'Kyrgyz Republic',
+    453: u'Macao',
+    455: u'Maldives',
+    457: u'Mongolia',
+    459: u'Nepal',
+    461: u'Oman',
+    463: u'Pakistan',
+    466: u'Qatar',
+    468: u'Syrian Arab Republic',
+    470: u'United Arab Emirates',
+    473: u'Yemen',
+    475: u'Yemen',
+    477: u'Hong Kong',
+    478: u'Bosnia and Herzegovina',
+    501: u'Adelie Land',
+    503: u'Australia',
+    506: u'Myanmar',
+    508: u'Brunei Darussalam',
+    510: u'Micronesia',
+    511: u'Palau',
+    512: u'New Zealand',
+    514: u'Cambodia',
+    515: u'Cambodia',
+    516: u'Christmas Island',
+    518: u'Cook Islands',
+    520: u'Fiji',
+    523: u'Cocos',
+    525: u'Indonesia',
+    529: u'Kiribati',
+    531: u"Lao People's Democratic Republic",
+    533: u'Malaysia',
+    536: u'Northern Mariana Islands',
+    538: u'Marshall Islands',
+    540: u'New Caledonia',
+    542: u'Niue',
+    544: u'Nauru',
+    546: u'French Polynesia',
+    548: u'Philippines',
+    553: u'Papua New Guinea',
+    555: u'Pitcairn Island',
+    557: u'Solomon Islands',
+    559: u'American Samoa',
+    561: u'Samoa',
+    563: u'Singapore',
+    564: u'Singapore',
+    565: u'Singapore',
+    567: u'Thailand',
+    570: u'Tonga',
+    572: u'Tuvalu',
+    574: u'Viet Nam',
+    576: u'Vanuatu',
+    578: u'Wallis and Futuna Islands',
+    601: u'South Africa',
+    603: u'Angola',
+    605: u'Algeria',
+    607: u'Saint Paul and Amsterdam Islands',
+    608: u'Ascension Island',
+    609: u'Burundi',
+    610: u'Benin',
+    611: u'Botswana',
+    612: u'Central African Republic',
+    613: u'Cameroon',
+    615: u'Congo',
+    616: u'Comoros',
+    617: u'Cape Verde',
+    618: u'Crozet Archipelago',
+    619: u"C\xc3\xb4te d'Ivoire",
+    621: u'Djibouti',
+    622: u'Egypt',
+    624: u'Ethiopia',
+    625: u'Eritrea',
+    626: u'Gabonese Republic',
+    627: u'Ghana',
+    629: u'Gambia',
+    630: u'Guinea-Bissau',
+    631: u'Equatorial Guinea',
+    632: u'Guinea',
+    633: u'Burkina Faso',
+    634: u'Kenya',
+    635: u'Kerguelen Islands',
+    636: u'Liberia',
+    637: u'Liberia',
+    642: u"Socialist People's Libyan Arab Jamahiriya",
+    644: u'Lesotho',
+    645: u'Mauritius',
+    647: u'Madagascar',
+    649: u'Mali',
+    650: u'Mozambique',
+    654: u'Mauritania',
+    655: u'Malawi',
+    656: u'Niger',
+    657: u'Nigeria',
+    659: u'Namibia',
+    660: u'Reunion',
+    661: u'Rwanda',
+    662: u'Sudan',
+    663: u'Senegal',
+    664: u'Seychelles',
+    665: u'Saint Helena',
+    666: u'Somali Democratic Republic',
+    667: u'Sierra Leone',
+    668: u'Sao Tome and Principe',
+    669: u'Swaziland',
+    670: u'Chad',
+    671: u'Togolese Republic',
+    672: u'Tunisia',
+    674: u'Tanzania',
+    675: u'Uganda',
+    676: u'Democratic Republic of the Congo',
+    677: u'Tanzania',
+    678: u'Zambia',
+    679: u'Zimbabwe',
+    701: u'Argentine Republic',
+    710: u'Brazil',
+    720: u'Bolivia',
+    725: u'Chile',
+    730: u'Colombia',
+    735: u'Ecuador',
+    740: u'Falkland Islands',
+    745: u'Guiana',
+    750: u'Guyana',
+    755: u'Paraguay',
+    760: u'Peru',
+    765: u'Suriname',
+    770: u'Uruguay',
+    775: u'Venezuela',
+}
+
+STATUS_CODES = {
+     0:  'Under way using engine',
+     1:  'At anchor',
+     2:  'Not under command',
+     3:  'Restricted manoeuverability',
+     4:  'Constrained by her draught',
+     5:  'Moored',
+     6:  'Aground',
+     7:  'Engaged in Fishing',
+     8:  'Under way sailing',
+     9:  '9 - Reserved for future amendment of Navigational Status for HSC',
+    10:  '10 - Reserved for future amendment of Navigational Status for WIG',
+    11:  '11 - Reserved for future use',
+    12:  '12 - Reserved for future use',
+    13:  '13 - Reserved for future use',
+    14:  '14 - Reserved for future use', # Land stations
+    15:  'Not defined', # default
+}
+
+SHIP_TYPES = {
+     0: 'Not available (default)',
+     1: 'Reserved for future use',
+     2: 'Reserved for future use',
+     3: 'Reserved for future use',
+     4: 'Reserved for future use',
+     5: 'Reserved for future use',
+     6: 'Reserved for future use',
+     7: 'Reserved for future use',
+     8: 'Reserved for future use',
+     9: 'Reserved for future use',
+    10: 'Reserved for future use',
+    11: 'Reserved for future use',
+    12: 'Reserved for future use',
+    13: 'Reserved for future use',
+    14: 'Reserved for future use',
+    15: 'Reserved for future use',
+    16: 'Reserved for future use',
+    17: 'Reserved for future use',
+    18: 'Reserved for future use',
+    19: 'Reserved for future use',
+    20: 'Wing in ground (WIG), all ships of this type',
+    21: 'Wing in ground (WIG), Hazardous category A',
+    22: 'Wing in ground (WIG), Hazardous category B',
+    23: 'Wing in ground (WIG), Hazardous category C',
+    24: 'Wing in ground (WIG), Hazardous category D',
+    25: 'Wing in ground (WIG), Reserved for future use',
+    26: 'Wing in ground (WIG), Reserved for future use',
+    27: 'Wing in ground (WIG), Reserved for future use',
+    28: 'Wing in ground (WIG), Reserved for future use',
+    29: 'Wing in ground (WIG), Reserved for future use',
+    30: 'Fishing',
+    31: 'Towing',
+    32: 'Towing: length exceeds 200m or breadth exceeds 25m',
+    33: 'Dredging or underwater ops',
+    34: 'Diving ops',
+    35: 'Military ops',
+    36: 'Sailing',
+    37: 'Pleasure Craft',
+    38: 'Reserved',
+    39: 'Reserved',
+    40: 'High speed craft (HSC), all ships of this type',
+    41: 'High speed craft (HSC), Hazardous category A',
+    42: 'High speed craft (HSC), Hazardous category B',
+    43: 'High speed craft (HSC), Hazardous category C',
+    44: 'High speed craft (HSC), Hazardous category D',
+    45: 'High speed craft (HSC), Reserved for future use',
+    46: 'High speed craft (HSC), Reserved for future use',
+    47: 'High speed craft (HSC), Reserved for future use',
+    48: 'High speed craft (HSC), Reserved for future use',
+    49: 'High speed craft (HSC), No additional information',
+    50: 'Pilot Vessel',
+    51: 'Search and Rescue vessel',
+    52: 'Tug',
+    53: 'Port Tender',
+    54: 'Anti-pollution equipment',
+    55: 'Law Enforcement',
+    56: 'Spare - Local Vessel',
+    57: 'Spare - Local Vessel',
+    58: 'Medical Transport',
+    59: 'Ship according to RR Resolution No. 18',
+    60: 'Passenger, all ships of this type',
+    61: 'Passenger, Hazardous category A',
+    62: 'Passenger, Hazardous category B',
+    63: 'Passenger, Hazardous category C',
+    64: 'Passenger, Hazardous category D',
+    65: 'Passenger, Reserved for future use',
+    66: 'Passenger, Reserved for future use',
+    67: 'Passenger, Reserved for future use',
+    68: 'Passenger, Reserved for future use',
+    69: 'Passenger, No additional information',
+    70: 'Cargo', # 'Cargo, all ships of this type',
+    71: 'Cargo, Hazardous category A',
+    72: 'Cargo, Hazardous category B',
+    73: 'Cargo, Hazardous category C',
+    74: 'Cargo, Hazardous category D',
+    75: 'Cargo', # 'Cargo, Reserved for future use',
+    76: 'Cargo', # 'Cargo, Reserved for future use',
+    77: 'Cargo', # 'Cargo, Reserved for future use',
+    78: 'Cargo', # 'Cargo, Reserved for future use',
+    79: 'Cargo', # 'Cargo, No additional information',
+    80: 'Tanker', # 'Tanker, all ships of this type',
+    81: 'Tanker, Hazardous category A',
+    82: 'Tanker, Hazardous category B',
+    83: 'Tanker, Hazardous category C',
+    84: 'Tanker, Hazardous category D',
+    85: 'Tanker', # 'Tanker, Reserved for future use',
+    86: 'Tanker', # 'Tanker, Reserved for future use',
+    87: 'Tanker', # 'Tanker, Reserved for future use',
+    88: 'Tanker', # 'Tanker, Reserved for future use',
+    89: 'Tanker, No additional information',
+    90: 'Other Type, all ships of this type',
+    91: 'Other Type, Hazardous category A',
+    92: 'Other Type, Hazardous category B',
+    93: 'Other Type, Hazardous category C',
+    94: 'Other Type, Hazardous category D',
+    95: 'Other Type, Reserved for future use',
+    96: 'Other Type, Reserved for future use',
+    97: 'Other Type, Reserved for future use',
+    98: 'Other Type, Reserved for future use',
+    99: 'Other Type, no additional information',
+    100: 'Default Navaid',
+    101: 'Reference point',
+    102: 'RACON',
+    103: 'Offshore Structure',
+    104: 'Spare',
+    105: 'Light, without sectors',
+    106: 'Light, with sectors',
+    107: 'Leading Light Front',
+    108: 'Leading Light Rear',
+    109: 'Beacon, Cardinal N',
+    110: 'Beacon, Cardinal E',
+    111: 'Beacon, Cardinal S',
+    112: 'Beacon, Cardinal W',
+    113: 'Beacon, Port hand',
+    114: 'Beacon, Starboard hand',
+    115: 'Beacon, Preferred Channel port hand',
+    116: 'Beacon, Preferred Channel starboard hand',
+    117: 'Beacon, Isolated danger',
+    118: 'Beacon, Safe water',
+    119: 'Beacon, Special mark',
+    120: 'Cardinal Mark N',
+    121: 'Cardinal Mark E',
+    122: 'Cardinal Mark S',
+    123: 'Cardinal Mark W',
+    124: 'Port hand Mark',
+    125: 'Starboard hand Mark',
+    126: 'Preferred Channel Port hand',
+    127: 'Preferred Channel Starboard hand',
+    128: 'Isolated danger',
+    129: 'Safe Water',
+    130: 'Manned VTS / Special Mark',
+    131: 'Light Vessel / LANBY',
+}
+
+AIS_STATUS_NOT_AVAILABLE = 15
+AIS_ROT_HARD_LEFT = -127
+AIS_ROT_HARD_RIGHT = 127
+AIS_ROT_NOT_AVAILABLE = -128 # not like gpsd
+
+AIS_LATLON_SCALE = 600000.0
+AIS_LON_NOT_AVAILABLE = 0x6791AC0
+AIS_LAT_NOT_AVAILABLE = 0x3412140
+AIS_COG_SCALE = 10.0
+AIS_COG_NOT_AVAILABLE = 3600
+AIS_NO_HEADING = 511
+AIS_SOG_SCALE = 10.0
+AIS_SOG_NOT_AVAILABLE = 1023
+AIS_SOG_FAST_MOVER = 1022
+AIS_SOG_MAX_SPEED = 1021
+
+
+def _hash3_pathfilename(filename):
+    """
+    Returns a level 3 directory hashed filename on that basis:
+    123456789 -> 1/12/123/123456789
+    """
+    return os.path.join(filename[0], filename[:2], filename[:3], filename)
+
+
+def db_bydate_addrecord(basefilename, record, timestamp):
+    strdt = datetime.utcfromtimestamp(timestamp).strftime('%Y%m%d')
+    filename = os.path.join(DBPATH, 'bydate', strdt, _hash3_pathfilename(basefilename))
+    f = open_with_mkdirs(filename, 'ab')
+    lockf(f, LOCK_EX)
+    #f.seek(0,2) # go to EOF
+    assert f.tell() % len(record) == 0, 'Invalid length for %s' % filename
+    f.write(record)
+    f.close()
+
+
+def db_lastinfo_setrecord_ifnewer(basefilename, record, timestamp):
+    '''
+    Overwrite last information if date is newer
+    Input record must be complete
+    '''
+    filename = DBPATH+'/last/'+_hash3_pathfilename(basefilename)
+
+    try:
+        f = open(filename, 'r+b')
+    except IOError, ioerr:
+        if ioerr.errno != 2:
+            raise
+        # File was not found? Ok, create it. FIXME: we should lock something...
+        f = open_with_mkdirs(filename, 'wb')
+        f.write(record)
+        updated = True
+    else:
+        lockf(f, LOCK_EX)
+        assert f.tell() == 0
+        oldrecord = f.read(4)
+        assert len(oldrecord) == 4
+        oldtimestamp = struct.unpack('I', oldrecord)[0]
+        f.seek(0)
+        assert f.tell() == 0
+        if timestamp > oldtimestamp:
+            f.write(record)
+            assert f.tell() == len(record), \
+                "tell=%s size=%s" % (f.tell(), len(record))
+            updated = True
+        else:
+            updated = False
+    f.close()
+    return updated
+
+
+def sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
+    dim_bow, dim_stern, dim_port, dim_starboard, \
+    eta_M, eta_D, eta_h, eta_m, draught, destination, source):
+    ''' Don't call directly '''
+    sqlinfo = {}
+    sqlinfo['mmsi'] = strmmsi_to_mmsi(strmmsi)
+    sqlinfo['updated'] = datetime.utcfromtimestamp(timestamp)
+    sqlinfo['imo'] = imo or None
+    sqlinfo['name'] = name or None
+    sqlinfo['callsign'] = callsign or None
+    sqlinfo['type'] = type
+    sqlinfo['destination'] = None
+    if destination:
+        destination = destination.replace('\0', ' ').rstrip(' @\0')
+    sqlinfo['destination'] = destination or None
+    sqlinfo['source'] = source
+    sqlexec(u'''INSERT INTO vessel (mmsi, updated) SELECT %(mmsi)s, '1970-01-01T00:00:00' WHERE NOT EXISTS (SELECT * FROM vessel WHERE mmsi=%(mmsi)s)''', sqlinfo)
+    if sqlinfo['imo']:
+        sqlexec(u'UPDATE vessel SET imo = %(imo)s WHERE mmsi=%(mmsi)s AND (imo IS NULL OR updated<%(updated)s)', sqlinfo)
+    if sqlinfo['name']:
+        sqlexec(u'UPDATE vessel SET name = %(name)s WHERE mmsi=%(mmsi)s AND (name IS NULL OR updated<%(updated)s)', sqlinfo)
+    if sqlinfo['callsign']:
+        sqlexec(u'UPDATE vessel SET callsign = %(callsign)s WHERE mmsi=%(mmsi)s AND (callsign IS NULL OR updated<%(updated)s)', sqlinfo)
+    if sqlinfo['type']:
+        sqlexec(u'UPDATE vessel SET type = %(type)s WHERE mmsi=%(mmsi)s AND (type IS NULL OR updated<%(updated)s)', sqlinfo)
+    if sqlinfo['destination']:
+        sqlexec(u'UPDATE vessel SET destination = %(destination)s WHERE mmsi=%(mmsi)s AND (destination IS NULL OR updated<%(updated)s)', sqlinfo)
+    sqlexec(u'UPDATE vessel SET (updated, source) = (%(updated)s, %(source)s) WHERE mmsi=%(mmsi)s AND updated<%(updated)s', sqlinfo)
+    dbcommit()
+
+
+
+
+aivdm_record123_format = 'IBbhiiII4s'
+aivdm_record123_length = struct.calcsize(aivdm_record123_format)
+aivdm_record5_format = 'II20s7sBHHBBBBBBH20s4s'
+aivdm_record5_length = struct.calcsize(aivdm_record5_format)
+
+
+def add_nmea1(strmmsi, timestamp, status, rot, sog, \
+              latitude, longitude, cog, heading, source):
+    '''
+    Input is raw data, unscaled
+    '''
+    record = struct.pack(aivdm_record123_format, timestamp, status, rot, sog, latitude, longitude, cog, heading, source)
+    #print repr(record)
+    filename = strmmsi+'.nmea1'
+    db_bydate_addrecord(filename, record, timestamp)
+    # There's no need to be smart: all the information are taken, or none.
+    return db_lastinfo_setrecord_ifnewer(filename, record, timestamp)
+
+
+def add_nmea5_full(strmmsi, timestamp, imo, name, callsign, type, \
+                   dim_bow, dim_stern, dim_port, dim_starboard, \
+                   eta_M, eta_D, eta_h, eta_m, draught, destination, source):
+    '''
+    Input is raw data, unscaled
+    All fields are set, and can be upgraded if the record is newer
+    '''
+    record = struct.pack(aivdm_record5_format, timestamp, imo, name, callsign, \
+                         type, dim_bow, dim_stern, dim_port, dim_starboard, \
+                         eta_M, eta_D, eta_h, eta_m, draught, destination, source)
+    #print repr(record)
+    filename = strmmsi+'.nmea5'
+    db_bydate_addrecord(filename, record, timestamp)
+    updated = db_lastinfo_setrecord_ifnewer(filename, record, timestamp)
+    if updated:
+        sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
+                      dim_bow, dim_stern, dim_port, dim_starboard, \
+                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
+    return updated
+
+def add_nmea5_partial(strmmsi, timestamp, imo, name, callsign, type, \
+                      dim_bow, dim_stern, dim_port, dim_starboard, \
+                      eta_M, eta_D, eta_h, eta_m, draught, destination, source):
+    '''
+    Input is raw data, unscaled
+    All fields are not set. Only some of them can be upgraded, if they're newer
+    '''
+    record = struct.pack(aivdm_record5_format, \
+                         timestamp, imo, name, callsign, type, \
+                         dim_bow, dim_stern, dim_port, dim_starboard, \
+                         eta_M, eta_D, eta_h, eta_m, draught, destination, \
+                         source)
+    #print repr(record)
+    filename = strmmsi + '.nmea5'
+    db_bydate_addrecord(filename, record, timestamp)
+
+    updated = False
+    filename = os.path.join(DBPATH, 'last', _hash3_pathfilename(filename))
+    try:
+        f = open(filename, 'r+b')
+    except IOError, ioerr:
+        if ioerr.errno != 2:
+            raise
+        # File was not found? Ok, create it. FIXME: we should lock something...
+        f = open_with_mkdirs(filename, 'wb')
+        lockf(f, LOCK_EX)
+        f.write(record)
+        # keep the lock
+        updated = True
+    else:
+        lockf(f, LOCK_EX)
+        oldrecord = f.read(aivdm_record5_length)
+        oldtimestamp, oldimo, oldname, oldcallsign, oldtype, \
+        olddim_bow, olddim_stern, olddim_port, olddim_starboard, \
+        oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
+        olddraught, olddestination, oldsource \
+                  = struct.unpack(aivdm_record5_format, oldrecord)
+        if timestamp > oldtimestamp:
+            # we have incoming recent information
+            if imo == 0:
+                imo = oldimo
+            if name == '':
+                name = oldname
+            if callsign == '':
+                callsign = oldcallsign
+            if type == 0:
+                type = oldtype
+            if dim_bow == 0:
+                dim_bow = olddim_bow
+            if dim_stern == 0:
+                dim_stern = olddim_stern
+            if dim_port == 0:
+                dim_port = olddim_port
+            if dim_starboard == 0:
+                dim_starboard = olddim_starboard
+            if eta_M == 0 or eta_D == 0 or eta_h == 24 or eta_m == 60 \
+                          or destination == '':
+                eta_M = oldeta_M
+                eta_D = oldeta_D
+                eta_h = oldeta_h
+                eta_m = oldeta_m
+                destination = olddestination
+            if draught == 0:
+                draught = olddraught
+            record = struct.pack(aivdm_record5_format, \
+                                 timestamp, imo, name, callsign, type, \
+                                 dim_bow, dim_stern, dim_port, dim_starboard, \
+                                 eta_M, eta_D, eta_h, eta_m, draught, \
+                                 destination, source)
+            f.seek(0)
+            f.write(record)
+            updated = True
+        else:
+            # we received an obsolete info, but maybe there are some new things in it
+            if oldimo == 0 and imo != 0:
+                oldimo = imo
+                updated = True
+            if oldname == '' and name != '':
+                oldname = name
+                updated = True
+            if oldcallsign == '' and callsign != '':
+                oldcallsign = callsign
+                updated = True
+            if oldtype == 0 and type != 0:
+                oldtype = type
+                updated = True
+            if olddim_bow == 0 and dim_bow != 0:
+                olddim_bow = dim_bow
+                updated = True
+            if olddim_stern == 0 and dim_stern != 0:
+                olddim_stern = dim_stern
+                updated = True
+            if olddim_port == 0 and dim_port != 0:
+                olddim_port = dim_port
+                updated = True
+            if olddim_starboard == 0 and dim_starboard != 0:
+                olddim_starboard = dim_starboard
+                updated = True
+            # FIXME
+            if (oldeta_M == 0 or oldeta_D == 0 or olddestination == '') \
+                    and ((eta_M != 0 and eta_D != 0) or destination!=''):
+                oldeta_M = eta_M
+                oldeta_D = eta_D
+                oldeta_h = eta_h
+                oldeta_m = eta_m
+                olddestination = destination
+                updated = True
+            if olddraught == 0 and draught != 0:
+                olddraught = draught
+                updated = True
+            if updated:
+                oldsource = source
+                record = struct.pack(aivdm_record5_format, \
+                                     oldtimestamp, oldimo, oldname, \
+                                     oldcallsign, oldtype, \
+                                     olddim_bow, olddim_stern, \
+                                     olddim_port, olddim_starboard, \
+                                     oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
+                                     olddraught, olddestination, oldsource)
+            
+                f.seek(0)
+                f.write(record)
+    # keep the file locked during SQL updates
+    if updated:
+        sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
+                      dim_bow, dim_stern, dim_port, dim_starboard, \
+                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
+    f.close()
+    return updated
+
+
+
+def strmmsi_to_mmsi(strmmsi):
+    """
+    Convert from str mmsi to sql-int mmsi
+    Special treatment manal input
+    """
+    if strmmsi.isdigit():
+        return int(strmmsi)
+    else:
+        assert strmmsi[3:5] == 'MI'
+        strmmsi = strmmsi[:3]+'00'+strmmsi[5:]
+        return int('-'+strmmsi)
+
+
+def mmsi_to_strmmsi(mmsi):
+    """
+    Convert from sql-into mmsi to str mmsi
+    Special treatment manal input
+    """
+    if mmsi >= 0:
+        return "%08d" % mmsi
+    strmmsi = "%08d" % -mmsi
+    assert strmmsi[3:5] == '00'
+    strmmsi = strmmsi[:3]+'MI'+strmmsi[5:]
+    return strmmsi
+
+
+__misources__ = {} # cache of manual source names
+def _get_mi_sourcename(id):
+    """
+    Get the nice name for sources whose id4 starts with 'MI'
+    """
+    global __misources__
+    if not __misources__:
+        sqlexec(u'SELECT id, name FROM mi_source')
+        while True:
+            row = get_common_cursor().fetchone()
+            if row is None:
+                break
+            __misources__[row[0]] = row[1]
+    result = __misources__.get(id, None)
+    if result is None:
+        return u"Manual input #%s" % id
+    return result
+
+
+class Nmea1:
+    def __init__(self, timestamp, status=AIS_STATUS_NOT_AVAILABLE, rot=AIS_ROT_NOT_AVAILABLE, sog=AIS_SOG_NOT_AVAILABLE, latitude=AIS_LAT_NOT_AVAILABLE, longitude=AIS_LON_NOT_AVAILABLE, cog=AIS_COG_NOT_AVAILABLE, heading=AIS_NO_HEADING, source='\x00\x00\x00\x00'):
+        self.timestamp_1 = timestamp
+        self.status      = status
+        self.rot         = rot
+        self.sog         = sog
+        self.latitude    = latitude
+        self.longitude   = longitude
+        self.cog         = cog
+        self.heading     = heading
+        self.source_1    = source
+
+    from_values = __init__
+
+    def to_values(self):
+        return self.timestamp_1, self.status, self.rot, self.sog, self.latitude, self.longitude, self.cog, self.heading, self.source_1
+
+    def from_record(self, record):
+        values = struct.unpack(aivdm_record123_format, record)
+        Nmea1.__init__(self, *values)
+
+    @staticmethod
+    def new_from_record(record):
+        values = struct.unpack(aivdm_record123_format, record)
+        return Nmea1(*values)
+
+    def to_record(self):
+        return struct.pack(aivdm_record123_format, *Nmea1.to_values())
+        
+    def from_file(self, file):
+        record = file.read(aivdm_record123_length)
+        Nmea1.from_record(self, record)
+
+    @staticmethod
+    def new_from_file(file):
+        record = file.read(aivdm_record123_length)
+        return Nmea1.new_from_record(record)
+
+    def from_lastinfo(self, strmmsi):
+        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
+        try:
+            f = file(filename_nmea1, 'rb')
+        except IOError:
+            logging.debug("file %s doesn't exists" % filename_nmea1)
+            return
+        lockf(f, LOCK_SH)
+        Nmea1.from_file(self, f)
+        f.close()
+
+    @staticmethod
+    def new_from_lastinfo(strmmsi):
+        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
+        try:
+            f = file(filename_nmea1, 'rb')
+        except IOError:
+            logging.debug("file %s doesn't exists" % filename_nmea1)
+            return None
+        lockf(f, LOCK_SH)
+        record = f.read(aivdm_record123_length)
+        f.close()
+        return Nmea1.new_from_record(record)
+
+
+    def dump_to_stdout(self):
+        """
+        Prints content to stdout
+        """
+        print datetime.utcfromtimestamp(self.timestamp_1), 
+        for i in (self.status, self.rot, self.sog, self.latitude/AIS_LATLON_SCALE, self.longitude/AIS_LATLON_SCALE, self.cog, self.heading, self.source_1):
+            print repr(i),
+        print
+    @staticmethod
+    def _clean_str(txt):
+        if txt is None:
+            return ''
+        return txt.replace('\0','').replace('@', '').strip()
+
+    def get_status(self, default='Unknown'):
+        return STATUS_CODES.get(self.status, default)
+    def get_sog_str(self, default='Unknown'):
+        if self.sog == AIS_SOG_NOT_AVAILABLE:
+            return default
+        if self.sog == AIS_SOG_FAST_MOVER:
+            return 'over 102.2 kts'
+        return '%.1f kts' % (self.sog/AIS_SOG_SCALE)
+
+    def get_rot_str(self, default='Unknown'):
+        if self.rot == AIS_ROT_NOT_AVAILABLE:
+            return default
+        if self.rot == 0:
+            return 'Not turning'
+        if self.rot < 0:
+            side = 'port'
+        else:
+            side = 'starboard'
+        rot = abs(self.rot)
+        if rot == 127:
+            result = 'To '
+        else:
+            result = '%d %% to ' % rot*100./127
+        return result + side
+
+    @staticmethod
+    def _decimaldegree_to_dms(f, emispheres):
+        if f >= 0:
+            e = emispheres[0]
+        else:
+            f = -f
+            e = emispheres[1]
+        result = '%d°' % int(f)
+        f = (f%1)*60
+        result += '%02.05f\' ' % f
+        result += e
+        return result
+
+    def get_latitude_str(self, default='Unknown'):
+        if self.latitude == AIS_LAT_NOT_AVAILABLE:
+            return default
+        return Nmea1._decimaldegree_to_dms(self.latitude / AIS_LATLON_SCALE, 'NS')
+
+    def get_longitude_str(self, default='Unknown'):
+        if self.longitude == AIS_LON_NOT_AVAILABLE:
+            return default
+        return Nmea1._decimaldegree_to_dms(self.longitude / AIS_LATLON_SCALE, 'EW')
+
+    def get_cog_str(self, default='Unknown'):
+        if self.cog == AIS_COG_NOT_AVAILABLE:
+            return default
+        return '%.1f°' % (self.cog/10.)
+
+    def get_heading_str(self, default='Unknown'):
+        if self.heading == AIS_NO_HEADING:
+            return default
+        return '%s°' % self.heading
+
+    def get_source_1_str(self):
+        return Nmea.format_source(self.source_1)
+
+class Nmea5:
+    def __init__(self, timestamp, imo=0, name='', callsign='', type=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
+        self.timestamp_5   = timestamp
+        self.imo           = imo
+        self.name          = name         
+        self.callsign      = callsign
+        self.type          = type
+        self.dim_bow       = dim_bow
+        self.dim_stern     = dim_stern
+        self.dim_port      = dim_port
+        self.dim_starboard = dim_starboard
+        self.eta_M         = eta_M
+        self.eta_D         = eta_D
+        self.eta_h         = eta_h
+        self.eta_m         = eta_m
+        self.draught       = draught
+        self.destination   = destination
+        self.source_5      = source
+
+    from_values = __init__
+
+    def merge_from_values(self, timestamp, imo=0, name='', callsign='', type=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
+        updated = False
+        if self.imo == 0 or imo != 0:
+            self.imo = imo
+            updated = True
+        if self.name == '' or name != '':
+            self.name = name
+            updated = True
+        if self.callsign == '' or callsign != '':
+            self.callsign = callsign
+            updated = True
+        if self.type == 0 or type != 0:
+            self.type = type
+            updated = True
+        if self.dim_bow == 0 or dim_bow != 0:
+            self.dim_bow = dim_bow
+            updated = True
+        if self.dim_stern == 0 or dim_stern != 0:
+            self.dim_stern = dim_stern
+            updated = True
+        if self.dim_port == 0 or dim_port != 0:
+            self.dim_port = dim_port
+            updated = True
+        if self.dim_starboard == 0 or dim_starboard != 0:
+            self.dim_starboard = dim_starboard
+            updated = True
+        if (self.eta_M == 0 and self.eta_D == 0 and self.eta_h == 24 and self.eta_m == 60) or eta_M != 0 or eta_D != 0 or eta_h != 24 or eta_m != 60:
+            self.eta_M = eta_M
+            self.eta_D = eta_D
+            self.eta_h = eta_h
+            self.eta_m = eta_m
+            updated = True
+        if self.draught == 0 or draught != 0:
+            self.draught = draught
+            updated = True
+        if self.destination == '' or destination != '':
+            self.destination = destination
+            updated = True
+        if updated:
+            self.timestamp_5 = timestamp
+            self.source_5 = source
+        return updated
+
+    def to_values(self):
+        return self.timestamp_5, self.imo, self.name, self.callsign, self.type, self.dim_bow, self.dim_stern, self.dim_port, self.dim_starboard, self.eta_M, self.eta_D, self.eta_h, self.eta_m, self.draught, self.destination, self.source_5
+
+    def from_record(self, record):
+        values = struct.unpack(aivdm_record5_format, record)
+        Nmea5.__init__(self, *values)
+
+    @staticmethod
+    def new_from_record(record):
+        values = struct.unpack(aivdm_record5_format, record)
+        return Nmea5(*values)
+
+    def to_record(self):
+        return struct.pack(aivdm_record5_format, *Nmea5.to_values(self))
+        
+    def from_file(self, file):
+        record = file.read(aivdm_record5_length)
+        Nmea5.from_record(self, record)
+
+    @staticmethod
+    def new_from_file(file):
+        record = file.read(aivdm_record5_length)
+        return Nmea5.new_from_record(record)
+
+    def from_lastinfo(self, strmmsi):
+        filename_nmea5 = os.path.join(DBPATH,
+                                      'last',
+                                      _hash3_pathfilename(strmmsi+'.nmea5'))
+        try:
+            f = file(filename_nmea5, 'rb')
+        except IOError:
+            logging.debug("file %s doesn't exists" % filename_nmea5)
+            return
+        lockf(f, LOCK_SH)
+        Nmea5.from_file(self, f)
+        f.close()
+
+    @staticmethod
+    def new_from_lastinfo(strmmsi):
+        filename_nmea5 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea5')
+        try:
+            f = file(filename_nmea5, 'rb')
+        except IOError:
+            logging.debug("file %s doesn't exists" % filename_nmea5)
+            return None
+        lockf(f, LOCK_SH)
+        record = f.read(aivdm_record5_length)
+        f.close()
+        return Nmea5.new_from_record(record)
+
+    @staticmethod
+    def _clean_str(txt):
+        if txt is None:
+            return ''
+        return txt.replace('\0','').replace('@', '').strip()
+
+    def get_name(self, default='Unknown'):
+        result = self._clean_str(self.name)
+        if result:
+            return result
+        return default
+
+    def get_callsign(self, default='Unknown'):
+        return self._clean_str(self.callsign) or default
+
+    def get_shiptype(self, default='Unknown'):
+        return SHIP_TYPES.get(self.type, default)
+
+    def get_length(self):
+        return self.dim_bow + self.dim_stern
+
+    def get_width(self):
+        return self.dim_port + self.dim_starboard
+
+    _monthes = 'Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec'.split(',')
+    def get_eta_str(self, default='Unknown'):
+        if not self.eta_M and not self.eta_D:
+            return default
+        result = ''
+        if self.eta_M:
+            if self.eta_M <= len(Nmea5._monthes):
+                result += Nmea5._monthes[self.eta_M - 1]
+            else:
+                result += '%02d' % self.eta_M
+        else:
+            result += '***'
+        result += ' '
+        if self.eta_D:
+            result += '%02d' % self.eta_D
+        else:
+            result += '**'
+        if self.eta_h != 24:
+            result += ' %02d' % self.eta_h
+            if self.eta_m == 60:
+                result += 'h'
+            else:
+                result += ':%02d' % self.eta_m
+        return result
+    
+    def get_draught_str(self, default='Unknown'):
+        if not self.draught:
+            return default
+        return '%.1f meters' % (self.draught/10.)
+
+    def get_destination(self, default='Unknown'):
+        return self._clean_str(self.destination) or default
+
+    def get_source_5_str(self):
+        return Nmea.format_source(self.source_5)
+
+class Nmea(Nmea1, Nmea5):
+    """
+    This is nmea info, a merge of nmea1 and nmea5 packets
+    """
+    def __init__(self, strmmsi):
+        self.strmmsi = strmmsi
+        Nmea1.__init__(self, timestamp=0)
+        Nmea5.__init__(self, timestamp=0)
+
+    ########################
+    # Because of multiple inheritance some functions are unavailable:
+    def _nmea_not_implemented(*args, **kargs):
+        # used to avoid conflicting inherited members
+        raise NotImplementedError
+    from_values = _nmea_not_implemented
+    to_values = _nmea_not_implemented
+    from_record = _nmea_not_implemented
+    new_from_record = _nmea_not_implemented
+    to_record = _nmea_not_implemented
+    from_file = _nmea_not_implemented
+    new_from_file = _nmea_not_implemented
+    ########################
+
+    def from_lastinfo(self, strmmsi):
+        Nmea1.from_lastinfo(self, strmmsi)
+        Nmea5.from_lastinfo(self, strmmsi)
+    
+    @staticmethod
+    def new_from_lastinfo(strmmsi):
+        # better than unimplemented, but not optimal
+        nmea = Nmea(strmmsi)
+        nmea.from_lastinfo(strmmsi)
+        return nmea
+
+
+    def get_flag(self, default=u'Unknown'):
+        if self.strmmsi.startswith('00') and self.strmmsi[3:5]!='MI':
+            ref_mmsi = self.strmmsi[2:]
+        else:
+            ref_mmsi = self.strmmsi
+        country_mid = int(ref_mmsi[0:3])
+        country_name = COUNTRIES_MID.get(country_mid, default)
+        return country_name
+
+    def get_mmsi_public(self, default='Unknown'):
+        if self.strmmsi.isdigit():
+            return self.strmmsi
+        return default
+
+    def get_title(self):
+        """
+        Returns the name of the ship if available
+        Or its mmsi
+        """
+        return self.get_name(None) or self.get_mmsi_public()
+
+    def get_last_timestamp(self):
+        """
+        Returns the most recent of update from timestamp1, timestamp5
+        """
+        if self.timestamp_1 > self.timestamp_5:
+            return self.timestamp_1
+        else:
+            return self.timestamp_5
+
+    def get_last_updated_str(self):
+        """
+        Returns a pretty formated update data as a string
+        """
+        lastupdate = self.get_last_timestamp()
+        if lastupdate == 0:
+            return u'Never'
+        dt_lastupdate = datetime.utcfromtimestamp(lastupdate)
+        delta = datetime.utcnow() - dt_lastupdate
+        def nice_timedelta_str(delta):
+            strdelta = ''
+            if delta.days:
+                strdelta += str(delta.days)
+                if delta.days > 1:
+                    strdelta += ' days '
+                else:
+                    strdelta += ' day '
+            delta_s = delta.seconds
+            delta_m = delta_s / 60
+            delta_s -= delta_m * 60
+            delta_h = delta_m / 60
+            delta_m -= delta_h * 60
+
+            if delta_h:
+                strdelta += str(delta_h)
+                if delta_h > 1:
+                    strdelta += ' hours '
+                else:
+                    strdelta += ' hour '
+            if delta_m:
+                strdelta += str(delta_m)
+                if delta_m > 1:
+                    strdelta += ' minutes '
+                else:
+                    strdelta += ' minute '
+            if delta_s:
+                strdelta += str(delta_s)
+                if delta_s > 1:
+                    strdelta += ' seconds '
+                else:
+                    strdelta += ' second '
+            if not strdelta:
+                strdelta = 'less than a second '
+            strdelta += ' ago'
+            return strdelta
+        return nice_timedelta_str(delta) + ' (' + dt_lastupdate.strftime('%Y-%m-%d %H:%M:%S GMT') + ')'
+
+    @staticmethod
+    def format_source(infosrc):
+        if infosrc == '\0\0\0\0':
+            return u'(empty)'
+        elif infosrc.startswith('MI'):
+            if len(infosrc) == 4:
+                return _get_mi_sourcename(struct.unpack('<2xH', infosrc)[0])
+            else:
+                return u'Manual input'
+        elif infosrc.startswith('U'):
+            return u'User input'
+        elif infosrc.startswith('NM'):
+            return u'NMEA packets from '+xml_escape(infosrc[2:])
+        elif infosrc.startswith('SP'):
+            return u"ShipPlotter user %s" % infosrc[2:]
+        elif infosrc == u'MTWW':
+            return u'MarineTraffic.com web site'
+        elif infosrc == u'MTTR':
+            return u'MarineTraffic.com track files'
+        else:
+            return infosrc
+
+    csv_headers = [
+        'mmsi',
+        'flag',
+        'name',
+        'imo',
+        'callsign',
+        'type',
+        'length',
+        'width',
+        'datetime',
+        'status',
+        'sog',
+        'latitude',
+        'longitude',
+        'cog',
+        'heading',
+        'destination',
+        'eta',
+        'draught',
+        ]
+
+    def get_dump_row(self):
+        result = []
+        def _clean(txt):
+            if txt is None:
+                return ''
+            return txt.replace('\0','').replace('@', '').strip()
+        result.append(self.strmmsi)
+        country_mid = int(self.strmmsi[:3])
+        country_name = COUNTRIES_MID.get(country_mid, u'unknown')
+        result.append(country_name.encode('utf-8'))
+        result.append(_clean(self.name))
+        result.append(str(self.imo))
+        result.append(_clean(self.callsign))
+        result.append(str(self.type) + '-' + SHIP_TYPES.get(self.type, 'unknown'))
+        d = self.dim_bow + self.dim_stern
+        if d:
+            result.append(d)
+        else:
+            result.append(None)
+        d = self.dim_port + self.dim_starboard
+        if d:
+            result.append(d)
+        else:
+            result.append(None)
+        result.append(datetime.utcfromtimestamp(self.timestamp_1).strftime('%Y-%m-%dT%H:%M:%SZ'))
+        result.append(STATUS_CODES.get(self.status, 'unknown'))
+        if self.sog != AIS_SOG_NOT_AVAILABLE:
+            result.append(str(self.sog/AIS_SOG_SCALE))
+        else:
+            result.append(None)
+        if self.latitude != AIS_LAT_NOT_AVAILABLE:
+            result.append(str(self.latitude/AIS_LATLON_SCALE))
+        else:
+            result.append(None)
+        if self.longitude != AIS_LON_NOT_AVAILABLE:
+            result.append(str(self.longitude/AIS_LATLON_SCALE))
+        else:
+            result.append(None)
+        if self.cog != AIS_COG_NOT_AVAILABLE:
+            result.append(str(self.cog/10.))
+        else:
+            result.append(None)
+        if self.heading != AIS_NO_HEADING:
+            result.append(str(self.heading))
+        else:
+            result.append(None)
+        result.append(self.get_destination(''))
+        result.append(self.get_eta_str(''))
+        result.append(self.draught)
+        result.append(self.source_5)
+        return result
+
+
+class BankNmea1(list):
+    """
+    That class handle a .nmea1 archive file
+    """
+    def __init__(self, strmmsi, dt):
+        list.__init__(self)
+        self.strmmsi = strmmsi
+        if isinstance(dt, date):
+            dt = dt.strftime('%Y%m%d')
+        self.date = dt
+
+    def get_filename(self):
+        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea1'))
+
+    def __load_from_file(self, file):
+        '''
+        Adds all record from opened file in this bank
+        File must be locked before call
+        '''
+        while True:
+            record = file.read(aivdm_record123_length)
+            if not record:
+                break
+            self.append(Nmea1.new_from_record(record))
+
+    def _write_in_file(self, file):
+        '''
+        Write all records from that bank in opened file
+        File must be locked before call
+        File should be truncated after call
+        '''
+        for nmea1 in self:
+            file.write(nmea1.to_record())
+
+    def __load(self):
+        try:
+            file = open(self.get_filename(), 'rb')
+            lockf(file, LOCK_SH)
+        except IOError, ioerr:
+            if ioerr.errno == 2: # No file
+                return
+            raise
+        self.__load_from_file(file)
+        file.close()
+        
+    def __iter__(self):
+        """
+        Each call reload the file
+        """
+        self.__load()
+        self.sort_by_date_reverse()
+        return list.__iter__(self)
+
+    def packday(remove_manual_input=False):
+        # FIXME broken
+        #print "MMSI", strmmsi
+
+        self = BankNmea1(self.strmmsi, self.date)
+        filename = self.get_filename()
+        try:
+            file = open(filename, 'r+b') # read/write binary
+        except IOError, ioerr:
+            if ioerr.errno != 2: # No file
+                raise
+            return self # no data
+        lockf(file, LOCK_EX)
+        self.__load_from_file(file)
+        self.sort_by_date()
+
+        file_has_changed = False
+        file_must_be_unlinked = False
+
+        #print "PACKING..."
+        file_has_changed = self.remove_duplicate_timestamp() or file_has_changed
+
+        if remove_manual_input:
+            #print "REMOVING MANUAL INPUT..."
+            file_has_changed = self.remove_manual_input() or file_has_changed
+
+        if file_has_changed:
+            file.seek(0)
+            self._write_in_file(file)
+            file.truncate()
+            if file.tell() == 0:
+                file_must_be_unlinked = True
+
+        file.close()
+        
+        if file_must_be_unlinked:
+            # FIXME we release the lock before unlinking
+            # another process might encounter an empty file (not handled)
+            logging.warning('file was truncated to size 0. unlinking')
+            os.unlink(filename) # we have the lock (!)
+
+    def dump_to_stdout(self):
+        """
+        Print contents to stdout
+        """
+        for nmea1 in self:
+            nmea1.dump_to_stdout()
+
+    def sort_by_date(self):
+        self.sort(lambda n1, n2: n1.timestamp_1 - n2.timestamp_1)
+
+    def sort_by_date_reverse(self):
+        self.sort(lambda n1, n2: n2.timestamp_1 - n1.timestamp_1)
+
+    def remove_duplicate_timestamp(self):
+        file_has_changed = False
+        if len(self) <= 1:
+            return file_has_changed
+        last_timestamp = self[0].timestamp_1
+        i = 1
+        while i < len(self):
+            if self[i].timestamp_1 == last_timestamp:
+                del self[i]
+                file_has_changed = True
+            else:
+                last_timestamp = self[i].timestamp_1
+                i += 1
+        return file_has_changed
+        
+    def remove_manual_input(self):
+        file_has_changed = False
+        i = 0
+        while i < len(self):
+            if self[i].source_1[:2] == 'MI':
+                del self[i]
+                file_has_changed = True
+            else:
+                i += 1
+        return file_has_changed
+
+class Nmea1Feeder:
+    """
+    Yields all nmea1 packets between two given datetimes
+    in REVERSE order (recent information first)
+    """
+    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
+        self.strmmsi = strmmsi
+        assert datetime_end is not None
+        self.datetime_end = datetime_end
+        self.datetime_begin = datetime_begin or DB_STARTDATE
+        self.max_count = max_count
+
+    def __iter__(self):
+        dt_end = self.datetime_end
+        d_end = dt_end.date()
+        ts_end = datetime_to_timestamp(dt_end)
+        if self.datetime_begin:
+            dt_begin = self.datetime_begin
+            d_begin = dt_begin.date()
+            ts_begin = datetime_to_timestamp(dt_begin)
+        else:
+            dt_begin = None
+            d_begin = None
+            ts_begin = None
+
+        d = d_end
+        count = 0
+        while True:
+            if d_begin is not None and d < d_begin:
+                return
+            bank = BankNmea1(self.strmmsi, d)
+            for nmea1 in bank:
+                if ts_begin is not None and nmea1.timestamp_1 < ts_begin:
+                    return
+                if nmea1.timestamp_1 > ts_end:
+                    continue
+                
+                yield nmea1
+               
+                count += 1
+                if self.max_count and count >= self.max_count:
+                    return
+            d += timedelta(-1)
+
+
+class BankNmea5(list):
+    """
+    That class handle a .nmea5 archive file
+    """
+    def __init__(self, strmmsi, dt):
+        list.__init__(self)
+        self.strmmsi = strmmsi
+        if isinstance(dt, date):
+            try:
+                dt = dt.strftime('%Y%m%d')
+            except ValueError:
+                logging.critical('dt=%s', dt)
+                raise
+        self.date = dt
+
+    def get_filename(self):
+        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea5'))
+
+    def __load_from_file(self, file):
+        '''
+        Adds all record from opened file in this bank
+        File must be locked before call
+        '''
+        while True:
+            record = file.read(aivdm_record5_length)
+            if not record:
+                break
+            self.append(Nmea5.new_from_record(record))
+
+    def _write_in_file(self, file):
+        '''
+        Write all records from that bank in opened file
+        File must be locked before call
+        File should be truncated after call
+        '''
+        for nmea5 in self:
+            file.write(nmea5.to_record())
+
+    def __load(self):
+        try:
+            file = open(self.get_filename(), 'rb')
+            lockf(file, LOCK_SH)
+        except IOError, ioerr:
+            if ioerr.errno == 2: # No file
+                return
+            raise
+        self.__load_from_file(file)
+        file.close()
+        
+    def __iter__(self):
+        """
+        Each call reload the file
+        """
+        self.__load()
+        self.sort_by_date_reverse()
+        return list.__iter__(self)
+
+    def sort_by_date(self):
+        self.sort(lambda n1, n2: n1.timestamp_5 - n2.timestamp_5)
+
+    def sort_by_date_reverse(self):
+        self.sort(lambda n1, n2: n2.timestamp_5 - n1.timestamp_5)
+
+class Nmea5Feeder:
+    """
+    Yields all nmea5 packets between two given datetimes
+    in REVERSE order (recent information first)
+    """
+    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
+        self.strmmsi = strmmsi
+        assert datetime_end is not None
+        self.datetime_end = datetime_end
+        self.datetime_begin = datetime_begin or DB_STARTDATE
+        self.max_count = max_count
+
+    def __iter__(self):
+        dt_end = self.datetime_end
+        d_end = dt_end.date()
+        ts_end = datetime_to_timestamp(dt_end)
+        if self.datetime_begin:
+            dt_begin = self.datetime_begin
+            d_begin = dt_begin.date()
+            ts_begin = datetime_to_timestamp(dt_begin)
+        else:
+            dt_begin = None
+            d_begin = None
+            ts_begin = None
+
+        d = d_end
+        count = 0
+        while True:
+            if d_begin is not None and d < d_begin:
+                return
+            bank = BankNmea5(self.strmmsi, d)
+            for nmea1 in bank:
+                if ts_begin is not None and nmea1.timestamp_5 < ts_begin:
+                    return
+                if nmea1.timestamp_5 > ts_end:
+                    continue
+                
+                yield nmea1
+               
+                count += 1
+                if self.max_count and count >= self.max_count:
+                    return
+            d += timedelta(-1)
+
+
+class NmeaFeeder:
+    """
+    Yields nmea packets matching criteria.
+    """
+    def __init__(self, strmmsi, datetime_end, datetime_begin=None, filters=None, granularity=1, max_count=None):
+        if granularity <= 0:
+            logging.warning('Granularity=%d generates duplicate entries', granularity)
+        self.strmmsi = strmmsi
+        assert datetime_end is not None
+        self.datetime_end = datetime_end
+        self.datetime_begin = datetime_begin or DB_STARTDATE
+        self.filters = filters or []
+        self.granularity = granularity
+        self.max_count = max_count
+
+    def __iter__(self):
+        nmea = Nmea(self.strmmsi)
+        if self.datetime_begin:
+            nmea5_datetime_begin = self.datetime_begin - timedelta(30) # go back up to 30 days to get a good nmea5 packet
+        else:
+            nmea5_datetime_begin = None
+        nmea5_iterator = Nmea5Feeder(self.strmmsi, self.datetime_end, nmea5_datetime_begin).__iter__()
+        nmea5 = Nmea5(self.strmmsi, sys.maxint)
+
+        count = 0
+        lasttimestamp = sys.maxint
+        for nmea1 in Nmea1Feeder(self.strmmsi, self.datetime_end, self.datetime_begin):
+            Nmea1.from_values(nmea, *nmea1.to_values())
+            
+            # try to get an nmea5 paket older
+            nmea5_updated = False
+            while nmea5 is not None and nmea5.timestamp_5 > nmea1.timestamp_1:
+                try:
+                    nmea5 = nmea5_iterator.next()
+                    nmea5_updated = True
+                except StopIteration:
+                    nmea5 = None
+            
+            if nmea5_updated and nmea5 is not None:
+                Nmea5.merge_from_values(nmea, *nmea5.to_values())
+
+            filtered_out = False
+            for is_ok in self.filters:
+                if not is_ok(nmea):
+                    filtered_out = True
+                    break
+            if filtered_out:
+                continue
+
+            if nmea.timestamp_1 <= lasttimestamp - self.granularity:
+                yield nmea
+                count += 1
+                if self.max_count and count >= self.max_count:
+                    return
+                lasttimestamp = nmea.timestamp_1
+
+
+def all_mmsi_generator():
+    """
+    Returns an array of all known strmmsi.
+    """
+    for dirname, dirs, fnames in os.walk(os.path.join(DBPATH, 'last')):
+        for fname in fnames:
+            if fname[-6:] == '.nmea1':
+                yield fname[:-6]
+
+
+def load_fleet_to_uset(fleetname):
+    """
+    Loads a fleet by name-id.
+    Returns an array of strmmsi.
+    """
+    result = []
+    sqlexec(u"SELECT mmsi FROM fleet_vessel WHERE fleet=%(fleetname)s", {'fleetname': fleetname})
+    cursor = get_common_cursor()
+    while True:
+        row = cursor.fetchone()
+        if not row:
+            break
+        mmsi = row[0]
+        result.append(mmsi_to_strmmsi(mmsi))
+    logging.debug('fleet=%s', result)
+    return result
+
+
+def filter_area(nmea, area):
+    """
+    Returns false if position is out of area.
+    """
+    if nmea.latitude == AIS_LAT_NOT_AVAILABLE or nmea.longitude == AIS_LON_NOT_AVAILABLE:
+        return False
+    if not area.contains((nmea.latitude/AIS_LATLON_SCALE, nmea.longitude/AIS_LATLON_SCALE)):
+        return False
+    return True
+
+def filter_knownposition(nmea):
+    """
+    Returns false if position is not fully known
+    """
+    # we are filtering out latitude=0 and longitude=0, that is not supposed to be necessary...
+    return nmea.latitude != AIS_LAT_NOT_AVAILABLE and nmea.longitude != AIS_LON_NOT_AVAILABLE and nmea.latitude != 0 and nmea.longitude != 0
+
+
+_filter_positioncheck_last_mmsi = None
+def filter_speedcheck(nmea, max_mps):
+    """
+    mps is miles per seconds
+    """
+    global _filter_positioncheck_last_mmsi
+    global _filter_positioncheck_last_time
+    global _filter_positioncheck_last_time_failed
+    global _filter_positioncheck_last_lat
+    global _filter_positioncheck_last_lon
+    global _filter_positioncheck_error_count
+    if nmea.strmmsi != _filter_positioncheck_last_mmsi:
+        _filter_positioncheck_last_time = None
+        _filter_positioncheck_last_mmsi = nmea.strmmsi
+        _filter_positioncheck_error_count = 0
+    if _filter_positioncheck_last_time is not None:
+        seconds = _filter_positioncheck_last_time - nmea.timestamp_1
+        distance = dist3_latlong_ais((_filter_positioncheck_last_lat, _filter_positioncheck_last_lon), (nmea.latitude, nmea.longitude))
+        if seconds:
+            speed = distance/seconds
+            if speed > max_mps:
+                if _filter_positioncheck_error_count < 10:
+                    logging.debug("Ignoring point: distance = %s, time = %s, speed = %s kt, source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
+                    if _filter_positioncheck_error_count == 0 or _filter_positioncheck_last_time_failed != nmea.timestamp_1:
+                        _filter_positioncheck_error_count += 1
+                        _filter_positioncheck_last_time_failed = nmea.timestamp_1
+                    return False
+                else:
+                    logging.warning("Discontinous position accepted after too many failures: %.2f nm in %s s (%.0f kt), source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
+            _filter_positioncheck_error_count = 0
+    _filter_positioncheck_last_time = nmea.timestamp_1
+    _filter_positioncheck_last_lat = nmea.latitude
+    _filter_positioncheck_last_lon = nmea.longitude
+    return True
+
+
+def main():
+    """
+    Perform various operation on the database
+    For usage, see "ais --help"
+    """
+    from optparse import OptionParser, OptionGroup
+    global DBPATH
+
+    parser = OptionParser(usage='%prog [options] { mmsi | @fleet }+ | all')
+
+    parser.add_option('-d', '--debug',
+        action='store_true', dest='debug', default=False,
+        help="debug mode")
+
+    parser.add_option('-e', '--end',
+        action='store', dest='sdt_end', metavar="'YYYYMMDD HHMMSS'",
+        help='End data processing on that GMT date time.'
+             'Default is now.'
+             'If a date is provided without time, time defaults to 235959.')
+    parser.add_option('-s', '--start',
+        action='store', dest='sdt_start', metavar="'YYYYMMDD HHMMSS'",
+        help='Start data processing on that date.'
+             'Using that option enables multiple output of the same boat.'
+             'Disabled by default.'
+             'If a date is provided without time, time default to 000000.'
+             'If other options enable multiple output, default to 1 day before'
+             ' --end date/time.')
+    parser.add_option('-g', '--granularity',
+        action='store', type='int', dest='granularity', metavar='SECONDS',
+        help='Dump only one position every granularity seconds.'
+             'Using that option enables multiple output of the same boat.'
+             'If other options enable multiple output, defaults to 600'
+             ' (10 minutes)')
+    parser.add_option('--max',
+        action='store', type='int', dest='max_count', metavar='NUMBER',
+        help='Dump a maximum of NUMBER positions every granularity seconds.'
+             'Using that option enables multiple output of the same boat.')
+
+    parser.add_option('--filter-knownposition',
+        action='store_true', dest='filter_knownposition', default=False,
+        help="Eliminate unknown positions from results.")
+
+    parser.add_option('--filter-speedcheck',
+        action='store', type='int', dest='speedcheck', default=200, metavar='KNOTS',
+        help='Eliminate erroneaous positions from results,' 
+             ' based on impossible speed.')
+
+    parser.add_option('--filter-type',
+        action='append', type='int', dest='type_list', metavar='TYPE',
+        help="process a specific ship type.")
+    parser.add_option('--help-types',
+        action='store_true', dest='help_types', default=False,
+        help="display list of available types")
+
+    parser.add_option('--filter-area',
+        action='store', type='str', dest='area_file', metavar="FILE.KML",
+        help="only process a specific area as defined in a kml polygon file.")
+
+    parser.add_option('--filter-destination',
+        action='store', type='str', dest='filter_destination', metavar="DESTINATION",
+        help="Only print ships with that destination.")
+
+    parser.add_option('--no-headers',
+        action='store_false', dest='csv_headers', default=True,
+        help="skip CSV headers")
+    #
+
+    expert_group = OptionGroup(parser, "Expert Options",
+        "You normaly don't need any of these")
+
+    expert_group.add_option('--db',
+        action='store', dest='db', default=DBPATH,
+        help="path to filesystem database. Default=%default")
+
+    expert_group.add_option('--debug-sql',
+        action='store_true', dest='debug_sql', default=False,
+        help="print all sql queries to stdout before running them")
+
+    expert_group.add_option('--action',
+        choices=('dump', 'removemanual', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump',
+        help='Possible values are:\n'
+            'dump: dump values in csv format. This is the default.\n'
+            'removemanual: Delete Manual Input entries from the database.\n'
+            'mmsidump: Dump mmsi')
+    parser.add_option_group(expert_group)
+
+    (options, args) = parser.parse_args()
+
+
+    if options.help_types:
+        print "Known ship types:"
+        keys = SHIP_TYPES.keys()
+        keys.sort()
+        for k in keys:
+            print k, SHIP_TYPES[k]
+        sys.exit(0)
+
+    DBPATH = options.db
+
+    if options.debug:
+        loglevel = logging.DEBUG
+    else:
+        loglevel = logging.INFO
+    logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s')
+
+    if options.debug_sql:
+        sql_setdebug(True)
+
+    #
+    # Ships selections
+    #
+
+    if len(args)==0:
+        print >> sys.stderr, "No ship to process"
+        sys.exit(1)
+
+    target_mmsi_iterator = []
+    all_targets = False
+    for arg in args:
+        if arg == 'all':
+            all_targets = True
+        elif arg.startswith('@'):
+            target_mmsi_iterator += load_fleet_to_uset(arg[1:])
+        else:
+            target_mmsi_iterator.append(arg)
+    if all_targets:
+        if target_mmsi_iterator:
+            logging.warning('Selecting all ships, ignoring other arguments')
+        target_mmsi_iterator = all_mmsi_generator()
+
+    #
+    # Dates selections
+    #
+
+    if options.sdt_end:
+        # remove non digit characters
+        options.sdt_end = "".join([ c for c in options.sdt_end if c.isdigit()])
+        if len(options.sdt_end)==14:
+            dt_end = datetime.strptime(options.sdt_end, '%Y%m%d%H%M%S')
+        elif len(options.sdt_end)==8:
+            dt_end = datetime.strptime(options.sdt_end, '%Y%m%d')
+            dt_end = datetime.combine(dt_end.date(), time(23,59,59))
+        else:
+            print >> sys.stderr, "Invalid format for --end option"
+            sys.exit(1)
+    else:
+        dt_end = datetime.utcnow()
+    logging.debug('--end is %s', dt_end)
+
+    if options.sdt_start or options.granularity is not None or options.max_count:
+        # time period is enabled
+        if options.sdt_start:
+            options.sdt_start = "".join([ c for c in options.sdt_start if c.isdigit()])
+            if len(options.sdt_start)==14:
+                dt_start = datetime.strptime(options.sdt_start, '%Y%m%d%H%M%S')
+            elif len(options.sdt_start)==8:
+                dt_start = datetime.strptime(options.sdt_start, '%Y%m%d')
+            else:
+                print >> sys.stderr, "Invalid format for --start option"
+                sys.exit(1)
+        else:
+            dt_start = dt_end - timedelta(1)
+        if options.granularity is None:
+            options.granularity = 600
+    else:
+        dt_start = None
+        options.max_count = 1
+        if options.granularity is None:
+            options.granularity = 600
+    logging.debug('--start is %s', dt_start)
+
+    #
+    # Filters
+    #
+
+    filters = []
+    
+    if options.filter_knownposition:
+        filters.append(filter_knownposition)
+
+    if options.speedcheck != 0:
+        maxmps = options.speedcheck / 3600. # from knots to NM per seconds
+        filters.append(lambda nmea: filter_speedcheck(nmea, maxmps))
+
+    if options.area_file:
+        area = load_area_from_kml_polygon(options.area_file)
+        filters.append(lambda nmea: filter_area(nmea, area))
+    
+    if options.type_list:
+        def filter_type(nmea):
+            return nmea.type in options.type_list
+        filters.append(filter_type)
+
+    if options.filter_destination:
+        filters.append(lambda nmea: nmea.destination.startswith(options.filter_destination))
+
+    #
+    # Processing
+    #
+
+    if options.action == 'dump':
+        output = csv.writer(sys.stdout)
+        if options.csv_headers:
+            output.writerow(Nmea.csv_headers)
+        for mmsi in target_mmsi_iterator:
+            logging.debug('Considering %s', repr(mmsi))
+            assert dt_end is not None
+            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
+                output.writerow(nmea.get_dump_row())
+
+    elif options.action == 'removemanual':
+        if filters:
+            print >> sys.stderr, "removemanual action doesn't support filters"
+            sys.exit(1)
+
+        for dt in dates:
+            logging.info("Processing date %s", dt)
+            for mmsi in target_mmsi_iterator:
+                BankNmea1(mmsi, dt).packday(remove_manual_input=True)
+    
+    elif options.action == 'mmsidump':
+        for strmmsi in target_mmsi_iterator :
+            print strmmsi
+
+    elif options.action == 'fixdestination':
+        for mmsi in target_mmsi_iterator:
+            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
+                destination = nmea.destination.rstrip(' @\0')
+                if destination:
+                    sqlexec(u'UPDATE vessel SET destination = %(destination)s WHERE mmsi=%(mmsi)s AND destination IS NULL', {'mmsi':strmmsi_to_mmsi(mmsi), 'destination':destination})
+                    logging.info('%s -> %s', mmsi, repr(destination))
+                    dbcommit()
+                    break # go to next mmsi
+
+
+if __name__ == '__main__':
+    main()
index 73dfdaff3dbf2e565e7de190d1c4d3fc4c1bc8e4..32f58831648dd2457432e0e788f2025a71d98325 100755 (executable)
--- a/bin/dj.py
+++ b/bin/dj.py
@@ -1,7 +1,7 @@
 #!/usr/bin/python
 from django.core.management import execute_manager
 try:
-    from djais import settings
+    from ais.djais import settings
 except ImportError:
     import sys
     sys.stderr.write("Error: Can't find the file 'settings.py' in the directory containing %r. It appears you've customized things.\nYou'll have to run django-admin.py, passing it your settings module.\n(If the file settings.py does indeed exist, it's causing an ImportError somehow.)\n" % __file__)
index beeebf43be14a15dd3e6882cce9716f94d98314e..9af87a8f89d0d395f091123d64d2ce9b8c9370f2 100644 (file)
@@ -1,15 +1,15 @@
 # -*- coding: utf-8 -*-
-#
-#Example usage:
-#def auth(username, password):
-#    return (username,password)==('me', 'secret')
-# FIX: should return User object
-#
-#@http_authenticate(auth, 'myrealm')
-#def myview(request):
-#    return HttpResponse("Hello world!")
+'''
+Example usage:
+def auth(username, password):
+    return (username,password)==('me', 'secret')
+ FIX: should return User object
 
-from django.http import *
+@http_authenticate(auth, 'myrealm')
+def myview(request):
+    return HttpResponse("Hello world!")
+'''
+from django.http import HttpResponse
 import base64
 
 
@@ -30,11 +30,12 @@ class HttpResponseAuthenticate(HttpResponse):
 
 
 class http_authenticate:
-    """ Decorator that check authorization.
+    '''
+    Decorator that check authorization.
         Parameters:
             passwd_checker(username,password): function that must return True if the username is recognised.
             realm: string with the realm. See rfc1945.
-    """
+    '''
     def __init__(self, passwd_checker, realm):
         self.passwd_checker = passwd_checker
         self.realm = realm
@@ -45,10 +46,12 @@ class http_authenticate:
             if not 'HTTP_AUTHORIZATION' in request.META:
                 username, password = "", ""
                 if not self.passwd_checker(username, password):
-                    return HttpResponseAuthenticate("Password required", realm=self.realm)
+                    return HttpResponseAuthenticate("Password required",
+                                                    realm=self.realm)
             else:
                 auth = request.META['HTTP_AUTHORIZATION']
-                assert auth.startswith('Basic '), "Invalid authentification scheme"
+                assert auth.startswith('Basic '), \
+                    'Invalid authentification scheme'
                 username, password = base64.decodestring(auth[len('Basic '):]).split(':', 2)
                 user =  self.passwd_checker(username, password)
                 if not user:
index 0dea2e74739cfcedc64b999cd7080964dd661a18..df45c79a9feb8671d382804a512a3f816b628519 100644 (file)
@@ -1,7 +1,8 @@
 # -*- coding: utf-8 -*-
 from django.db import models
 from django.contrib.auth.models import get_hexdigest
-from ais import Nmea, mmsi_to_strmmsi
+
+from ais.common import Nmea, mmsi_to_strmmsi
 
 class User(models.Model):
     id = models.AutoField(primary_key=True)
index 0b59f30e345f4146e9e2cd4f5293fad4e9e5efc2..35dc2d02cd6580cb5f13fcef7a189b64a8a181f0 100644 (file)
@@ -1,6 +1,6 @@
 # -*- encofing: utf8 -*-
 from django import template
-from ais import mmsi_to_strmmsi
+from ais.common import mmsi_to_strmmsi
 
 register = template.Library()
 
index 7749632716ce4d3778fd767411911027ed652da0..9ecdb996910ba808811a990c171b91736094418b 100644 (file)
@@ -1,5 +1,5 @@
 from django.conf.urls.defaults import *
-import djais
+import ais.djais
 
 # Uncomment the next two lines to enable the admin:
 # from django.contrib import admin
index 1336842952fc7819eb0d83f0030252c745d3b8bb..7734659abe84c7716cee226e4f25b9925d2417e8 100644 (file)
@@ -14,12 +14,12 @@ from django.db import IntegrityError
 
 from decoratedstr import remove_decoration
 
-from basicauth import http_authenticate
-from models import *
-from show_targets_ships import *
-from ais import Nmea, NmeaFeeder, strmmsi_to_mmsi, SHIP_TYPES, STATUS_CODES, AIS_STATUS_NOT_AVAILABLE, AIS_ROT_NOT_AVAILABLE, AIS_LATLON_SCALE, AIS_LON_NOT_AVAILABLE, AIS_LAT_NOT_AVAILABLE, AIS_COG_SCALE, AIS_COG_NOT_AVAILABLE, AIS_NO_HEADING, AIS_SOG_SCALE, AIS_SOG_NOT_AVAILABLE, AIS_SOG_MAX_SPEED, add_nmea1, add_nmea5_partial, load_fleet_to_uset
-from ntools import datetime_to_timestamp, clean_ais_charset
-from inputs.stats import STATS_DIR
+from ais.djais.basicauth import http_authenticate
+from ais.djais.models import *
+from ais.show_targets_ships import *
+from ais.common import Nmea, NmeaFeeder, strmmsi_to_mmsi, SHIP_TYPES, STATUS_CODES, AIS_STATUS_NOT_AVAILABLE, AIS_ROT_NOT_AVAILABLE, AIS_LATLON_SCALE, AIS_LON_NOT_AVAILABLE, AIS_LAT_NOT_AVAILABLE, AIS_COG_SCALE, AIS_COG_NOT_AVAILABLE, AIS_NO_HEADING, AIS_SOG_SCALE, AIS_SOG_NOT_AVAILABLE, AIS_SOG_MAX_SPEED, add_nmea1, add_nmea5_partial, load_fleet_to_uset
+from ais.ntools import datetime_to_timestamp, clean_ais_charset
+from ais.inputs.common import STATS_DIR
 
 def auth(username, raw_password):
     try:
@@ -76,7 +76,6 @@ def vessel_search(request):
 
 @http_authenticate(auth, 'ais')
 def vessel(request, strmmsi):
-    os.chdir('/home/nirgal/ais.nirgal.com/ais/') # FIXME
     mmsi = strmmsi_to_mmsi(strmmsi)
     vessel = get_object_or_404(Vessel, pk=mmsi)
     nmea = Nmea.new_from_lastinfo(strmmsi)
@@ -340,7 +339,6 @@ def vessel_track(request, strmmsi):
         grain = int(grain)
     except ValueError:
         grain = 3600
-    os.chdir('/home/nirgal/ais.nirgal.com/ais/') # FIXME
     date_end = datetime.utcnow()
     date_start = date_end - timedelta(ndays)
     nmea_iterator = NmeaFeeder(strmmsi, date_end, date_start, granularity=grain)
@@ -362,7 +360,6 @@ def vessel_animation(request, strmmsi):
         grain = int(grain)
     except ValueError:
         grain = 3600
-    os.chdir('/home/nirgal/ais.nirgal.com/ais/') # FIXME
     date_end = datetime.utcnow()
     date_start = date_end - timedelta(ndays)
     nmea_iterator = NmeaFeeder(strmmsi, date_end, date_start, granularity=grain)
@@ -510,7 +507,6 @@ def fleet_lastpos(request, fleetname):
         return HttpResponseForbidden('<h1>Forbidden</h1>')
     fleet_uset = load_fleet_to_uset(fleetname)
     # = set([mmsi_to_strmmsi(vessel.mmsi) for vessel in fleet.vessel.all()])
-    os.chdir('/home/nirgal/ais.nirgal.com/ais/') # FIXME
     value = kml_to_kmz(format_fleet(fleet_uset, document_name=fleetname+' fleet').encode('utf-8'))
     response = HttpResponse(value, mimetype="application/vnd.google-earth.kml")
     response['Content-Disposition'] = 'attachment; filename=%s.kmz' % fleetname
@@ -633,7 +629,6 @@ def logout(request):
 
 @http_authenticate(auth, 'ais')
 def sources(request):
-    os.chdir('/home/nirgal/ais.nirgal.com/ais/') # FIXME
     sources = ( 'NMMT', 'NMKT', 'NMRW', 'NMEZ', 'NMAS', 'NMAH', 'NMBB' )
     now = int(get_timestamp())
     periods = ({
index 547b55395b6295fa591365fd6882af21737af4a0..7512998f81235844ce55f82f3572be30232bd078 100755 (executable)
@@ -2,10 +2,10 @@
 # -*- coding: utf-8 -*-
 
 import sys, struct
-from ais import *
-
 from datetime import datetime
-from ntools import datetime_to_timestamp
+
+from ais.common import *
+from ais.ntools import datetime_to_timestamp
 
 
 BLACKLIST_MMSI_SYNC = set( (2717205, 2717213, 2717216, 2391300, 2393200, ) )
index b96b5eb94e04b74b78b1680116c7159d3d186e84..6467674b7437ba3dceb904cbca885304ddf7ddb5 100644 (file)
@@ -19,9 +19,9 @@ import logging
 from datetime import datetime
 from time import time as get_timestamp
 
-from stats import STATS_RATE, InStats
-from peers import SOURCES
-from outpeer import outpeers_from_config
+from ais.inputs.stats import STATS_RATE, InStats
+from ais.inputs.peers import SOURCES
+from ais.inputs.outpeer import outpeers_from_config
 
 
 NMEA_DIR = '/var/lib/ais/nmea'
index cef132e0f79bdd5f6993d4ff51df123425b8e014..e06c6b97a3e9cfd68da1ccfaf49982e4115c53c3 100644 (file)
@@ -5,9 +5,9 @@ Peers definition
 
 import logging
 import pprint
-from config import SOURCES
-#from udp import UdpService
-#from serialin import SerialService
+from ais.inputs.config import SOURCES
+#from ais.inputs.udp import UdpService
+#from ais.inputs.serialin import SerialService
 
 UDPIN_HLPORT = {}
 __source_normalized__ = False
index 877b5c0a7dc9c74de95c476e40a0d00f2ee25f5a..7f718d4289d212547abe5f113c21ee25f3d332a7 100755 (executable)
@@ -9,8 +9,8 @@ import sys
 import serial
 import logging
 
-from common import Service, get_source_by_id4
-from stats import STATS_RATE
+from ais.inputs.common import Service, get_source_by_id4
+from ais.inputs.stats import STATS_RATE
 
 DEFAULT_SPEED = 38400
 
index db5a00fcfd4f56e8febdb0f12f8b0a6b2f96f75d..39a0b10cf0a14958f514e86de5c2f56b01717c40 100644 (file)
@@ -11,7 +11,6 @@ __all__ = [
 import os
 from time import time as get_timestamp
 import logging
-
 import rrdtool
 
 STATS_DIR = '/var/lib/ais/stats'
index 0bbb0fca2f59f720e0af9269859e0f67e663a40c..4f9d6e65f40ab35163a7d7a2eeb285b6d6390153 100755 (executable)
@@ -7,8 +7,8 @@ Module for receiving AIVDM data from outbound TCP connection.
 import logging
 import socket
 
-from common import DEFAULT_MTU, Service, get_source_by_id4
-from stats import STATS_RATE
+from ais.inputs.common import DEFAULT_MTU, Service, get_source_by_id4
+from ais.inputs.stats import STATS_RATE
 
 TCP_HEADER_SIZE = 52
 
index 28fc5bdab1d52bc92f3e1ea7c2dca8b9a444a3fc..c039e722b59e22708da2d61944de385c48f65852 100755 (executable)
@@ -9,9 +9,10 @@ import sys
 import logging
 import socket
 
-from common import DEFAULT_MTU, formataddr, Service, get_source_by_id4
-from stats import STATS_RATE
-from peers import udpin_get_id4_from_recvinfo
+from ais.inputs.common import DEFAULT_MTU, formataddr, Service, get_source_by_id4
+from ais.inputs.stats import STATS_RATE
+from ais.inputs.peers import udpin_get_id4_from_recvinfo
+
 UDP_HEADER_SIZE = 28
 
 
index 7a92415cba6c239ba7e9fc076e879dcee58b10d4..34d181728beba8fcc43f2d174e92541286d5ca63 100755 (executable)
@@ -5,7 +5,7 @@
 #
 # taken from http://www.itu.int/cgi-bin/htsh/glad/cga_mids.sh?lng=E
 #
-s='''
+s = '''
 201 Albania (Republic of)
 202 Andorra (Principality of)
 203 Austria
@@ -263,7 +263,7 @@ for line in s.split('\n'):
     while True:
         codes.append(line[:3])
         line = line[3:]
-        if line[0:2]==', ':
+        if line[0:2] == ', ':
             line = line[2:]
         else:
             break
index 84581b15d4127a6093a4c9369e26f9f31a5d483e..8e251e21e9d1f9f50639421f7647f3c80364cb8f 100644 (file)
@@ -1,3 +1,5 @@
+# -*- coding: utf-8 -*-
+#
 #MESSAGE_TYPES = {
 #     1: 'Position Report Class A',
 #     2: 'Position Report Class A (Assigned schedule)',
index bcf1bffa2a5e4241a5068b072cabd7188b6feada..a4368882f950af4658519682a68027c2bba6e98e 100755 (executable)
@@ -4,7 +4,7 @@
 import sys
 from datetime import datetime, timedelta
 
-from db import *
+from ais.db import *
 
 def main():
     #from optparse import OptionParser
index 52526d31bc7fa09ffbb0a7c74d40d0b7c48a8c13..c1ffe87ceaba722c8b7120c71e2ad5d29e658627 100755 (executable)
@@ -8,9 +8,9 @@ from StringIO import StringIO # TODO use python 2.6 io.BufferedWrite(sys.stdout,
 from datetime import datetime, timedelta, time
 import copy
 
-from ais import *
-from area import load_area_from_kml_polygon
-from ntools import datetime_to_timestamp, xml_escape
+from ais.common import *
+from ais.area import load_area_from_kml_polygon
+from ais.ntools import datetime_to_timestamp, xml_escape
 
 
 KML_DISPLAYOPT_NONAMES = 1 # don't print ship name