[ais.git] / bin / common.py

#!/usr/bin/env python
# -*- coding: utf-8 -*-

from __future__ import division
import sys
import os
import struct
import logging
from datetime import datetime, timedelta, date, time
from fcntl import lockf, LOCK_EX, LOCK_UN, LOCK_SH
import csv

from ais.ntools import *
from ais.db import *
from ais.area import load_area_from_kml_polygon
from ais.earth3d import dist3_latlong_ais, dist3_xyz, latlon_to_xyz_deg, latlon_to_xyz_ais

import ais.inputs.config

__all__ = [
    'DB_STARTDATE', 'DBPATH',
    'COUNTRIES_MID', 'STATUS_CODES', 'SHIP_TYPES',
    'AIS_STATUS_NOT_AVAILABLE',
    'AIS_ROT_HARD_LEFT', 'AIS_ROT_HARD_RIGHT', 'AIS_ROT_NOT_AVAILABLE',
    'AIS_LATLON_SCALE', 'AIS_LON_NOT_AVAILABLE', 'AIS_LAT_NOT_AVAILABLE',
    'AIS_COG_SCALE', 'AIS_COG_NOT_AVAILABLE',
    'AIS_NO_HEADING',
    'AIS_SOG_SCALE', 'AIS_SOG_NOT_AVAILABLE', 'AIS_SOG_FAST_MOVER', 'AIS_SOG_MAX_SPEED',
    #'_hash3_pathfilename',
    'db_bydate_addrecord',
    'db_lastinfo_setrecord_ifnewer',
    'add_nmea1',
    'add_nmea5_full',
    'add_nmea5_partial',
    'Nmea1',
    'Nmea5',
    'Nmea',
    'BankNmea1',
    'Nmea1Feeder',
    'BankNmea5',
    'Nmea5Feeder',
    'NmeaFeeder',
    'nice_timedelta_str',
    'all_mmsi_generator',
    'load_fleet_to_uset',
    'fleetname_to_fleetid',
    'mmsiiterator_nohiddenship',
    'filter_area',
    'filter_close_to',
    'filter_far_from',
    'filter_sog_le',
    'filter_sog_ge',
    'filter_knownposition',
    'filter_speedcheck',
    ]
            
DB_STARTDATE = datetime(2008, 6, 1)

# This is the location of the filesystem database
DBPATH = '/var/lib/ais/db'

# see make-countries.py
COUNTRIES_MID = {
    201: u'Albania',
    202: u'Andorra',
    203: u'Austria',
    204: u'Azores',
    205: u'Belgium',
    206: u'Belarus',
    207: u'Bulgaria',
    208: u'Vatican City State',
    209: u'Cyprus',
    210: u'Cyprus',
    211: u'Germany',
    212: u'Cyprus',
    213: u'Georgia',
    214: u'Moldova',
    215: u'Malta',
    216: u'Armenia',
    218: u'Germany',
    219: u'Denmark',
    220: u'Denmark',
    224: u'Spain',
    225: u'Spain',
    226: u'France',
    227: u'France',
    228: u'France',
    230: u'Finland',
    231: u'Faroe Islands',
    232: u'United Kingdom',
    233: u'United Kingdom',
    234: u'United Kingdom',
    235: u'United Kingdom',
    236: u'Gibraltar',
    237: u'Greece',
    238: u'Croatia',
    239: u'Greece',
    240: u'Greece',
    242: u'Morocco',
    243: u'Hungary',
    244: u'Netherlands',
    245: u'Netherlands',
    246: u'Netherlands',
    247: u'Italy',
    248: u'Malta',
    249: u'Malta',
    250: u'Ireland',
    251: u'Iceland',
    252: u'Liechtenstein',
    253: u'Luxembourg',
    254: u'Monaco',
    255: u'Madeira',
    256: u'Malta',
    257: u'Norway',
    258: u'Norway',
    259: u'Norway',
    261: u'Poland',
    262: u'Montenegro',
    263: u'Portugal',
    264: u'Romania',
    265: u'Sweden',
    266: u'Sweden',
    267: u'Slovak Republic',
    268: u'San Marino',
    269: u'Switzerland',
    270: u'Czech Republic',
    271: u'Turkey',
    272: u'Ukraine',
    273: u'Russian Federation',
    274: u'The Former Yugoslav Republic of Macedonia',
    275: u'Latvia',
    276: u'Estonia',
    277: u'Lithuania',
    278: u'Slovenia',
    279: u'Serbia',
    301: u'Anguilla',
    303: u'Alaska',
    304: u'Antigua and Barbuda',
    305: u'Antigua and Barbuda',
    306: u'Netherlands Antilles',
    307: u'Aruba',
    308: u'Bahamas',
    309: u'Bahamas',
    310: u'Bermuda',
    311: u'Bahamas',
    312: u'Belize',
    314: u'Barbados',
    316: u'Canada',
    319: u'Cayman Islands',
    321: u'Costa Rica',
    323: u'Cuba',
    325: u'Dominica',
    327: u'Dominican Republic',
    329: u'Guadeloupe',
    330: u'Grenada',
    331: u'Greenland',
    332: u'Guatemala',
    334: u'Honduras',
    336: u'Haiti',
    338: u'United States of America',
    339: u'Jamaica',
    341: u'Saint Kitts and Nevis',
    343: u'Saint Lucia',
    345: u'Mexico',
    347: u'Martinique',
    348: u'Montserrat',
    350: u'Nicaragua',
    351: u'Panama',
    352: u'Panama',
    353: u'Panama',
    354: u'Panama',
    355: u'Panama',
    356: u'Panama',
    357: u'Panama',
    358: u'Puerto Rico',
    359: u'El Salvador',
    361: u'Saint Pierre and Miquelon',
    362: u'Trinidad and Tobago',
    364: u'Turks and Caicos Islands',
    366: u'United States of America',
    367: u'United States of America',
    368: u'United States of America',
    369: u'United States of America',
    370: u'Panama',
    371: u'Panama',
    372: u'Panama',
    375: u'Saint Vincent and the Grenadines',
    376: u'Saint Vincent and the Grenadines',
    377: u'Saint Vincent and the Grenadines',
    378: u'British Virgin Islands',
    379: u'United States Virgin Islands',
    401: u'Afghanistan',
    403: u'Saudi Arabia',
    405: u'Bangladesh',
    408: u'Bahrain',
    410: u'Bhutan',
    412: u'China',
    413: u'China',
    416: u'Taiwan',
    417: u'Sri Lanka',
    419: u'India',
    422: u'Iran',
    423: u'Azerbaijani Republic',
    425: u'Iraq',
    428: u'Israel',
    431: u'Japan',
    432: u'Japan',
    434: u'Turkmenistan',
    436: u'Kazakhstan',
    437: u'Uzbekistan',
    438: u'Jordan',
    440: u'Korea',
    441: u'Korea',
    443: u'Palestine',
    445: u"Democratic People's Republic of Korea",
    447: u'Kuwait',
    450: u'Lebanon',
    451: u'Kyrgyz Republic',
    453: u'Macao',
    455: u'Maldives',
    457: u'Mongolia',
    459: u'Nepal',
    461: u'Oman',
    463: u'Pakistan',
    466: u'Qatar',
    468: u'Syrian Arab Republic',
    470: u'United Arab Emirates',
    473: u'Yemen',
    475: u'Yemen',
    477: u'Hong Kong',
    478: u'Bosnia and Herzegovina',
    501: u'Adelie Land',
    503: u'Australia',
    506: u'Myanmar',
    508: u'Brunei Darussalam',
    510: u'Micronesia',
    511: u'Palau',
    512: u'New Zealand',
    514: u'Cambodia',
    515: u'Cambodia',
    516: u'Christmas Island',
    518: u'Cook Islands',
    520: u'Fiji',
    523: u'Cocos',
    525: u'Indonesia',
    529: u'Kiribati',
    531: u"Lao People's Democratic Republic",
    533: u'Malaysia',
    536: u'Northern Mariana Islands',
    538: u'Marshall Islands',
    540: u'New Caledonia',
    542: u'Niue',
    544: u'Nauru',
    546: u'French Polynesia',
    548: u'Philippines',
    553: u'Papua New Guinea',
    555: u'Pitcairn Island',
    557: u'Solomon Islands',
    559: u'American Samoa',
    561: u'Samoa',
    563: u'Singapore',
    564: u'Singapore',
    565: u'Singapore',
    567: u'Thailand',
    570: u'Tonga',
    572: u'Tuvalu',
    574: u'Viet Nam',
    576: u'Vanuatu',
    578: u'Wallis and Futuna Islands',
    601: u'South Africa',
    603: u'Angola',
    605: u'Algeria',
    607: u'Saint Paul and Amsterdam Islands',
    608: u'Ascension Island',
    609: u'Burundi',
    610: u'Benin',
    611: u'Botswana',
    612: u'Central African Republic',
    613: u'Cameroon',
    615: u'Congo',
    616: u'Comoros',
    617: u'Cape Verde',
    618: u'Crozet Archipelago',
    619: u"Côte d'Ivoire",
    621: u'Djibouti',
    622: u'Egypt',
    624: u'Ethiopia',
    625: u'Eritrea',
    626: u'Gabonese Republic',
    627: u'Ghana',
    629: u'Gambia',
    630: u'Guinea-Bissau',
    631: u'Equatorial Guinea',
    632: u'Guinea',
    633: u'Burkina Faso',
    634: u'Kenya',
    635: u'Kerguelen Islands',
    636: u'Liberia',
    637: u'Liberia',
    642: u"Socialist People's Libyan Arab Jamahiriya",
    644: u'Lesotho',
    645: u'Mauritius',
    647: u'Madagascar',
    649: u'Mali',
    650: u'Mozambique',
    654: u'Mauritania',
    655: u'Malawi',
    656: u'Niger',
    657: u'Nigeria',
    659: u'Namibia',
    660: u'Reunion',
    661: u'Rwanda',
    662: u'Sudan',
    663: u'Senegal',
    664: u'Seychelles',
    665: u'Saint Helena',
    666: u'Somali Democratic Republic',
    667: u'Sierra Leone',
    668: u'Sao Tome and Principe',
    669: u'Swaziland',
    670: u'Chad',
    671: u'Togolese Republic',
    672: u'Tunisia',
    674: u'Tanzania',
    675: u'Uganda',
    676: u'Democratic Republic of the Congo',
    677: u'Tanzania',
    678: u'Zambia',
    679: u'Zimbabwe',
    701: u'Argentine Republic',
    710: u'Brazil',
    720: u'Bolivia',
    725: u'Chile',
    730: u'Colombia',
    735: u'Ecuador',
    740: u'Falkland Islands',
    745: u'Guiana',
    750: u'Guyana',
    755: u'Paraguay',
    760: u'Peru',
    765: u'Suriname',
    770: u'Uruguay',
    775: u'Venezuela',
}

STATUS_CODES = {
     0:  'Under way using engine',
     1:  'At anchor',
     2:  'Not under command',
     3:  'Restricted manoeuverability',
     4:  'Constrained by her draught',
     5:  'Moored',
     6:  'Aground',
     7:  'Engaged in Fishing',
     8:  'Under way sailing',
     9:  '9 - Reserved for future amendment of Navigational Status for HSC',
    10:  '10 - Reserved for future amendment of Navigational Status for WIG',
    11:  '11 - Reserved for future use',
    12:  '12 - Reserved for future use',
    13:  '13 - Reserved for future use',
    14:  '14 - Reserved for future use', # Land stations
    15:  'Not defined', # default
}

SHIP_TYPES = {
     0: 'Not available (default)',
     1: 'Reserved for future use',
     2: 'Reserved for future use',
     3: 'Reserved for future use',
     4: 'Reserved for future use',
     5: 'Reserved for future use',
     6: 'Reserved for future use',
     7: 'Reserved for future use',
     8: 'Reserved for future use',
     9: 'Reserved for future use',
    10: 'Reserved for future use',
    11: 'Reserved for future use',
    12: 'Reserved for future use',
    13: 'Reserved for future use',
    14: 'Reserved for future use',
    15: 'Reserved for future use',
    16: 'Reserved for future use',
    17: 'Reserved for future use',
    18: 'Reserved for future use',
    19: 'Reserved for future use',
    20: 'Wing in ground (WIG), all ships of this type',
    21: 'Wing in ground (WIG), Hazardous category A',
    22: 'Wing in ground (WIG), Hazardous category B',
    23: 'Wing in ground (WIG), Hazardous category C',
    24: 'Wing in ground (WIG), Hazardous category D',
    25: 'Wing in ground (WIG), Reserved for future use',
    26: 'Wing in ground (WIG), Reserved for future use',
    27: 'Wing in ground (WIG), Reserved for future use',
    28: 'Wing in ground (WIG), Reserved for future use',
    29: 'Wing in ground (WIG), Reserved for future use',
    30: 'Fishing',
    31: 'Towing',
    32: 'Towing: length exceeds 200m or breadth exceeds 25m',
    33: 'Dredging or underwater ops',
    34: 'Diving ops',
    35: 'Military ops',
    36: 'Sailing',
    37: 'Pleasure Craft',
    38: 'Reserved',
    39: 'Reserved',
    40: 'High speed craft (HSC), all ships of this type',
    41: 'High speed craft (HSC), Hazardous category A',
    42: 'High speed craft (HSC), Hazardous category B',
    43: 'High speed craft (HSC), Hazardous category C',
    44: 'High speed craft (HSC), Hazardous category D',
    45: 'High speed craft (HSC), Reserved for future use',
    46: 'High speed craft (HSC), Reserved for future use',
    47: 'High speed craft (HSC), Reserved for future use',
    48: 'High speed craft (HSC), Reserved for future use',
    49: 'High speed craft (HSC), No additional information',
    50: 'Pilot Vessel',
    51: 'Search and Rescue vessel',
    52: 'Tug',
    53: 'Port Tender',
    54: 'Anti-pollution equipment',
    55: 'Law Enforcement',
    56: 'Spare - Local Vessel',
    57: 'Spare - Local Vessel',
    58: 'Medical Transport',
    59: 'Ship according to RR Resolution No. 18',
    60: 'Passenger, all ships of this type',
    61: 'Passenger, Hazardous category A',
    62: 'Passenger, Hazardous category B',
    63: 'Passenger, Hazardous category C',
    64: 'Passenger, Hazardous category D',
    65: 'Passenger, Reserved for future use',
    66: 'Passenger, Reserved for future use',
    67: 'Passenger, Reserved for future use',
    68: 'Passenger, Reserved for future use',
    69: 'Passenger, No additional information',
    70: 'Cargo', # 'Cargo, all ships of this type',
    71: 'Cargo, Hazardous category A',
    72: 'Cargo, Hazardous category B',
    73: 'Cargo, Hazardous category C',
    74: 'Cargo, Hazardous category D',
    75: 'Cargo', # 'Cargo, Reserved for future use',
    76: 'Cargo', # 'Cargo, Reserved for future use',
    77: 'Cargo', # 'Cargo, Reserved for future use',
    78: 'Cargo', # 'Cargo, Reserved for future use',
    79: 'Cargo', # 'Cargo, No additional information',
    80: 'Tanker', # 'Tanker, all ships of this type',
    81: 'Tanker, Hazardous category A',
    82: 'Tanker, Hazardous category B',
    83: 'Tanker, Hazardous category C',
    84: 'Tanker, Hazardous category D',
    85: 'Tanker', # 'Tanker, Reserved for future use',
    86: 'Tanker', # 'Tanker, Reserved for future use',
    87: 'Tanker', # 'Tanker, Reserved for future use',
    88: 'Tanker', # 'Tanker, Reserved for future use',
    89: 'Tanker, No additional information',
    90: 'Other Type, all ships of this type',
    91: 'Other Type, Hazardous category A',
    92: 'Other Type, Hazardous category B',
    93: 'Other Type, Hazardous category C',
    94: 'Other Type, Hazardous category D',
    95: 'Other Type, Reserved for future use',
    96: 'Other Type, Reserved for future use',
    97: 'Other Type, Reserved for future use',
    98: 'Other Type, Reserved for future use',
    99: 'Other Type, no additional information',
    100: 'Default Navaid',
    101: 'Reference point',
    102: 'RACON',
    103: 'Offshore Structure',
    104: 'Spare',
    105: 'Light, without sectors',
    106: 'Light, with sectors',
    107: 'Leading Light Front',
    108: 'Leading Light Rear',
    109: 'Beacon, Cardinal N',
    110: 'Beacon, Cardinal E',
    111: 'Beacon, Cardinal S',
    112: 'Beacon, Cardinal W',
    113: 'Beacon, Port hand',
    114: 'Beacon, Starboard hand',
    115: 'Beacon, Preferred Channel port hand',
    116: 'Beacon, Preferred Channel starboard hand',
    117: 'Beacon, Isolated danger',
    118: 'Beacon, Safe water',
    119: 'Beacon, Special mark',
    120: 'Cardinal Mark N',
    121: 'Cardinal Mark E',
    122: 'Cardinal Mark S',
    123: 'Cardinal Mark W',
    124: 'Port hand Mark',
    125: 'Starboard hand Mark',
    126: 'Preferred Channel Port hand',
    127: 'Preferred Channel Starboard hand',
    128: 'Isolated danger',
    129: 'Safe Water',
    130: 'Manned VTS / Special Mark',
    131: 'Light Vessel / LANBY',
}

AIS_STATUS_NOT_AVAILABLE = 15
AIS_ROT_HARD_LEFT = -127
AIS_ROT_HARD_RIGHT = 127
AIS_ROT_NOT_AVAILABLE = -128 # not like gpsd

AIS_LATLON_SCALE = 600000.0
AIS_LON_NOT_AVAILABLE = 0x6791AC0
AIS_LAT_NOT_AVAILABLE = 0x3412140
AIS_COG_SCALE = 10.0
AIS_COG_NOT_AVAILABLE = 3600
AIS_NO_HEADING = 511
AIS_SOG_SCALE = 10.0
AIS_SOG_NOT_AVAILABLE = 1023
AIS_SOG_FAST_MOVER = 1022
AIS_SOG_MAX_SPEED = 1021


def _hash3_pathfilename(filename):
    """
    Returns a level 3 directory hashed filename on that basis:
    123456789 -> 1/12/123/123456789
    """
    return os.path.join(filename[0], filename[:2], filename[:3], filename)


def db_bydate_addrecord(basefilename, record, timestamp):
    strdt = datetime.utcfromtimestamp(timestamp).strftime('%Y%m%d')
    filename = os.path.join(DBPATH, 'bydate', strdt, _hash3_pathfilename(basefilename))
    f = open_with_mkdirs(filename, 'ab')
    lockf(f, LOCK_EX)
    #f.seek(0,2) # go to EOF
    assert f.tell() % len(record) == 0, 'Invalid length for %s' % filename
    f.write(record)
    f.close()


def db_lastinfo_setrecord_ifnewer(basefilename, record, timestamp):
    '''
    Overwrite last information if date is newer
    Input record must be complete
    '''
    filename = DBPATH+'/last/'+_hash3_pathfilename(basefilename)

    try:
        f = open(filename, 'r+b')
    except IOError as ioerr:
        if ioerr.errno != 2:
            raise
        # File was not found? Ok, create it. FIXME: we should lock something...
        f = open_with_mkdirs(filename, 'wb')
        f.write(record)
        updated = True
    else:
        lockf(f, LOCK_EX)
        assert f.tell() == 0
        oldrecord = f.read(4)
        assert len(oldrecord) == 4
        oldtimestamp = struct.unpack('I', oldrecord)[0]
        f.seek(0)
        assert f.tell() == 0
        if timestamp > oldtimestamp:
            f.write(record)
            assert f.tell() == len(record), \
                "tell=%s size=%s" % (f.tell(), len(record))
            updated = True
        else:
            updated = False
    f.close()
    return updated


def _sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
    dim_bow, dim_stern, dim_port, dim_starboard, \
    eta_M, eta_D, eta_h, eta_m, draught, destination, source):
    ''' Don't call directly '''
    sqlinfo = {}
    sqlinfo['mmsi'] = strmmsi_to_mmsi(strmmsi)
    sqlinfo['updated'] = datetime.utcfromtimestamp(timestamp)
    sqlinfo['imo'] = imo or None
    sqlinfo['name'] = name or None
    sqlinfo['callsign'] = callsign or None
    sqlinfo['type'] = type
    sqlinfo['dim_bow'] = dim_bow
    sqlinfo['dim_stern'] = dim_stern
    sqlinfo['dim_port'] = dim_port
    sqlinfo['dim_starboard'] = dim_starboard
    sqlinfo['destination'] = None
    eta = '%02d%02d%02d%02d' % ( eta_M, eta_D, eta_h, eta_m)
    if eta == '00000000':
        # FIXME tempory hack for corrupted db/latest/*.nmea5 file
        eta = '00002460'
    sqlinfo['eta'] = eta
    if destination:
        destination = destination.replace('\0', ' ').rstrip(' @\0')
    sqlinfo['destination'] = destination or None
    sqlinfo['source'] = source
    sqlexec(u'''INSERT INTO vessel (mmsi, updated) SELECT %(mmsi)s, '1970-01-01T00:00:00' WHERE NOT EXISTS (SELECT * FROM vessel WHERE mmsi=%(mmsi)s)''', sqlinfo)
    if sqlinfo['imo']:
        sqlexec(u'UPDATE vessel SET imo = %(imo)s WHERE mmsi=%(mmsi)s AND (imo IS NULL OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['name']:
        sqlexec(u'UPDATE vessel SET name = %(name)s WHERE mmsi=%(mmsi)s AND (name IS NULL OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['callsign']:
        sqlexec(u'UPDATE vessel SET callsign = %(callsign)s WHERE mmsi=%(mmsi)s AND (callsign IS NULL OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['type']:
        sqlexec(u'UPDATE vessel SET type = %(type)s WHERE mmsi=%(mmsi)s AND (type IS NULL OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['dim_bow'] or sqlinfo['dim_stern']:
        sqlexec(u'UPDATE vessel SET dim_bow = %(dim_bow)s, dim_stern = %(dim_stern)s WHERE mmsi=%(mmsi)s AND ((dim_port = 0 OR dim_stern=0) OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['dim_port'] or sqlinfo['dim_starboard']:
        sqlexec(u'UPDATE vessel SET dim_port = %(dim_port)s, dim_starboard = %(dim_starboard)s WHERE mmsi=%(mmsi)s AND ((dim_port = 0 OR dim_starboard=0) OR updated<%(updated)s)', sqlinfo)
    if sqlinfo['destination'] or sqlinfo['eta'] != '00002460':
        sqlexec(u"UPDATE vessel SET destination = %(destination)s, eta = %(eta)s WHERE mmsi=%(mmsi)s AND (destination IS NULL OR eta = '00002460' OR updated<%(updated)s)", sqlinfo)
    sqlexec(u'UPDATE vessel SET (updated, source) = (%(updated)s, %(source)s) WHERE mmsi=%(mmsi)s AND updated<%(updated)s', sqlinfo)
    dbcommit()




AIVDM_RECORD123_FORMAT = 'IBbhiiII4s'
AIVDM_RECORD123_LENGTH = struct.calcsize(AIVDM_RECORD123_FORMAT)
AIVDM_RECORD5_FORMAT = 'II20s7sBHHBBBBBBH20s4s'
AIVDM_RECORD5_LENGTH = struct.calcsize(AIVDM_RECORD5_FORMAT)


def add_nmea1(strmmsi, timestamp, status, rot, sog, \
              latitude, longitude, cog, heading, source):
    '''
    Input is raw data, unscaled
    FIXME: lat & lon are inverted compared to raw aivdm structure
    '''
    record = struct.pack(AIVDM_RECORD123_FORMAT, timestamp, status, rot, sog, latitude, longitude, cog, heading, source)
    #print repr(record)
    filename = strmmsi+'.nmea1'
    db_bydate_addrecord(filename, record, timestamp)
    # There's no need to be smart: all the information are taken, or none.
    return db_lastinfo_setrecord_ifnewer(filename, record, timestamp)


def add_nmea5_full(strmmsi, timestamp, imo, name, callsign, type, \
                   dim_bow, dim_stern, dim_port, dim_starboard, \
                   eta_M, eta_D, eta_h, eta_m, draught, destination, source):
    '''
    Input is raw data, unscaled
    All fields are set, and can be upgraded if the record is newer
    FIXME: name & callsign are inverted compared to raw aivdm structure
    '''
    record = struct.pack(AIVDM_RECORD5_FORMAT, timestamp, imo, name, callsign, \
                         type, dim_bow, dim_stern, dim_port, dim_starboard, \
                         eta_M, eta_D, eta_h, eta_m, draught, destination, source)
    #print repr(record)
    filename = strmmsi+'.nmea5'
    db_bydate_addrecord(filename, record, timestamp)
    updated = db_lastinfo_setrecord_ifnewer(filename, record, timestamp)
    if updated:
        _sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
                      dim_bow, dim_stern, dim_port, dim_starboard, \
                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
    return updated

def add_nmea5_partial(strmmsi, timestamp, imo, name, callsign, type, \
                      dim_bow, dim_stern, dim_port, dim_starboard, \
                      eta_M, eta_D, eta_h, eta_m, draught, destination, source):
    '''
    Input is raw data, unscaled
    All fields are not set. Only some of them can be upgraded, if they're newer
    '''
    record = struct.pack(AIVDM_RECORD5_FORMAT, \
                         timestamp, imo, name, callsign, type, \
                         dim_bow, dim_stern, dim_port, dim_starboard, \
                         eta_M, eta_D, eta_h, eta_m, draught, destination, \
                         source)
    #print repr(record)
    filename = strmmsi + '.nmea5'
    db_bydate_addrecord(filename, record, timestamp)

    updated = False
    filename = os.path.join(DBPATH, 'last', _hash3_pathfilename(filename))
    try:
        f = open(filename, 'r+b')
    except IOError as ioerr:
        if ioerr.errno != 2:
            raise
        # File was not found? Ok, create it. FIXME: we should lock something...
        f = open_with_mkdirs(filename, 'wb')
        lockf(f, LOCK_EX)
        f.write(record)
        # keep the lock
        updated = True
    else:
        lockf(f, LOCK_EX)
        oldrecord = f.read(AIVDM_RECORD5_LENGTH)
        oldtimestamp, oldimo, oldname, oldcallsign, oldtype, \
        olddim_bow, olddim_stern, olddim_port, olddim_starboard, \
        oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
        olddraught, olddestination, oldsource \
                  = struct.unpack(AIVDM_RECORD5_FORMAT, oldrecord)
        if timestamp > oldtimestamp:
            # we have incoming recent information
            if imo == 0:
                imo = oldimo
            if name == '':
                name = oldname
            if callsign == '':
                callsign = oldcallsign
            if type == 0:
                type = oldtype
            if dim_bow == 0:
                dim_bow = olddim_bow
            if dim_stern == 0:
                dim_stern = olddim_stern
            if dim_port == 0:
                dim_port = olddim_port
            if dim_starboard == 0:
                dim_starboard = olddim_starboard
            if eta_M == 0 or eta_D == 0 or eta_h == 24 or eta_m == 60 \
                          or destination == '':
                eta_M = oldeta_M
                eta_D = oldeta_D
                eta_h = oldeta_h
                eta_m = oldeta_m
                destination = olddestination
            if draught == 0:
                draught = olddraught
            record = struct.pack(AIVDM_RECORD5_FORMAT, \
                                 timestamp, imo, name, callsign, type, \
                                 dim_bow, dim_stern, dim_port, dim_starboard, \
                                 eta_M, eta_D, eta_h, eta_m, draught, \
                                 destination, source)
            f.seek(0)
            f.write(record)
            updated = True
        else:
            # we received an obsolete info, but maybe there are some new things in it
            if oldimo == 0 and imo != 0:
                oldimo = imo
                updated = True
            if oldname == '' and name != '':
                oldname = name
                updated = True
            if oldcallsign == '' and callsign != '':
                oldcallsign = callsign
                updated = True
            if oldtype == 0 and type != 0:
                oldtype = type
                updated = True
            if olddim_bow == 0 and dim_bow != 0:
                olddim_bow = dim_bow
                updated = True
            if olddim_stern == 0 and dim_stern != 0:
                olddim_stern = dim_stern
                updated = True
            if olddim_port == 0 and dim_port != 0:
                olddim_port = dim_port
                updated = True
            if olddim_starboard == 0 and dim_starboard != 0:
                olddim_starboard = dim_starboard
                updated = True
            # FIXME
            if (oldeta_M == 0 or oldeta_D == 0 or olddestination == '') \
                    and ((eta_M != 0 and eta_D != 0) or destination!=''):
                oldeta_M = eta_M
                oldeta_D = eta_D
                oldeta_h = eta_h
                oldeta_m = eta_m
                olddestination = destination
                updated = True
            if olddraught == 0 and draught != 0:
                olddraught = draught
                updated = True
            if updated:
                oldsource = source
                record = struct.pack(AIVDM_RECORD5_FORMAT, \
                                     oldtimestamp, oldimo, oldname, \
                                     oldcallsign, oldtype, \
                                     olddim_bow, olddim_stern, \
                                     olddim_port, olddim_starboard, \
                                     oldeta_M, oldeta_D, oldeta_h, oldeta_m, \
                                     olddraught, olddestination, oldsource)
            
                f.seek(0)
                f.write(record)
    # keep the file locked during SQL updates
    if updated:
        _sql_add_nmea5(strmmsi, timestamp, imo, name, callsign, type, \
                      dim_bow, dim_stern, dim_port, dim_starboard, \
                      eta_M, eta_D, eta_h, eta_m, draught, destination, source)
    f.close()
    return updated



__misources__ = {} # cache of manual source names
def _get_mi_sourcename(id):
    """
    Get the nice name for sources whose id4 starts with 'MI'
    """
    global __misources__
    if not __misources__:
        sqlexec(u'SELECT id, name FROM mi_source')
        while True:
            row = get_common_cursor().fetchone()
            if row is None:
                break
            __misources__[row[0]] = row[1]
    result = __misources__.get(id, None)
    if result is None:
        return u"Manual input #%s" % id
    return result


class Nmea1:
    def __init__(self, timestamp, status=AIS_STATUS_NOT_AVAILABLE, rot=AIS_ROT_NOT_AVAILABLE, sog=AIS_SOG_NOT_AVAILABLE, latitude=AIS_LAT_NOT_AVAILABLE, longitude=AIS_LON_NOT_AVAILABLE, cog=AIS_COG_NOT_AVAILABLE, heading=AIS_NO_HEADING, source='\x00\x00\x00\x00'):
        self.timestamp_1 = timestamp
        self.status      = status
        self.rot         = rot
        self.sog         = sog
        self.latitude    = latitude
        self.longitude   = longitude
        self.cog         = cog
        self.heading     = heading
        self.source_1    = source

    from_values = __init__

    def to_values(self):
        return self.timestamp_1, self.status, self.rot, self.sog, self.latitude, self.longitude, self.cog, self.heading, self.source_1

    def from_record(self, record):
        values = struct.unpack(AIVDM_RECORD123_FORMAT, record)
        Nmea1.__init__(self, *values)

    @staticmethod
    def new_from_record(record):
        values = struct.unpack(AIVDM_RECORD123_FORMAT, record)
        return Nmea1(*values)

    def to_record(self):
        return struct.pack(AIVDM_RECORD123_FORMAT, *Nmea1.to_values(self))
        
    def from_file(self, file):
        record = file.read(AIVDM_RECORD123_LENGTH)
        Nmea1.from_record(self, record)

    @staticmethod
    def new_from_file(file):
        record = file.read(AIVDM_RECORD123_LENGTH)
        return Nmea1.new_from_record(record)

    def from_lastinfo(self, strmmsi):
        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
        try:
            f = file(filename_nmea1, 'rb')
        except IOError:
            logging.debug("file %s doesn't exists" % filename_nmea1)
            return
        lockf(f, LOCK_SH)
        Nmea1.from_file(self, f)
        f.close()

    @staticmethod
    def new_from_lastinfo(strmmsi):
        filename_nmea1 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea1')
        try:
            f = file(filename_nmea1, 'rb')
        except IOError:
            logging.debug("file %s doesn't exists" % filename_nmea1)
            return None
        lockf(f, LOCK_SH)
        record = f.read(AIVDM_RECORD123_LENGTH)
        f.close()
        return Nmea1.new_from_record(record)


    def dump_to_stdout(self):
        """
        Prints content to stdout
        """
        print datetime.utcfromtimestamp(self.timestamp_1), 
        for i in (self.status, self.rot, self.sog, self.latitude/AIS_LATLON_SCALE, self.longitude/AIS_LATLON_SCALE, self.cog, self.heading, self.source_1):
            print repr(i),
        print
 
    @staticmethod
    def _clean_str(txt):
        if txt is None:
            return ''
        return txt.replace('\0','').replace('@', '').strip()

    def get_status(self, default='Unknown'):
        return STATUS_CODES.get(self.status, default)
 
    def get_sog_str(self, default='Unknown'):
        if self.sog == AIS_SOG_NOT_AVAILABLE:
            return default
        if self.sog == AIS_SOG_FAST_MOVER:
            return 'over 102.2 kts'
        return '%.1f kts' % (self.sog/AIS_SOG_SCALE)

    def get_rot_str(self, default='Unknown'):
        if self.rot == AIS_ROT_NOT_AVAILABLE:
            return default
        if self.rot == 0:
            return 'Not turning'
        if self.rot < 0:
            side = 'port'
        else:
            side = 'starboard'
        rot = abs(self.rot)
        if rot == 127:
            result = 'To '
        else:
            result = '%d %% to ' % rot*100./127
        return result + side

    @staticmethod
    def _decimaldegree_to_dms(f, emispheres):
        if f >= 0:
            e = emispheres[0]
        else:
            f = -f
            e = emispheres[1]
        result = '%d°' % int(f)
        f = (f%1)*60
        result += '%02.05f\' ' % f
        result += e
        return result

    def get_latitude_str(self, default='Unknown'):
        if self.latitude == AIS_LAT_NOT_AVAILABLE:
            return default
        return Nmea1._decimaldegree_to_dms(self.latitude / AIS_LATLON_SCALE, 'NS')

    def get_longitude_str(self, default='Unknown'):
        if self.longitude == AIS_LON_NOT_AVAILABLE:
            return default
        return Nmea1._decimaldegree_to_dms(self.longitude / AIS_LATLON_SCALE, 'EW')

    def get_cog_str(self, default='Unknown'):
        if self.cog == AIS_COG_NOT_AVAILABLE:
            return default
        return '%.1f°' % (self.cog/10.)

    def get_heading_str(self, default='Unknown'):
        if self.heading == AIS_NO_HEADING:
            return default
        return '%s°' % self.heading

    def get_source_1_str(self):
        return Nmea.format_source(self.source_1)

class Nmea5:
    def __init__(self, timestamp, imo=0, name='', callsign='', type=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
        self.timestamp_5   = timestamp
        self.imo           = imo
        self.name          = name         
        self.callsign      = callsign
        self.type          = type
        self.dim_bow       = dim_bow
        self.dim_stern     = dim_stern
        self.dim_port      = dim_port
        self.dim_starboard = dim_starboard
        self.eta_M         = eta_M
        self.eta_D         = eta_D
        self.eta_h         = eta_h
        self.eta_m         = eta_m
        self.draught       = draught
        self.destination   = destination
        self.source_5      = source

    from_values = __init__

    def merge_from_values(self, timestamp, imo=0, name='', callsign='', type_=0, dim_bow=0, dim_stern=0, dim_port=0, dim_starboard=0, eta_M=0, eta_D=0, eta_h=24, eta_m=60, draught=0, destination='', source=''):
        updated = False
        if self.imo == 0 or imo != 0:
            self.imo = imo
            updated = True
        if self.name == '' or name != '':
            self.name = name
            updated = True
        if self.callsign == '' or callsign != '':
            self.callsign = callsign
            updated = True
        if self.type == 0 or type_ != 0:
            self.type = type_
            updated = True
        if self.dim_bow == 0 or dim_bow != 0:
            self.dim_bow = dim_bow
            updated = True
        if self.dim_stern == 0 or dim_stern != 0:
            self.dim_stern = dim_stern
            updated = True
        if self.dim_port == 0 or dim_port != 0:
            self.dim_port = dim_port
            updated = True
        if self.dim_starboard == 0 or dim_starboard != 0:
            self.dim_starboard = dim_starboard
            updated = True
        if (self.eta_M == 0 and self.eta_D == 0 and self.eta_h == 24 and self.eta_m == 60) or eta_M != 0 or eta_D != 0 or eta_h != 24 or eta_m != 60:
            self.eta_M = eta_M
            self.eta_D = eta_D
            self.eta_h = eta_h
            self.eta_m = eta_m
            updated = True
        if self.draught == 0 or draught != 0:
            self.draught = draught
            updated = True
        if self.destination == '' or destination != '':
            self.destination = destination
            updated = True
        if updated:
            self.timestamp_5 = timestamp
            self.source_5 = source
        return updated

    def to_values(self):
        return self.timestamp_5, self.imo, self.name, self.callsign, self.type, self.dim_bow, self.dim_stern, self.dim_port, self.dim_starboard, self.eta_M, self.eta_D, self.eta_h, self.eta_m, self.draught, self.destination, self.source_5

    def from_record(self, record):
        values = struct.unpack(AIVDM_RECORD5_FORMAT, record)
        Nmea5.__init__(self, *values)

    @staticmethod
    def new_from_record(record):
        values = struct.unpack(AIVDM_RECORD5_FORMAT, record)
        return Nmea5(*values)

    def to_record(self):
        return struct.pack(AIVDM_RECORD5_FORMAT, *Nmea5.to_values(self))
        
    def from_file(self, file):
        record = file.read(AIVDM_RECORD5_LENGTH)
        Nmea5.from_record(self, record)

    @staticmethod
    def new_from_file(file):
        record = file.read(AIVDM_RECORD5_LENGTH)
        return Nmea5.new_from_record(record)

    def from_lastinfo(self, strmmsi):
        filename_nmea5 = os.path.join(DBPATH,
                                      'last',
                                      _hash3_pathfilename(strmmsi+'.nmea5'))
        try:
            f = file(filename_nmea5, 'rb')
        except IOError:
            logging.debug("file %s doesn't exists" % filename_nmea5)
            return
        lockf(f, LOCK_SH)
        Nmea5.from_file(self, f)
        f.close()

    @staticmethod
    def new_from_lastinfo(strmmsi):
        filename_nmea5 = DBPATH+'/last/'+_hash3_pathfilename(strmmsi+'.nmea5')
        try:
            f = file(filename_nmea5, 'rb')
        except IOError:
            logging.debug("file %s doesn't exists" % filename_nmea5)
            return None
        lockf(f, LOCK_SH)
        record = f.read(AIVDM_RECORD5_LENGTH)
        f.close()
        return Nmea5.new_from_record(record)

    @staticmethod
    def _clean_str(txt):
        if txt is None:
            return ''
        return txt.replace('\0','').replace('@', '').strip()

    def get_name(self, default='Unknown'):
        result = self._clean_str(self.name)
        if result:
            return result
        return default

    def get_callsign(self, default='Unknown'):
        return self._clean_str(self.callsign) or default

    def get_shiptype(self, default='Unknown'):
        return SHIP_TYPES.get(self.type, default)

    def get_length(self):
        return self.dim_bow + self.dim_stern

    def get_width(self):
        return self.dim_port + self.dim_starboard

    _monthes = 'Jan,Feb,Mar,Apr,May,Jun,Jul,Aug,Sep,Oct,Nov,Dec'.split(',')
    def get_eta_str(self, default='Unknown'):
        if not self.eta_M and not self.eta_D:
            return default
        result = ''
        if self.eta_M:
            if self.eta_M <= len(Nmea5._monthes):
                result += Nmea5._monthes[self.eta_M - 1]
            else:
                result += '%02d' % self.eta_M
        else:
            result += '***'
        result += ' '
        if self.eta_D:
            result += '%02d' % self.eta_D
        else:
            result += '**'
        if self.eta_h != 24:
            result += ' %02d' % self.eta_h
            if self.eta_m == 60:
                result += 'h'
            else:
                result += ':%02d' % self.eta_m
        return result
    
    def get_draught_str(self, default='Unknown'):
        if not self.draught:
            return default
        return '%.1f meters' % (self.draught/10.)

    def get_destination(self, default='Unknown'):
        return self._clean_str(self.destination) or default

    def get_source_5_str(self):
        return Nmea.format_source(self.source_5)

class Nmea(Nmea1, Nmea5):
    """
    This is nmea info, a merge of nmea1 and nmea5 packets
    """
    def __init__(self, strmmsi):
        self.strmmsi = strmmsi
        Nmea1.__init__(self, timestamp=0)
        Nmea5.__init__(self, timestamp=0)

    ########################
    # Because of multiple inheritance some functions are unavailable:
    def _nmea_not_implemented(*args, **kargs):
        # used to avoid conflicting inherited members
        raise NotImplementedError
    from_values = _nmea_not_implemented
    to_values = _nmea_not_implemented
    from_record = _nmea_not_implemented
    new_from_record = _nmea_not_implemented
    to_record = _nmea_not_implemented
    from_file = _nmea_not_implemented
    new_from_file = _nmea_not_implemented
    ########################

    def from_lastinfo(self, strmmsi):
        Nmea1.from_lastinfo(self, strmmsi)
        Nmea5.from_lastinfo(self, strmmsi)
    
    @staticmethod
    def new_from_lastinfo(strmmsi):
        # better than unimplemented, but not optimal
        nmea = Nmea(strmmsi)
        nmea.from_lastinfo(strmmsi)
        return nmea


    def get_flag(self, default=u'Unknown'):
        if self.strmmsi.startswith('00') and self.strmmsi[3:5]!='MI':
            ref_mmsi = self.strmmsi[2:]
        else:
            ref_mmsi = self.strmmsi
        country_mid = int(ref_mmsi[0:3])
        country_name = COUNTRIES_MID.get(country_mid, default)
        return country_name

    def get_mmsi_public(self, default='Unknown'):
        if self.strmmsi.isdigit():
            return self.strmmsi
        return default

    def get_title(self):
        """
        Returns the name of the ship if available
        Or its mmsi
        """
        return self.get_name(None) or self.get_mmsi_public()

    def get_last_timestamp(self):
        """
        Returns the most recent of update from timestamp1, timestamp5
        """
        if self.timestamp_1 > self.timestamp_5:
            return self.timestamp_1
        else:
            return self.timestamp_5

    def get_last_updated_delta_str(self):
        """
        Returns a pretty formated update data as a string
        """
        lastupdate = self.get_last_timestamp()
        if lastupdate == 0:
            return u'Never'
        dt_lastupdate = datetime.utcfromtimestamp(lastupdate)
        delta = datetime.utcnow() - dt_lastupdate
        return nice_timedelta_str(delta) + u' ago'

    def get_last_updated_str(self):
        """
        Returns a pretty formated update data as a string
        """
        lastupdate = self.get_last_timestamp()
        if lastupdate == 0:
            return u'Never'
        dt_lastupdate = datetime.utcfromtimestamp(lastupdate)
        delta = datetime.utcnow() - dt_lastupdate
        return dt_lastupdate.strftime('%Y-%m-%d %H:%M:%S GMT') + ' (' +  nice_timedelta_str(delta) + ' ago)'

    @staticmethod
    def format_source(infosrc):
        if infosrc == '\0\0\0\0':
            return u'(empty)'
        elif infosrc.startswith('MI'):
            if len(infosrc) == 4:
                return _get_mi_sourcename(struct.unpack('<2xH', infosrc)[0])
            else:
                return u'Manual input'
        elif infosrc.startswith('U'):
            return u'User input'
        elif infosrc.startswith('NM'):
            return u'NMEA packets from '+xml_escape(infosrc[2:])
        elif infosrc.startswith('SP'):
            return u"ShipPlotter user %s" % infosrc[2:]
        elif infosrc.startswith('ST'):
            return u"Spot track %s" % infosrc[2:]
        elif infosrc == u'MTWW':
            return u'MarineTraffic.com web site'
        elif infosrc == u'MTTR':
            return u'MarineTraffic.com track files'
        else:
            return infosrc

    '''
    Maps the csv header name to matching function to call to get the data.
    '''
    csv_name_to_function = {
        'mmsi': lambda nmea: nmea.strmmsi,
        'flag': get_flag,
        'name': Nmea5.get_name,
        'imo': lambda nmea: str(nmea.imo),
        'callsign': Nmea5.get_callsign,
        'type': lambda nmea: str(nmea.type) + '-' + nmea.get_shiptype(),
        'length':lambda nmea: str(nmea.get_length()),
        'width': lambda nmea: str(nmea.get_width()),
        'datetime': lambda nmea: datetime.utcfromtimestamp(nmea.get_last_timestamp()).strftime('%Y-%m-%dT%H:%M:%SZ'),
        'status': Nmea1.get_status,
        'sog': Nmea1.get_sog_str,
        'latitude': Nmea1.get_latitude_str,
        'longitude': Nmea1.get_longitude_str,
        'cog': Nmea1.get_cog_str,
        'heading': Nmea1.get_heading_str,
        'destination': Nmea5.get_destination,
        'eta': Nmea5.get_eta_str,
        'draught': Nmea5.get_draught_str,
    }

    def get_dump_row(self, fieldnames):
        result = []
        for fieldname in fieldnames:
            f = self.csv_name_to_function[fieldname]
            result.append(f(self))
        return result

    #def get_dump_row(self):
    #    result = []
    #    def _clean(txt):
    #        if txt is None:
    #            return ''
    #        return txt.replace('\0','').replace('@', '').strip()

    #    result.append(self.strmmsi)
    #    result.append(self.get_flag().encode('utf-8'))
    #    result.append(self.get_name())
    #    result.append(str(self.imo))
    #    result.append(_clean(self.callsign))
    #    result.append(str(self.type) + '-' + SHIP_TYPES.get(self.type, 'unknown'))
    #    d = self.dim_bow + self.dim_stern
    #    if d:
    #        result.append(d)
    #    else:
    #        result.append(None)
    #    d = self.dim_port + self.dim_starboard
    #    if d:
    #        result.append(d)
    #    else:
    #        result.append(None)
    #    result.append(datetime.utcfromtimestamp(self.timestamp_1).strftime('%Y-%m-%dT%H:%M:%SZ'))
    #    result.append(STATUS_CODES.get(self.status, 'unknown'))
    #    if self.sog != AIS_SOG_NOT_AVAILABLE:
    #        result.append(str(self.sog/AIS_SOG_SCALE))
    #    else:
    #        result.append(None)
    #    if self.latitude != AIS_LAT_NOT_AVAILABLE:
    #        result.append(str(self.latitude/AIS_LATLON_SCALE))
    #    else:
    #        result.append(None)
    #    if self.longitude != AIS_LON_NOT_AVAILABLE:
    #        result.append(str(self.longitude/AIS_LATLON_SCALE))
    #    else:
    #        result.append(None)
    #    if self.cog != AIS_COG_NOT_AVAILABLE:
    #        result.append(str(self.cog/10.))
    #    else:
    #        result.append(None)
    #    if self.heading != AIS_NO_HEADING:
    #        result.append(str(self.heading))
    #    else:
    #        result.append(None)
    #    result.append(self.get_destination(''))
    #    result.append(self.get_eta_str(''))
    #    result.append(self.draught)
    #    result.append(self.source_5)
    #    return result


class BankNmea1(list):
    """
    That class handle a .nmea1 archive file
    """
    def __init__(self, strmmsi, dt):
        list.__init__(self)
        self.strmmsi = strmmsi
        if isinstance(dt, date):
            dt = dt.strftime('%Y%m%d')
        self.date = dt

    def get_filename(self):
        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea1'))

    def __load_from_file(self, file):
        '''
        Adds all record from opened file in this bank
        File must be locked before call
        '''
        while True:
            record = file.read(AIVDM_RECORD123_LENGTH)
            if not record:
                break
            self.append(Nmea1.new_from_record(record))

    def _write_in_file(self, file):
        '''
        Write all records from that bank in opened file
        File must be locked before call
        File should be truncated after call
        '''
        for nmea1 in list.__iter__(self): # self.__iter__ reload the bank, we don't want that
            file.write(nmea1.to_record())

    def __load(self):
        try:
            file = open(self.get_filename(), 'rb')
            lockf(file, LOCK_SH)
        except IOError as ioerr:
            if ioerr.errno == 2: # No file
                return
            raise
        self.__load_from_file(file)
        file.close()
        
    def __iter__(self):
        """
        Each call reload the file
        """
        self.__load()
        self.sort_by_date_reverse()
        return list.__iter__(self)

    def packday(self, remove_manual_input=False, remove_source_name=None):
        #print "MMSI", strmmsi

        filename = self.get_filename()
        try:
            file = open(filename, 'r+b') # read/write binary
        except IOError as ioerr:
            if ioerr.errno != 2: # No file
                raise
            return self # no data
        lockf(file, LOCK_EX)
        self.__load_from_file(file)
        self.sort_by_date()

        file_has_changed = False
        file_must_be_unlinked = False

        logging.debug('PACKING...')
        file_has_changed = self.remove_duplicate_timestamp() or file_has_changed

        if remove_manual_input:
            logging.debug('REMOVING MANUAL INPUT...')
            file_has_changed = self.remove_manual_input() or file_has_changed

        if remove_source_name:
            logging.debug('REMOVING SOURCES STARTING BY %s', remove_source_name)
            file_has_changed = self.remove_by_source(source_name_start=remove_source_name) or file_has_changed

        if file_has_changed:
            logging.debug('SAVING CHANGES')
            file.seek(0)
            self._write_in_file(file)
            file.truncate()
            if file.tell() == 0:
                file_must_be_unlinked = True

        file.close()
        
        if file_must_be_unlinked:
            # FIXME we release the lock before unlinking
            # another process might encounter an empty file (not handled)
            logging.warning('file was truncated to size 0. unlinking')
            os.unlink(filename) # we have the lock (!)

    def dump_to_stdout(self):
        """
        Print contents to stdout
        """
        for nmea1 in self:
            nmea1.dump_to_stdout()

    def sort_by_date(self):
        self.sort(lambda n1, n2: n1.timestamp_1 - n2.timestamp_1)

    def sort_by_date_reverse(self):
        self.sort(lambda n1, n2: n2.timestamp_1 - n1.timestamp_1)

    def remove_duplicate_timestamp(self):
        file_has_changed = False
        if len(self) <= 1:
            return file_has_changed
        last_timestamp = self[0].timestamp_1
        i = 1
        while i < len(self):
            if self[i].timestamp_1 == last_timestamp:
                del self[i]
                file_has_changed = True
            else:
                last_timestamp = self[i].timestamp_1
                i += 1
        return file_has_changed
        
    def remove_manual_input(self):
        file_has_changed = False
        i = 0
        while i < len(self):
            if self[i].source_1[:2] == 'MI':
                del self[i]
                file_has_changed = True
            else:
                i += 1
        return file_has_changed

    def remove_by_source(self, source_name_start):
        file_has_changed = False
        i = 0
        while i < len(self):
            #logging.debug('Testing %s ...', self[i].source_1)
            if self[i].source_1.startswith(source_name_start):
                #logging.debug('Deleting ...')
                del self[i]
                file_has_changed = True
            else:
                #logging.debug('Keeping ...')
                i += 1
        return file_has_changed

class Nmea1Feeder:
    """
    Yields all nmea1 packets between two given datetimes
    in REVERSE order (recent information first)
    """
    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
        self.strmmsi = strmmsi
        assert datetime_end is not None
        self.datetime_end = datetime_end
        self.datetime_begin = datetime_begin or DB_STARTDATE
        self.max_count = max_count

    def __iter__(self):
        dt_end = self.datetime_end
        d_end = dt_end.date()
        ts_end = datetime_to_timestamp(dt_end)
        if self.datetime_begin:
            dt_begin = self.datetime_begin
            d_begin = dt_begin.date()
            ts_begin = datetime_to_timestamp(dt_begin)
        else:
            dt_begin = None
            d_begin = None
            ts_begin = None

        d = d_end
        count = 0
        while True:
            if d_begin is not None and d < d_begin:
                return
            bank = BankNmea1(self.strmmsi, d)
            for nmea1 in bank:
                if ts_begin is not None and nmea1.timestamp_1 < ts_begin:
                    return
                if nmea1.timestamp_1 > ts_end:
                    continue
                
                yield nmea1
               
                count += 1
                if self.max_count and count >= self.max_count:
                    return
            d += timedelta(-1)


class BankNmea5(list):
    """
    That class handle a .nmea5 archive file
    """
    def __init__(self, strmmsi, dt):
        list.__init__(self)
        self.strmmsi = strmmsi
        if isinstance(dt, date):
            try:
                dt = dt.strftime('%Y%m%d')
            except ValueError:
                logging.critical('dt=%s', dt)
                raise
        self.date = dt

    def get_filename(self):
        return os.path.join(DBPATH, 'bydate', self.date, _hash3_pathfilename(self.strmmsi+'.nmea5'))

    def __load_from_file(self, file):
        '''
        Adds all record from opened file in this bank
        File must be locked before call
        '''
        while True:
            record = file.read(AIVDM_RECORD5_LENGTH)
            if not record:
                break
            self.append(Nmea5.new_from_record(record))

    def _write_in_file(self, file):
        '''
        Write all records from that bank in opened file
        File must be locked before call
        File should be truncated after call
        '''
        for nmea5 in self:
            file.write(nmea5.to_record())

    def __load(self):
        try:
            file = open(self.get_filename(), 'rb')
            lockf(file, LOCK_SH)
        except IOError as ioerr:
            if ioerr.errno == 2: # No file
                return
            raise
        self.__load_from_file(file)
        file.close()
        
    def __iter__(self):
        """
        Each call reload the file
        """
        self.__load()
        self.sort_by_date_reverse()
        return list.__iter__(self)

    def sort_by_date(self):
        self.sort(lambda n1, n2: n1.timestamp_5 - n2.timestamp_5)

    def sort_by_date_reverse(self):
        self.sort(lambda n1, n2: n2.timestamp_5 - n1.timestamp_5)

class Nmea5Feeder:
    """
    Yields all nmea5 packets between two given datetimes
    in REVERSE order (recent information first)
    """
    def __init__(self, strmmsi, datetime_end, datetime_begin=None, max_count=0):
        self.strmmsi = strmmsi
        assert datetime_end is not None
        self.datetime_end = datetime_end
        self.datetime_begin = datetime_begin or DB_STARTDATE
        self.max_count = max_count

    def __iter__(self):
        dt_end = self.datetime_end
        d_end = dt_end.date()
        ts_end = datetime_to_timestamp(dt_end)
        if self.datetime_begin:
            dt_begin = self.datetime_begin
            d_begin = dt_begin.date()
            ts_begin = datetime_to_timestamp(dt_begin)
        else:
            dt_begin = None
            d_begin = None
            ts_begin = None

        d = d_end
        count = 0
        while True:
            if d_begin is not None and d < d_begin:
                return
            bank = BankNmea5(self.strmmsi, d)
            for nmea1 in bank:
                if ts_begin is not None and nmea1.timestamp_5 < ts_begin:
                    return
                if nmea1.timestamp_5 > ts_end:
                    continue
                
                yield nmea1
               
                count += 1
                if self.max_count and count >= self.max_count:
                    return
            d += timedelta(-1)


class NmeaFeeder:
    """
    Yields nmea packets matching criteria.
    """
    def __init__(self, strmmsi, datetime_end, datetime_begin=None, filters=None, granularity=1, max_count=None):
        if granularity <= 0:
            logging.warning('Granularity=%d generates duplicate entries', granularity)
        self.strmmsi = strmmsi
        assert datetime_end is not None
        self.datetime_end = datetime_end
        self.datetime_begin = datetime_begin or DB_STARTDATE
        self.filters = filters or []
        self.granularity = granularity
        self.max_count = max_count

    def __iter__(self):
        nmea = Nmea(self.strmmsi)
        if self.datetime_begin:
            nmea5_datetime_begin = self.datetime_begin - timedelta(30) # go back up to 30 days to get a good nmea5 packet
        else:
            nmea5_datetime_begin = None
        nmea5_iterator = Nmea5Feeder(self.strmmsi, self.datetime_end, nmea5_datetime_begin).__iter__()
        nmea5 = Nmea5(self.strmmsi, sys.maxint)

        count = 0
        lasttimestamp = sys.maxint
        for nmea1 in Nmea1Feeder(self.strmmsi, self.datetime_end, self.datetime_begin):
            Nmea1.from_values(nmea, *nmea1.to_values())
            
            # try to get an nmea5 paket older
            nmea5_updated = False
            while nmea5 is not None and nmea5.timestamp_5 > nmea1.timestamp_1:
                try:
                    nmea5 = nmea5_iterator.next()
                    nmea5_updated = True
                except StopIteration:
                    nmea5 = None
            
            if nmea5_updated and nmea5 is not None:
                Nmea5.merge_from_values(nmea, *nmea5.to_values())

            filtered_out = False
            for is_ok in self.filters:
                if not is_ok(nmea):
                    filtered_out = True
                    break
            if filtered_out:
                continue

            if nmea.timestamp_1 <= lasttimestamp - self.granularity:
                yield nmea
                count += 1
                if self.max_count and count >= self.max_count:
                    return
                lasttimestamp = nmea.timestamp_1


def nice_timedelta_str(delta):
    strdelta = ''
    disprank = None # first item type displayed
    if delta.days:
        strdelta += str(delta.days)
        if delta.days > 1:
            strdelta += ' days '
        else:
            strdelta += ' day '
        disprank = 0
    delta_s = delta.seconds
    delta_m = delta_s // 60
    delta_s -= delta_m * 60
    delta_h = delta_m // 60
    delta_m -= delta_h * 60

    if delta_h:
        strdelta += str(delta_h)
        if delta_h > 1:
            strdelta += ' hours '
        else:
            strdelta += ' hour '
        if disprank is None:
            disprank = 1
    if delta_m and (disprank is None or disprank >= 1):
        strdelta += str(delta_m)
        if delta_m > 1:
            strdelta += ' minutes '
        else:
            strdelta += ' minute '
        if disprank is None:
            disprank = 2
    if delta_s and (disprank is None or disprank >= 2):
        strdelta += str(delta_s)
        if delta_s > 1:
            strdelta += ' seconds '
        else:
            strdelta += ' second '
        if disprank is None:
            disprank = 3
    if not strdelta:
        strdelta = 'less than a second '
    return strdelta

def all_mmsi_generator():
    """
    Returns an array of all known strmmsi.
    """
    for dirname, dirs, fnames in os.walk(os.path.join(DBPATH, 'last')):
        for fname in fnames:
            if fname[-6:] == '.nmea1':
                yield fname[:-6]


def load_fleet_to_uset(fleetid):
    """
    Loads a fleet by id.
    Returns an array of strmmsi.
    """
    result = []
    sqlexec(u"SELECT mmsi FROM fleet_vessel WHERE fleet_id=" + unicode(fleetid))
    cursor = get_common_cursor()
    while True:
        row = cursor.fetchone()
        if not row:
            break
        mmsi = row[0]
        result.append(mmsi_to_strmmsi(mmsi))
    logging.debug('fleet=%s', result)
    return result


def fleetname_to_fleetid(fleetname):
    sqlexec(u"SELECT id FROM fleet WHERE name=%(fleetname)s", {'fleetname': fleetname})
    cursor = get_common_cursor()
    row = cursor.fetchone()
    return row[0]



def mmsiiterator_nohiddenship(mmsiiterator):
    '''
    filters strmmsi from an strmmsi iterator
    '''
    for strmmsi in mmsiiterator:
        mmsi = strmmsi_to_mmsi(strmmsi)
        if mmsi not in ais.inputs.config.get_hidden_mmsi():
            yield strmmsi

def filter_area(nmea, area):
    """
    Returns false if position is out of area.
    """
    if nmea.latitude == AIS_LAT_NOT_AVAILABLE or nmea.longitude == AIS_LON_NOT_AVAILABLE:
        return False
    if not area.contains((nmea.latitude/AIS_LATLON_SCALE, nmea.longitude/AIS_LATLON_SCALE)):
        return False
    return True

def filter_close_to(nmea, lat, lon, miles=1.0):
    '''
    Returns true if position is closer than miles from (lat, lon)
    '''
    return dist3_xyz(latlon_to_xyz_deg(lat, lon), latlon_to_xyz_ais(nmea.latitude, nmea.longitude)) <= miles


def filter_far_from(nmea, lat, lon, miles=1.0):
    '''
    Returns true if position is farther than miles from (lat, lon)
    '''
    return dist3_xyz(latlon_to_xyz_deg(lat, lon), latlon_to_xyz_ais(nmea.latitude, nmea.longitude)) >= miles


def filter_sog_le(nmea, max_knts):
    '''
    Returns true if speed over ground is less than max_knts
    '''
    return nmea.sog/AIS_SOG_SCALE <= max_knts


def filter_sog_ge(nmea, min_knts):
    '''
    Returns true if speed over ground is less than min_knts
    '''
    return nmea.sog/AIS_SOG_SCALE >= min_knts


def filter_knownposition(nmea):
    """
    Returns false if position is not fully known
    """
    # we are filtering out latitude=0 and longitude=0, that is not supposed to be necessary...
    return nmea.latitude != AIS_LAT_NOT_AVAILABLE and nmea.longitude != AIS_LON_NOT_AVAILABLE and nmea.latitude != 0 and nmea.longitude != 0


_filter_positioncheck_last_mmsi = None
def filter_speedcheck(nmea, max_mps):
    """
    mps is miles per seconds
    """
    global _filter_positioncheck_last_mmsi
    global _filter_positioncheck_last_time
    global _filter_positioncheck_last_time_failed
    global _filter_positioncheck_last_lat
    global _filter_positioncheck_last_lon
    global _filter_positioncheck_error_count
    if nmea.strmmsi != _filter_positioncheck_last_mmsi:
        _filter_positioncheck_last_time = None
        _filter_positioncheck_last_mmsi = nmea.strmmsi
        _filter_positioncheck_error_count = 0
    if _filter_positioncheck_last_time is not None:
        seconds = _filter_positioncheck_last_time - nmea.timestamp_1
        distance = dist3_latlong_ais((_filter_positioncheck_last_lat, _filter_positioncheck_last_lon), (nmea.latitude, nmea.longitude))
        if seconds:
            speed = distance/seconds
            if speed > max_mps:
                if _filter_positioncheck_error_count < 10:
                    logging.debug("Ignoring point: distance = %s, time = %s, speed = %s kt, source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
                    if _filter_positioncheck_error_count == 0 or _filter_positioncheck_last_time_failed != nmea.timestamp_1:
                        _filter_positioncheck_error_count += 1
                        _filter_positioncheck_last_time_failed = nmea.timestamp_1
                    return False
                else:
                    logging.warning("Discontinous position accepted after too many failures: %.2f nm in %s s (%.0f kt), source = %s", distance, seconds, distance/seconds*3600, repr(nmea.source_1))
            _filter_positioncheck_error_count = 0
    _filter_positioncheck_last_time = nmea.timestamp_1
    _filter_positioncheck_last_lat = nmea.latitude
    _filter_positioncheck_last_lon = nmea.longitude
    return True


def main():
    """
    Perform various operation on the database
    For usage, see "ais --help"
    """
    from optparse import OptionParser, OptionGroup
    global DBPATH

    parser = OptionParser(usage='%prog [options] { mmsi | @fleetname | ^fleetid }+ | all')

    parser.add_option('-d', '--debug',
        action='store_true', dest='debug', default=False,
        help="debug mode")

    parser.add_option('-e', '--end',
        action='store', dest='sdt_end', metavar="'YYYYMMDD HHMMSS'",
        help='End data processing on that GMT date time.'
             ' Default is now.'
             ' If a date is provided without time, time defaults to 235959.')
    parser.add_option('-s', '--start',
        action='store', dest='sdt_start', metavar="'YYYYMMDD HHMMSS'",
        help='Start data processing on that date.'
             ' Using that option enables multiple output of the same boat.'
             ' Disabled by default.'
             ' If a date is provided without time, time default to 000000.'
             ' If other options enable multiple output, default to 1 day before'
             ' --end date/time.')
    parser.add_option('--duration',
        action='store', dest='sdt_duration', metavar="DURATION",
        help='Duration of reference period.'
             ' Last character may be S for seconds, M(inutes), D(ays), W(eeks)'
             ' Default is seconds.'
             ' This is the time length bewteen --start and --end above.'
             ' If you want multiple output of the same boat, you may use '
             ' --start, --end or --duration, 2 of them, but not 3 of them.')
    parser.add_option('-g', '--granularity',
        action='store', type='int', dest='granularity', metavar='SECONDS',
        help='Dump only one position every granularity seconds.'
             ' Using that option enables multiple output of the same boat.'
             ' If other options enable multiple output, defaults to 600'
             ' (10 minutes)')
    parser.add_option('--max',
        action='store', type='int', dest='max_count', metavar='NUMBER',
        help='Dump a maximum of NUMBER positions every granularity seconds.'
             'Using that option enables multiple output of the same boat.')

    parser.add_option('--show-hidden-ships',
        action='store_true', dest='show_hidden_ships', default=False,
        help='Include hidden ships in results')

    parser.add_option('--filter-knownposition',
        action='store_true', dest='filter_knownposition', default=False,
        help="Eliminate unknown positions from results.")

    parser.add_option('--filter-speedcheck',
        action='store', type='int', dest='speedcheck', default=200, metavar='KNOTS',
        help='Eliminate erroneaous positions from results,' 
             ' based on impossible speed.')

    parser.add_option('--filter-type',
        action='append', type='int', dest='type_list', metavar='TYPE',
        help="process a specific ship type.")
    parser.add_option('--help-types',
        action='store_true', dest='help_types', default=False,
        help="display list of available types")

    parser.add_option('--filter-area',
        action='store', type='str', dest='area_file', metavar="FILE.KML",
        help="only process a specific area as defined in a kml polygon file.")
    parser.add_option('--filter-farfrom',
        action='store', dest='far_from', nargs=3, metavar='LAT LONG MILES',
        help="only show ships farther than MILES miles from LAT,LONG")
    parser.add_option('--filter-closeto',
        action='store', dest='close_to', nargs=3, metavar='LAT LONG MILES',
        help="only show ships closer than MILES miles from LAT,LONG")
    parser.add_option('--filter-sog-le',
        action='store', dest='sog_le', metavar='KNOTS',
        help='only show ships when speed over ground is less or equal than KNOTS.')
    parser.add_option('--filter-sog-ge',
        action='store', dest='sog_ge', metavar='KNOTS',
        help='only show ships when speed over ground is greater or equal than KNOTS.')

    parser.add_option('--filter-destination',
        action='store', type='str', dest='filter_destination', metavar="DESTINATION",
        help="Only print ships with that destination.")

    parser.add_option('--no-headers',
        action='store_false', dest='csv_headers', default=True,
        help="skip CSV headers")

    parser.add_option('--csv-fields',
        action='store', type='str', dest='csv_fields',
        default='mmsi,flag,name,imo,callsign,type,length,width,datetime,status,sog,latitude,longitude,cog,heading,destination,eta,draught',
        help='Which fields should be extracted for csv output. Default=%default')
    #

    expert_group = OptionGroup(parser, "Expert Options",
        "You normaly don't need any of these")

    expert_group.add_option('--db',
        action='store', dest='db', default=DBPATH,
        help="path to filesystem database. Default=%default")

    expert_group.add_option('--debug-sql',
        action='store_true', dest='debug_sql', default=False,
        help="print all sql queries to stdout before running them")

    expert_group.add_option('--action',
        choices=('dump', 'removemanual', 'removebysource', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump',
        help='Possible values are:\n'
            'dump: dump values in csv format. This is the default.\n'
            'removemanual: Delete Manual Input entries from the database.\n'
            'mmsidump: Dump mmsi')
    parser.add_option_group(expert_group)

    (options, args) = parser.parse_args()


    if options.help_types:
        print "Known ship types:"
        keys = SHIP_TYPES.keys()
        keys.sort()
        for k in keys:
            print k, SHIP_TYPES[k]
        sys.exit(0)

    DBPATH = options.db

    if options.debug:
        loglevel = logging.DEBUG
    else:
        loglevel = logging.INFO
    logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s')

    if options.debug_sql:
        sql_setdebug(True)

    #
    # Ships selections
    #

    if len(args)==0:
        print >> sys.stderr, "No ship to process"
        sys.exit(1)

    target_mmsi_iterator = [] # strmmsi
    all_targets = False
    for arg in args:
        if arg == 'all':
            all_targets = True
        elif arg.startswith('@'):
            target_mmsi_iterator += load_fleet_to_uset(fleetname_to_fleetid(arg[1:]))
        elif arg.startswith('^'):
            target_mmsi_iterator += load_fleet_to_uset(int(arg[1:]))
        else:
            target_mmsi_iterator.append(arg)
    if all_targets:
        if target_mmsi_iterator:
            logging.warning('Selecting all ships, ignoring other arguments')
        target_mmsi_iterator = all_mmsi_generator()

    if not options.show_hidden_ships:
        target_mmsi_iterator = mmsiiterator_nohiddenship(target_mmsi_iterator)

    #
    # Dates selections
    #

    if options.sdt_start:
        # remove non digit characters
        options.sdt_start = "".join([ c for c in options.sdt_start if c.isdigit()])
        if len(options.sdt_start)==14:
            options.sdt_start = datetime.strptime(options.sdt_start, '%Y%m%d%H%M%S')
        elif len(options.sdt_start)==8:
            options.sdt_start = datetime.strptime(options.sdt_start, '%Y%m%d')
        else:
            print >> sys.stderr, "Invalid format for --start option"
            sys.exit(1)

    if options.sdt_end:
        # remove non digit characters
        options.sdt_end = "".join([ c for c in options.sdt_end if c.isdigit()])
        if len(options.sdt_end)==14:
            options.sdt_end = datetime.strptime(options.sdt_end, '%Y%m%d%H%M%S')
        elif len(options.sdt_end)==8:
            options.sdt_end = datetime.strptime(options.sdt_end, '%Y%m%d')
            options.sdt_end = datetime.combine(options.sdt_end.date(), time(23, 59, 59))
        else:
            print >> sys.stderr, "Invalid format for --end option"
            sys.exit(1)
    
    if options.sdt_duration:
        # remove spaces
        options.sdt_duration = options.sdt_duration.replace(' ', '')
        # use upercase
        options.sdt_duration = options.sdt_duration.upper()
        if options.sdt_duration[-1] == 'S':
            options.sdt_duration = options.sdt_duration[:-1]
            duration_unit = 1
        elif options.sdt_duration[-1] == 'M':
            options.sdt_duration = options.sdt_duration[:-1]
            duration_unit = 60
        elif options.sdt_duration[-1] == 'H':
            options.sdt_duration = options.sdt_duration[:-1]
            duration_unit = 60*60
        elif options.sdt_duration[-1] == 'D':
            options.sdt_duration = options.sdt_duration[:-1]
            duration_unit = 24*60*60
        elif options.sdt_duration[-1] == 'W':
            options.sdt_duration = options.sdt_duration[:-1]
            duration_unit = 7*24*60*60
        else:
            duration_unit = 1
        try:
            options.sdt_duration = long(options.sdt_duration)
        except ValueError:
            print >> sys.stderr, "Can't parse duration"
            sys.exit(1)
        options.sdt_duration = timedelta(0, options.sdt_duration * duration_unit)

    if options.sdt_start or options.sdt_duration or options.granularity is not None or options.max_count:
        # Time period is enabled (note that date_end only defaults to one day archives ending then)
        if not options.sdt_start and not options.sdt_end and not options.sdt_duration:
            options.sdt_duration = timedelta(1) # One day
        # continue without else
        if not options.sdt_start and not options.sdt_end and options.sdt_duration:
            dt_end = datetime.utcnow()
            dt_start = dt_end - options.sdt_duration
        #elif not options.sdt_start and options.sdt_end and not options.sdt_duration:
            # never reached
        elif not options.sdt_start and options.sdt_end and options.sdt_duration:
            dt_end = options.sdt_end
            dt_start = dt_end - options.sdt_duration
        elif options.sdt_start and not options.sdt_end and not options.sdt_duration:
            dt_start = options.sdt_start
            dt_end = datetime.utcnow()
        elif options.sdt_start and not options.sdt_end and options.sdt_duration:
            dt_start = options.sdt_start
            dt_end = dt_start + options.sdt_duration
        elif options.sdt_start and options.sdt_end and not options.sdt_duration:
            dt_start = options.sdt_start
            dt_end = options.sdt_end
        else:
            assert options.sdt_start and options.sdt_end and options.sdt_duration, 'Internal error'
            print >> sys.stderr, "You can't have all 3 --start --end and --duration"
            sys.exit(1)
        if options.granularity is None:
            options.granularity = 600
    else:
        # Only get one position
        dt_start = None
        if options.sdt_end:
            dt_end = options.sdt_end
        else:
            dt_end = datetime.utcnow()
        options.max_count = 1
        if options.granularity is None:
            options.granularity = 600
            
    logging.debug('--start is %s', dt_start)
    logging.debug('--end is %s', dt_end)

    #
    # Filters
    #

    filters = []
    
    if options.filter_knownposition:
        filters.append(filter_knownposition)

    if options.speedcheck != 0:
        maxmps = options.speedcheck / 3600. # from knots to NM per seconds
        filters.append(lambda nmea: filter_speedcheck(nmea, maxmps))

    if options.area_file:
        area = load_area_from_kml_polygon(options.area_file)
        filters.append(lambda nmea: filter_area(nmea, area))
    
    if options.close_to:
        try:
            lat = clean_latitude(unicode(options.close_to[0], 'utf-8'))
            lon = clean_longitude(unicode(options.close_to[1], 'utf-8'))
        except LatLonFormatError as err:
            print >> sys.stderr, err.args
            sys.exit(1)
        miles = float(options.close_to[2])
        filters.append(lambda nmea: filter_close_to(nmea, lat, lon, miles))

    if options.far_from:
        try:
            lat = clean_latitude(unicode(options.far_from[0], 'utf-8'))
            lon = clean_longitude(unicode(options.far_from[1], 'utf-8'))
        except LatLonFormatError as err:
            print >> sys.stderr, err.args
            sys.exit(1)
        miles = float(options.far_from[2])
        filters.append(lambda nmea: filter_far_from(nmea, lat, lon, miles))
    
    if options.sog_le:
        filters.append(lambda nmea: filter_sog_le(nmea, float(options.sog_le)))
    if options.sog_ge:
        filters.append(lambda nmea: filter_sog_ge(nmea, float(options.sog_ge)))

    if options.type_list:
        def filter_type(nmea):
            return nmea.type in options.type_list
        filters.append(filter_type)

    if options.filter_destination:
        filters.append(lambda nmea: nmea.destination.startswith(options.filter_destination))

    #
    # Processing
    #

    if options.action == 'dump':
        fields = options.csv_fields.split(',')
        output = csv.writer(sys.stdout)
        if options.csv_headers:
            output.writerow(fields)
        for mmsi in target_mmsi_iterator:
            logging.debug('Considering %s', repr(mmsi))
            assert dt_end is not None
            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
                output.writerow(nmea.get_dump_row(fields))

    elif options.action == 'removemanual':
        if filters:
            print >> sys.stderr, "removemanual action doesn't support filters"
            sys.exit(1)

        # TODO: dates = range dt_start, dt_end
        dt = dt_start.date()
        while dt < dt_end.date():
            logging.info("Processing date %s", dt)
            for mmsi in target_mmsi_iterator:
                BankNmea1(mmsi, dt).packday(remove_manual_input=True)
            dt = dt + timedelta(1)
    
    elif options.action == 'removebysource':
        if filters:
            print >> sys.stderr, "removebysource action doesn't support filters"
            sys.exit(1)

        # TODO: dates = range dt_start, dt_end
        dt = dt_start.date()
        while dt <= dt_end.date():
            logging.info("Processing date %s", dt)
            for mmsi in target_mmsi_iterator:
                if BankNmea1(mmsi, dt).packday(remove_source_name='MT'):
                    logging.info('File was modified. mmsi=%s dt=%s', mmsi, dt)
            dt = dt + timedelta(1)
    
    elif options.action == 'mmsidump':
        for strmmsi in target_mmsi_iterator :
            print strmmsi

    elif options.action == 'fixdestination':
        for mmsi in target_mmsi_iterator:
            for nmea in NmeaFeeder(mmsi, dt_end, dt_start, filters, granularity=options.granularity, max_count=options.max_count):
                destination = nmea.destination.rstrip(' @\0')
                if destination:
                    sqlexec(u'UPDATE vessel SET destination = %(destination)s WHERE mmsi=%(mmsi)s AND destination IS NULL', {'mmsi':strmmsi_to_mmsi(mmsi), 'destination':destination})
                    logging.info('%s -> %s', mmsi, repr(destination))
                    dbcommit()
                    break # go to next mmsi


if __name__ == '__main__':
    main()