From: Jean-Michel Nirgal Vourgère Date: Tue, 12 Apr 2011 08:15:37 +0000 (+0000) Subject: New sog_ge speed filter X-Git-Url: https://git.nirgal.com/?p=ais.git;a=commitdiff_plain;h=c1812259ec8589ab707f885c964da7c92e978916;hp=f674c3f9b98004a0e484e81d6c8118ff45cc58c9 New sog_ge speed filter Improved remove_by_source tool for data erasing --- diff --git a/bin/common.py b/bin/common.py index 3102153..66bfc6e 100755 --- a/bin/common.py +++ b/bin/common.py @@ -48,6 +48,7 @@ __all__ = [ 'filter_close_to', 'filter_far_from', 'filter_sog_le', + 'filter_sog_ge', 'filter_knownposition', 'filter_speedcheck', ] @@ -857,7 +858,7 @@ class Nmea1: return Nmea1(*values) def to_record(self): - return struct.pack(AIVDM_RECORD123_FORMAT, *Nmea1.to_values()) + return struct.pack(AIVDM_RECORD123_FORMAT, *Nmea1.to_values(self)) def from_file(self, file): record = file.read(AIVDM_RECORD123_LENGTH) @@ -1369,7 +1370,7 @@ class BankNmea1(list): File must be locked before call File should be truncated after call ''' - for nmea1 in self: + for nmea1 in list.__iter__(self): # self.__iter__ reload the bank, we don't want that file.write(nmea1.to_record()) def __load(self): @@ -1391,11 +1392,9 @@ class BankNmea1(list): self.sort_by_date_reverse() return list.__iter__(self) - def packday(remove_manual_input=False): - # FIXME broken + def packday(self, remove_manual_input=False, remove_source_name=None): #print "MMSI", strmmsi - self = BankNmea1(self.strmmsi, self.date) filename = self.get_filename() try: file = open(filename, 'r+b') # read/write binary @@ -1410,14 +1409,19 @@ class BankNmea1(list): file_has_changed = False file_must_be_unlinked = False - #print "PACKING..." + logging.debug('PACKING...') file_has_changed = self.remove_duplicate_timestamp() or file_has_changed if remove_manual_input: - #print "REMOVING MANUAL INPUT..." + logging.debug('REMOVING MANUAL INPUT...') file_has_changed = self.remove_manual_input() or file_has_changed + if remove_source_name: + logging.debug('REMOVING SOURCES STARTING BY %s', remove_source_name) + file_has_changed = self.remove_by_source(source_name_start=remove_source_name) or file_has_changed + if file_has_changed: + logging.debug('SAVING CHANGES') file.seek(0) self._write_in_file(file) file.truncate() @@ -1471,6 +1475,20 @@ class BankNmea1(list): i += 1 return file_has_changed + def remove_by_source(self, source_name_start): + file_has_changed = False + i = 0 + while i < len(self): + #logging.debug('Testing %s ...', self[i].source_1) + if self[i].source_1.startswith(source_name_start): + #logging.debug('Deleting ...') + del self[i] + file_has_changed = True + else: + #logging.debug('Keeping ...') + i += 1 + return file_has_changed + class Nmea1Feeder: """ Yields all nmea1 packets between two given datetimes @@ -1791,6 +1809,13 @@ def filter_sog_le(nmea, max_knts): return nmea.sog/AIS_SOG_SCALE <= max_knts +def filter_sog_ge(nmea, min_knts): + ''' + Returns true if speed over ground is less than min_knts + ''' + return nmea.sog/AIS_SOG_SCALE >= min_knts + + def filter_knownposition(nmea): """ Returns false if position is not fully known @@ -1908,7 +1933,10 @@ def main(): help="only show ships closer than MILES miles from LAT,LONG") parser.add_option('--filter-sog-le', action='store', dest='sog_le', metavar='KNOTS', - help='only show ships when speed over ground is less or equal to KNOTS.') + help='only show ships when speed over ground is less or equal than KNOTS.') + parser.add_option('--filter-sog-ge', + action='store', dest='sog_ge', metavar='KNOTS', + help='only show ships when speed over ground is greater or equal than KNOTS.') parser.add_option('--filter-destination', action='store', type='str', dest='filter_destination', metavar="DESTINATION", @@ -1936,7 +1964,7 @@ def main(): help="print all sql queries to stdout before running them") expert_group.add_option('--action', - choices=('dump', 'removemanual', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump', + choices=('dump', 'removemanual', 'removebysource', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump', help='Possible values are:\n' 'dump: dump values in csv format. This is the default.\n' 'removemanual: Delete Manual Input entries from the database.\n' @@ -2126,6 +2154,8 @@ def main(): if options.sog_le: filters.append(lambda nmea: filter_sog_le(nmea, float(options.sog_le))) + if options.sog_ge: + filters.append(lambda nmea: filter_sog_ge(nmea, float(options.sog_ge))) if options.type_list: def filter_type(nmea): @@ -2156,10 +2186,26 @@ def main(): sys.exit(1) # TODO: dates = range dt_start, dt_end - for dt in dates: + dt = dt_start.date() + while dt < dt_end.date(): logging.info("Processing date %s", dt) for mmsi in target_mmsi_iterator: BankNmea1(mmsi, dt).packday(remove_manual_input=True) + dt = dt + timedelta(1) + + elif options.action == 'removebysource': + if filters: + print >> sys.stderr, "removebysource action doesn't support filters" + sys.exit(1) + + # TODO: dates = range dt_start, dt_end + dt = dt_start.date() + while dt <= dt_end.date(): + logging.info("Processing date %s", dt) + for mmsi in target_mmsi_iterator: + if BankNmea1(mmsi, dt).packday(remove_source_name='MT'): + logging.info('File was modified. mmsi=%s dt=%s', mmsi, dt) + dt = dt + timedelta(1) elif options.action == 'mmsidump': for strmmsi in target_mmsi_iterator :