New sog_ge speed filter

Improved remove_by_source tool for data erasing

diff --git a/bin/common.py b/bin/common.py
index 3102153056f2bee4f8309412d77b04ce545741ac..66bfc6e76ff90e7a259f7cd1bcca79f2f28ad5da 100755 (executable)
--- a/bin/common.py
+++ b/bin/common.py
@@ -48,6 +48,7 @@ __all__ = [
     'filter_close_to',
     'filter_far_from',
     'filter_sog_le',
+    'filter_sog_ge',
     'filter_knownposition',
     'filter_speedcheck',
     ]
@@ -857,7 +858,7 @@ class Nmea1:
         return Nmea1(*values)
 
     def to_record(self):
-        return struct.pack(AIVDM_RECORD123_FORMAT, *Nmea1.to_values())
+        return struct.pack(AIVDM_RECORD123_FORMAT, *Nmea1.to_values(self))
         
     def from_file(self, file):
         record = file.read(AIVDM_RECORD123_LENGTH)
@@ -1369,7 +1370,7 @@ class BankNmea1(list):
         File must be locked before call
         File should be truncated after call
         '''
-        for nmea1 in self:
+        for nmea1 in list.__iter__(self): # self.__iter__ reload the bank, we don't want that
             file.write(nmea1.to_record())
 
     def __load(self):
@@ -1391,11 +1392,9 @@ class BankNmea1(list):
         self.sort_by_date_reverse()
         return list.__iter__(self)
 
-    def packday(remove_manual_input=False):
-        # FIXME broken
+    def packday(self, remove_manual_input=False, remove_source_name=None):
         #print "MMSI", strmmsi
 
-        self = BankNmea1(self.strmmsi, self.date)
         filename = self.get_filename()
         try:
             file = open(filename, 'r+b') # read/write binary
@@ -1410,14 +1409,19 @@ class BankNmea1(list):
         file_has_changed = False
         file_must_be_unlinked = False
 
-        #print "PACKING..."
+        logging.debug('PACKING...')
         file_has_changed = self.remove_duplicate_timestamp() or file_has_changed
 
         if remove_manual_input:
-            #print "REMOVING MANUAL INPUT..."
+            logging.debug('REMOVING MANUAL INPUT...')
             file_has_changed = self.remove_manual_input() or file_has_changed
 
+        if remove_source_name:
+            logging.debug('REMOVING SOURCES STARTING BY %s', remove_source_name)
+            file_has_changed = self.remove_by_source(source_name_start=remove_source_name) or file_has_changed
+
         if file_has_changed:
+            logging.debug('SAVING CHANGES')
             file.seek(0)
             self._write_in_file(file)
             file.truncate()
@@ -1471,6 +1475,20 @@ class BankNmea1(list):
                 i += 1
         return file_has_changed
 
+    def remove_by_source(self, source_name_start):
+        file_has_changed = False
+        i = 0
+        while i < len(self):
+            #logging.debug('Testing %s ...', self[i].source_1)
+            if self[i].source_1.startswith(source_name_start):
+                #logging.debug('Deleting ...')
+                del self[i]
+                file_has_changed = True
+            else:
+                #logging.debug('Keeping ...')
+                i += 1
+        return file_has_changed
+
 class Nmea1Feeder:
     """
     Yields all nmea1 packets between two given datetimes
@@ -1791,6 +1809,13 @@ def filter_sog_le(nmea, max_knts):
     return nmea.sog/AIS_SOG_SCALE <= max_knts
 
 
+def filter_sog_ge(nmea, min_knts):
+    '''
+    Returns true if speed over ground is less than min_knts
+    '''
+    return nmea.sog/AIS_SOG_SCALE >= min_knts
+
+
 def filter_knownposition(nmea):
     """
     Returns false if position is not fully known
@@ -1908,7 +1933,10 @@ def main():
         help="only show ships closer than MILES miles from LAT,LONG")
     parser.add_option('--filter-sog-le',
         action='store', dest='sog_le', metavar='KNOTS',
-        help='only show ships when speed over ground is less or equal to KNOTS.')
+        help='only show ships when speed over ground is less or equal than KNOTS.')
+    parser.add_option('--filter-sog-ge',
+        action='store', dest='sog_ge', metavar='KNOTS',
+        help='only show ships when speed over ground is greater or equal than KNOTS.')
 
     parser.add_option('--filter-destination',
         action='store', type='str', dest='filter_destination', metavar="DESTINATION",
@@ -1936,7 +1964,7 @@ def main():
         help="print all sql queries to stdout before running them")
 
     expert_group.add_option('--action',
-        choices=('dump', 'removemanual', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump',
+        choices=('dump', 'removemanual', 'removebysource', 'mmsidump', 'nirgaldebug', 'fixdestination'), default='dump',
         help='Possible values are:\n'
             'dump: dump values in csv format. This is the default.\n'
             'removemanual: Delete Manual Input entries from the database.\n'
@@ -2126,6 +2154,8 @@ def main():
     
     if options.sog_le:
         filters.append(lambda nmea: filter_sog_le(nmea, float(options.sog_le)))
+    if options.sog_ge:
+        filters.append(lambda nmea: filter_sog_ge(nmea, float(options.sog_ge)))
 
     if options.type_list:
         def filter_type(nmea):
@@ -2156,10 +2186,26 @@ def main():
             sys.exit(1)
 
         # TODO: dates = range dt_start, dt_end
-        for dt in dates:
+        dt = dt_start.date()
+        while dt < dt_end.date():
             logging.info("Processing date %s", dt)
             for mmsi in target_mmsi_iterator:
                 BankNmea1(mmsi, dt).packday(remove_manual_input=True)
+            dt = dt + timedelta(1)
+    
+    elif options.action == 'removebysource':
+        if filters:
+            print >> sys.stderr, "removebysource action doesn't support filters"
+            sys.exit(1)
+
+        # TODO: dates = range dt_start, dt_end
+        dt = dt_start.date()
+        while dt <= dt_end.date():
+            logging.info("Processing date %s", dt)
+            for mmsi in target_mmsi_iterator:
+                if BankNmea1(mmsi, dt).packday(remove_source_name='MT'):
+                    logging.info('File was modified. mmsi=%s dt=%s', mmsi, dt)
+            dt = dt + timedelta(1)
     
     elif options.action == 'mmsidump':
         for strmmsi in target_mmsi_iterator :