Fix for marinetraffic web change
[ais.git] / bin / extras / marinetraffic.py
index 3d721b7fa58ab5454c9dc73c51cba9c81b803bd7..87a34860a655d36d193285a31309b4242e226ea4 100755 (executable)
@@ -1,10 +1,7 @@
 #!/usr/bin/env python
 # -*- coding: utf-8 -*-
 
-DOWNLOAD_SLEEP_TIME = 10
-DISABLE_DOWNLOAD = False
-MARINETRAFFIC_DIR = '/var/lib/ais/marinetraffic/'
-
+from __future__ import division
 import sys, os, urllib2, time
 from pprint import pprint
 from datetime import datetime, date
@@ -16,13 +13,17 @@ from ais.ntools import clean_alnum, clean_ais_charset, open_with_mkdirs, datetim
 from ais.common import *
 from ais.html_parser import *
 
+DOWNLOAD_SLEEP_TIME = 10
+DISABLE_DOWNLOAD = False
+MARINETRAFFIC_DIR = '/var/lib/ais/marinetraffic/'
+
 
 def go_summary(reference_date, mmsi):
     def get_raw_summary(html):
         root = html_parse(html)
         divs = get_elem(root, 'div')
         divdetail = divs[3]
-        
+        #print_idented_tree(divdetail, 0)
         info_raw = {}
         section = u''
         boldtext = u''
@@ -33,7 +34,10 @@ def go_summary(reference_date, mmsi):
                 continue
             if isinstance(node, Tag) and node.name == 'h2':
                 if boldtext or text:
-                    info_raw[section][boldtext] = text
+                    if not section:
+                        print >> sys.stderr, "WARNING: section is empty for setting", boldtext, "=", text
+                    else:
+                        info_raw[section][boldtext] = text
                     boldtext = text = u''
                 section = get_inner_text(node)
                 info_raw[section] = {}
@@ -45,7 +49,10 @@ def go_summary(reference_date, mmsi):
                     boldtext = text = u''
             elif isinstance(node, Tag) and node.name == 'b':
                 if boldtext or text:
-                    info_raw[section][boldtext] = text
+                    if not section:
+                        print >> sys.stderr, "WARNING: section is empty for setting", boldtext, "=", text
+                    else:
+                        info_raw[section][boldtext] = text
                     boldtext = text = u''
                 boldtext = get_inner_text(node)
             else:
@@ -180,7 +187,7 @@ def go_summary(reference_date, mmsi):
 
     info_raw = get_raw_summary(html)
     info = qualify_summary(info_raw)
-    #pprint(info)
+    pprint(info)
  
     assert info['mmsi'] == mmsi
     
@@ -274,7 +281,7 @@ def import_last_pos(reference_date, mmsi, page=None):
         latlong = get_merged_leaf_content(line[5])
         latlong = latlong.replace(u'\xa0', u'').strip()
         if latlong:
-            lon, lat = latlong.split(' ')
+            lat, lon = latlong.split(' ')
         
         show_on_map = line[8]
         assert get_merged_leaf_content(show_on_map).strip() == 'Show on Map'
@@ -308,6 +315,7 @@ def import_last_pos(reference_date, mmsi, page=None):
             cog = int(course*AIS_COG_SCALE)
             heading = AIS_NO_HEADING
             source = 'MTWW'
+            print strmmsi, timestamp, status, rot, sog, latitude, longitude, cog, heading, source
             add_nmea1(strmmsi, timestamp, status, rot, sog, latitude, longitude, cog, heading, source)
 
         import_track(mmsi, dt, isodt)
@@ -363,10 +371,10 @@ def import_track(mmsi, dt, isodt):
             cog = int(info['course'])*AIS_COG_SCALE
             heading = AIS_NO_HEADING
             source = 'MTTR'
-            #print datetime.utcfromtimestamp(timestamp),
-            #for i in strmmsi, timestamp, status, rot, sog, latitude, longitude, cog, heading, source:
-            #    print repr(i),
-            #print
+            print datetime.utcfromtimestamp(timestamp),
+            for i in strmmsi, timestamp, status, rot, sog, latitude, longitude, cog, heading, source:
+                print repr(i),
+            print
             add_nmea1(strmmsi, timestamp, status, rot, sog, latitude, longitude, cog, heading, source)
     #dbcommit()