Improved ec.europa.eu web grabbing
[ais.git] / bin / extras / shipplotter_coaa.py
1 #!/usr/bin/env python
2 # -*- coding: utf-8 -*-
3
4 from __future__ import division
5 import urllib2, StringIO, gzip, time, random, os
6 #from urlgrabber.keepalive import HTTPHandler
7 from datetime import datetime
8 #from shipplotter_parselogs import parse_file
9
10 from ais.ntools import read_cfg
11
12 __config_filename__ = '/etc/ais/shipplotter'
13
14 #Warning: keep-alive handler does mixes with User-Agent!
15 #keepalive_handler = HTTPHandler()
16 #opener = urllib2.build_opener(keepalive_handler)
17 #urllib2.install_opener(opener)
18
19 request = urllib2.Request('http://www.coaa.co.uk/shipinfo.php')
20 request.add_header('User-Agent', 'ShipPlotter')
21 request.add_header('Accept-encoding', 'gzip, deflate')
22 request.add_header('Cache-Control', 'no-cache')
23
24
25 def get_httpdata():
26     cfg = read_cfg(__config_filename__)
27     reg = cfg['reg']
28     extracode = cfg['extracode']
29
30     # If sharing 1 + 2 + 1->2, Fwd(1)=10, Fwd(2)=0
31     # If sharing 1 + 2, Fwd(1)=0, Fwd(2)=0
32     mouset = random.randint(1,60)
33     tt = int(time.time())
34     
35     # tt must be UTC
36     uo = urllib2.urlopen(request, 'Lines=0&Reg=' + reg + '&Extracode=' + extracode + '&Uponly=0&Mouset='+str(mouset)+'&LatN=161.549709&LatS=-161.548334&LonE=+266.288827&LonW=-245.287049&Fwd=0&Ver=12.4.3&Tt='+str(tt))
37     httpdata = uo.read()
38     uo.close()
39     if uo.headers.get('Content-Encoding')=='gzip':
40         compressedstream = StringIO.StringIO(httpdata)
41         gzipper = gzip.GzipFile(fileobj=compressedstream)
42         httpdata = gzipper.read()
43     return httpdata
44
45 def save_httpdata(httpdata):
46     now = datetime.utcnow()
47     foldername = 'shipplotter/'+now.strftime('%Y%m%d')
48     try:
49         os.mkdir(foldername)
50     except OSError, oserr:
51         if oserr.errno != 17:
52             raise
53         # else folder allrady exists, perfect
54     filename = foldername+'/'+now.strftime('%H%M%S')
55     file = open(filename, 'wb')
56     file.write(httpdata)
57     file.close()
58     return filename
59
60
61 if __name__ == '__main__':
62     from optparse import OptionParser
63     parser = OptionParser()
64     #parser.add_option('--db-path', help="set db path. default=%default", action='store', dest='dbpath', default='db')
65     options, args = parser.parse_args()
66     #for i in range(10):
67     #    data = get_httpdata()
68     #    filename = save_httpdata(data)
69     #    print filename
70     #    parse_file(options.dbpath, filename)
71     #    print 'parsed ok'
72     #    time.sleep(60)
73     data = get_httpdata()
74     print data
75     #keepalive_handler.close_all()