UTC encode datetime in results.csv
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime, tzinfo, timedelta
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO, GRADE
24
25 Exemples:
26 NOOP,,,,,,,,,,Pas de changement dans le TdC
27 JOIN,Nirgal,Recrue,0,571628,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,Nick1 est maintenant en vacances
36 COLON,Nick1,Grade1,False,Nick2,,,,,,Nick1 s'est fait coloniser par Nick2
37 COLON,Nick1,Grade1,Nick2,False,,,,,,Nick1 n'est plus colonisé par Nick2
38
39 ANTHILL,Nick1,Grade1,199,200,,,,,,La fourmilière de Nick1 a atteint le niveau 200
40 TECHNO,Nick1,Grade1,199,200,,,,,,La technologie Nick1 a atteint le niveau 200
41
42 GRADE,Nick1,Recrue,Recrue,Passer 806,,,,,,Nick1 est maintenant Passeur 806
43
44 Note that in last 2 FLOOD lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
45 """
46
47 # globals:
48 #BASE_DIRECTORY
49 #SERVER
50 #LOGIN
51 #PASSWORD
52 #BASE_URL
53 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
54 TIMEDELTA0 = timedelta(0)
55
56 class Utc(tzinfo):
57     """UTC"""
58
59     def utcoffset(self, dt):
60         return TIMEDELTA0
61
62     def tzname(self, dt):
63         return "UTC"
64
65     def dst(self, dt):
66         return TIMEDELTA0
67
68 UTC = Utc()
69
70
71 USER_AGENT="Mozilla/5.0 (X11; U; Linux.i686; en-US; rv:1.9.1.16) Gecko/20121020 Iceweasel/3.5.16 (like Firefox/3.5.16)"
72
73 def read_config(config_dir):
74     '''
75     That function will read config.py in .fourmizzz directory
76     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
77     '''
78     global SERVER, LOGIN, PASSWORD
79     global BASE_URL
80     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
81 SERVER = 's1.fourmizzz.fr'
82 LOGIN = 'MonIdentifiant'
83 PASSWORD = 'MonMotDePasse'
84     '''
85
86     sys.path.append(config_dir)
87     try:
88         import config
89     except ImportError:
90         config_filename = os.path.join(config_dir, 'config.py')
91         logging.fatal("No configuration file. Creating %s", config_filename)
92         f = open(config_filename, mode='w+', encoding='utf-8')
93         f.write(CONFIG_TEMPLATE)
94         f.close()
95         logging.fatal("Please update this file with your details.")
96         sys.exit(1)
97     SERVER = config.SERVER
98     LOGIN = config.LOGIN
99     PASSWORD = config.PASSWORD
100     BASE_URL = 'http://%s' % SERVER
101
102 def hide_password_from_log(txt):
103     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
104
105 __opener__ = None
106 def httpopen(url, post_data=None):
107     if post_data:
108         log_post_data = hide_password_from_log(post_data)
109         post_data = post_data.encode('utf-8') # str->bytes
110         logging.debug('HTTP POST %s %s', url, log_post_data)
111     else:
112         logging.debug('HTTP GET %s', url)
113     global __opener__
114     if __opener__ is None:
115         cookiejar = CookieJar()
116         __opener__ = urllib.request.build_opener()
117         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
118         __opener__.addheaders = [('User-agent', USER_AGENT)]
119     http_response = __opener__.open(url, post_data)
120     return http_response
121
122 def sleep(seconds):
123     logging.debug('Waiting %s seconds', seconds)
124     time.sleep(seconds)
125
126 def timestamp():
127     return calendar.timegm(datetime.now().timetuple())
128
129 def number_format(i):
130     result = ''
131     while i:
132         if result:
133             result = ' ' + result
134         d3 = i % 1000
135         i = i//1000
136         if i: # there will be something else left of that
137             result = ('%03d' % d3) + result
138         else:
139             result = ('%d' % d3) + result
140     return result
141
142
143 def members_fetch(alliance_tag=None):
144     #httpresponse = httpopen(BASE_URL + '/')
145     #html = httpresponse.read().decode('utf-8')
146     #print(html)
147    
148     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
149     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
150     html = httpresponse.read().decode('utf-8')
151     if "redirectReine" not in html:
152         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
153         sys.exit(1)
154     #print('-'*80)
155     #print(html)
156     
157     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
158     #html = httpresponse.read().decode('utf-8')
159     #print('-'*80)
160     #print(html)
161
162     if not alliance_tag:
163         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
164             'xajax=membre&xajaxr='+str(timestamp()))
165         html = httpresponse.read().decode('utf-8')
166         #print('-'*80)
167         #print(html)
168
169         root = html_parse(html)
170         table = get_elem(root, 'table')[0]
171         td = get_elem(table, 'td')[3]
172         table = get_elem(td, 'table')[0]
173         #print('-'*80)
174         #print_idented_tree(table)
175     else:
176         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
177         html = httpresponse.read().decode('utf-8')
178         
179         root = html_parse(html)
180         table = get_elem(root, 'table')[2]
181         #print('-'*80)
182         #print_idented_tree(table)
183
184     members = {} # NEW
185     for tr in get_elem(table, 'tr'):
186         row = get_elem(tr, 'td')
187         #print('-'*10)
188         #print_idented_tree(tr)
189         if not alliance_tag:
190             if len(row) != 10:
191                 continue
192             grade = get_merged_leaf_content(row[2])
193             nick = get_merged_leaf_content(row[3])
194             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
195             techno = int(get_merged_leaf_content(row[5]))
196             anthill = int(get_merged_leaf_content(row[6]))
197             active = get_elem(row[7], 'img')[0].attributes['alt']
198             colonized = len(get_elem(row[8], 'img')) == 1 # True will become a string, see bellow
199             #print(nick, grade, field, techno, anthill, active, colonized)
200
201         else:
202             if len(row) != 6:
203                 continue
204             grade = get_merged_leaf_content(row[1])
205             nick = get_merged_leaf_content(row[2])
206             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
207             techno = int(get_merged_leaf_content(row[4]))
208             anthill = int(get_merged_leaf_content(row[5]))
209             active = None
210             colonized = None
211             #print(nick, grade, field, techno, anthill)
212
213         members[nick] = {
214             'grade': grade,
215             'field': field,
216             'techno': techno,
217             'anthill': anthill,
218             'active': active,
219             'colonized': colonized,
220         }
221         #print(members[nick])
222
223     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
224     return members
225
226
227 def members_fill_positions(newmembers):
228     for nick, data in newmembers.items():
229         if data['id'] is not None and data['colonized'] != True:
230             continue
231
232         httpresponse = httpopen(BASE_URL + '/Membre.php?Pseudo=' + nick)
233         html = httpresponse.read().decode('utf-8')
234
235         f = open("debug.html", mode='w+', encoding='utf-8')
236         f.write(html)
237
238         match = re.match('.*x=([0-9]+) et y=([0-9]+)', html, re.DOTALL|re.MULTILINE)
239         x = int(match.group(1))
240         y = int(match.group(2))
241         newmembers[nick]['x'] = x
242         newmembers[nick]['y'] = y
243
244         match=re.match('.*commerce.php\\?ID=([0-9]+).*', html, re.DOTALL|re.MULTILINE)
245         _id = int(match.group(1))
246         newmembers[nick]['id'] = _id
247
248         match=re.match('.*Fourmilière soumise par <a href="Membre.php\\?Pseudo=([^"]+)', html, re.DOTALL|re.MULTILINE)
249         if match:
250             newmembers[nick]['colonized'] = match.group(1)
251         else:
252             newmembers[nick]['colonized'] = False
253
254         logging.debug('%s has position %s %s and ID %s. Colonized:%s', nick, x, y, _id, newmembers[nick]['colonized'])
255
256         delay = random.randint(0,2)
257         sleep(delay)
258         #break # only one at a time (debug)
259
260 def copy_existing_data(oldmembers, newmembers):
261     '''
262     Check if newmembers contains some keys and if missing, copy them from
263     oldmembers:
264     colonized, active: if alliance is forced, these keys are not fetched
265     x, y, id: only fetch them when neded.
266     '''
267     for nick, data in newmembers.items():
268         for key in ['colonized', 'active', 'x', 'y', 'id' ]:
269             if data.get(key, None) is None:
270                 # missing or None
271                 value = oldmembers.get(nick, {}).get(key, None)
272                 newmembers[nick][key] = value
273         if data.get('colonized', None) == True:
274             value = oldmembers.get(nick, {}).get('colonized', True)
275             if isinstance(value, str):
276                 newmembers[nick]['colonized'] = value
277
278 def members_to_field(members):
279     """
280     Takes a full information dict "members"
281     Returns a simple dict with nick->field
282     """
283     fields  = {}
284     for nick, data in members.items():
285         fields[nick] = data['field']
286     return fields
287
288
289 def publish_line(csvfile, text, **args):
290     csvline = text.format(**args)
291     logging.info(csvline.split(',')[-1])
292     if not options.dryrun:
293         csvfile.write(datetime.now(UTC).strftime(DATETIME_FORMAT)+','+csvline+'\n')
294
295 def field_compare(oldmembers, newmembers, csvfile):
296     
297     oldfield = members_to_field(oldmembers)
298     newfield = members_to_field(newmembers)
299
300     changes = {}
301
302     for nick in newfield.keys():
303         told = oldfield.get(nick, 0)
304         tnew = newfield[nick]
305         if told == tnew:
306             continue
307         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
308     
309     for nick in oldfield.keys():
310         if nick not in newfield:
311             told = oldfield[nick]
312             tnew = 0
313             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
314     #for nick, change in changes.items():
315     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
316     #print
317
318     if not len(changes):
319         publish_line(csvfile, 'NOOP,,,,,,,,,,No changes')
320  
321     for nick in changes.keys():
322         for nick2 in changes.keys():
323             delta = changes[nick]['delta']
324             if delta <= 0:
325                 continue # 0 is already process, <0 will be processed when nick swaps with nick2
326             if changes[nick2]['delta'] == 0:
327                 continue # already done
328             if changes[nick2]['delta'] != -delta:
329                 continue # not the good one
330             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
331             publish_line(csvfile, 
332                 'FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)',
333                 nick1=nick,
334                 grade1=newmembers[nick].get('grade', ''),
335                 oldfield1=oldfield[nick],
336                 newfield1=newfield[nick],
337                 nick2=nick2,
338                 grade2=newmembers[nick2].get('grade', ''),
339                 oldfield2=oldfield[nick2],
340                 newfield2=newfield[nick2],
341                 deltafield=delta,
342                 nicedeltafield= number_format(delta),
343                 percent=percent,
344                 )
345             changes[nick]['delta'] = 0
346             changes[nick2]['delta'] = 0
347
348     for nick, change in changes.items():
349         delta = change['delta']
350         if delta < 0:
351             if nick in newfield:
352                 percent = float(-delta) / oldfield[nick] * 100.
353                 publish_line(csvfile, 
354                     'LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)',
355                     nick1=nick,
356                     grade1=newmembers[nick].get('grade', ''),
357                     oldfield1=oldfield[nick],
358                     newfield1=newfield[nick],
359                     deltafield=-delta,
360                     nicedeltafield=number_format(-delta),
361                     percent=percent,
362                     )
363             else:
364                 publish_line(csvfile, 
365                     'QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},{nick1} a quité l\'alliance avec {nicedeltafield} cm²',
366                     nick1=nick,
367                     grade1=oldmembers[nick].get('grade', ''),
368                     oldfield1=oldfield[nick],
369                     deltafield=-delta,
370                     nicedeltafield=number_format(-delta),
371                     )
372         elif delta > 0:
373             if nick in oldfield:
374                 publish_line(csvfile, 
375                     'WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a gagné {nicedeltafield} cm²',
376                     nick1=nick,
377                     grade1=newmembers[nick].get('grade', ''),
378                     oldfield1=oldfield[nick],
379                     newfield1=newfield[nick],
380                     deltafield=delta,
381                     nicedeltafield=number_format(delta),
382                     )
383             else:
384                 publish_line(csvfile, 
385                     'JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²',
386                     nick1=nick,
387                     grade1=newmembers[nick].get('grade', ''),
388                     newfield1=newfield[nick],
389                     deltafield=delta,
390                     nicedeltafield=number_format(delta),
391                     )
392
393 def activity_compare(oldmembers, newmembers, csvfile):
394     for nick, data in newmembers.items():
395         newa = data.get('active', None)
396         olda = oldmembers.get(nick, {}).get('active', None)
397         if newa and olda and newa != olda:
398             publish_line(csvfile, 
399                 'ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est maintenant {lowernewdata1}',
400                 nick1=nick,
401                 grade1=newmembers[nick]['grade'],
402                 olddata1=olda,
403                 newdata1=newa,
404                 lowernewdata1=newa.lower(),
405                 )
406
407 def grade_compare(oldmembers, newmembers, csvfile):
408     for nick, data in newmembers.items():
409         newdata = data.get('grade', None)
410         olddata = oldmembers.get(nick, {}).get('grade', None)
411         if newdata is not None and olddata is not None and newdata != olddata:
412             publish_line(csvfile, 
413                 'GRADE,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est désormais {grade1}',
414                 nick1=nick,
415                 grade1=newmembers[nick]['grade'],
416                 olddata1=olddata,
417                 newdata1=newdata,
418                 )
419
420 def colonized_compare(oldmembers, newmembers, csvfile):
421     for nick, data in newmembers.items():
422         newc = data.get('colonized', None)
423         oldc = oldmembers.get(nick, {}).get('colonized', None)
424         if newc is not None and oldc is not None and newc != oldc:
425             if newc:
426                 if oldc == True and isinstance(newc, str):
427                     continue # old versions of script had only True/False
428                 txt = "s'est fait coloniser"
429                 if newc != True:
430                     txt += " par " + newc
431             else:
432                 txt = "n'est plus colonisé"
433                 if oldc != True:
434                     txt += " par " + oldc
435             publish_line(csvfile, 
436                 'COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} {txt}',
437                 nick1=nick,
438                 grade1=newmembers[nick]['grade'],
439                 olddata1=str(oldc),
440                 newdata1=str(newc),
441                 txt=txt,
442                 )
443
444 def evolution_compare(oldmembers, newmembers, csvfile):
445     for nick, data in newmembers.items():
446         newdata = data['anthill']
447         olddata = oldmembers.get(nick, {}).get('anthill', None)
448         if newdata is not None and olddata is not None and newdata != olddata:
449             publish_line(csvfile, 
450                 'ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}',
451                 nick1=nick,
452                 grade1=newmembers[nick]['grade'],
453                 olddata1=olddata,
454                 newdata1=newdata,
455                 )
456         newdata = data['techno']
457         olddata = oldmembers.get(nick, {}).get('techno', None)
458         if newdata is not None and olddata is not None and newdata != olddata:
459             publish_line(csvfile, 
460                 'TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,La technologie de {nick1} a atteint le niveau {newdata1}',
461                 nick1=nick,
462                 grade1=newmembers[nick]['grade'],
463                 olddata1=olddata,
464                 newdata1=newdata,
465                 )
466
467 if __name__ == '__main__':
468     from optparse import OptionParser
469     parser = OptionParser()
470     parser.add_option('-d', '-v', '--debug', '--verbose',
471         action='store_true', dest='debug', default=False,
472         help="debug mode")
473     parser.add_option('-q', '--quiet',
474         action='store_true', dest='quiet', default=False,
475         help="quiet mode")
476     parser.add_option('-a', '--alliance',
477         dest='alliance', metavar="TAG",
478         help="Default is to process player own alliance.")
479     parser.add_option('--dir',
480         dest='directory', default='.fourmizzz',
481         help="Working directory, including config.py, archives and results. "
482              "Relative to HOME unless it starts with /. "
483              "Default is '%default'.")
484     parser.add_option('--quick',
485         action='store_true', dest='quick', default=False,
486         help="Don't sleep a random delay before hiting the server. "
487              "The programme usually wait between 0 and 59s, so that "
488              "there is no pick usage on a round minute")
489     parser.add_option('--members-name',
490         dest='archive_filename', metavar='FILENAME', default="archive.py",
491         help="Override filename containing old data. "
492         "Default is '%default' in Working directory. "
493         "See --dir.")
494     parser.add_option('--new-members-name',
495         dest='new_archive_filename', metavar='FILENAME',
496         help="Read new members data from a file. "
497         "Default is to fetch data from antzzz server.")
498     parser.add_option('--dry-run',
499         action='store_true', dest='dryrun', default=False,
500         help="don't store result in archives.")
501     options, args = parser.parse_args()
502
503     if options.debug:
504         loglevel = logging.DEBUG
505     elif options.quiet:
506         loglevel = logging.WARNING
507     else:
508         loglevel = logging.INFO
509     
510     HOME = os.environ['HOME']
511     if options.directory[0] == os.path.sep: # '/' on linux
512         BASE_DIRECTORY = options.directory
513     else:
514         BASE_DIRECTORY = os.path.join(HOME, options.directory)
515     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
516
517     if not options.quick:
518         delay = random.randint(0,59)
519         sleep(delay)
520
521     read_config(BASE_DIRECTORY)
522
523     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
524     oldmembers = None
525     try:
526         f = open(members_filename, mode='r+', encoding='utf-8')
527     except IOError as err:
528         if err.errno == 2: # No such file or directory
529             logging.warning("No archive file, creating one.")
530             f = open(members_filename, mode='w+', encoding='utf-8')
531         else:
532             raise
533     else:
534         oldmembers = eval(f.read())
535     
536     if options.new_archive_filename:
537         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
538         newf = open(new_archive_filename, mode='r', encoding='utf8')
539         newmembers = eval(newf.read())
540     else:
541         newmembers = members_fetch(options.alliance)
542
543     if oldmembers is not None:
544         copy_existing_data(oldmembers, newmembers)
545         if not options.new_archive_filename:
546             members_fill_positions(newmembers)
547         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
548         csvfile = open(csvfilename, mode='a', encoding='utf-8')
549         field_compare(oldmembers, newmembers, csvfile)
550         grade_compare(oldmembers, newmembers, csvfile)
551         activity_compare(oldmembers, newmembers, csvfile)
552         colonized_compare(oldmembers, newmembers, csvfile)
553         evolution_compare(oldmembers, newmembers, csvfile)
554     
555     # Save archive only after processing, just in case it crashes
556     if not options.dryrun:
557         f.seek(0)
558         f.write(repr(newmembers))
559         f.truncate()
560         f.close()
561
562         # We work in members.csv.new file then rename it
563         # In the unlikely event that some script is reading it
564         # while we updated it
565         members_csvfilename = os.path.join(BASE_DIRECTORY, "members.csv")
566         members_csvfilename_new = members_csvfilename + '.new'
567         f = open(members_csvfilename_new, mode='w', encoding='utf-8')
568         f.write('grade,nick,field,techno,anthill,active,colonized,x,y,id\n')
569         for nick, data in newmembers.items():
570             data = [
571                 data['grade'],
572                 nick,
573                 data['field'],
574                 data['techno'],
575                 data['anthill'],
576                 data['active'],
577                 data['colonized'],
578                 data['x'],
579                 data['y'],
580                 data['id'],
581                 ]
582             data = [ str(x) for x in data ]
583             f.write(','.join(data) + '\n')
584         f.close()
585         os.rename(members_csvfilename_new, members_csvfilename)