87b3876cb61735610a0680acec41c811ead247f0
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO, GRADE
24
25 Exemples:
26 NOOP,,,,,,,,,,Pas de changement dans le TdC
27 JOIN,Nirgal,Recrue,0,571628,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,Nick1 est maintenant en vacances
36 COLON,Nick1,Grade1,False,Nick2,,,,,,Nick1 s'est fait coloniser par Nick2
37 COLON,Nick1,Grade1,Nick2,False,,,,,,Nick1 n'est plus colonisé par Nick2
38
39 ANTHILL,Nick1,Grade1,199,200,,,,,,La fourmilière de Nick1 a atteint le niveau 200
40 TECHNO,Nick1,Grade1,199,200,,,,,,La technologie Nick1 a atteint le niveau 200
41
42 GRADE,Nick1,Recrue,Recrue,Passer 806,,,,,,Nick1 est maintenant Passeur 806
43
44 Note that in last 2 FLOOD lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
45 """
46
47 # globals:
48 #BASE_DIRECTORY
49 #SERVER
50 #LOGIN
51 #PASSWORD
52 #BASE_URL
53 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
54
55 USER_AGENT="Mozilla/5.0 (X11; U; Linux.i686; en-US; rv:1.9.1.16) Gecko/20121020 Iceweasel/3.5.16 (like Firefox/3.5.16)"
56
57 def read_config(config_dir):
58     '''
59     That function will read config.py in .fourmizzz directory
60     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
61     '''
62     global SERVER, LOGIN, PASSWORD
63     global BASE_URL
64     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
65 SERVER = 's1.fourmizzz.fr'
66 LOGIN = 'MonIdentifiant'
67 PASSWORD = 'MonMotDePasse'
68     '''
69
70     sys.path.append(config_dir)
71     try:
72         import config
73     except ImportError:
74         config_filename = os.path.join(config_dir, 'config.py')
75         logging.fatal("No configuration file. Creating %s", config_filename)
76         f = open(config_filename, mode='w+', encoding='utf-8')
77         f.write(CONFIG_TEMPLATE)
78         f.close()
79         logging.fatal("Please update this file with your details.")
80         sys.exit(1)
81     SERVER = config.SERVER
82     LOGIN = config.LOGIN
83     PASSWORD = config.PASSWORD
84     BASE_URL = 'http://%s' % SERVER
85
86 def hide_password_from_log(txt):
87     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
88
89 __opener__ = None
90 def httpopen(url, post_data=None):
91     if post_data:
92         log_post_data = hide_password_from_log(post_data)
93         post_data = post_data.encode('utf-8') # str->bytes
94         logging.debug('HTTP POST %s %s', url, log_post_data)
95     else:
96         logging.debug('HTTP GET %s', url)
97     global __opener__
98     if __opener__ is None:
99         cookiejar = CookieJar()
100         __opener__ = urllib.request.build_opener()
101         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
102         __opener__.addheaders = [('User-agent', USER_AGENT)]
103     http_response = __opener__.open(url, post_data)
104     return http_response
105
106 def sleep(seconds):
107     logging.debug('Waiting %s seconds', seconds)
108     time.sleep(seconds)
109
110 def timestamp():
111     return calendar.timegm(datetime.now().timetuple())
112
113 def number_format(i):
114     result = ''
115     while i:
116         if result:
117             result = ' ' + result
118         d3 = i % 1000
119         i = i//1000
120         if i: # there will be something else left of that
121             result = ('%03d' % d3) + result
122         else:
123             result = ('%d' % d3) + result
124     return result
125
126
127 def members_fetch(alliance_tag=None):
128     #httpresponse = httpopen(BASE_URL + '/')
129     #html = httpresponse.read().decode('utf-8')
130     #print(html)
131    
132     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
133     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
134     html = httpresponse.read().decode('utf-8')
135     if "redirectReine" not in html:
136         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
137         sys.exit(1)
138     #print('-'*80)
139     #print(html)
140     
141     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
142     #html = httpresponse.read().decode('utf-8')
143     #print('-'*80)
144     #print(html)
145
146     if not alliance_tag:
147         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
148             'xajax=membre&xajaxr='+str(timestamp()))
149         html = httpresponse.read().decode('utf-8')
150         #print('-'*80)
151         #print(html)
152
153         root = html_parse(html)
154         table = get_elem(root, 'table')[0]
155         td = get_elem(table, 'td')[3]
156         table = get_elem(td, 'table')[0]
157         #print('-'*80)
158         #print_idented_tree(table)
159     else:
160         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
161         html = httpresponse.read().decode('utf-8')
162         
163         root = html_parse(html)
164         table = get_elem(root, 'table')[2]
165         #print('-'*80)
166         #print_idented_tree(table)
167
168     members = {} # NEW
169     for tr in get_elem(table, 'tr'):
170         row = get_elem(tr, 'td')
171         #print('-'*10)
172         #print_idented_tree(tr)
173         if not alliance_tag:
174             if len(row) != 10:
175                 continue
176             grade = get_merged_leaf_content(row[2])
177             nick = get_merged_leaf_content(row[3])
178             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
179             techno = int(get_merged_leaf_content(row[5]))
180             anthill = int(get_merged_leaf_content(row[6]))
181             active = get_elem(row[7], 'img')[0].attributes['alt']
182             colonized = len(get_elem(row[8], 'img')) == 1 # True will become a string, see bellow
183             #print(nick, grade, field, techno, anthill, active, colonized)
184
185         else:
186             if len(row) != 6:
187                 continue
188             grade = get_merged_leaf_content(row[1])
189             nick = get_merged_leaf_content(row[2])
190             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
191             techno = int(get_merged_leaf_content(row[4]))
192             anthill = int(get_merged_leaf_content(row[5]))
193             active = None
194             colonized = None
195             #print(nick, grade, field, techno, anthill)
196
197         members[nick] = {
198             'grade': grade,
199             'field': field,
200             'techno': techno,
201             'anthill': anthill,
202             'active': active,
203             'colonized': colonized,
204         }
205         #print(members[nick])
206
207     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
208     return members
209
210
211 def members_fill_positions(newmembers):
212     for nick, data in newmembers.items():
213         if data['id'] is not None and data['colonized'] != True:
214             continue
215
216         httpresponse = httpopen(BASE_URL + '/Membre.php?Pseudo=' + nick)
217         html = httpresponse.read().decode('utf-8')
218
219         f = open("debug.html", mode='w+', encoding='utf-8')
220         f.write(html)
221
222         match = re.match('.*x=([0-9]+) et y=([0-9]+)', html, re.DOTALL|re.MULTILINE)
223         x = int(match.group(1))
224         y = int(match.group(2))
225         newmembers[nick]['x'] = x
226         newmembers[nick]['y'] = y
227
228         match=re.match('.*commerce.php\\?ID=([0-9]+).*', html, re.DOTALL|re.MULTILINE)
229         _id = int(match.group(1))
230         newmembers[nick]['id'] = _id
231
232         match=re.match('.*Fourmilière soumise par <a href="Membre.php\\?Pseudo=([^"]+)', html, re.DOTALL|re.MULTILINE)
233         if match:
234             newmembers[nick]['colonized'] = match.group(1)
235         else:
236             newmembers[nick]['colonized'] = False
237
238         logging.debug('%s has position %s %s and ID %s. Colonized:%s', nick, x, y, _id, newmembers[nick]['colonized'])
239
240         delay = random.randint(0,2)
241         sleep(delay)
242         #break # only one at a time (debug)
243
244 def copy_existing_data(oldmembers, newmembers):
245     '''
246     Check if newmembers contains some keys and if missing, copy them from
247     oldmembers:
248     colonized, active: if alliance is forced, these keys are not fetched
249     x, y, id: only fetch them when neded.
250     '''
251     for nick, data in newmembers.items():
252         for key in ['colonized', 'active', 'x', 'y', 'id' ]:
253             if data.get(key, None) is None:
254                 # missing or None
255                 value = oldmembers.get(nick, {}).get(key, None)
256                 newmembers[nick][key] = value
257         if data.get('colonized', None) == True:
258             value = oldmembers.get(nick, {}).get('colonized', True)
259             if isinstance(value, str):
260                 newmembers[nick]['colonized'] = value
261
262 def members_to_field(members):
263     """
264     Takes a full information dict "members"
265     Returns a simple dict with nick->field
266     """
267     fields  = {}
268     for nick, data in members.items():
269         fields[nick] = data['field']
270     return fields
271
272
273 def publish_line(csvfile, text, **args):
274     csvline = text.format(**args)
275     logging.info(csvline.split(',')[-1])
276     if not options.dryrun:
277         csvfile.write(time.strftime(DATETIME_FORMAT)+','+csvline+'\n')
278
279 def field_compare(oldmembers, newmembers, csvfile):
280     
281     oldfield = members_to_field(oldmembers)
282     newfield = members_to_field(newmembers)
283
284     changes = {}
285
286     for nick in newfield.keys():
287         told = oldfield.get(nick, 0)
288         tnew = newfield[nick]
289         if told == tnew:
290             continue
291         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
292     
293     for nick in oldfield.keys():
294         if nick not in newfield:
295             told = oldfield[nick]
296             tnew = 0
297             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
298     #for nick, change in changes.items():
299     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
300     #print
301
302     if not len(changes):
303         publish_line(csvfile, 'NOOP,,,,,,,,,,No changes')
304  
305     for nick in changes.keys():
306         for nick2 in changes.keys():
307             delta = changes[nick]['delta']
308             if delta <= 0:
309                 continue # 0 is already process, <0 will be processed when nick swaps with nick2
310             if changes[nick2]['delta'] == 0:
311                 continue # already done
312             if changes[nick2]['delta'] != -delta:
313                 continue # not the good one
314             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
315             publish_line(csvfile, 
316                 'FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)',
317                 nick1=nick,
318                 grade1=newmembers[nick].get('grade', ''),
319                 oldfield1=oldfield[nick],
320                 newfield1=newfield[nick],
321                 nick2=nick2,
322                 grade2=newmembers[nick2].get('grade', ''),
323                 oldfield2=oldfield[nick2],
324                 newfield2=newfield[nick2],
325                 deltafield=delta,
326                 nicedeltafield= number_format(delta),
327                 percent=percent,
328                 )
329             changes[nick]['delta'] = 0
330             changes[nick2]['delta'] = 0
331
332     for nick, change in changes.items():
333         delta = change['delta']
334         if delta < 0:
335             if nick in newfield:
336                 percent = float(-delta) / oldfield[nick] * 100.
337                 publish_line(csvfile, 
338                     'LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)',
339                     nick1=nick,
340                     grade1=newmembers[nick].get('grade', ''),
341                     oldfield1=oldfield[nick],
342                     newfield1=newfield[nick],
343                     deltafield=-delta,
344                     nicedeltafield=number_format(-delta),
345                     percent=percent,
346                     )
347             else:
348                 publish_line(csvfile, 
349                     'QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},{nick1} a quité l\'alliance avec {nicedeltafield} cm²',
350                     nick1=nick,
351                     grade1=oldmembers[nick].get('grade', ''),
352                     oldfield1=oldfield[nick],
353                     deltafield=-delta,
354                     nicedeltafield=number_format(-delta),
355                     )
356         elif delta > 0:
357             if nick in oldfield:
358                 publish_line(csvfile, 
359                     'WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a gagné {nicedeltafield} cm²',
360                     nick1=nick,
361                     grade1=newmembers[nick].get('grade', ''),
362                     oldfield1=oldfield[nick],
363                     newfield1=newfield[nick],
364                     deltafield=delta,
365                     nicedeltafield=number_format(delta),
366                     )
367             else:
368                 publish_line(csvfile, 
369                     'JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²',
370                     nick1=nick,
371                     grade1=newmembers[nick].get('grade', ''),
372                     newfield1=newfield[nick],
373                     deltafield=delta,
374                     nicedeltafield=number_format(delta),
375                     )
376
377 def activity_compare(oldmembers, newmembers, csvfile):
378     for nick, data in newmembers.items():
379         newa = data.get('active', None)
380         olda = oldmembers.get(nick, {}).get('active', None)
381         if newa and olda and newa != olda:
382             publish_line(csvfile, 
383                 'ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est maintenant {lowernewdata1}',
384                 nick1=nick,
385                 grade1=newmembers[nick]['grade'],
386                 olddata1=olda,
387                 newdata1=newa,
388                 lowernewdata1=newa.lower(),
389                 )
390
391 def grade_compare(oldmembers, newmembers, csvfile):
392     for nick, data in newmembers.items():
393         newdata = data.get('grade', None)
394         olddata = oldmembers.get(nick, {}).get('grade', None)
395         if newdata is not None and olddata is not None and newdata != olddata:
396             publish_line(csvfile, 
397                 'GRADE,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est désormais {grade1}',
398                 nick1=nick,
399                 grade1=newmembers[nick]['grade'],
400                 olddata1=olddata,
401                 newdata1=newdata,
402                 )
403
404 def colonized_compare(oldmembers, newmembers, csvfile):
405     for nick, data in newmembers.items():
406         newc = data.get('colonized', None)
407         oldc = oldmembers.get(nick, {}).get('colonized', None)
408         if newc is not None and oldc is not None and newc != oldc:
409             if newc:
410                 if oldc == True and isinstance(newc, str):
411                     continue # old versions of script had only True/False
412                 txt = "s'est fait coloniser"
413                 if newc != True:
414                     txt += " par " + newc
415             else:
416                 txt = "n'est plus colonisé"
417                 if oldc != True:
418                     txt += " par " + oldc
419             publish_line(csvfile, 
420                 'COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} {txt}',
421                 nick1=nick,
422                 grade1=newmembers[nick]['grade'],
423                 olddata1=str(oldc),
424                 newdata1=str(newc),
425                 txt=txt,
426                 )
427
428 def evolution_compare(oldmembers, newmembers, csvfile):
429     for nick, data in newmembers.items():
430         newdata = data['anthill']
431         olddata = oldmembers.get(nick, {}).get('anthill', None)
432         if newdata is not None and olddata is not None and newdata != olddata:
433             publish_line(csvfile, 
434                 'ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}',
435                 nick1=nick,
436                 grade1=newmembers[nick]['grade'],
437                 olddata1=olddata,
438                 newdata1=newdata,
439                 )
440         newdata = data['techno']
441         olddata = oldmembers.get(nick, {}).get('techno', None)
442         if newdata is not None and olddata is not None and newdata != olddata:
443             publish_line(csvfile, 
444                 'TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,La technologie de {nick1} a atteint le niveau {newdata1}',
445                 nick1=nick,
446                 grade1=newmembers[nick]['grade'],
447                 olddata1=olddata,
448                 newdata1=newdata,
449                 )
450
451 if __name__ == '__main__':
452     from optparse import OptionParser
453     parser = OptionParser()
454     parser.add_option('-d', '-v', '--debug', '--verbose',
455         action='store_true', dest='debug', default=False,
456         help="debug mode")
457     parser.add_option('-q', '--quiet',
458         action='store_true', dest='quiet', default=False,
459         help="quiet mode")
460     parser.add_option('-a', '--alliance',
461         dest='alliance', metavar="TAG",
462         help="Default is to process player own alliance.")
463     parser.add_option('--dir',
464         dest='directory', default='.fourmizzz',
465         help="Working directory, including config.py, archives and results. "
466              "Relative to HOME unless it starts with /. "
467              "Default is '%default'.")
468     parser.add_option('--quick',
469         action='store_true', dest='quick', default=False,
470         help="Don't sleep a random delay before hiting the server. "
471              "The programme usually wait between 0 and 59s, so that "
472              "there is no pick usage on a round minute")
473     parser.add_option('--members-name',
474         dest='archive_filename', metavar='FILENAME', default="archive.py",
475         help="Override filename containing old data. "
476         "Default is '%default' in Working directory. "
477         "See --dir.")
478     parser.add_option('--new-members-name',
479         dest='new_archive_filename', metavar='FILENAME',
480         help="Read new members data from a file. "
481         "Default is to fetch data from antzzz server.")
482     parser.add_option('--dry-run',
483         action='store_true', dest='dryrun', default=False,
484         help="don't store result in archives.")
485     options, args = parser.parse_args()
486
487     if options.debug:
488         loglevel = logging.DEBUG
489     elif options.quiet:
490         loglevel = logging.WARNING
491     else:
492         loglevel = logging.INFO
493     
494     HOME = os.environ['HOME']
495     if options.directory[0] == os.path.sep: # '/' on linux
496         BASE_DIRECTORY = options.directory
497     else:
498         BASE_DIRECTORY = os.path.join(HOME, options.directory)
499     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
500
501     if not options.quick:
502         delay = random.randint(0,59)
503         sleep(delay)
504
505     read_config(BASE_DIRECTORY)
506
507     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
508     oldmembers = None
509     try:
510         f = open(members_filename, mode='r+', encoding='utf-8')
511     except IOError as err:
512         if err.errno == 2: # No such file or directory
513             logging.warning("No archive file, creating one.")
514             f = open(members_filename, mode='w+', encoding='utf-8')
515         else:
516             raise
517     else:
518         oldmembers = eval(f.read())
519     
520     if options.new_archive_filename:
521         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
522         newf = open(new_archive_filename, mode='r', encoding='utf8')
523         newmembers = eval(newf.read())
524     else:
525         newmembers = members_fetch(options.alliance)
526
527     if oldmembers is not None:
528         copy_existing_data(oldmembers, newmembers)
529         if not options.new_archive_filename:
530             members_fill_positions(newmembers)
531         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
532         csvfile = open(csvfilename, mode='a', encoding='utf-8')
533         field_compare(oldmembers, newmembers, csvfile)
534         grade_compare(oldmembers, newmembers, csvfile)
535         activity_compare(oldmembers, newmembers, csvfile)
536         colonized_compare(oldmembers, newmembers, csvfile)
537         evolution_compare(oldmembers, newmembers, csvfile)
538     
539     # Save archive only after processing, just in case it crashes
540     if not options.dryrun:
541         f.seek(0)
542         f.write(repr(newmembers))
543         f.truncate()
544         f.close()
545
546         # We work in members.csv.new file then rename it
547         # In the unlikely event that some script is reading it
548         # while we updated it
549         members_csvfilename = os.path.join(BASE_DIRECTORY, "members.csv")
550         members_csvfilename_new = members_csvfilename + '.new'
551         f = open(members_csvfilename_new, mode='w', encoding='utf-8')
552         f.write('grade,nick,field,techno,anthill,active,colonized,x,y,id\n')
553         for nick, data in newmembers.items():
554             data = [
555                 data['grade'],
556                 nick,
557                 data['field'],
558                 data['techno'],
559                 data['anthill'],
560                 data['active'],
561                 data['colonized'],
562                 data['x'],
563                 data['y'],
564                 data['id'],
565                 ]
566             data = [ str(x) for x in data ]
567             f.write(','.join(data) + '\n')
568         f.close()
569         os.rename(members_csvfilename_new, members_csvfilename)