Added TECHNO and ANTHILL events to track member progression
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO
24
25 Exemples:
26 NOOP,,,,,,,,,,,Pas de changement dans le TdC
27 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,,Nick1 est maintenant en vacances
36 COLON,Nick1,Grade1,False,True,,,,,,,Nick1 s'est fait colonisé
37 COLON,Nick1,Grade1,True,False,,,,,,,Nick1 n'est plus colonisé
38
39 ANTHILL,Nick1,Grade1,199,200,,,,,,,La fourmilière de Nick1 a atteint le niveau 200
40 TECHNO,Nick1,Grade1,199,200,,,,,,,La technologie Nick1 a atteint le niveau 200
41
42 Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
43 """
44
45 # globals:
46 #BASE_DIRECTORY
47 #SERVER
48 #LOGIN
49 #PASSWORD
50 #BASE_URL
51 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
52
53 def read_config(config_dir):
54     '''
55     That function will read config.py in .fourmizzz directory
56     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
57     '''
58     global SERVER, LOGIN, PASSWORD
59     global BASE_URL
60     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
61 SERVER = 's1.fourmizzz.fr'
62 LOGIN = 'MonIdentifiant'
63 PASSWORD = 'MonMotDePasse'
64     '''
65
66     sys.path.append(config_dir)
67     try:
68         import config
69     except ImportError:
70         config_filename = os.path.join(config_dir, 'config.py')
71         logging.fatal("No configuration file. Creating %s", config_filename)
72         f = open(config_filename, mode='w+', encoding='utf-8')
73         f.write(CONFIG_TEMPLATE)
74         f.close()
75         logging.fatal("Please update this file with your details.")
76         sys.exit(1)
77     SERVER = config.SERVER
78     LOGIN = config.LOGIN
79     PASSWORD = config.PASSWORD
80     BASE_URL = 'http://%s' % SERVER
81
82 def hide_password_from_log(txt):
83     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
84
85 __opener__ = None
86 def httpopen(url, post_data=None):
87     if post_data:
88         log_post_data = hide_password_from_log(post_data)
89         post_data = post_data.encode('utf-8') # str->bytes
90         logging.debug('HTTP POST %s %s', url, log_post_data)
91     else:
92         logging.debug('HTTP GET %s', url)
93     global __opener__
94     if __opener__ is None:
95         cookiejar = CookieJar()
96         __opener__ = urllib.request.build_opener()
97         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
98     http_response = __opener__.open(url, post_data)
99     return http_response
100
101 def sleep(seconds):
102     logging.debug('Waiting %s seconds', seconds)
103     time.sleep(seconds)
104
105 def timestamp():
106     return calendar.timegm(datetime.now().timetuple())
107
108 def number_format(i):
109     result = ''
110     while i:
111         if result:
112             result = ' ' + result
113         d3 = i % 1000
114         i = i//1000
115         if i: # there will be something else left of that
116             result = ('%03d' % d3) + result
117         else:
118             result = ('%d' % d3) + result
119     return result
120
121
122 def members_fetch(alliance_tag=None):
123     #httpresponse = httpopen(BASE_URL + '/')
124     #html = httpresponse.read().decode('utf-8')
125     #print(html)
126    
127     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
128     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
129     html = httpresponse.read().decode('utf-8')
130     if "redirectReine" not in html:
131         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
132         sys.exit(1)
133     #print('-'*80)
134     #print(html)
135     
136     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
137     #html = httpresponse.read().decode('utf-8')
138     #print('-'*80)
139     #print(html)
140
141     if not alliance_tag:
142         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
143             'xajax=membre&xajaxr='+str(timestamp()))
144         html = httpresponse.read().decode('utf-8')
145         #print('-'*80)
146         #print(html)
147
148         root = html_parse(html)
149         table = get_elem(root, 'table')[0]
150         td = get_elem(table, 'td')[3]
151         table = get_elem(td, 'table')[0]
152         #print('-'*80)
153         #print_idented_tree(table)
154     else:
155         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
156         html = httpresponse.read().decode('utf-8')
157         
158         root = html_parse(html)
159         table = get_elem(root, 'table')[2]
160         #print('-'*80)
161         #print_idented_tree(table)
162
163     members = {} # NEW
164     for tr in get_elem(table, 'tr'):
165         row = get_elem(tr, 'td')
166         #print('-'*10)
167         #print_idented_tree(tr)
168         if not alliance_tag:
169             if len(row) != 10:
170                 continue
171             grade = get_merged_leaf_content(row[2])
172             nick = get_merged_leaf_content(row[3])
173             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
174             techno = int(get_merged_leaf_content(row[5]))
175             anthill = int(get_merged_leaf_content(row[6]))
176             active = get_elem(row[7], 'img')[0].attributes['alt']
177             colonized = len(get_elem(row[8], 'img')) == 1
178             #print(nick, grade, field, techno, anthill, active, colonized)
179
180         else:
181             if len(row) != 6:
182                 continue
183             grade = get_merged_leaf_content(row[1])
184             nick = get_merged_leaf_content(row[2])
185             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
186             techno = int(get_merged_leaf_content(row[4]))
187             anthill = int(get_merged_leaf_content(row[5]))
188             active = None
189             colonized = None
190             #print(nick, grade, field, techno, anthill)
191
192         members[nick] = {
193             'grade': grade,
194             'field': field,
195             'techno': techno,
196             'anthill': anthill,
197             'active': active,
198             'colonized': colonized,
199         }
200         #print(members[nick])
201
202     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
203     return members
204
205
206 def members_to_field(members):
207     """
208     Takes a full information dict "members"
209     Returns a simple dict with nick->field
210     """
211     fields  = {}
212     for nick, data in members.items():
213         fields[nick] = data['field']
214     return fields
215
216
217 def field_compare(oldmembers, newmembers, csvfile):
218     
219     oldfield = members_to_field(oldmembers)
220     newfield = members_to_field(newmembers)
221
222     changes = {}
223     txtchanges2 = []
224
225     for nick in newfield.keys():
226         told = oldfield.get(nick, 0)
227         tnew = newfield[nick]
228         if told == tnew:
229             continue
230         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
231     
232     for nick in oldfield.keys():
233         if nick not in newfield:
234             told = oldfield[nick]
235             tnew = 0
236             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
237     #for nick, change in changes.items():
238     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
239     #print
240
241     if not len(changes):
242         txtchanges2.append('NOOP,,,,,,,,,,,No changes')
243  
244     for nick in changes.keys():
245         delta = changes[nick]['delta']
246         if delta <= 0:
247             continue # 0 is already process, <0 will be processed when nick swaps with nick2
248         for nick2 in changes.keys():
249             if changes[nick2]['delta'] == 0:
250                 continue # already done
251             if changes[nick2]['delta'] != -delta:
252                 continue # not the good one
253             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
254             txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)'.format(
255                 nick1=nick,
256                 grade1=newmembers[nick].get('grade', ''),
257                 oldfield1=oldfield[nick],
258                 newfield1=newfield[nick],
259                 nick2=nick2,
260                 grade2=newmembers[nick2].get('grade', ''),
261                 oldfield2=oldfield[nick2],
262                 newfield2=newfield[nick2],
263                 deltafield=delta,
264                 nicedeltafield= number_format(delta),
265                 percent=percent,
266                 ))
267             changes[nick]['delta'] = 0
268             changes[nick2]['delta'] = 0
269
270     for nick, change in changes.items():
271         delta = change['delta']
272         if delta < 0:
273             if nick in newfield:
274                 percent = float(-delta) / oldfield[nick] * 100.
275                 txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
276                     nick1=nick,
277                     grade1=newmembers[nick].get('grade', ''),
278                     oldfield1=oldfield[nick],
279                     newfield1=newfield[nick],
280                     deltafield=-delta,
281                     nicedeltafield=number_format(-delta),
282                     percent=percent,
283                     ))
284             else:
285                 txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
286                     nick1=nick,
287                     grade1=oldmembers[nick].get('grade', ''),
288                     oldfield1=oldfield[nick],
289                     deltafield=-delta,
290                     nicedeltafield=number_format(-delta),
291                     ))
292         elif delta > 0:
293             if nick in oldfield:
294                 txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
295                     nick1=nick,
296                     grade1=newmembers[nick].get('grade', ''),
297                     oldfield1=oldfield[nick],
298                     newfield1=newfield[nick],
299                     deltafield=delta,
300                     nicedeltafield=number_format(delta),
301                     ))
302             else:
303                 txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
304                     nick1=nick,
305                     grade1=newmembers[nick].get('grade', ''),
306                     newfield1=newfield[nick],
307                     deltafield=delta,
308                     nicedeltafield=number_format(delta),
309                     ))
310
311     for txtchange in txtchanges2:
312         logging.info(txtchange.split(',')[-1])
313         if not options.dryrun:
314             csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
315
316 def activity_compare(oldmembers, newmembers, csvfile):
317     txtchanges2 = []
318     for nick, data in newmembers.items():
319         newa = data.get('active', None)
320         olda = oldmembers.get(nick, {}).get('active', None)
321         if newa and olda and newa != olda:
322             txtchanges2.append('ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est maintenant {lowernewdata1}'.format(
323                 nick1=nick,
324                 grade1=newmembers[nick]['grade'],
325                 olddata1=olda,
326                 newdata1=newa,
327                 lowernewdata1=newa.lower(),
328                 ))
329
330     for txtchange in txtchanges2:
331         logging.info(txtchange.split(',')[-1])
332         if not options.dryrun:
333             csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
334
335 def colonized_compare(oldmembers, newmembers, csvfile):
336     txtchanges2 = []
337     for nick, data in newmembers.items():
338         newc = data.get('colonized', None)
339         oldc = oldmembers.get(nick, {}).get('colonized', None)
340         if newc is not None and oldc is not None and newc != oldc:
341             if newc:
342                 txt = "s'est fait colonisé"
343             else:
344                 txt = "n'est plus colonisé"
345             txtchanges2.append('COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} {txt}'.format(
346                 nick1=nick,
347                 grade1=newmembers[nick]['grade'],
348                 olddata1=str(oldc),
349                 newdata1=str(newc),
350                 txt=txt,
351                 ))
352
353     for txtchange in txtchanges2:
354         logging.info(txtchange.split(',')[-1])
355         if not options.dryrun:
356             csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
357
358
359 def evolution_compare(oldmembers, newmembers, csvfile):
360     txtchanges2 = []
361     for nick, data in newmembers.items():
362         newdata = data['anthill']
363         olddata = oldmembers.get(nick, {}).get('anthill', None)
364         if newdata is not None and olddata is not None and newdata != olddata:
365             txtchanges2.append('ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}'.format(
366                 nick1=nick,
367                 grade1=newmembers[nick]['grade'],
368                 olddata1=olddata,
369                 newdata1=newdata,
370                 ))
371         newdata = data['techno']
372         olddata = oldmembers.get(nick, {}).get('techno', None)
373         if newdata is not None and olddata is not None and newdata != olddata:
374             txtchanges2.append('TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,,La technologie de {nick1} a atteint le niveau {newdata1}'.format(
375                 nick1=nick,
376                 grade1=newmembers[nick]['grade'],
377                 olddata1=olddata,
378                 newdata1=newdata,
379                 ))
380
381     for txtchange in txtchanges2:
382         logging.info(txtchange.split(',')[-1])
383         if not options.dryrun:
384             csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
385
386 if __name__ == '__main__':
387     from optparse import OptionParser
388     parser = OptionParser()
389     parser.add_option('-d', '-v', '--debug', '--verbose',
390         action='store_true', dest='debug', default=False,
391         help="debug mode")
392     parser.add_option('-q', '--quiet',
393         action='store_true', dest='quiet', default=False,
394         help="quiet mode")
395     parser.add_option('-a', '--alliance',
396         dest='alliance', metavar="TAG",
397         help="Default is to process player own alliance.")
398     parser.add_option('--dir',
399         dest='directory', default='.fourmizzz',
400         help="Working directory, including config.py, archives and results. "
401              "Relative to HOME unless it starts with /. "
402              "Default is '%default'.")
403     parser.add_option('--quick',
404         action='store_true', dest='quick', default=False,
405         help="Don't sleep a random delay before hiting the server. "
406              "The programme usually wait between 0 and 59s, so that "
407              "there is no pick usage on a round minute")
408     parser.add_option('--members-name',
409         dest='archive_filename', metavar='FILENAME', default="archive.py",
410         help="Override filename containing old data. "
411         "Default is '%default' in Working directory. "
412         "See --dir.")
413     parser.add_option('--new-members-name',
414         dest='new_archive_filename', metavar='FILENAME',
415         help="Read new members data from a file. "
416         "Default is to fetch data from antzzz server.")
417     parser.add_option('--dry-run',
418         action='store_true', dest='dryrun', default=False,
419         help="don't store result in archives.")
420     options, args = parser.parse_args()
421
422     if options.debug:
423         loglevel = logging.DEBUG
424     elif options.quiet:
425         loglevel = logging.WARNING
426     else:
427         loglevel = logging.INFO
428     
429     HOME = os.environ['HOME']
430     if options.directory[0] == os.path.sep: # '/' on linux
431         BASE_DIRECTORY = options.directory
432     else:
433         BASE_DIRECTORY = os.path.join(HOME, options.directory)
434     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
435
436     if not options.quick:
437         delay = random.randint(0,59)
438         sleep(delay)
439
440     read_config(BASE_DIRECTORY)
441
442     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
443     oldmembers = None
444     try:
445         f = open(members_filename, mode='r+', encoding='utf-8')
446     except IOError as err:
447         if err.errno == 2: # No such file or directory
448             logging.warning("No archive file, creating one.")
449             f = open(members_filename, mode='w+', encoding='utf-8')
450         else:
451             raise
452     else:
453         oldmembers = eval(f.read())
454     
455     if options.new_archive_filename:
456         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
457         newf = open(new_archive_filename, mode='r', encoding='utf8')
458         newmembers = eval(newf.read())
459     else:
460         newmembers = members_fetch(options.alliance)
461
462     if oldmembers is not None:
463         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
464         csvfile = open(csvfilename, mode='a', encoding='utf-8')
465         field_compare(oldmembers, newmembers, csvfile)
466         activity_compare(oldmembers, newmembers, csvfile)
467         colonized_compare(oldmembers, newmembers, csvfile)
468         evolution_compare(oldmembers, newmembers, csvfile)
469     
470     # Save archive only after processing, just in case it crashes
471     if not options.dryrun:
472         f.seek(0)
473         f.write(repr(newmembers))
474         f.truncate()