Now also monitor active/inactive status change
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT
24
25 Exemples:
26 NOOP,,,,,,,,,,,No changes
27 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,100,100,,,,,,,Nick1 est en vacances
36
37 Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
38 """
39
40 # globals:
41 #BASE_DIRECTORY
42 #SERVER
43 #LOGIN
44 #PASSWORD
45 #BASE_URL
46 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
47
48 def read_config(config_dir):
49     '''
50     That function will read config.py in .fourmizzz directory
51     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
52     '''
53     global SERVER, LOGIN, PASSWORD
54     global BASE_URL
55     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
56 SERVER = 's1.fourmizzz.fr'
57 LOGIN = 'MonIdentifiant'
58 PASSWORD = 'MonMotDePasse'
59     '''
60
61     sys.path.append(config_dir)
62     try:
63         import config
64     except ImportError:
65         config_filename = os.path.join(config_dir, 'config.py')
66         logging.fatal("No configuration file. Creating %s", config_filename)
67         f = open(config_filename, mode='w+', encoding='utf-8')
68         f.write(CONFIG_TEMPLATE)
69         f.close()
70         logging.fatal("Please update this file with your details.")
71         sys.exit(1)
72     SERVER = config.SERVER
73     LOGIN = config.LOGIN
74     PASSWORD = config.PASSWORD
75     BASE_URL = 'http://%s' % SERVER
76
77 def hide_password_from_log(txt):
78     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
79
80 __opener__ = None
81 def httpopen(url, post_data=None):
82     if post_data:
83         log_post_data = hide_password_from_log(post_data)
84         post_data = post_data.encode('utf-8') # str->bytes
85         logging.debug('HTTP POST %s %s', url, log_post_data)
86     else:
87         logging.debug('HTTP GET %s', url)
88     global __opener__
89     if __opener__ is None:
90         cookiejar = CookieJar()
91         __opener__ = urllib.request.build_opener()
92         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
93     http_response = __opener__.open(url, post_data)
94     return http_response
95
96 def sleep(seconds):
97     logging.debug('Waiting %s seconds', seconds)
98     time.sleep(seconds)
99
100 def timestamp():
101     return calendar.timegm(datetime.now().timetuple())
102
103 def number_format(i):
104     result = ''
105     while i:
106         if result:
107             result = ' ' + result
108         d3 = i % 1000
109         i = i//1000
110         if i: # there will be something else left of that
111             result = ('%03d' % d3) + result
112         else:
113             result = ('%d' % d3) + result
114     return result
115
116
117 def members_fetch(alliance_tag=None):
118     #httpresponse = httpopen(BASE_URL + '/')
119     #html = httpresponse.read().decode('utf-8')
120     #print(html)
121    
122     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
123     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
124     html = httpresponse.read().decode('utf-8')
125     if "redirectReine" not in html:
126         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
127         sys.exit(1)
128     #print('-'*80)
129     #print(html)
130     
131     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
132     #html = httpresponse.read().decode('utf-8')
133     #print('-'*80)
134     #print(html)
135
136     if not alliance_tag:
137         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
138             'xajax=membre&xajaxr='+str(timestamp()))
139         html = httpresponse.read().decode('utf-8')
140         #print('-'*80)
141         #print(html)
142
143         root = html_parse(html)
144         table = get_elem(root, 'table')[0]
145         td = get_elem(table, 'td')[3]
146         table = get_elem(td, 'table')[0]
147         #print('-'*80)
148         #print_idented_tree(table)
149     else:
150         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
151         html = httpresponse.read().decode('utf-8')
152         
153         root = html_parse(html)
154         table = get_elem(root, 'table')[2]
155         #print('-'*80)
156         #print_idented_tree(table)
157
158     members = {} # NEW
159     for tr in get_elem(table, 'tr'):
160         row = get_elem(tr, 'td')
161         #print('-'*10)
162         #print_idented_tree(tr)
163         if not alliance_tag:
164             if len(row) != 10:
165                 continue
166             grade = get_merged_leaf_content(row[2])
167             nick = get_merged_leaf_content(row[3])
168             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
169             techno = int(get_merged_leaf_content(row[5]))
170             anthill = int(get_merged_leaf_content(row[6]))
171             active = get_elem(row[7], 'img')[0].attributes['alt']
172             colonized = len(get_elem(row[8], 'img')) == 1
173             #print(nick, grade, field, techno, anthill, active, colonized)
174
175         else:
176             if len(row) != 6:
177                 continue
178             grade = get_merged_leaf_content(row[1])
179             nick = get_merged_leaf_content(row[2])
180             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
181             techno = int(get_merged_leaf_content(row[4]))
182             anthill = int(get_merged_leaf_content(row[5]))
183             active = None
184             colonized = None
185             #print(nick, grade, field, techno, anthill)
186
187         members[nick] = {
188             'grade': grade,
189             'field': field,
190             'techno': techno,
191             'anthill': anthill,
192             'active': active,
193             'colonized': colonized,
194         }
195         #print(members[nick])
196
197     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
198     return members
199
200
201 def members_to_field(members):
202     """
203     Takes a full information dict "members"
204     Returns a simple dict with nick->field
205     """
206     fields  = {}
207     for nick, data in members.items():
208         fields[nick] = data['field']
209     return fields
210
211
212 def field_compare(oldmembers, newmembers, csvfile):
213     
214     oldfield = members_to_field(oldmembers)
215     newfield = members_to_field(newmembers)
216
217     changes = {}
218     txtchanges = []
219     txtchanges2 = []
220
221     for nick in newfield.keys():
222         told = oldfield.get(nick, 0)
223         tnew = newfield[nick]
224         if told == tnew:
225             continue
226         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
227     
228     for nick in oldfield.keys():
229         if nick not in newfield:
230             told = oldfield[nick]
231             tnew = 0
232             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
233     #for nick, change in changes.items():
234     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
235     #print
236
237     if not len(changes):
238         txtchanges.append('No changes')
239         txtchanges2.append('NOOP,,,,,,,,,,,No changes')
240  
241     for nick in changes.keys():
242         delta = changes[nick]['delta']
243         if delta <= 0:
244             continue # 0 is already process, <0 will be processed when nick swaps with nick2
245         for nick2 in changes.keys():
246             if changes[nick2]['delta'] == 0:
247                 continue # already done
248             if changes[nick2]['delta'] != -delta:
249                 continue # not the good one
250             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
251             txtchanges.append('%s a pris %s cm² à %s (%.2f%%)' % (nick, number_format(delta), nick2, percent))
252             txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)'.format(
253                 nick1=nick,
254                 grade1=newmembers[nick].get('grade', ''),
255                 oldfield1=oldfield[nick],
256                 newfield1=newfield[nick],
257                 nick2=nick2,
258                 grade2=newmembers[nick2].get('grade', ''),
259                 oldfield2=oldfield[nick2],
260                 newfield2=newfield[nick2],
261                 deltafield=delta,
262                 nicedeltafield= number_format(delta),
263                 percent=percent,
264                 ))
265             changes[nick]['delta'] = 0
266             changes[nick2]['delta'] = 0
267
268     for nick, change in changes.items():
269         delta = change['delta']
270         if delta < 0:
271             if nick in newfield:
272                 percent = float(-delta) / oldfield[nick] * 100.
273                 txtchanges.append('%s a perdu %s cm² (%.2f%%)' % (nick, number_format(-delta), percent))
274                 txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
275                     nick1=nick,
276                     grade1=newmembers[nick].get('grade', ''),
277                     oldfield1=oldfield[nick],
278                     newfield1=newfield[nick],
279                     deltafield=-delta,
280                     nicedeltafield=number_format(-delta),
281                     percent=percent,
282                     ))
283             else:
284                 txtchanges.append("%s a quité l'alliance avec %s cm²" % (nick, number_format(-delta)))
285                 txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
286                     nick1=nick,
287                     grade1=oldmembers[nick].get('grade', ''),
288                     oldfield1=oldfield[nick],
289                     deltafield=-delta,
290                     nicedeltafield=number_format(-delta),
291                     ))
292         elif delta > 0:
293             if nick in oldfield:
294                 txtchanges.append('%s a gagné %s cm²' % (nick, number_format(delta)))
295                 txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
296                     nick1=nick,
297                     grade1=newmembers[nick].get('grade', ''),
298                     oldfield1=oldfield[nick],
299                     newfield1=newfield[nick],
300                     deltafield=delta,
301                     nicedeltafield=number_format(delta),
302                     ))
303             else:
304                 txtchanges.append("%s a rejoint l'alliance avec %s cm²" % (nick, number_format(delta)))
305                 txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
306                     nick1=nick,
307                     grade1=newmembers[nick].get('grade', ''),
308                     newfield1=newfield[nick],
309                     deltafield=delta,
310                     nicedeltafield=number_format(delta),
311                     ))
312
313     for txtchange in txtchanges:
314         logging.info(txtchange)
315
316     for txtchange in txtchanges2:
317         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
318
319 def activity_compare(oldmembers, newmembers, csvfile):
320     txtchanges2 = []
321     for nick, data in newmembers.items():
322         newa = data.get('active', None)
323         olda = oldmembers.get(nick, {}).get('active', None)
324         if newa and olda and newa != olda:
325             txtchanges2.append('ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est maintenant {lowernewdata1}'.format(
326                 nick1=nick,
327                 grade1=newmembers[nick]['grade'],
328                 olddata1=olda,
329                 newdata1=newa,
330                 lowernewdata1=newa.lower(),
331                 ))
332
333     for txtchange in txtchanges2:
334         logging.info(txtchange.split(',')[-1])
335         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
336
337 if __name__ == '__main__':
338     from optparse import OptionParser
339     parser = OptionParser()
340     parser.add_option('-d', '-v', '--debug', '--verbose',
341         action='store_true', dest='debug', default=False,
342         help="debug mode")
343     parser.add_option('-q', '--quiet',
344         action='store_true', dest='quiet', default=False,
345         help="quiet mode")
346     parser.add_option('-a', '--alliance',
347         dest='alliance', metavar="TAG",
348         help="Default is to process player own alliance.")
349     parser.add_option('--dir',
350         dest='directory', default='.fourmizzz',
351         help="Working directory, including config.py, archives and results. "
352              "Relative to HOME unless it starts with /. "
353              "Default is '%default'.")
354     parser.add_option('--quick',
355         action='store_true', dest='quick', default=False,
356         help="Don't sleep a random delay before hiting the server. "
357              "The programme usually wait between 0 and 59s, so that "
358              "there is no pick usage on a round minute")
359     parser.add_option('--members-name',
360         dest='archive_filename', metavar='FILENAME', default="archive.py",
361         help="Override filename containing old data. "
362         "Default is '%default' in Working directory. "
363         "See --dir.")
364     parser.add_option('--new-members-name',
365         dest='new_archive_filename', metavar='FILENAME',
366         help="Read new members data from a file. "
367         "Default is to fetch data from antzzz server.")
368     parser.add_option('--dry-run',
369         action='store_true', dest='dryrun', default=False,
370         help="don't store result in archives.")
371     options, args = parser.parse_args()
372
373     if options.debug:
374         loglevel = logging.DEBUG
375     elif options.quiet:
376         loglevel = logging.WARNING
377     else:
378         loglevel = logging.INFO
379     
380     HOME = os.environ['HOME']
381     if options.directory[0] == os.path.sep: # '/' on linux
382         BASE_DIRECTORY = options.directory
383     else:
384         BASE_DIRECTORY = os.path.join(HOME, options.directory)
385     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
386
387     if not options.quick:
388         delay = random.randint(0,59)
389         sleep(delay)
390
391     read_config(BASE_DIRECTORY)
392
393     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
394     oldmembers = None
395     try:
396         f = open(members_filename, mode='r+', encoding='utf-8')
397     except IOError as err:
398         if err.errno == 2: # No such file or directory
399             logging.warning("No archive file, creating one.")
400             f = open(members_filename, mode='w+', encoding='utf-8')
401         else:
402             raise
403     else:
404         oldmembers = eval(f.read())
405     
406     if options.new_archive_filename:
407         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
408         newf = open(new_archive_filename, mode='r', encoding='utf8')
409         newmembers = eval(newf.read())
410     else:
411         newmembers = members_fetch(options.alliance)
412
413     if oldmembers is not None:
414         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
415         csvfile = open(csvfilename, mode='a', encoding='utf-8')
416         field_compare(oldmembers, newmembers, csvfile)
417         activity_compare(oldmembers, newmembers, csvfile)
418     
419     # Save archive only after processing, just in case it crashes
420     if not options.dryrun:
421         f.seek(0)
422         f.write(repr(newmembers))
423         f.truncate()