Added NOOP to csv file
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import urllib.request
14 from http.cookiejar import CookieJar
15
16 from html_parser import *
17 import htmlentities
18
19 """
20 csv file format:
21 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
22 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP
23
24 Exemples:
25 NOOP,,,,,,,,,,,No changes
26 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
27 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
28 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
29 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
30 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
31 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
33
34 Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
35 """
36
37 # globals:
38 #BASE_DIRECTORY
39 #SERVER
40 #LOGIN
41 #PASSWORD
42 #BASE_URL
43 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
44
45 def read_config(config_dir):
46     '''
47     That function will read config.py in .fourmizzz directory
48     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
49     '''
50     global SERVER, LOGIN, PASSWORD
51     global BASE_URL
52     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
53 SERVER = 's1.fourmizzz.fr'
54 LOGIN = 'MonIdentifiant'
55 PASSWORD = 'MonMotDePasse'
56     '''
57
58     sys.path.append(config_dir)
59     try:
60         import config
61     except ImportError:
62         config_filename = os.path.join(config_dir, 'config.py')
63         logging.fatal("No configuration file. Creating %s", config_filename)
64         f = open(config_filename, mode='w+', encoding='utf-8')
65         f.write(CONFIG_TEMPLATE)
66         f.close()
67         logging.fatal("Please update this file with your details.")
68         sys.exit(1)
69     SERVER = config.SERVER
70     LOGIN = config.LOGIN
71     PASSWORD = config.PASSWORD
72     BASE_URL = 'http://%s' % SERVER
73
74 def hide_password_from_log(txt):
75     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
76
77 __opener__ = None
78 def httpopen(url, post_data=None):
79     if post_data:
80         log_post_data = hide_password_from_log(post_data)
81         post_data = post_data.encode('utf-8') # str->bytes
82         logging.debug('HTTP POST %s %s', url, log_post_data)
83     else:
84         logging.debug('HTTP GET %s', url)
85     global __opener__
86     if __opener__ is None:
87         cookiejar = CookieJar()
88         __opener__ = urllib.request.build_opener()
89         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
90     http_response = __opener__.open(url, post_data)
91     return http_response
92
93 def sleep(seconds):
94     logging.debug('Waiting %s seconds', seconds)
95     time.sleep(seconds)
96
97 def timestamp():
98     return calendar.timegm(datetime.now().timetuple())
99
100 def number_format(i):
101     result = ''
102     while i:
103         if result:
104             result = ' ' + result
105         d3 = i % 1000
106         i = i//1000
107         if i: # there will be something else left of that
108             result = ('%03d' % d3) + result
109         else:
110             result = ('%d' % d3) + result
111     return result
112
113
114 def members_fetch(alliance_tag=None):
115     #httpresponse = httpopen(BASE_URL + '/')
116     #html = httpresponse.read().decode('utf-8')
117     #print(html)
118    
119     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
120     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
121     html = httpresponse.read().decode('utf-8')
122     if "redirectReine" not in html:
123         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
124         sys.exit(1)
125     #print('-'*80)
126     #print(html)
127     
128     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
129     #html = httpresponse.read().decode('utf-8')
130     #print('-'*80)
131     #print(html)
132
133     if not alliance_tag:
134         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
135             'xajax=membre&xajaxr='+str(timestamp()))
136         html = httpresponse.read().decode('utf-8')
137         #print('-'*80)
138         #print(html)
139
140         root = html_parse(html)
141         table = get_elem(root, 'table')[0]
142         td = get_elem(table, 'td')[3]
143         table = get_elem(td, 'table')[0]
144         #print('-'*80)
145         #print_idented_tree(table)
146     else:
147         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
148         html = httpresponse.read().decode('utf-8')
149         
150         root = html_parse(html)
151         table = get_elem(root, 'table')[2]
152         #print('-'*80)
153         #print_idented_tree(table)
154
155     members = {} # NEW
156     for tr in get_elem(table, 'tr'):
157         row = get_elem(tr, 'td')
158         #print('-'*10)
159         #print_idented_tree(tr)
160         if not alliance_tag:
161             if len(row) != 10:
162                 continue
163             grade = get_merged_leaf_content(row[2])
164             nick = get_merged_leaf_content(row[3])
165             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
166             techno = int(get_merged_leaf_content(row[5]))
167             anthill = int(get_merged_leaf_content(row[6]))
168             active = get_elem(row[7], 'img')[0].attributes['alt']
169             colonized = len(get_elem(row[8], 'img')) == 1
170             #print(nick, grade, field, techno, anthill, active, colonized)
171
172         else:
173             if len(row) != 6:
174                 continue
175             grade = get_merged_leaf_content(row[1])
176             nick = get_merged_leaf_content(row[2])
177             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
178             techno = int(get_merged_leaf_content(row[4]))
179             anthill = int(get_merged_leaf_content(row[5]))
180             active = None
181             colonized = None
182             #print(nick, grade, field, techno, anthill)
183
184         members[nick] = {
185             'grade': grade,
186             'field': field,
187             'techno': techno,
188             'anthill': anthill,
189             'active': active,
190             'colonized': colonized,
191         }
192         #print(members[nick])
193
194     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
195     return members
196
197
198 def members_to_field(members):
199     """
200     Takes a full information dict "members"
201     Returns a simple dict with nick->field
202     """
203     fields  = {}
204     for nick, data in members.items():
205         fields[nick] = data['field']
206     return fields
207
208
209 def field_compare(oldmembers, newmembers, csvfile):
210     
211     oldfield = members_to_field(oldmembers)
212     newfield = members_to_field(newmembers)
213
214     changes = {}
215     txtchanges = []
216     txtchanges2 = []
217
218     for nick in newfield.keys():
219         told = oldfield.get(nick, 0)
220         tnew = newfield[nick]
221         if told == tnew:
222             continue
223         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
224     
225     for nick in oldfield.keys():
226         if nick not in newfield:
227             told = oldfield[nick]
228             tnew = 0
229             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
230     #for nick, change in changes.items():
231     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
232     #print
233
234     if not len(changes):
235         txtchanges.append('No changes')
236         txtchanges2.append('NOOP,,,,,,,,,,,No changes')
237  
238     for nick in changes.keys():
239         delta = changes[nick]['delta']
240         if delta <= 0:
241             continue # 0 is already process, <0 will be processed when nick swaps with nick2
242         for nick2 in changes.keys():
243             if changes[nick2]['delta'] == 0:
244                 continue # already done
245             if changes[nick2]['delta'] != -delta:
246                 continue # not the good one
247             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
248             txtchanges.append('%s a pris %s cm² à %s (%.2f%%)' % (nick, number_format(delta), nick2, percent))
249             txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {deltafield} cm² à {nick2} ({percent:.2f}%)'.format(
250                 nick1=nick,
251                 grade1=newmembers[nick].get('grade', ''),
252                 oldfield1=oldfield[nick],
253                 newfield1=newfield[nick],
254                 nick2=nick2,
255                 grade2=newmembers[nick2].get('grade', ''),
256                 oldfield2=oldfield[nick2],
257                 newfield2=newfield[nick2],
258                 deltafield=delta,
259                 nicedeltafield= number_format(delta),
260                 percent=percent,
261                 ))
262             changes[nick]['delta'] = 0
263             changes[nick2]['delta'] = 0
264
265     for nick, change in changes.items():
266         delta = change['delta']
267         if delta < 0:
268             if nick in newfield:
269                 percent = float(-delta) / oldfield[nick] * 100.
270                 txtchanges.append('%s a perdu %s cm² (%.2f%%)' % (nick, number_format(-delta), percent))
271                 txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
272                     nick1=nick,
273                     grade1=newmembers[nick].get('grade', ''),
274                     oldfield1=oldfield[nick],
275                     newfield1=newfield[nick],
276                     deltafield=-delta,
277                     nicedeltafield=number_format(-delta),
278                     percent=percent,
279                     ))
280             else:
281                 txtchanges.append("%s a quité l'alliance avec %s cm²" % (nick, number_format(-delta)))
282                 txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
283                     nick1=nick,
284                     grade1=oldmembers[nick].get('grade', ''),
285                     oldfield1=oldfield[nick],
286                     deltafield=-delta,
287                     nicedeltafield=number_format(-delta),
288                     ))
289         elif delta > 0:
290             if nick in oldfield:
291                 txtchanges.append('%s a gagné %s cm²' % (nick, number_format(delta)))
292                 txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
293                     nick1=nick,
294                     grade1=newmembers[nick].get('grade', ''),
295                     oldfield1=oldfield[nick],
296                     newfield1=newfield[nick],
297                     deltafield=delta,
298                     nicedeltafield=number_format(delta),
299                     percent=percent,
300                     ))
301             else:
302                 txtchanges.append("%s a rejoint l'alliance avec %s cm²" % (nick, number_format(delta)))
303                 txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
304                     nick1=nick,
305                     grade1=newmembers[nick].get('grade', ''),
306                     newfield1=newfield[nick],
307                     deltafield=delta,
308                     nicedeltafield=number_format(delta),
309                     ))
310
311     for txtchange in txtchanges:
312         logging.info(txtchange)
313
314     for txtchange in txtchanges2:
315         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
316
317 if __name__ == '__main__':
318     from optparse import OptionParser
319     parser = OptionParser()
320     parser.add_option('-d', '-v', '--debug', '--verbose',
321         action='store_true', dest='debug', default=False,
322         help="debug mode")
323     parser.add_option('-q', '--quiet',
324         action='store_true', dest='quiet', default=False,
325         help="quiet mode")
326     parser.add_option('-a', '--alliance',
327         dest='alliance', metavar="TAG",
328         help="Default is to process player own alliance.")
329     parser.add_option('--dir',
330         dest='directory', default='.fourmizzz',
331         help="Working directory, including config.py, archives and results. "
332              "Relative to HOME unless it starts with /. "
333              "Default is '%default'.")
334     parser.add_option('--members-name',
335         dest='archive_filename', metavar='FILENAME', default="archive.py",
336         help="Override filename containing old data. "
337         "Default is '%default' in Working directory. "
338         "See --dir.")
339     parser.add_option('--new-members-name',
340         dest='new_archive_filename', metavar='FILENAME',
341         help="Read new members data from a file. "
342         "Default is to fetch data from antzzz server.")
343     parser.add_option('--dry-run',
344         action='store_true', dest='dryrun', default=False,
345         help="don't store result in archives.")
346     parser.add_option('--stdout',
347         action='store_true', dest='stdout', default=False,
348         help="Print result on stdout rather that in results.log")
349     options, args = parser.parse_args()
350
351     if options.debug:
352         loglevel = logging.DEBUG
353     elif options.quiet:
354         loglevel = logging.WARNING
355     else:
356         loglevel = logging.INFO
357     
358     HOME = os.environ['HOME']
359     if options.directory[0] == '/':
360         BASE_DIRECTORY = options.directory
361     else:
362         BASE_DIRECTORY = os.path.join(HOME, options.directory)
363
364     if options.stdout:
365         logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
366     else:
367         result_filename = os.path.join(BASE_DIRECTORY, 'results.log')
368         logging.basicConfig(filename=result_filename, level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
369
370     read_config(BASE_DIRECTORY)
371
372     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
373     oldmembers = None
374     try:
375         f = open(members_filename, mode='r+', encoding='utf-8')
376     except IOError as err:
377         if err.errno == 2: # No such file or directory
378             logging.warning("No archive file, creating one.")
379             f = open(members_filename, mode='w+', encoding='utf-8')
380         else:
381             raise
382     else:
383         oldmembers = eval(f.read())
384     
385     if options.new_archive_filename:
386         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
387         newf = open(new_archive_filename, mode='r', encoding='utf8')
388         newmembers = eval(newf.read())
389     else:
390         newmembers = members_fetch(options.alliance)
391
392     if oldmembers is not None:
393         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
394         csvfile = open(csvfilename, mode='a', encoding='utf-8')
395         field_compare(oldmembers, newmembers, csvfile)
396     
397     # Save archive only after processing, just in case it crashes
398     if not options.dryrun:
399         f.seek(0)
400         f.write(repr(newmembers))
401         f.truncate()