56dccfba8d266c5afee8ec0e7cbe5b270317a075
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT
24
25 Exemples:
26 NOOP,,,,,,,,,,,Pas de changement dans le TdC
27 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,,Nick1 est maintenant en vacances
36 COLON,Nick1,Grade1,False,True,,,,,,,Nick1 s'est fait colonisé
37 COLON,Nick1,Grade1,True,False,,,,,,,Nick1 n'est plus colonisé
38
39 Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
40 """
41
42 # globals:
43 #BASE_DIRECTORY
44 #SERVER
45 #LOGIN
46 #PASSWORD
47 #BASE_URL
48 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
49
50 def read_config(config_dir):
51     '''
52     That function will read config.py in .fourmizzz directory
53     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
54     '''
55     global SERVER, LOGIN, PASSWORD
56     global BASE_URL
57     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
58 SERVER = 's1.fourmizzz.fr'
59 LOGIN = 'MonIdentifiant'
60 PASSWORD = 'MonMotDePasse'
61     '''
62
63     sys.path.append(config_dir)
64     try:
65         import config
66     except ImportError:
67         config_filename = os.path.join(config_dir, 'config.py')
68         logging.fatal("No configuration file. Creating %s", config_filename)
69         f = open(config_filename, mode='w+', encoding='utf-8')
70         f.write(CONFIG_TEMPLATE)
71         f.close()
72         logging.fatal("Please update this file with your details.")
73         sys.exit(1)
74     SERVER = config.SERVER
75     LOGIN = config.LOGIN
76     PASSWORD = config.PASSWORD
77     BASE_URL = 'http://%s' % SERVER
78
79 def hide_password_from_log(txt):
80     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
81
82 __opener__ = None
83 def httpopen(url, post_data=None):
84     if post_data:
85         log_post_data = hide_password_from_log(post_data)
86         post_data = post_data.encode('utf-8') # str->bytes
87         logging.debug('HTTP POST %s %s', url, log_post_data)
88     else:
89         logging.debug('HTTP GET %s', url)
90     global __opener__
91     if __opener__ is None:
92         cookiejar = CookieJar()
93         __opener__ = urllib.request.build_opener()
94         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
95     http_response = __opener__.open(url, post_data)
96     return http_response
97
98 def sleep(seconds):
99     logging.debug('Waiting %s seconds', seconds)
100     time.sleep(seconds)
101
102 def timestamp():
103     return calendar.timegm(datetime.now().timetuple())
104
105 def number_format(i):
106     result = ''
107     while i:
108         if result:
109             result = ' ' + result
110         d3 = i % 1000
111         i = i//1000
112         if i: # there will be something else left of that
113             result = ('%03d' % d3) + result
114         else:
115             result = ('%d' % d3) + result
116     return result
117
118
119 def members_fetch(alliance_tag=None):
120     #httpresponse = httpopen(BASE_URL + '/')
121     #html = httpresponse.read().decode('utf-8')
122     #print(html)
123    
124     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
125     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
126     html = httpresponse.read().decode('utf-8')
127     if "redirectReine" not in html:
128         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
129         sys.exit(1)
130     #print('-'*80)
131     #print(html)
132     
133     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
134     #html = httpresponse.read().decode('utf-8')
135     #print('-'*80)
136     #print(html)
137
138     if not alliance_tag:
139         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
140             'xajax=membre&xajaxr='+str(timestamp()))
141         html = httpresponse.read().decode('utf-8')
142         #print('-'*80)
143         #print(html)
144
145         root = html_parse(html)
146         table = get_elem(root, 'table')[0]
147         td = get_elem(table, 'td')[3]
148         table = get_elem(td, 'table')[0]
149         #print('-'*80)
150         #print_idented_tree(table)
151     else:
152         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
153         html = httpresponse.read().decode('utf-8')
154         
155         root = html_parse(html)
156         table = get_elem(root, 'table')[2]
157         #print('-'*80)
158         #print_idented_tree(table)
159
160     members = {} # NEW
161     for tr in get_elem(table, 'tr'):
162         row = get_elem(tr, 'td')
163         #print('-'*10)
164         #print_idented_tree(tr)
165         if not alliance_tag:
166             if len(row) != 10:
167                 continue
168             grade = get_merged_leaf_content(row[2])
169             nick = get_merged_leaf_content(row[3])
170             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
171             techno = int(get_merged_leaf_content(row[5]))
172             anthill = int(get_merged_leaf_content(row[6]))
173             active = get_elem(row[7], 'img')[0].attributes['alt']
174             colonized = len(get_elem(row[8], 'img')) == 1
175             #print(nick, grade, field, techno, anthill, active, colonized)
176
177         else:
178             if len(row) != 6:
179                 continue
180             grade = get_merged_leaf_content(row[1])
181             nick = get_merged_leaf_content(row[2])
182             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
183             techno = int(get_merged_leaf_content(row[4]))
184             anthill = int(get_merged_leaf_content(row[5]))
185             active = None
186             colonized = None
187             #print(nick, grade, field, techno, anthill)
188
189         members[nick] = {
190             'grade': grade,
191             'field': field,
192             'techno': techno,
193             'anthill': anthill,
194             'active': active,
195             'colonized': colonized,
196         }
197         #print(members[nick])
198
199     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
200     return members
201
202
203 def members_to_field(members):
204     """
205     Takes a full information dict "members"
206     Returns a simple dict with nick->field
207     """
208     fields  = {}
209     for nick, data in members.items():
210         fields[nick] = data['field']
211     return fields
212
213
214 def field_compare(oldmembers, newmembers, csvfile):
215     
216     oldfield = members_to_field(oldmembers)
217     newfield = members_to_field(newmembers)
218
219     changes = {}
220     txtchanges2 = []
221
222     for nick in newfield.keys():
223         told = oldfield.get(nick, 0)
224         tnew = newfield[nick]
225         if told == tnew:
226             continue
227         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
228     
229     for nick in oldfield.keys():
230         if nick not in newfield:
231             told = oldfield[nick]
232             tnew = 0
233             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
234     #for nick, change in changes.items():
235     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
236     #print
237
238     if not len(changes):
239         txtchanges2.append('NOOP,,,,,,,,,,,No changes')
240  
241     for nick in changes.keys():
242         delta = changes[nick]['delta']
243         if delta <= 0:
244             continue # 0 is already process, <0 will be processed when nick swaps with nick2
245         for nick2 in changes.keys():
246             if changes[nick2]['delta'] == 0:
247                 continue # already done
248             if changes[nick2]['delta'] != -delta:
249                 continue # not the good one
250             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
251             txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)'.format(
252                 nick1=nick,
253                 grade1=newmembers[nick].get('grade', ''),
254                 oldfield1=oldfield[nick],
255                 newfield1=newfield[nick],
256                 nick2=nick2,
257                 grade2=newmembers[nick2].get('grade', ''),
258                 oldfield2=oldfield[nick2],
259                 newfield2=newfield[nick2],
260                 deltafield=delta,
261                 nicedeltafield= number_format(delta),
262                 percent=percent,
263                 ))
264             changes[nick]['delta'] = 0
265             changes[nick2]['delta'] = 0
266
267     for nick, change in changes.items():
268         delta = change['delta']
269         if delta < 0:
270             if nick in newfield:
271                 percent = float(-delta) / oldfield[nick] * 100.
272                 txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
273                     nick1=nick,
274                     grade1=newmembers[nick].get('grade', ''),
275                     oldfield1=oldfield[nick],
276                     newfield1=newfield[nick],
277                     deltafield=-delta,
278                     nicedeltafield=number_format(-delta),
279                     percent=percent,
280                     ))
281             else:
282                 txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
283                     nick1=nick,
284                     grade1=oldmembers[nick].get('grade', ''),
285                     oldfield1=oldfield[nick],
286                     deltafield=-delta,
287                     nicedeltafield=number_format(-delta),
288                     ))
289         elif delta > 0:
290             if nick in oldfield:
291                 txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
292                     nick1=nick,
293                     grade1=newmembers[nick].get('grade', ''),
294                     oldfield1=oldfield[nick],
295                     newfield1=newfield[nick],
296                     deltafield=delta,
297                     nicedeltafield=number_format(delta),
298                     ))
299             else:
300                 txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
301                     nick1=nick,
302                     grade1=newmembers[nick].get('grade', ''),
303                     newfield1=newfield[nick],
304                     deltafield=delta,
305                     nicedeltafield=number_format(delta),
306                     ))
307
308     for txtchange in txtchanges2:
309         logging.info(txtchange.split(',')[-1])
310         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
311
312 def activity_compare(oldmembers, newmembers, csvfile):
313     txtchanges2 = []
314     for nick, data in newmembers.items():
315         newa = data.get('active', None)
316         olda = oldmembers.get(nick, {}).get('active', None)
317         if newa and olda and newa != olda:
318             txtchanges2.append('ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est maintenant {lowernewdata1}'.format(
319                 nick1=nick,
320                 grade1=newmembers[nick]['grade'],
321                 olddata1=olda,
322                 newdata1=newa,
323                 lowernewdata1=newa.lower(),
324                 ))
325
326     for txtchange in txtchanges2:
327         logging.info(txtchange.split(',')[-1])
328         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
329
330 def colonized_compare(oldmembers, newmembers, csvfile):
331     txtchanges2 = []
332     for nick, data in newmembers.items():
333         newc = data.get('colonized', None)
334         oldc = oldmembers.get(nick, {}).get('colonized', None)
335         if newc is not None and oldc is not None and newc != oldc:
336             if newc:
337                 txt = "s'est fait colonisé"
338             else:
339                 txt = "n'est plus colonisé"
340             txtchanges2.append('COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} {txt}'.format(
341                 nick1=nick,
342                 grade1=newmembers[nick]['grade'],
343                 olddata1=str(oldc),
344                 newdata1=str(newc),
345                 txt=txt,
346                 ))
347
348     for txtchange in txtchanges2:
349         logging.info(txtchange.split(',')[-1])
350         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
351
352 if __name__ == '__main__':
353     from optparse import OptionParser
354     parser = OptionParser()
355     parser.add_option('-d', '-v', '--debug', '--verbose',
356         action='store_true', dest='debug', default=False,
357         help="debug mode")
358     parser.add_option('-q', '--quiet',
359         action='store_true', dest='quiet', default=False,
360         help="quiet mode")
361     parser.add_option('-a', '--alliance',
362         dest='alliance', metavar="TAG",
363         help="Default is to process player own alliance.")
364     parser.add_option('--dir',
365         dest='directory', default='.fourmizzz',
366         help="Working directory, including config.py, archives and results. "
367              "Relative to HOME unless it starts with /. "
368              "Default is '%default'.")
369     parser.add_option('--quick',
370         action='store_true', dest='quick', default=False,
371         help="Don't sleep a random delay before hiting the server. "
372              "The programme usually wait between 0 and 59s, so that "
373              "there is no pick usage on a round minute")
374     parser.add_option('--members-name',
375         dest='archive_filename', metavar='FILENAME', default="archive.py",
376         help="Override filename containing old data. "
377         "Default is '%default' in Working directory. "
378         "See --dir.")
379     parser.add_option('--new-members-name',
380         dest='new_archive_filename', metavar='FILENAME',
381         help="Read new members data from a file. "
382         "Default is to fetch data from antzzz server.")
383     parser.add_option('--dry-run',
384         action='store_true', dest='dryrun', default=False,
385         help="don't store result in archives.")
386     options, args = parser.parse_args()
387
388     if options.debug:
389         loglevel = logging.DEBUG
390     elif options.quiet:
391         loglevel = logging.WARNING
392     else:
393         loglevel = logging.INFO
394     
395     HOME = os.environ['HOME']
396     if options.directory[0] == os.path.sep: # '/' on linux
397         BASE_DIRECTORY = options.directory
398     else:
399         BASE_DIRECTORY = os.path.join(HOME, options.directory)
400     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
401
402     if not options.quick:
403         delay = random.randint(0,59)
404         sleep(delay)
405
406     read_config(BASE_DIRECTORY)
407
408     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
409     oldmembers = None
410     try:
411         f = open(members_filename, mode='r+', encoding='utf-8')
412     except IOError as err:
413         if err.errno == 2: # No such file or directory
414             logging.warning("No archive file, creating one.")
415             f = open(members_filename, mode='w+', encoding='utf-8')
416         else:
417             raise
418     else:
419         oldmembers = eval(f.read())
420     
421     if options.new_archive_filename:
422         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
423         newf = open(new_archive_filename, mode='r', encoding='utf8')
424         newmembers = eval(newf.read())
425     else:
426         newmembers = members_fetch(options.alliance)
427
428     if oldmembers is not None:
429         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
430         csvfile = open(csvfilename, mode='a', encoding='utf-8')
431         field_compare(oldmembers, newmembers, csvfile)
432         activity_compare(oldmembers, newmembers, csvfile)
433         colonized_compare(oldmembers, newmembers, csvfile)
434     
435     # Save archive only after processing, just in case it crashes
436     if not options.dryrun:
437         f.seek(0)
438         f.write(repr(newmembers))
439         f.truncate()