Automatically fills x & y position from server
[fourmizzz.git] / members.py
1 #!/usr/bin/env python3
2 # -*- coding: utf-8 -*-
3 #
4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
5 # Licensed under GPL
6
7 import os
8 import time
9 import re
10 import logging
11 from datetime import datetime
12 import calendar
13 import random
14 import urllib.request
15 from http.cookiejar import CookieJar
16
17 from html_parser import *
18 import htmlentities
19
20 """
21 csv file format:
22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO, GRADE
24
25 Exemples:
26 NOOP,,,,,,,,,,,Pas de changement dans le TdC
27 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
29 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
34
35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,,Nick1 est maintenant en vacances
36 COLON,Nick1,Grade1,False,True,,,,,,,Nick1 s'est fait colonisé
37 COLON,Nick1,Grade1,True,False,,,,,,,Nick1 n'est plus colonisé
38
39 ANTHILL,Nick1,Grade1,199,200,,,,,,,La fourmilière de Nick1 a atteint le niveau 200
40 TECHNO,Nick1,Grade1,199,200,,,,,,,La technologie Nick1 a atteint le niveau 200
41
42 GRADE,Nick1,Recrue,Recrue,Passer 806,,,,,,,Nick1 est maintenant Passeur 806
43
44 Note that in last 2 FLOOD lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
45 """
46
47 # globals:
48 #BASE_DIRECTORY
49 #SERVER
50 #LOGIN
51 #PASSWORD
52 #BASE_URL
53 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
54
55 def read_config(config_dir):
56     '''
57     That function will read config.py in .fourmizzz directory
58     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
59     '''
60     global SERVER, LOGIN, PASSWORD
61     global BASE_URL
62     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
63 SERVER = 's1.fourmizzz.fr'
64 LOGIN = 'MonIdentifiant'
65 PASSWORD = 'MonMotDePasse'
66     '''
67
68     sys.path.append(config_dir)
69     try:
70         import config
71     except ImportError:
72         config_filename = os.path.join(config_dir, 'config.py')
73         logging.fatal("No configuration file. Creating %s", config_filename)
74         f = open(config_filename, mode='w+', encoding='utf-8')
75         f.write(CONFIG_TEMPLATE)
76         f.close()
77         logging.fatal("Please update this file with your details.")
78         sys.exit(1)
79     SERVER = config.SERVER
80     LOGIN = config.LOGIN
81     PASSWORD = config.PASSWORD
82     BASE_URL = 'http://%s' % SERVER
83
84 def hide_password_from_log(txt):
85     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
86
87 __opener__ = None
88 def httpopen(url, post_data=None):
89     if post_data:
90         log_post_data = hide_password_from_log(post_data)
91         post_data = post_data.encode('utf-8') # str->bytes
92         logging.debug('HTTP POST %s %s', url, log_post_data)
93     else:
94         logging.debug('HTTP GET %s', url)
95     global __opener__
96     if __opener__ is None:
97         cookiejar = CookieJar()
98         __opener__ = urllib.request.build_opener()
99         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
100     http_response = __opener__.open(url, post_data)
101     return http_response
102
103 def sleep(seconds):
104     logging.debug('Waiting %s seconds', seconds)
105     time.sleep(seconds)
106
107 def timestamp():
108     return calendar.timegm(datetime.now().timetuple())
109
110 def number_format(i):
111     result = ''
112     while i:
113         if result:
114             result = ' ' + result
115         d3 = i % 1000
116         i = i//1000
117         if i: # there will be something else left of that
118             result = ('%03d' % d3) + result
119         else:
120             result = ('%d' % d3) + result
121     return result
122
123
124 def members_fetch(alliance_tag=None):
125     #httpresponse = httpopen(BASE_URL + '/')
126     #html = httpresponse.read().decode('utf-8')
127     #print(html)
128    
129     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
130     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
131     html = httpresponse.read().decode('utf-8')
132     if "redirectReine" not in html:
133         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
134         sys.exit(1)
135     #print('-'*80)
136     #print(html)
137     
138     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
139     #html = httpresponse.read().decode('utf-8')
140     #print('-'*80)
141     #print(html)
142
143     if not alliance_tag:
144         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
145             'xajax=membre&xajaxr='+str(timestamp()))
146         html = httpresponse.read().decode('utf-8')
147         #print('-'*80)
148         #print(html)
149
150         root = html_parse(html)
151         table = get_elem(root, 'table')[0]
152         td = get_elem(table, 'td')[3]
153         table = get_elem(td, 'table')[0]
154         #print('-'*80)
155         #print_idented_tree(table)
156     else:
157         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
158         html = httpresponse.read().decode('utf-8')
159         
160         root = html_parse(html)
161         table = get_elem(root, 'table')[2]
162         #print('-'*80)
163         #print_idented_tree(table)
164
165     members = {} # NEW
166     for tr in get_elem(table, 'tr'):
167         row = get_elem(tr, 'td')
168         #print('-'*10)
169         #print_idented_tree(tr)
170         if not alliance_tag:
171             if len(row) != 10:
172                 continue
173             grade = get_merged_leaf_content(row[2])
174             nick = get_merged_leaf_content(row[3])
175             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
176             techno = int(get_merged_leaf_content(row[5]))
177             anthill = int(get_merged_leaf_content(row[6]))
178             active = get_elem(row[7], 'img')[0].attributes['alt']
179             colonized = len(get_elem(row[8], 'img')) == 1
180             #print(nick, grade, field, techno, anthill, active, colonized)
181
182         else:
183             if len(row) != 6:
184                 continue
185             grade = get_merged_leaf_content(row[1])
186             nick = get_merged_leaf_content(row[2])
187             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
188             techno = int(get_merged_leaf_content(row[4]))
189             anthill = int(get_merged_leaf_content(row[5]))
190             active = None
191             colonized = None
192             #print(nick, grade, field, techno, anthill)
193
194         members[nick] = {
195             'grade': grade,
196             'field': field,
197             'techno': techno,
198             'anthill': anthill,
199             'active': active,
200             'colonized': colonized,
201         }
202         #print(members[nick])
203
204     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
205     return members
206
207
208 def members_fill_positions(newmembers):
209     for nick, data in newmembers.items():
210         if data['x'] is not None:
211             continue
212
213         httpresponse = httpopen(BASE_URL + '/Membre.php?Pseudo=' + nick)
214         html = httpresponse.read().decode('utf-8')
215
216         match = re.match('.*x=([0-9]+) et y=([0-9]+)', html, re.DOTALL|re.MULTILINE)
217         x = int(match.group(1))
218         y = int(match.group(2))
219         newmembers[nick]['x'] = x
220         newmembers[nick]['y'] = y
221         logging.debug('%s has position %s %s', nick, x, y)
222         break # XXX
223
224 def copy_existing_data(oldmembers, newmembers):
225     '''
226     Check if newmembers contains some keys and if missing, copy them from
227     oldmembers:
228     colonized, active: if alliance is forced, these keys are not fetched
229     x, y: only fetch them when neded.
230     '''
231     for nick, data in newmembers.items():
232         for key in ['colonized', 'active', 'x', 'y' ]:
233             if data.get(key, None) is None:
234                 # missing or None
235                 value = oldmembers.get(nick, {}).get(key, None)
236                 newmembers[nick][key] = value
237
238 def members_to_field(members):
239     """
240     Takes a full information dict "members"
241     Returns a simple dict with nick->field
242     """
243     fields  = {}
244     for nick, data in members.items():
245         fields[nick] = data['field']
246     return fields
247
248
249 def publish_line(csvfile, text, **args):
250     csvline = text.format(**args)
251     logging.info(csvline.split(',')[-1])
252     if not options.dryrun:
253         csvfile.write(time.strftime(DATETIME_FORMAT)+','+csvline+'\n')
254
255 def field_compare(oldmembers, newmembers, csvfile):
256     
257     oldfield = members_to_field(oldmembers)
258     newfield = members_to_field(newmembers)
259
260     changes = {}
261
262     for nick in newfield.keys():
263         told = oldfield.get(nick, 0)
264         tnew = newfield[nick]
265         if told == tnew:
266             continue
267         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
268     
269     for nick in oldfield.keys():
270         if nick not in newfield:
271             told = oldfield[nick]
272             tnew = 0
273             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
274     #for nick, change in changes.items():
275     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
276     #print
277
278     if not len(changes):
279         publish_line(csvfile, 'NOOP,,,,,,,,,,,No changes')
280  
281     for nick in changes.keys():
282         delta = changes[nick]['delta']
283         if delta <= 0:
284             continue # 0 is already process, <0 will be processed when nick swaps with nick2
285         for nick2 in changes.keys():
286             if changes[nick2]['delta'] == 0:
287                 continue # already done
288             if changes[nick2]['delta'] != -delta:
289                 continue # not the good one
290             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
291             publish_line(csvfile, 
292                 'FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)',
293                 nick1=nick,
294                 grade1=newmembers[nick].get('grade', ''),
295                 oldfield1=oldfield[nick],
296                 newfield1=newfield[nick],
297                 nick2=nick2,
298                 grade2=newmembers[nick2].get('grade', ''),
299                 oldfield2=oldfield[nick2],
300                 newfield2=newfield[nick2],
301                 deltafield=delta,
302                 nicedeltafield= number_format(delta),
303                 percent=percent,
304                 )
305             changes[nick]['delta'] = 0
306             changes[nick2]['delta'] = 0
307
308     for nick, change in changes.items():
309         delta = change['delta']
310         if delta < 0:
311             if nick in newfield:
312                 percent = float(-delta) / oldfield[nick] * 100.
313                 publish_line(csvfile, 
314                     'LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)',
315                     nick1=nick,
316                     grade1=newmembers[nick].get('grade', ''),
317                     oldfield1=oldfield[nick],
318                     newfield1=newfield[nick],
319                     deltafield=-delta,
320                     nicedeltafield=number_format(-delta),
321                     percent=percent,
322                     )
323             else:
324                 publish_line(csvfile, 
325                     'QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²',
326                     nick1=nick,
327                     grade1=oldmembers[nick].get('grade', ''),
328                     oldfield1=oldfield[nick],
329                     deltafield=-delta,
330                     nicedeltafield=number_format(-delta),
331                     )
332         elif delta > 0:
333             if nick in oldfield:
334                 publish_line(csvfile, 
335                     'WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²',
336                     nick1=nick,
337                     grade1=newmembers[nick].get('grade', ''),
338                     oldfield1=oldfield[nick],
339                     newfield1=newfield[nick],
340                     deltafield=delta,
341                     nicedeltafield=number_format(delta),
342                     )
343             else:
344                 publish_line(csvfile, 
345                     'JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²',
346                     nick1=nick,
347                     grade1=newmembers[nick].get('grade', ''),
348                     newfield1=newfield[nick],
349                     deltafield=delta,
350                     nicedeltafield=number_format(delta),
351                     )
352
353 def activity_compare(oldmembers, newmembers, csvfile):
354     for nick, data in newmembers.items():
355         newa = data.get('active', None)
356         olda = oldmembers.get(nick, {}).get('active', None)
357         if newa and olda and newa != olda:
358             publish_line(csvfile, 
359                 'ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est maintenant {lowernewdata1}',
360                 nick1=nick,
361                 grade1=newmembers[nick]['grade'],
362                 olddata1=olda,
363                 newdata1=newa,
364                 lowernewdata1=newa.lower(),
365                 )
366
367 def grade_compare(oldmembers, newmembers, csvfile):
368     for nick, data in newmembers.items():
369         newdata = data.get('grade', None)
370         olddata = oldmembers.get(nick, {}).get('grade', None)
371         if newdata is not None and olddata is not None and newdata != olddata:
372             publish_line(csvfile, 
373                 'GRADE,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est désormais {grade1}',
374                 nick1=nick,
375                 grade1=newmembers[nick]['grade'],
376                 olddata1=olddata,
377                 newdata1=newdata,
378                 )
379
380 def colonized_compare(oldmembers, newmembers, csvfile):
381     for nick, data in newmembers.items():
382         newc = data.get('colonized', None)
383         oldc = oldmembers.get(nick, {}).get('colonized', None)
384         if newc is not None and oldc is not None and newc != oldc:
385             if newc:
386                 txt = "s'est fait colonisé"
387             else:
388                 txt = "n'est plus colonisé"
389             publish_line(csvfile, 
390                 'COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} {txt}',
391                 nick1=nick,
392                 grade1=newmembers[nick]['grade'],
393                 olddata1=str(oldc),
394                 newdata1=str(newc),
395                 txt=txt,
396                 )
397
398 def evolution_compare(oldmembers, newmembers, csvfile):
399     for nick, data in newmembers.items():
400         newdata = data['anthill']
401         olddata = oldmembers.get(nick, {}).get('anthill', None)
402         if newdata is not None and olddata is not None and newdata != olddata:
403             publish_line(csvfile, 
404                 'ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}',
405                 nick1=nick,
406                 grade1=newmembers[nick]['grade'],
407                 olddata1=olddata,
408                 newdata1=newdata,
409                 )
410         newdata = data['techno']
411         olddata = oldmembers.get(nick, {}).get('techno', None)
412         if newdata is not None and olddata is not None and newdata != olddata:
413             publish_line(csvfile, 
414                 'TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,,La technologie de {nick1} a atteint le niveau {newdata1}',
415                 nick1=nick,
416                 grade1=newmembers[nick]['grade'],
417                 olddata1=olddata,
418                 newdata1=newdata,
419                 )
420
421 if __name__ == '__main__':
422     from optparse import OptionParser
423     parser = OptionParser()
424     parser.add_option('-d', '-v', '--debug', '--verbose',
425         action='store_true', dest='debug', default=False,
426         help="debug mode")
427     parser.add_option('-q', '--quiet',
428         action='store_true', dest='quiet', default=False,
429         help="quiet mode")
430     parser.add_option('-a', '--alliance',
431         dest='alliance', metavar="TAG",
432         help="Default is to process player own alliance.")
433     parser.add_option('--dir',
434         dest='directory', default='.fourmizzz',
435         help="Working directory, including config.py, archives and results. "
436              "Relative to HOME unless it starts with /. "
437              "Default is '%default'.")
438     parser.add_option('--quick',
439         action='store_true', dest='quick', default=False,
440         help="Don't sleep a random delay before hiting the server. "
441              "The programme usually wait between 0 and 59s, so that "
442              "there is no pick usage on a round minute")
443     parser.add_option('--members-name',
444         dest='archive_filename', metavar='FILENAME', default="archive.py",
445         help="Override filename containing old data. "
446         "Default is '%default' in Working directory. "
447         "See --dir.")
448     parser.add_option('--new-members-name',
449         dest='new_archive_filename', metavar='FILENAME',
450         help="Read new members data from a file. "
451         "Default is to fetch data from antzzz server.")
452     parser.add_option('--dry-run',
453         action='store_true', dest='dryrun', default=False,
454         help="don't store result in archives.")
455     options, args = parser.parse_args()
456
457     if options.debug:
458         loglevel = logging.DEBUG
459     elif options.quiet:
460         loglevel = logging.WARNING
461     else:
462         loglevel = logging.INFO
463     
464     HOME = os.environ['HOME']
465     if options.directory[0] == os.path.sep: # '/' on linux
466         BASE_DIRECTORY = options.directory
467     else:
468         BASE_DIRECTORY = os.path.join(HOME, options.directory)
469     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
470
471     if not options.quick:
472         delay = random.randint(0,59)
473         sleep(delay)
474
475     read_config(BASE_DIRECTORY)
476
477     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
478     oldmembers = None
479     try:
480         f = open(members_filename, mode='r+', encoding='utf-8')
481     except IOError as err:
482         if err.errno == 2: # No such file or directory
483             logging.warning("No archive file, creating one.")
484             f = open(members_filename, mode='w+', encoding='utf-8')
485         else:
486             raise
487     else:
488         oldmembers = eval(f.read())
489     
490     if options.new_archive_filename:
491         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
492         newf = open(new_archive_filename, mode='r', encoding='utf8')
493         newmembers = eval(newf.read())
494     else:
495         newmembers = members_fetch(options.alliance)
496
497     if oldmembers is not None:
498         copy_existing_data(oldmembers, newmembers)
499         if not options.new_archive_filename:
500             members_fill_positions(newmembers)
501         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
502         csvfile = open(csvfilename, mode='a', encoding='utf-8')
503         field_compare(oldmembers, newmembers, csvfile)
504         grade_compare(oldmembers, newmembers, csvfile)
505         activity_compare(oldmembers, newmembers, csvfile)
506         colonized_compare(oldmembers, newmembers, csvfile)
507         evolution_compare(oldmembers, newmembers, csvfile)
508     
509     # Save archive only after processing, just in case it crashes
510     if not options.dryrun:
511         f.seek(0)
512         f.write(repr(newmembers))
513         f.truncate()
514         f.close()
515
516         # We work in members.csv.new file then rename it
517         # In the unlikely event that some script is reading it
518         # while we updated it
519         members_csvfilename = os.path.join(BASE_DIRECTORY, "members.csv")
520         members_csvfilename_new = members_csvfilename + '.new'
521         f = open(members_csvfilename_new, mode='w', encoding='utf-8')
522         f.write('grade,nick,field,techno,anthill,active,colonized,x,y\n')
523         for nick, data in newmembers.items():
524             data = [
525                 data['grade'],
526                 nick,
527                 data['field'],
528                 data['techno'],
529                 data['anthill'],
530                 data['active'],
531                 data['colonized'],
532                 data['x'],
533                 data['y'],
534                 ]
535             data = [ str(x) for x in data ]
536             f.write(','.join(data) + '\n')
537         f.close()
538         os.rename(members_csvfilename_new, members_csvfilename)