members.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
   5 # Licensed under GPL
   6
   7 import os
   8 import time
   9 import re
  10 import logging
  11 from datetime import datetime
  12 import calendar
  13 import random
  14 import urllib.request
  15 from http.cookiejar import CookieJar
  16
  17 from html_parser import *
  18 import htmlentities
  19
  20 """
  21 csv file format:
  22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
  23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT
  24
  25 Exemples:
  26 NOOP,,,,,,,,,,,Pas de changement dans le TdC
  27 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm²
  28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm²
  29 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm²
  30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%)
  31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
  32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
  33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
  34
  35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,,Nick1 est maintenant en vacances
  36
  37 Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
  38 """
  39
  40 # globals:
  41 #BASE_DIRECTORY
  42 #SERVER
  43 #LOGIN
  44 #PASSWORD
  45 #BASE_URL
  46 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
  47
  48 def read_config(config_dir):
  49     '''
  50     That function will read config.py in .fourmizzz directory
  51     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
  52     '''
  53     global SERVER, LOGIN, PASSWORD
  54     global BASE_URL
  55     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
  56 SERVER = 's1.fourmizzz.fr'
  57 LOGIN = 'MonIdentifiant'
  58 PASSWORD = 'MonMotDePasse'
  59     '''
  60
  61     sys.path.append(config_dir)
  62     try:
  63         import config
  64     except ImportError:
  65         config_filename = os.path.join(config_dir, 'config.py')
  66         logging.fatal("No configuration file. Creating %s", config_filename)
  67         f = open(config_filename, mode='w+', encoding='utf-8')
  68         f.write(CONFIG_TEMPLATE)
  69         f.close()
  70         logging.fatal("Please update this file with your details.")
  71         sys.exit(1)
  72     SERVER = config.SERVER
  73     LOGIN = config.LOGIN
  74     PASSWORD = config.PASSWORD
  75     BASE_URL = 'http://%s' % SERVER
  76
  77 def hide_password_from_log(txt):
  78     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
  79
  80 __opener__ = None
  81 def httpopen(url, post_data=None):
  82     if post_data:
  83         log_post_data = hide_password_from_log(post_data)
  84         post_data = post_data.encode('utf-8') # str->bytes
  85         logging.debug('HTTP POST %s %s', url, log_post_data)
  86     else:
  87         logging.debug('HTTP GET %s', url)
  88     global __opener__
  89     if __opener__ is None:
  90         cookiejar = CookieJar()
  91         __opener__ = urllib.request.build_opener()
  92         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
  93     http_response = __opener__.open(url, post_data)
  94     return http_response
  95
  96 def sleep(seconds):
  97     logging.debug('Waiting %s seconds', seconds)
  98     time.sleep(seconds)
  99
 100 def timestamp():
 101     return calendar.timegm(datetime.now().timetuple())
 102
 103 def number_format(i):
 104     result = ''
 105     while i:
 106         if result:
 107             result = ' ' + result
 108         d3 = i % 1000
 109         i = i//1000
 110         if i: # there will be something else left of that
 111             result = ('%03d' % d3) + result
 112         else:
 113             result = ('%d' % d3) + result
 114     return result
 115
 116
 117 def members_fetch(alliance_tag=None):
 118     #httpresponse = httpopen(BASE_URL + '/')
 119     #html = httpresponse.read().decode('utf-8')
 120     #print(html)
 121
 122     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
 123     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
 124     html = httpresponse.read().decode('utf-8')
 125     if "redirectReine" not in html:
 126         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
 127         sys.exit(1)
 128     #print('-'*80)
 129     #print(html)
 130
 131     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
 132     #html = httpresponse.read().decode('utf-8')
 133     #print('-'*80)
 134     #print(html)
 135
 136     if not alliance_tag:
 137         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
 138             'xajax=membre&xajaxr='+str(timestamp()))
 139         html = httpresponse.read().decode('utf-8')
 140         #print('-'*80)
 141         #print(html)
 142
 143         root = html_parse(html)
 144         table = get_elem(root, 'table')[0]
 145         td = get_elem(table, 'td')[3]
 146         table = get_elem(td, 'table')[0]
 147         #print('-'*80)
 148         #print_idented_tree(table)
 149     else:
 150         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
 151         html = httpresponse.read().decode('utf-8')
 152
 153         root = html_parse(html)
 154         table = get_elem(root, 'table')[2]
 155         #print('-'*80)
 156         #print_idented_tree(table)
 157
 158     members = {} # NEW
 159     for tr in get_elem(table, 'tr'):
 160         row = get_elem(tr, 'td')
 161         #print('-'*10)
 162         #print_idented_tree(tr)
 163         if not alliance_tag:
 164             if len(row) != 10:
 165                 continue
 166             grade = get_merged_leaf_content(row[2])
 167             nick = get_merged_leaf_content(row[3])
 168             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
 169             techno = int(get_merged_leaf_content(row[5]))
 170             anthill = int(get_merged_leaf_content(row[6]))
 171             active = get_elem(row[7], 'img')[0].attributes['alt']
 172             colonized = len(get_elem(row[8], 'img')) == 1
 173             #print(nick, grade, field, techno, anthill, active, colonized)
 174
 175         else:
 176             if len(row) != 6:
 177                 continue
 178             grade = get_merged_leaf_content(row[1])
 179             nick = get_merged_leaf_content(row[2])
 180             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
 181             techno = int(get_merged_leaf_content(row[4]))
 182             anthill = int(get_merged_leaf_content(row[5]))
 183             active = None
 184             colonized = None
 185             #print(nick, grade, field, techno, anthill)
 186
 187         members[nick] = {
 188             'grade': grade,
 189             'field': field,
 190             'techno': techno,
 191             'anthill': anthill,
 192             'active': active,
 193             'colonized': colonized,
 194         }
 195         #print(members[nick])
 196
 197     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
 198     return members
 199
 200
 201 def members_to_field(members):
 202     """
 203     Takes a full information dict "members"
 204     Returns a simple dict with nick->field
 205     """
 206     fields  = {}
 207     for nick, data in members.items():
 208         fields[nick] = data['field']
 209     return fields
 210
 211
 212 def field_compare(oldmembers, newmembers, csvfile):
 213
 214     oldfield = members_to_field(oldmembers)
 215     newfield = members_to_field(newmembers)
 216
 217     changes = {}
 218     txtchanges2 = []
 219
 220     for nick in newfield.keys():
 221         told = oldfield.get(nick, 0)
 222         tnew = newfield[nick]
 223         if told == tnew:
 224             continue
 225         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
 226
 227     for nick in oldfield.keys():
 228         if nick not in newfield:
 229             told = oldfield[nick]
 230             tnew = 0
 231             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
 232     #for nick, change in changes.items():
 233     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
 234     #print
 235
 236     if not len(changes):
 237         txtchanges2.append('NOOP,,,,,,,,,,,No changes')
 238
 239     for nick in changes.keys():
 240         delta = changes[nick]['delta']
 241         if delta <= 0:
 242             continue # 0 is already process, <0 will be processed when nick swaps with nick2
 243         for nick2 in changes.keys():
 244             if changes[nick2]['delta'] == 0:
 245                 continue # already done
 246             if changes[nick2]['delta'] != -delta:
 247                 continue # not the good one
 248             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
 249             txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)'.format(
 250                 nick1=nick,
 251                 grade1=newmembers[nick].get('grade', ''),
 252                 oldfield1=oldfield[nick],
 253                 newfield1=newfield[nick],
 254                 nick2=nick2,
 255                 grade2=newmembers[nick2].get('grade', ''),
 256                 oldfield2=oldfield[nick2],
 257                 newfield2=newfield[nick2],
 258                 deltafield=delta,
 259                 nicedeltafield= number_format(delta),
 260                 percent=percent,
 261                 ))
 262             changes[nick]['delta'] = 0
 263             changes[nick2]['delta'] = 0
 264
 265     for nick, change in changes.items():
 266         delta = change['delta']
 267         if delta < 0:
 268             if nick in newfield:
 269                 percent = float(-delta) / oldfield[nick] * 100.
 270                 txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
 271                     nick1=nick,
 272                     grade1=newmembers[nick].get('grade', ''),
 273                     oldfield1=oldfield[nick],
 274                     newfield1=newfield[nick],
 275                     deltafield=-delta,
 276                     nicedeltafield=number_format(-delta),
 277                     percent=percent,
 278                     ))
 279             else:
 280                 txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
 281                     nick1=nick,
 282                     grade1=oldmembers[nick].get('grade', ''),
 283                     oldfield1=oldfield[nick],
 284                     deltafield=-delta,
 285                     nicedeltafield=number_format(-delta),
 286                     ))
 287         elif delta > 0:
 288             if nick in oldfield:
 289                 txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
 290                     nick1=nick,
 291                     grade1=newmembers[nick].get('grade', ''),
 292                     oldfield1=oldfield[nick],
 293                     newfield1=newfield[nick],
 294                     deltafield=delta,
 295                     nicedeltafield=number_format(delta),
 296                     ))
 297             else:
 298                 txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
 299                     nick1=nick,
 300                     grade1=newmembers[nick].get('grade', ''),
 301                     newfield1=newfield[nick],
 302                     deltafield=delta,
 303                     nicedeltafield=number_format(delta),
 304                     ))
 305
 306     for txtchange in txtchanges2:
 307         logging.info(txtchange.split(',')[-1])
 308         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
 309
 310 def activity_compare(oldmembers, newmembers, csvfile):
 311     txtchanges2 = []
 312     for nick, data in newmembers.items():
 313         newa = data.get('active', None)
 314         olda = oldmembers.get(nick, {}).get('active', None)
 315         if newa and olda and newa != olda:
 316             txtchanges2.append('ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,,{nick1} est maintenant {lowernewdata1}'.format(
 317                 nick1=nick,
 318                 grade1=newmembers[nick]['grade'],
 319                 olddata1=olda,
 320                 newdata1=newa,
 321                 lowernewdata1=newa.lower(),
 322                 ))
 323
 324     for txtchange in txtchanges2:
 325         logging.info(txtchange.split(',')[-1])
 326         csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
 327
 328 if __name__ == '__main__':
 329     from optparse import OptionParser
 330     parser = OptionParser()
 331     parser.add_option('-d', '-v', '--debug', '--verbose',
 332         action='store_true', dest='debug', default=False,
 333         help="debug mode")
 334     parser.add_option('-q', '--quiet',
 335         action='store_true', dest='quiet', default=False,
 336         help="quiet mode")
 337     parser.add_option('-a', '--alliance',
 338         dest='alliance', metavar="TAG",
 339         help="Default is to process player own alliance.")
 340     parser.add_option('--dir',
 341         dest='directory', default='.fourmizzz',
 342         help="Working directory, including config.py, archives and results. "
 343              "Relative to HOME unless it starts with /. "
 344              "Default is '%default'.")
 345     parser.add_option('--quick',
 346         action='store_true', dest='quick', default=False,
 347         help="Don't sleep a random delay before hiting the server. "
 348              "The programme usually wait between 0 and 59s, so that "
 349              "there is no pick usage on a round minute")
 350     parser.add_option('--members-name',
 351         dest='archive_filename', metavar='FILENAME', default="archive.py",
 352         help="Override filename containing old data. "
 353         "Default is '%default' in Working directory. "
 354         "See --dir.")
 355     parser.add_option('--new-members-name',
 356         dest='new_archive_filename', metavar='FILENAME',
 357         help="Read new members data from a file. "
 358         "Default is to fetch data from antzzz server.")
 359     parser.add_option('--dry-run',
 360         action='store_true', dest='dryrun', default=False,
 361         help="don't store result in archives.")
 362     options, args = parser.parse_args()
 363
 364     if options.debug:
 365         loglevel = logging.DEBUG
 366     elif options.quiet:
 367         loglevel = logging.WARNING
 368     else:
 369         loglevel = logging.INFO
 370
 371     HOME = os.environ['HOME']
 372     if options.directory[0] == os.path.sep: # '/' on linux
 373         BASE_DIRECTORY = options.directory
 374     else:
 375         BASE_DIRECTORY = os.path.join(HOME, options.directory)
 376     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
 377
 378     if not options.quick:
 379         delay = random.randint(0,59)
 380         sleep(delay)
 381
 382     read_config(BASE_DIRECTORY)
 383
 384     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
 385     oldmembers = None
 386     try:
 387         f = open(members_filename, mode='r+', encoding='utf-8')
 388     except IOError as err:
 389         if err.errno == 2: # No such file or directory
 390             logging.warning("No archive file, creating one.")
 391             f = open(members_filename, mode='w+', encoding='utf-8')
 392         else:
 393             raise
 394     else:
 395         oldmembers = eval(f.read())
 396
 397     if options.new_archive_filename:
 398         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
 399         newf = open(new_archive_filename, mode='r', encoding='utf8')
 400         newmembers = eval(newf.read())
 401     else:
 402         newmembers = members_fetch(options.alliance)
 403
 404     if oldmembers is not None:
 405         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
 406         csvfile = open(csvfilename, mode='a', encoding='utf-8')
 407         field_compare(oldmembers, newmembers, csvfile)
 408         activity_compare(oldmembers, newmembers, csvfile)
 409
 410     # Save archive only after processing, just in case it crashes
 411     if not options.dryrun:
 412         f.seek(0)
 413         f.write(repr(newmembers))
 414         f.truncate()