members.py

   1 #!/usr/bin/env python3
   2 # -*- coding: utf-8 -*-
   3 #
   4 # Last version is available at svn://svn.nirgal.com/kod/fourmizzz/
   5 # Licensed under GPL
   6
   7 import os
   8 import time
   9 import re
  10 import logging
  11 from datetime import datetime
  12 import calendar
  13 import random
  14 import urllib.request
  15 from http.cookiejar import CookieJar
  16
  17 from html_parser import *
  18 import htmlentities
  19
  20 """
  21 csv file format:
  22 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,Text
  23 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO, GRADE
  24
  25 Exemples:
  26 NOOP,,,,,,,,,,Pas de changement dans le TdC
  27 JOIN,Nirgal,Recrue,0,571628,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm²
  28 QUIT,oncleblu,Chasseur,9940181,0,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm²
  29 WIN,Nirgal,Chasseur,200000,257900,,,,,,Nirgal a gagné 57 900 cm²
  30 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,Nirgal a perdu 110 724 cm² (20.00%)
  31 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,filoux a pris 18 411 228 cm² à yosemite (36.00%)
  32 FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,Nick1 a pris 1 cm² à Nick2 (1.00%)
  33 FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,Nick1 a pris 1 cm² à Nick3 (1.00%)
  34
  35 ACT,Nick1,Grade1,Actif,En vacances,,,,,,Nick1 est maintenant en vacances
  36 COLON,Nick1,Grade1,False,True,,,,,,Nick1 s'est fait colonisé
  37 COLON,Nick1,Grade1,True,False,,,,,,Nick1 n'est plus colonisé
  38
  39 ANTHILL,Nick1,Grade1,199,200,,,,,,La fourmilière de Nick1 a atteint le niveau 200
  40 TECHNO,Nick1,Grade1,199,200,,,,,,La technologie Nick1 a atteint le niveau 200
  41
  42 GRADE,Nick1,Recrue,Recrue,Passer 806,,,,,,Nick1 est maintenant Passeur 806
  43
  44 Note that in last 2 FLOOD lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
  45 """
  46
  47 # globals:
  48 #BASE_DIRECTORY
  49 #SERVER
  50 #LOGIN
  51 #PASSWORD
  52 #BASE_URL
  53 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
  54
  55 def read_config(config_dir):
  56     '''
  57     That function will read config.py in .fourmizzz directory
  58     and set up globals SERVER, LOGIN, PASSWORD, and BASE_URL.
  59     '''
  60     global SERVER, LOGIN, PASSWORD
  61     global BASE_URL
  62     CONFIG_TEMPLATE='''# Veuillez modifier les lignes ci dessous avec vos paramètres:
  63 SERVER = 's1.fourmizzz.fr'
  64 LOGIN = 'MonIdentifiant'
  65 PASSWORD = 'MonMotDePasse'
  66     '''
  67
  68     sys.path.append(config_dir)
  69     try:
  70         import config
  71     except ImportError:
  72         config_filename = os.path.join(config_dir, 'config.py')
  73         logging.fatal("No configuration file. Creating %s", config_filename)
  74         f = open(config_filename, mode='w+', encoding='utf-8')
  75         f.write(CONFIG_TEMPLATE)
  76         f.close()
  77         logging.fatal("Please update this file with your details.")
  78         sys.exit(1)
  79     SERVER = config.SERVER
  80     LOGIN = config.LOGIN
  81     PASSWORD = config.PASSWORD
  82     BASE_URL = 'http://%s' % SERVER
  83
  84 def hide_password_from_log(txt):
  85     return re.sub('mot_passe=.*', 'mot_passe=********', txt)
  86
  87 __opener__ = None
  88 def httpopen(url, post_data=None):
  89     if post_data:
  90         log_post_data = hide_password_from_log(post_data)
  91         post_data = post_data.encode('utf-8') # str->bytes
  92         logging.debug('HTTP POST %s %s', url, log_post_data)
  93     else:
  94         logging.debug('HTTP GET %s', url)
  95     global __opener__
  96     if __opener__ is None:
  97         cookiejar = CookieJar()
  98         __opener__ = urllib.request.build_opener()
  99         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
 100     http_response = __opener__.open(url, post_data)
 101     return http_response
 102
 103 def sleep(seconds):
 104     logging.debug('Waiting %s seconds', seconds)
 105     time.sleep(seconds)
 106
 107 def timestamp():
 108     return calendar.timegm(datetime.now().timetuple())
 109
 110 def number_format(i):
 111     result = ''
 112     while i:
 113         if result:
 114             result = ' ' + result
 115         d3 = i % 1000
 116         i = i//1000
 117         if i: # there will be something else left of that
 118             result = ('%03d' % d3) + result
 119         else:
 120             result = ('%d' % d3) + result
 121     return result
 122
 123
 124 def members_fetch(alliance_tag=None):
 125     #httpresponse = httpopen(BASE_URL + '/')
 126     #html = httpresponse.read().decode('utf-8')
 127     #print(html)
 128
 129     httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s' % (SERVER, LOGIN, PASSWORD))
 130     #httpresponse = httpopen(BASE_URL + '/index.php?connexion=1', 'serveur=%s&pseudo=%s&mot_passe=%s&connexion=Connexion' % (SERVER, LOGIN, PASSWORD))
 131     html = httpresponse.read().decode('utf-8')
 132     if "redirectReine" not in html:
 133         logging.fatal("Impossible de s'authentifier. Vérifiez vos paramètres dans config.py")
 134         sys.exit(1)
 135     #print('-'*80)
 136     #print(html)
 137
 138     #httpresponse = httpopen(BASE_URL + '/alliance.php?Membres')
 139     #html = httpresponse.read().decode('utf-8')
 140     #print('-'*80)
 141     #print(html)
 142
 143     if not alliance_tag:
 144         httpresponse = httpopen(BASE_URL + '/alliance.php?Membres',
 145             'xajax=membre&xajaxr='+str(timestamp()))
 146         html = httpresponse.read().decode('utf-8')
 147         #print('-'*80)
 148         #print(html)
 149
 150         root = html_parse(html)
 151         table = get_elem(root, 'table')[0]
 152         td = get_elem(table, 'td')[3]
 153         table = get_elem(td, 'table')[0]
 154         #print('-'*80)
 155         #print_idented_tree(table)
 156     else:
 157         httpresponse = httpopen(BASE_URL + '/classementAlliance.php?alliance=%s' % alliance_tag)
 158         html = httpresponse.read().decode('utf-8')
 159
 160         root = html_parse(html)
 161         table = get_elem(root, 'table')[2]
 162         #print('-'*80)
 163         #print_idented_tree(table)
 164
 165     members = {} # NEW
 166     for tr in get_elem(table, 'tr'):
 167         row = get_elem(tr, 'td')
 168         #print('-'*10)
 169         #print_idented_tree(tr)
 170         if not alliance_tag:
 171             if len(row) != 10:
 172                 continue
 173             grade = get_merged_leaf_content(row[2])
 174             nick = get_merged_leaf_content(row[3])
 175             field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
 176             techno = int(get_merged_leaf_content(row[5]))
 177             anthill = int(get_merged_leaf_content(row[6]))
 178             active = get_elem(row[7], 'img')[0].attributes['alt']
 179             colonized = len(get_elem(row[8], 'img')) == 1
 180             #print(nick, grade, field, techno, anthill, active, colonized)
 181
 182         else:
 183             if len(row) != 6:
 184                 continue
 185             grade = get_merged_leaf_content(row[1])
 186             nick = get_merged_leaf_content(row[2])
 187             field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
 188             techno = int(get_merged_leaf_content(row[4]))
 189             anthill = int(get_merged_leaf_content(row[5]))
 190             active = None
 191             colonized = None
 192             #print(nick, grade, field, techno, anthill)
 193
 194         members[nick] = {
 195             'grade': grade,
 196             'field': field,
 197             'techno': techno,
 198             'anthill': anthill,
 199             'active': active,
 200             'colonized': colonized,
 201         }
 202         #print(members[nick])
 203
 204     logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
 205     return members
 206
 207
 208 def members_fill_positions(newmembers):
 209     for nick, data in newmembers.items():
 210         if data['x'] is not None:
 211             continue
 212
 213         httpresponse = httpopen(BASE_URL + '/Membre.php?Pseudo=' + nick)
 214         html = httpresponse.read().decode('utf-8')
 215
 216         match = re.match('.*x=([0-9]+) et y=([0-9]+)', html, re.DOTALL|re.MULTILINE)
 217         x = int(match.group(1))
 218         y = int(match.group(2))
 219         newmembers[nick]['x'] = x
 220         newmembers[nick]['y'] = y
 221         logging.debug('%s has position %s %s', nick, x, y)
 222
 223 def copy_existing_data(oldmembers, newmembers):
 224     '''
 225     Check if newmembers contains some keys and if missing, copy them from
 226     oldmembers:
 227     colonized, active: if alliance is forced, these keys are not fetched
 228     x, y: only fetch them when neded.
 229     '''
 230     for nick, data in newmembers.items():
 231         for key in ['colonized', 'active', 'x', 'y' ]:
 232             if data.get(key, None) is None:
 233                 # missing or None
 234                 value = oldmembers.get(nick, {}).get(key, None)
 235                 newmembers[nick][key] = value
 236
 237 def members_to_field(members):
 238     """
 239     Takes a full information dict "members"
 240     Returns a simple dict with nick->field
 241     """
 242     fields  = {}
 243     for nick, data in members.items():
 244         fields[nick] = data['field']
 245     return fields
 246
 247
 248 def publish_line(csvfile, text, **args):
 249     csvline = text.format(**args)
 250     logging.info(csvline.split(',')[-1])
 251     if not options.dryrun:
 252         csvfile.write(time.strftime(DATETIME_FORMAT)+','+csvline+'\n')
 253
 254 def field_compare(oldmembers, newmembers, csvfile):
 255
 256     oldfield = members_to_field(oldmembers)
 257     newfield = members_to_field(newmembers)
 258
 259     changes = {}
 260
 261     for nick in newfield.keys():
 262         told = oldfield.get(nick, 0)
 263         tnew = newfield[nick]
 264         if told == tnew:
 265             continue
 266         changes[nick] = { 'old': told, 'new': tnew, 'delta': tnew-told }
 267
 268     for nick in oldfield.keys():
 269         if nick not in newfield:
 270             told = oldfield[nick]
 271             tnew = 0
 272             changes[nick] = { 'old': told, 'new': 0, 'delta': -told }
 273     #for nick, change in changes.items():
 274     #    print(nick, ' - ', change['old'], ' - ', change['new'], ' - ', change['delta'])
 275     #print
 276
 277     if not len(changes):
 278         publish_line(csvfile, 'NOOP,,,,,,,,,,No changes')
 279
 280     for nick in changes.keys():
 281         delta = changes[nick]['delta']
 282         if delta <= 0:
 283             continue # 0 is already process, <0 will be processed when nick swaps with nick2
 284         for nick2 in changes.keys():
 285             if changes[nick2]['delta'] == 0:
 286                 continue # already done
 287             if changes[nick2]['delta'] != -delta:
 288                 continue # not the good one
 289             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
 290             publish_line(csvfile,
 291                 'FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)',
 292                 nick1=nick,
 293                 grade1=newmembers[nick].get('grade', ''),
 294                 oldfield1=oldfield[nick],
 295                 newfield1=newfield[nick],
 296                 nick2=nick2,
 297                 grade2=newmembers[nick2].get('grade', ''),
 298                 oldfield2=oldfield[nick2],
 299                 newfield2=newfield[nick2],
 300                 deltafield=delta,
 301                 nicedeltafield= number_format(delta),
 302                 percent=percent,
 303                 )
 304             changes[nick]['delta'] = 0
 305             changes[nick2]['delta'] = 0
 306
 307     for nick, change in changes.items():
 308         delta = change['delta']
 309         if delta < 0:
 310             if nick in newfield:
 311                 percent = float(-delta) / oldfield[nick] * 100.
 312                 publish_line(csvfile,
 313                     'LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)',
 314                     nick1=nick,
 315                     grade1=newmembers[nick].get('grade', ''),
 316                     oldfield1=oldfield[nick],
 317                     newfield1=newfield[nick],
 318                     deltafield=-delta,
 319                     nicedeltafield=number_format(-delta),
 320                     percent=percent,
 321                     )
 322             else:
 323                 publish_line(csvfile,
 324                     'QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},{nick1} a quité l\'alliance avec {nicedeltafield} cm²',
 325                     nick1=nick,
 326                     grade1=oldmembers[nick].get('grade', ''),
 327                     oldfield1=oldfield[nick],
 328                     deltafield=-delta,
 329                     nicedeltafield=number_format(-delta),
 330                     )
 331         elif delta > 0:
 332             if nick in oldfield:
 333                 publish_line(csvfile,
 334                     'WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a gagné {nicedeltafield} cm²',
 335                     nick1=nick,
 336                     grade1=newmembers[nick].get('grade', ''),
 337                     oldfield1=oldfield[nick],
 338                     newfield1=newfield[nick],
 339                     deltafield=delta,
 340                     nicedeltafield=number_format(delta),
 341                     )
 342             else:
 343                 publish_line(csvfile,
 344                     'JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²',
 345                     nick1=nick,
 346                     grade1=newmembers[nick].get('grade', ''),
 347                     newfield1=newfield[nick],
 348                     deltafield=delta,
 349                     nicedeltafield=number_format(delta),
 350                     )
 351
 352 def activity_compare(oldmembers, newmembers, csvfile):
 353     for nick, data in newmembers.items():
 354         newa = data.get('active', None)
 355         olda = oldmembers.get(nick, {}).get('active', None)
 356         if newa and olda and newa != olda:
 357             publish_line(csvfile,
 358                 'ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est maintenant {lowernewdata1}',
 359                 nick1=nick,
 360                 grade1=newmembers[nick]['grade'],
 361                 olddata1=olda,
 362                 newdata1=newa,
 363                 lowernewdata1=newa.lower(),
 364                 )
 365
 366 def grade_compare(oldmembers, newmembers, csvfile):
 367     for nick, data in newmembers.items():
 368         newdata = data.get('grade', None)
 369         olddata = oldmembers.get(nick, {}).get('grade', None)
 370         if newdata is not None and olddata is not None and newdata != olddata:
 371             publish_line(csvfile,
 372                 'GRADE,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est désormais {grade1}',
 373                 nick1=nick,
 374                 grade1=newmembers[nick]['grade'],
 375                 olddata1=olddata,
 376                 newdata1=newdata,
 377                 )
 378
 379 def colonized_compare(oldmembers, newmembers, csvfile):
 380     for nick, data in newmembers.items():
 381         newc = data.get('colonized', None)
 382         oldc = oldmembers.get(nick, {}).get('colonized', None)
 383         if newc is not None and oldc is not None and newc != oldc:
 384             if newc:
 385                 txt = "s'est fait colonisé"
 386             else:
 387                 txt = "n'est plus colonisé"
 388             publish_line(csvfile,
 389                 'COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} {txt}',
 390                 nick1=nick,
 391                 grade1=newmembers[nick]['grade'],
 392                 olddata1=str(oldc),
 393                 newdata1=str(newc),
 394                 txt=txt,
 395                 )
 396
 397 def evolution_compare(oldmembers, newmembers, csvfile):
 398     for nick, data in newmembers.items():
 399         newdata = data['anthill']
 400         olddata = oldmembers.get(nick, {}).get('anthill', None)
 401         if newdata is not None and olddata is not None and newdata != olddata:
 402             publish_line(csvfile,
 403                 'ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}',
 404                 nick1=nick,
 405                 grade1=newmembers[nick]['grade'],
 406                 olddata1=olddata,
 407                 newdata1=newdata,
 408                 )
 409         newdata = data['techno']
 410         olddata = oldmembers.get(nick, {}).get('techno', None)
 411         if newdata is not None and olddata is not None and newdata != olddata:
 412             publish_line(csvfile,
 413                 'TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,La technologie de {nick1} a atteint le niveau {newdata1}',
 414                 nick1=nick,
 415                 grade1=newmembers[nick]['grade'],
 416                 olddata1=olddata,
 417                 newdata1=newdata,
 418                 )
 419
 420 if __name__ == '__main__':
 421     from optparse import OptionParser
 422     parser = OptionParser()
 423     parser.add_option('-d', '-v', '--debug', '--verbose',
 424         action='store_true', dest='debug', default=False,
 425         help="debug mode")
 426     parser.add_option('-q', '--quiet',
 427         action='store_true', dest='quiet', default=False,
 428         help="quiet mode")
 429     parser.add_option('-a', '--alliance',
 430         dest='alliance', metavar="TAG",
 431         help="Default is to process player own alliance.")
 432     parser.add_option('--dir',
 433         dest='directory', default='.fourmizzz',
 434         help="Working directory, including config.py, archives and results. "
 435              "Relative to HOME unless it starts with /. "
 436              "Default is '%default'.")
 437     parser.add_option('--quick',
 438         action='store_true', dest='quick', default=False,
 439         help="Don't sleep a random delay before hiting the server. "
 440              "The programme usually wait between 0 and 59s, so that "
 441              "there is no pick usage on a round minute")
 442     parser.add_option('--members-name',
 443         dest='archive_filename', metavar='FILENAME', default="archive.py",
 444         help="Override filename containing old data. "
 445         "Default is '%default' in Working directory. "
 446         "See --dir.")
 447     parser.add_option('--new-members-name',
 448         dest='new_archive_filename', metavar='FILENAME',
 449         help="Read new members data from a file. "
 450         "Default is to fetch data from antzzz server.")
 451     parser.add_option('--dry-run',
 452         action='store_true', dest='dryrun', default=False,
 453         help="don't store result in archives.")
 454     options, args = parser.parse_args()
 455
 456     if options.debug:
 457         loglevel = logging.DEBUG
 458     elif options.quiet:
 459         loglevel = logging.WARNING
 460     else:
 461         loglevel = logging.INFO
 462
 463     HOME = os.environ['HOME']
 464     if options.directory[0] == os.path.sep: # '/' on linux
 465         BASE_DIRECTORY = options.directory
 466     else:
 467         BASE_DIRECTORY = os.path.join(HOME, options.directory)
 468     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
 469
 470     if not options.quick:
 471         delay = random.randint(0,59)
 472         sleep(delay)
 473
 474     read_config(BASE_DIRECTORY)
 475
 476     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
 477     oldmembers = None
 478     try:
 479         f = open(members_filename, mode='r+', encoding='utf-8')
 480     except IOError as err:
 481         if err.errno == 2: # No such file or directory
 482             logging.warning("No archive file, creating one.")
 483             f = open(members_filename, mode='w+', encoding='utf-8')
 484         else:
 485             raise
 486     else:
 487         oldmembers = eval(f.read())
 488
 489     if options.new_archive_filename:
 490         new_archive_filename = os.path.join(BASE_DIRECTORY, options.new_archive_filename)
 491         newf = open(new_archive_filename, mode='r', encoding='utf8')
 492         newmembers = eval(newf.read())
 493     else:
 494         newmembers = members_fetch(options.alliance)
 495
 496     if oldmembers is not None:
 497         copy_existing_data(oldmembers, newmembers)
 498         if not options.new_archive_filename:
 499             members_fill_positions(newmembers)
 500         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
 501         csvfile = open(csvfilename, mode='a', encoding='utf-8')
 502         field_compare(oldmembers, newmembers, csvfile)
 503         grade_compare(oldmembers, newmembers, csvfile)
 504         activity_compare(oldmembers, newmembers, csvfile)
 505         colonized_compare(oldmembers, newmembers, csvfile)
 506         evolution_compare(oldmembers, newmembers, csvfile)
 507
 508     # Save archive only after processing, just in case it crashes
 509     if not options.dryrun:
 510         f.seek(0)
 511         f.write(repr(newmembers))
 512         f.truncate()
 513         f.close()
 514
 515         # We work in members.csv.new file then rename it
 516         # In the unlikely event that some script is reading it
 517         # while we updated it
 518         members_csvfilename = os.path.join(BASE_DIRECTORY, "members.csv")
 519         members_csvfilename_new = members_csvfilename + '.new'
 520         f = open(members_csvfilename_new, mode='w', encoding='utf-8')
 521         f.write('grade,nick,field,techno,anthill,active,colonized,x,y\n')
 522         for nick, data in newmembers.items():
 523             data = [
 524                 data['grade'],
 525                 nick,
 526                 data['field'],
 527                 data['techno'],
 528                 data['anthill'],
 529                 data['active'],
 530                 data['colonized'],
 531                 data['x'],
 532                 data['y'],
 533                 ]
 534             data = [ str(x) for x in data ]
 535             f.write(','.join(data) + '\n')
 536         f.close()
 537         os.rename(members_csvfilename_new, members_csvfilename)