Spoof User-agent:
[fourmizzz.git] / members.py
index 7842a3e5e50a8ddb5e3ba81fdd2ffeaf35c7719e..8c7aac217a0ed1e2769848909453a4ca375bc37e 100755 (executable)
@@ -10,6 +10,7 @@ import re
 import logging
 from datetime import datetime
 import calendar
+import random
 import urllib.request
 from http.cookiejar import CookieJar
 
@@ -18,20 +19,29 @@ import htmlentities
 
 """
 csv file format:
-DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
-Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP
+DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,Text
+Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD, NOOP, ACT, ANTHILL, TECHNO, GRADE
 
 Exemples:
-NOOP,,,,,,,,,,,No changes
-JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
-QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
-WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
-LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
-FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
-FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
-FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
-
-Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
+NOOP,,,,,,,,,,Pas de changement dans le TdC
+JOIN,Nirgal,Recrue,0,571628,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
+QUIT,oncleblu,Chasseur,9940181,0,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
+WIN,Nirgal,Chasseur,200000,257900,,,,,,Nirgal a gagné 57 900 cm² 
+LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,Nirgal a perdu 110 724 cm² (20.00%) 
+FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,filoux a pris 18 411 228 cm² à yosemite (36.00%)
+FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,Nick1 a pris 1 cm² à Nick2 (1.00%)
+FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,Nick1 a pris 1 cm² à Nick3 (1.00%)
+
+ACT,Nick1,Grade1,Actif,En vacances,,,,,,Nick1 est maintenant en vacances
+COLON,Nick1,Grade1,False,True,,,,,,Nick1 s'est fait coloniser
+COLON,Nick1,Grade1,True,False,,,,,,Nick1 n'est plus colonisé
+
+ANTHILL,Nick1,Grade1,199,200,,,,,,La fourmilière de Nick1 a atteint le niveau 200
+TECHNO,Nick1,Grade1,199,200,,,,,,La technologie Nick1 a atteint le niveau 200
+
+GRADE,Nick1,Recrue,Recrue,Passer 806,,,,,,Nick1 est maintenant Passeur 806
+
+Note that in last 2 FLOOD lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
 """
 
 # globals:
@@ -42,6 +52,8 @@ Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1
 #BASE_URL
 DATETIME_FORMAT = '%Y-%m-%d %H:%M:%S %Z'
 
+USER_AGENT="Mozilla/5.0 (X11; U; Linux.i686; en-US; rv:1.9.1.16) Gecko/20121020 Iceweasel/3.5.16 (like Firefox/3.5.16)"
+
 def read_config(config_dir):
     '''
     That function will read config.py in .fourmizzz directory
@@ -87,6 +99,7 @@ def httpopen(url, post_data=None):
         cookiejar = CookieJar()
         __opener__ = urllib.request.build_opener()
         __opener__.add_handler(urllib.request.HTTPCookieProcessor(cookiejar))
+        __opener__.addheaders = [('User-agent', USER_AGENT)]
     http_response = __opener__.open(url, post_data)
     return http_response
 
@@ -195,6 +208,45 @@ def members_fetch(alliance_tag=None):
     return members
 
 
+def members_fill_positions(newmembers):
+    for nick, data in newmembers.items():
+        if data['id'] is not None:
+            continue
+
+        httpresponse = httpopen(BASE_URL + '/Membre.php?Pseudo=' + nick)
+        html = httpresponse.read().decode('utf-8')
+
+        f = open("debug.html", mode='w+', encoding='utf-8')
+        f.write(html)
+
+        match = re.match('.*x=([0-9]+) et y=([0-9]+)', html, re.DOTALL|re.MULTILINE)
+        x = int(match.group(1))
+        y = int(match.group(2))
+        newmembers[nick]['x'] = x
+        newmembers[nick]['y'] = y
+
+        match=re.match('.*commerce.php\\?ID=([0-9]+).*', html, re.DOTALL|re.MULTILINE)
+        _id = int(match.group(1))
+        newmembers[nick]['id'] = _id
+
+        logging.debug('%s has position %s %s and ID %s', nick, x, y, _id)
+
+        break # only one at a time (debug)
+
+def copy_existing_data(oldmembers, newmembers):
+    '''
+    Check if newmembers contains some keys and if missing, copy them from
+    oldmembers:
+    colonized, active: if alliance is forced, these keys are not fetched
+    x, y, id: only fetch them when neded.
+    '''
+    for nick, data in newmembers.items():
+        for key in ['colonized', 'active', 'x', 'y', 'id' ]:
+            if data.get(key, None) is None:
+                # missing or None
+                value = oldmembers.get(nick, {}).get(key, None)
+                newmembers[nick][key] = value
+
 def members_to_field(members):
     """
     Takes a full information dict "members"
@@ -206,14 +258,18 @@ def members_to_field(members):
     return fields
 
 
+def publish_line(csvfile, text, **args):
+    csvline = text.format(**args)
+    logging.info(csvline.split(',')[-1])
+    if not options.dryrun:
+        csvfile.write(time.strftime(DATETIME_FORMAT)+','+csvline+'\n')
+
 def field_compare(oldmembers, newmembers, csvfile):
     
     oldfield = members_to_field(oldmembers)
     newfield = members_to_field(newmembers)
 
     changes = {}
-    txtchanges = []
-    txtchanges2 = []
 
     for nick in newfield.keys():
         told = oldfield.get(nick, 0)
@@ -232,8 +288,7 @@ def field_compare(oldmembers, newmembers, csvfile):
     #print
 
     if not len(changes):
-        txtchanges.append('No changes')
-        txtchanges2.append('NOOP,,,,,,,,,,,No changes')
+        publish_line(csvfile, 'NOOP,,,,,,,,,,No changes')
  
     for nick in changes.keys():
         delta = changes[nick]['delta']
@@ -245,8 +300,8 @@ def field_compare(oldmembers, newmembers, csvfile):
             if changes[nick2]['delta'] != -delta:
                 continue # not the good one
             percent = float(oldfield[nick2] - newfield[nick2]) / oldfield[nick2] * 100.
-            txtchanges.append('%s a pris %s cm² à %s (%.2f%%)' % (nick, number_format(delta), nick2, percent))
-            txtchanges2.append('FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{percent},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)'.format(
+            publish_line(csvfile, 
+                'FLOOD,{nick1},{grade1},{oldfield1},{newfield1},{nick2},{grade2},{oldfield2},{newfield2},{deltafield},{nick1} a pris {nicedeltafield} cm² à {nick2} ({percent:.2f}%)',
                 nick1=nick,
                 grade1=newmembers[nick].get('grade', ''),
                 oldfield1=oldfield[nick],
@@ -258,7 +313,7 @@ def field_compare(oldmembers, newmembers, csvfile):
                 deltafield=delta,
                 nicedeltafield= number_format(delta),
                 percent=percent,
-                ))
+                )
             changes[nick]['delta'] = 0
             changes[nick2]['delta'] = 0
 
@@ -267,8 +322,8 @@ def field_compare(oldmembers, newmembers, csvfile):
         if delta < 0:
             if nick in newfield:
                 percent = float(-delta) / oldfield[nick] * 100.
-                txtchanges.append('%s a perdu %s cm² (%.2f%%)' % (nick, number_format(-delta), percent))
-                txtchanges2.append('LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{percent},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)'.format(
+                publish_line(csvfile, 
+                    'LOSS,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a perdu {nicedeltafield} cm² ({percent:.2f}%)',
                     nick1=nick,
                     grade1=newmembers[nick].get('grade', ''),
                     oldfield1=oldfield[nick],
@@ -276,43 +331,104 @@ def field_compare(oldmembers, newmembers, csvfile):
                     deltafield=-delta,
                     nicedeltafield=number_format(-delta),
                     percent=percent,
-                    ))
+                    )
             else:
-                txtchanges.append("%s a quité l'alliance avec %s cm²" % (nick, number_format(-delta)))
-                txtchanges2.append('QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},,{nick1} a quité l\'alliance avec {nicedeltafield} cm²'.format(
+                publish_line(csvfile, 
+                    'QUIT,{nick1},{grade1},{oldfield1},0,,,,,{deltafield},{nick1} a quité l\'alliance avec {nicedeltafield} cm²',
                     nick1=nick,
                     grade1=oldmembers[nick].get('grade', ''),
                     oldfield1=oldfield[nick],
                     deltafield=-delta,
                     nicedeltafield=number_format(-delta),
-                    ))
+                    )
         elif delta > 0:
             if nick in oldfield:
-                txtchanges.append('%s a gagné %s cm²' % (nick, number_format(delta)))
-                txtchanges2.append('WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},,{nick1} a gagné {nicedeltafield} cm²'.format(
+                publish_line(csvfile, 
+                    'WIN,{nick1},{grade1},{oldfield1},{newfield1},,,,,{deltafield},{nick1} a gagné {nicedeltafield} cm²',
                     nick1=nick,
                     grade1=newmembers[nick].get('grade', ''),
                     oldfield1=oldfield[nick],
                     newfield1=newfield[nick],
                     deltafield=delta,
                     nicedeltafield=number_format(delta),
-                    percent=percent,
-                    ))
+                    )
             else:
-                txtchanges.append("%s a rejoint l'alliance avec %s cm²" % (nick, number_format(delta)))
-                txtchanges2.append('JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},,{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²'.format(
+                publish_line(csvfile, 
+                    'JOIN,{nick1},{grade1},0,{newfield1},,,,,{deltafield},{nick1} a rejoint l\'alliance avec {nicedeltafield} cm²',
                     nick1=nick,
                     grade1=newmembers[nick].get('grade', ''),
                     newfield1=newfield[nick],
                     deltafield=delta,
                     nicedeltafield=number_format(delta),
-                    ))
-
-    for txtchange in txtchanges:
-        logging.info(txtchange)
-
-    for txtchange in txtchanges2:
-        csvfile.write(time.strftime(DATETIME_FORMAT)+','+txtchange+'\n')
+                    )
+
+def activity_compare(oldmembers, newmembers, csvfile):
+    for nick, data in newmembers.items():
+        newa = data.get('active', None)
+        olda = oldmembers.get(nick, {}).get('active', None)
+        if newa and olda and newa != olda:
+            publish_line(csvfile, 
+                'ACT,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est maintenant {lowernewdata1}',
+                nick1=nick,
+                grade1=newmembers[nick]['grade'],
+                olddata1=olda,
+                newdata1=newa,
+                lowernewdata1=newa.lower(),
+                )
+
+def grade_compare(oldmembers, newmembers, csvfile):
+    for nick, data in newmembers.items():
+        newdata = data.get('grade', None)
+        olddata = oldmembers.get(nick, {}).get('grade', None)
+        if newdata is not None and olddata is not None and newdata != olddata:
+            publish_line(csvfile, 
+                'GRADE,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} est désormais {grade1}',
+                nick1=nick,
+                grade1=newmembers[nick]['grade'],
+                olddata1=olddata,
+                newdata1=newdata,
+                )
+
+def colonized_compare(oldmembers, newmembers, csvfile):
+    for nick, data in newmembers.items():
+        newc = data.get('colonized', None)
+        oldc = oldmembers.get(nick, {}).get('colonized', None)
+        if newc is not None and oldc is not None and newc != oldc:
+            if newc:
+                txt = "s'est fait colonisé"
+            else:
+                txt = "n'est plus colonisé"
+            publish_line(csvfile, 
+                'COLON,{nick1},{grade1},{olddata1},{newdata1},,,,,,{nick1} {txt}',
+                nick1=nick,
+                grade1=newmembers[nick]['grade'],
+                olddata1=str(oldc),
+                newdata1=str(newc),
+                txt=txt,
+                )
+
+def evolution_compare(oldmembers, newmembers, csvfile):
+    for nick, data in newmembers.items():
+        newdata = data['anthill']
+        olddata = oldmembers.get(nick, {}).get('anthill', None)
+        if newdata is not None and olddata is not None and newdata != olddata:
+            publish_line(csvfile, 
+                'ANTHILL,{nick1},{grade1},{olddata1},{newdata1},,,,,,La fourmilière de {nick1} a atteint le niveau {newdata1}',
+                nick1=nick,
+                grade1=newmembers[nick]['grade'],
+                olddata1=olddata,
+                newdata1=newdata,
+                )
+        newdata = data['techno']
+        olddata = oldmembers.get(nick, {}).get('techno', None)
+        if newdata is not None and olddata is not None and newdata != olddata:
+            publish_line(csvfile, 
+                'TECHNO,{nick1},{grade1},{olddata1},{newdata1},,,,,,La technologie de {nick1} a atteint le niveau {newdata1}',
+                nick1=nick,
+                grade1=newmembers[nick]['grade'],
+                olddata1=olddata,
+                newdata1=newdata,
+                )
 
 if __name__ == '__main__':
     from optparse import OptionParser
@@ -331,6 +447,11 @@ if __name__ == '__main__':
         help="Working directory, including config.py, archives and results. "
              "Relative to HOME unless it starts with /. "
              "Default is '%default'.")
+    parser.add_option('--quick',
+        action='store_true', dest='quick', default=False,
+        help="Don't sleep a random delay before hiting the server. "
+             "The programme usually wait between 0 and 59s, so that "
+             "there is no pick usage on a round minute")
     parser.add_option('--members-name',
         dest='archive_filename', metavar='FILENAME', default="archive.py",
         help="Override filename containing old data. "
@@ -353,12 +474,16 @@ if __name__ == '__main__':
         loglevel = logging.INFO
     
     HOME = os.environ['HOME']
-    if options.directory[0] == '/':
+    if options.directory[0] == os.path.sep: # '/' on linux
         BASE_DIRECTORY = options.directory
     else:
         BASE_DIRECTORY = os.path.join(HOME, options.directory)
     logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt=DATETIME_FORMAT)
 
+    if not options.quick:
+        delay = random.randint(0,59)
+        sleep(delay)
+
     read_config(BASE_DIRECTORY)
 
     members_filename = os.path.join(BASE_DIRECTORY, options.archive_filename)
@@ -382,12 +507,45 @@ if __name__ == '__main__':
         newmembers = members_fetch(options.alliance)
 
     if oldmembers is not None:
+        copy_existing_data(oldmembers, newmembers)
+        if not options.new_archive_filename:
+            members_fill_positions(newmembers)
         csvfilename = os.path.join(BASE_DIRECTORY, 'results.csv')
         csvfile = open(csvfilename, mode='a', encoding='utf-8')
         field_compare(oldmembers, newmembers, csvfile)
+        grade_compare(oldmembers, newmembers, csvfile)
+        activity_compare(oldmembers, newmembers, csvfile)
+        colonized_compare(oldmembers, newmembers, csvfile)
+        evolution_compare(oldmembers, newmembers, csvfile)
     
     # Save archive only after processing, just in case it crashes
     if not options.dryrun:
         f.seek(0)
         f.write(repr(newmembers))
         f.truncate()
+        f.close()
+
+        # We work in members.csv.new file then rename it
+        # In the unlikely event that some script is reading it
+        # while we updated it
+        members_csvfilename = os.path.join(BASE_DIRECTORY, "members.csv")
+        members_csvfilename_new = members_csvfilename + '.new'
+        f = open(members_csvfilename_new, mode='w', encoding='utf-8')
+        f.write('grade,nick,field,techno,anthill,active,colonized,x,y,id\n')
+        for nick, data in newmembers.items():
+            data = [
+                data['grade'],
+                nick,
+                data['field'],
+                data['techno'],
+                data['anthill'],
+                data['active'],
+                data['colonized'],
+                data['x'],
+                data['y'],
+                data['id'],
+                ]
+            data = [ str(x) for x in data ]
+            f.write(','.join(data) + '\n')
+        f.close()
+        os.rename(members_csvfilename_new, members_csvfilename)