Web scratching now recovers all data, including grade, techno, ...

Archive data format change (BW incompatible).

diff --git a/members.py b/members.py
index 29a939ef43e3592c8c9f1d11a691d03c5edc72c1..1e5b09bc7407cc02cd2adadab03ed84ff11aeeb4 100755 (executable)
--- a/members.py
+++ b/members.py
@@ -17,12 +17,17 @@ import htmlentities
 csv file format:
 DateTime,Action,Nick1,Grade1,OldField1,NewField1,Nick2,Grade2,OldField2,NewField2,FloodField,FieldLossPercent,Text
 Action is one of: JOIN, QUIT, WIN, LOSS, FLOOD
+
 Exemples:
 JOIN,Nirgal,Recrue,0,571628,,,,,,,Nirgal a rejoint l'alliance avec 571 628 cm² 
 QUIT,oncleblu,Chasseur,9940181,0,,,,,,,oncleblu a quité l'alliance avec 9 940 181 cm² 
 WIN,Nirgal,Chasseur,200000,257900,,,,,,,Nirgal a gagné 57 900 cm² 
 LOSS,Nirgal,Passeur 806,553624,442900,,,,,110724,20.00,Nirgal a perdu 110 724 cm² (20.00%) 
 FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,32731076,18411228,36.00,filoux a pris 18 411 228 cm² à yosemite (36.00%)
+FLOOD,Nick1,Grade1,100,102,Nick2,Grade2,100,99,1,1.00,Nick1 a pris 1 cm² à Nick2 (1.00%)
+FLOOD,Nick1,Grade1,100,102,Nick3,Grade3,100,99,1,1.00,Nick1 a pris 1 cm² à Nick3 (1.00%)
+
+Note that in last 2 lines exemple, NewField1 is 102 that is different from 100+1 because of multiple floods.
 """
 
 
@@ -32,8 +37,6 @@ FLOOD,filoux,Passeur 003,55813865,74225093,yosemite,Passeur 004,51142304,3273107
 #PASSWORD
 #BASE_URL
 HOME = os.environ['HOME']
-ARCHIVE = HOME + '/.fourmizzz/archive.py'
-RESULT = HOME + '/.fourmizzz/results.log'
 
 def read_config():
     '''
@@ -145,27 +148,58 @@ def tdc_get(alliance_tag=None):
         #print('-'*80)
         #print_idented_tree(table)
 
-    summary = {}
+    members = {} # NEW
     for tr in get_elem(table, 'tr'):
         row = get_elem(tr, 'td')
-        if not alliance_tag and len(row) != 10:
-            continue
-        if alliance_tag and len(row) != 6:
-            continue
         #print('-'*10)
         #print_idented_tree(tr)
         if not alliance_tag:
+            if len(row) != 10:
+                continue
+            grade = get_merged_leaf_content(row[2])
             nick = get_merged_leaf_content(row[3])
-            tdc = get_merged_leaf_content(row[4]).replace(' ', '')
+            field = int(get_merged_leaf_content(row[4]).replace(' ', ''))
+            techno = int(get_merged_leaf_content(row[5]))
+            anthill = int(get_merged_leaf_content(row[6]))
+            active = get_elem(row[7], 'img')[0].attributes['alt']
+            colonized = len(get_elem(row[8], 'img')) == 1
+            #print(nick, grade, field, techno, anthill, active, colonized)
+
         else:
+            if len(row) != 6:
+                continue
+            grade = get_merged_leaf_content(row[1])
             nick = get_merged_leaf_content(row[2])
-            tdc = get_merged_leaf_content(row[3]).replace(' ', '')
+            field = int(get_merged_leaf_content(row[3]).replace(' ', ''))
+            techno = int(get_merged_leaf_content(row[4]))
+            anthill = int(get_merged_leaf_content(row[5]))
+            active = None
+            colonized = None
+            #print(nick, grade, field, techno, anthill)
 
-        tdc = int(tdc)
-        #print(nick, tdc)
-        summary[nick] = tdc
-    logging.debug('%s members - total tdc = %s cm²', len(summary), number_format(sum(summary.values())))
-    return summary
+        members[nick] = {
+            'grade': grade,
+            'field': field,
+            'techno': techno,
+            'anthill': anthill,
+            'active': active,
+            'colonized': colonized,
+        }
+        #print(members[nick])
+
+    logging.debug('%s members - total field = %s cm²', len(members), number_format(sum([x['field'] for x in members.values()])))
+    return members
+
+
+def members_to_field(members):
+    """
+    Takes a full information dict "members"
+    Returns a simple dict with nick->field
+    """
+    fields  = {}
+    for nick, data in members.items():
+        fields[nick] = data['field']
+    return fields
 
 
 def tdc_compare(oldtdc, newtdc):
@@ -247,11 +281,14 @@ if __name__ == '__main__':
         loglevel = logging.WARNING
     else:
         loglevel = logging.INFO
+    
+    RESULT = HOME + '/.fourmizzz/results.log'
     logging.basicConfig(filename=RESULT, level=loglevel, format='%(asctime)s %(levelname)s %(message)s', datefmt='%Y-%m-%d %H:%M:%S %Z')
 
     read_config()
 
-    oldtdc = None
+    ARCHIVE = HOME + '/.fourmizzz/archive.py'
+    oldmembers = None
     try:
         f = open(ARCHIVE, mode='r+', encoding='utf-8')
     except IOError as err:
@@ -261,14 +298,17 @@ if __name__ == '__main__':
         else:
             raise
     else:
-        oldtdc = eval(f.read())
+        oldmembers = eval(f.read())
+        oldtdc = members_to_field(oldmembers)
     
-    newtdc = tdc_get(options.alliance)
+    newmembers = tdc_get(options.alliance)
+    newtdc = members_to_field(newmembers)
+
     if oldtdc is not None:
         tdc_compare(oldtdc, newtdc)
     
     # Save archive only after processing, just in case it crashes
     if not options.dryrun:
         f.seek(0)
-        f.write(repr(newtdc))
+        f.write(repr(newmembers))
         f.truncate()