from datetime import datetime
import urllib.request
from http.cookiejar import CookieJar
-from subprocess import Popen, PIPE
+from subprocess import Popen, PIPE, call
import html_parser
import htmlentities
logging.debug('Waiting %s seconds', seconds)
time.sleep(seconds)
-def main():
+def download():
'''
Download all the accounts csv data and store them in LOCAL_DIR
Return a list of filenames
cpttype, cptnum, searchtype = match.group(1), match.group(2), match.group(3)
logging.info('Found account type %s: %s' % (cpttype, cptnum))
+ result.append(cptnum)
httpresponse = httpopen(BASE_URL + '/voscomptes/canalXHTML/' + href[len('../../'):])
html = httpresponse.read().decode('iso8859-1')
open(filename, 'w', encoding='utf-8').write(csvdata)
sleep(9)
+ lastfilename = LOCAL_DIR + cptnum + '.last.csv'
+ try:
+ os.unlink(lastfilename)
+ except OSError as err:
+ if err.errno == 2: #No such file or directory
+ logging.warning('Could not find last csv link. Running for the first time?')
+ else:
+ raise
+ os.symlink(filename, lastfilename)
+
logging.info('Disconnecting')
httpresponse = httpopen(BASE_URL + '/voscomptes/canalXHTML/securite/deconnexion/init-deconnexion.ea')
html = httpresponse.read().decode('iso8859-1')
logging.info('Disconnected')
return result
+
+def agregate():
+ CSV_HEADER = 'Date;Libellé;Montant(EUROS);Montant(FRANCS)'
+
+ account_files = os.listdir(LOCAL_DIR)
+ for account_file in account_files:
+ if not account_file.endswith('.last.csv'):
+ continue
+
+ account = account_file[:-len('.last.csv')]
+ logging.debug('Agregating %s', account)
+
+ if os.access(LOCAL_DIR + account + '.csv', os.F_OK):
+ pass
+ else:
+ logging.warning('Master csv file not found for %s: creating', account)
+ masterfile = open(LOCAL_DIR + account + '.csv', 'w', encoding='utf-8')
+
+ past_headers = False
+ for line in open(LOCAL_DIR + account_file).read().split('\n'):
+ if not line:
+ continue
+ if past_headers:
+ masterfile.write(line + '\n')
+ elif line == CSV_HEADER:
+ past_headers = True
+
+
if __name__ == '__main__':
from optparse import OptionParser
parser = OptionParser()
loglevel = logging.INFO
logging.basicConfig(level=loglevel, format='%(asctime)s %(levelname)s %(message)s')
- os.umask(0o077)
+ os.umask(0o077) # this is really private
+
TMP_DIR = LOCAL_DIR + 'tmp/'
try:
os.mkdir(TMP_DIR)
raise
os.chdir(TMP_DIR)
- main()
+ download()
+ agregate()