Save job logs, repport the size

Update comment in input daemon

diff --git a/bin/djais/models.py b/bin/djais/models.py
index be413912a2734860e3c0b2b189d72bf15a3fedf9..397e3106ec545475031d28675dc0e71bbeefef8a 100644 (file)
--- a/bin/djais/models.py
+++ b/bin/djais/models.py
@@ -243,6 +243,15 @@ class Job(models.Model):
         filename = '/var/lib/ais/jobs/%s%s' % (self.id, extension)
         return os.path.getsize(filename)
 
+    def get_log_size(self):
+        filename = '/var/lib/ais/jobs/%s.log' % self.id
+        try:
+            return os.path.getsize(filename)
+        except OSError, err:
+            if err.errno == 2:
+                return 0
+            raise
+
     class Meta:
         db_table = u'job'
         ordering = ('queue_time',)

diff --git a/bin/djais/views.py b/bin/djais/views.py
index fcf80cc17daa793612f2b2051159e3edac76a4f5..ab346df3ab8c7716549df43d058efc2c0496fde3 100644 (file)
--- a/bin/djais/views.py
+++ b/bin/djais/views.py
@@ -904,7 +904,6 @@ def job_get(request, jobid):
         total_wait += 1
         if total_wait > 15:
             request.user.info('Your query is slow. Please be patient.')
-            # TODO: Do not overwrite job.finishtime / result
             if not job.notify:
                 # careful not to modify other unschronized columns
                 Job.objects.filter(id=jobid).update(notify = 'W') # Web

diff --git a/bin/inputs/run.py b/bin/inputs/run.py
index 461036555083e1d91c19eb8c2e2287b83f7d209b..662f9e9e0f764d8f25d683428c189d35efa7513b 100644 (file)
--- a/bin/inputs/run.py
+++ b/bin/inputs/run.py
@@ -108,7 +108,7 @@ def mainloop(options, args):
                         # dump the line to file
                         channel.source.logger.log_line(timestamp, from_, line)
 
-                        # process the line internally (WIP)
+                        # process the line internally
                         if options.todb:
                             try:
                                 proc = channel.aivdm_processor

diff --git a/bin/jobrunner.py b/bin/jobrunner.py
index d9a62f8148866bc143009302e8ea0b3903489438..85d2fdae06dc1c14c8dca60bb2d0a61ec2c21735 100755 (executable)
--- a/bin/jobrunner.py
+++ b/bin/jobrunner.py
@@ -61,9 +61,9 @@ def addjob(user_id, command, friendly_filename, notify=None):
     return jobid
 
 
-def jobid_ext_to_filename(jobid, friendly_filename):
+def jobid_ext_to_filenames(jobid, friendly_filename):
     extension = os.path.splitext(friendly_filename)[-1]
-    return RESULT_DIR + jobid + extension
+    return RESULT_DIR + jobid + extension, RESULT_DIR + jobid + '.log'
     
 def startup_clean():
     sqlexec(u'SELECT id, pid, start_time, finish_time FROM job WHERE pid IS NOT NULL OR (start_time IS NOT NULL AND finish_time IS NULL)')
@@ -102,11 +102,15 @@ def runjob():
         row = get_common_cursor().fetchone()
         if row is None:
             break
-        filename = jobid_ext_to_filename(*row)
+        filename, logfilename = jobid_ext_to_filenames(*row)
         try:
             os.unlink(filename)
         except OSError, err:
             logging.error('unlinking %s: %s', filename, err)
+        try:
+            os.unlink(logfilename)
+        except OSError, err:
+            logging.error('unlinking %s: %s', logfilename, err)
         deleted_jobs.append(row[0])
 
     for jobid in deleted_jobs:
@@ -127,10 +131,11 @@ def runjob():
     sqlexec(u"UPDATE job SET start_time=now() AT TIME ZONE 'GMT' WHERE id=%(jobid)s", {'jobid': jobid})
     dbcommit()
 
-    filename = jobid_ext_to_filename(jobid, friendly_filename)
+    filename, logfilename = jobid_ext_to_filenames(jobid, friendly_filename)
     output = file(filename, 'wb')
+    logfile = file(logfilename, 'wb')
 
-    p = subprocess.Popen(command, stdout=output, shell=True)
+    p = subprocess.Popen(command, stdout=output, stderr=logfile, shell=True)
     logging.debug('System process id = %s', p.pid)
     sqlexec(u'UPDATE job SET pid=' + unicode(p.pid) + ' WHERE id=%(jobid)s', {'jobid': jobid})
     dbcommit()

diff --git a/www_templates/job.html b/www_templates/job.html
index 6a8f848687d0090480d51f6804d3f2807f717a51..c07525e8b61687187d131b4ccef6b40ecc82a071 100644 (file)
--- a/www_templates/job.html
+++ b/www_templates/job.html
@@ -17,6 +17,7 @@ Job {{ job.id }} details
 Status: <b>Completed</b> at {{ job.finish_time|date:"Y-m-d H:i:s" }} UTC in {{ job.process_time }}<br>
 Result: {% if job.result %}<b>Error {{ job.result }}</b>{% else %}<b>Success</b><br>
 Result size: {{ job.get_sucess_size|filesizeformat }}<br>
+Log size: {{ job.get_log_size|filesizeformat }}<br>
 <a href="/job/{{ job.id }}/download" class=button>download</a>{% endif %}<br>
 {% else %}
     {% if job.start_time %}