Merge pull request #16 from tsshimo/feature/jobs_status

Improvement Job Logging

automan/api/projects/annotations/annotation_manager.py

@@ -64,8 +64,6 @@ def list_annotations(
                 Q(project_id=project_id),
                 Q(delete_flag=False),
                 Q(name__contains=search_keyword))[begin:begin + per_page]
-        if len(annotations) == 0:
-            raise ObjectDoesNotExist()
 
         records = []
         for annotation in annotations:

automan/api/projects/jobs/const.py

@@ -5,3 +5,5 @@
     'succeeded': 'SUCCEEDED',
     'unknown': 'UNKNOWN'
 }
+
+UNKNOWN_LIMIT_TIME = 1000

automan/api/projects/jobs/migrations/0002_auto_20190902_0214.py

@@ -0,0 +1,23 @@
+# Generated by Django 2.2.2 on 2019-09-02 02:14
+
+from django.db import migrations, models
+
+
+class Migration(migrations.Migration):
+
+    dependencies = [
+        ('jobs', '0001_initial'),
+    ]
+
+    operations = [
+        migrations.AddField(
+            model_name='job',
+            name='pod_log',
+            field=models.CharField(default='', max_length=1023),
+        ),
+        migrations.AddField(
+            model_name='job',
+            name='unknown_started_at',
+            field=models.DateTimeField(null=True),
+        ),
+    ]

automan/api/projects/jobs/models.py

@@ -12,6 +12,8 @@ class Job(models.Model):
     registered_at = models.DateTimeField(default=timezone.now)
     project = models.ForeignKey(Projects, on_delete=models.CASCADE)
     job_config = models.CharField(max_length=1023)
+    pod_log = models.CharField(max_length=1023, default='')
     status = models.CharField(max_length=45, default=STATUS_MAP['submitted'])
     started_at = models.DateTimeField(null=True)
     completed_at = models.DateTimeField(null=True)
+    unknown_started_at = models.DateTimeField(null=True)

automan/api/projects/jobs/serializer.py

@@ -9,9 +9,9 @@
 from libs.k8s.jobs.annotation_archiver import AnnotationArchiver
 from libs.k8s.jobs.rosbag_extractor import RosbagExtractor
 from libs.k8s.jobs.rosbag_analyzer import RosbagAnalyzer
-from datetime import datetime
+from datetime import datetime, timezone
 from projects.jobs.models import Job
-from projects.jobs.const import STATUS_MAP
+from projects.jobs.const import STATUS_MAP, UNKNOWN_LIMIT_TIME
 from projects.project_manager import ProjectManager
 from projects.originals.original_manager import OriginalManager
 from projects.datasets.dataset_manager import DatasetManager
@@ -53,15 +53,24 @@ def list_jobs(cls, project_id, sort_key, is_reverse=False, per_page=PER_PAGE, pa
             record['job_type'] = job.job_type
             if job.status not in [STATUS_MAP['succeeded'], STATUS_MAP['failed']]:
                 status, start_time, completion_time = cls.__get_job_status(job.id, job.job_type)
+                if job.status != STATUS_MAP['unknown'] and status == STATUS_MAP['unknown']:
+                    job.unknown_started_at = datetime.now(timezone.utc)
                 job.status = status
                 job.started_at = start_time
                 job.completed_at = completion_time
+                if job.status == STATUS_MAP['unknown'] and cls.__is_unknown_time_limit(job.unknown_started_at):
+                    job.status = STATUS_MAP['failed']
+                if job.status == STATUS_MAP['failed']:
+                    namespace = cls.__generate_job_namespace()
+                    pod_log = BaseJob().logs(cls.__generate_job_name(job.id, job.job_type), namespace)
+                    job.pod_log = pod_log
                 job.save()
             record['status'] = job.status
             record['started_at'] = str(job.started_at) if job.started_at else ''
             record['completed_at'] = str(job.completed_at) if job.completed_at else ''
             record['registered_at'] = str(job.registered_at)
             record['job_config'] = job.job_config
+            record['pod_log'] = job.pod_log
             record['user_id'] = job.user_id
             records.append(record)
         contents = {}
@@ -216,6 +225,15 @@ def __get_raw_data_config(project_id, original_id, candidates):
         }
         return raw_data_config
 
+    @classmethod
+    def __is_unknown_time_limit(cls, unknown_started):
+        if not unknown_started:
+            return False
+        time = datetime.now(timezone.utc) - unknown_started
+        if time.seconds > UNKNOWN_LIMIT_TIME:
+            return True
+        return False
+
     @classmethod
     def __get_job_status(cls, id, job_type):
         namespace = cls.__generate_job_namespace()

automan/libs/k8s/jobs/__init__.py

@@ -2,6 +2,7 @@
 import json
 from kubernetes import client, config
 from kubernetes.client.rest import ApiException
+from utility.service_log import ServiceLog
 
 
 class BaseJob(object):
@@ -13,6 +14,7 @@ def __init__(self, k8s_config_path=None):
         except Exception:
             config.load_incluster_config()  # in kubernetes
         self.batch_client = client.BatchV1Api()
+        self.core_client = client.CoreV1Api()
 
     def create(self):
         raise NotImplementedError
@@ -39,6 +41,25 @@ def list(self, namespace):
         except ApiException as e:
             print("Exception when calling BatchV1Api->list_namespaced_job: %s\n" % e)
 
+    def logs(self, name, namespace):
+        label_selector = 'job-name=' + name
+
+        try:
+            pods = self.core_client.list_namespaced_pod(namespace, label_selector=label_selector)
+        except ApiException as e:
+            print("Exception when calling CoreV1Api->list_namespaced_pod: %s\n" % e)
+            return
+        try:
+            for pod in pods.items:
+                pod_log = self.core_client.read_namespaced_pod_log(pod.metadata.name, namespace)
+                ServiceLog.error('pod-name=' + pod.metadata.name, detail_msg=pod_log)
+            if pods.items == []:
+                pod_log = 'no logs'
+                ServiceLog.error(label_selector, detail_msg='no logs')
+            return pod_log
+        except ApiException as e:
+            print("Exception when calling CoreV1Api->read_namespaced_pod_log: %s\n" % e)
+
     def fetch(self, name, namespace):
         pretty = 'pretty_example'
 

front/app/assets/main-style.js

@@ -75,6 +75,21 @@ export const mainStyle = theme => ({
   tableWrapper: {
     padding: theme.spacing.unit * 2
   },
+  popover: {
+    pointerEvents: 'none'
+  },
+  popoverText: {
+    pointerEvents: 'none',
+    padding: 3
+  },
+  tableActionButton: {
+    marginTop: -10,
+    marginBottom: -10,
+    marginLeft: 1,
+    marginRight:1,
+    minHeight: '30px',
+    padding: '0 0px'
+  },
   fab: {
     margin: theme.spacing.unit
   },

front/app/dashboard/components/annotation/table.jsx

@@ -7,6 +7,10 @@ import Typography from '@material-ui/core/Typography';
 
 import ResizableTable from 'automan/dashboard/components/parts/resizable_table';
 import { mainStyle } from 'automan/assets/main-style';
+import Button from '@material-ui/core/Button';
+import Tooltip from '@material-ui/core/Tooltip';
+import Archive from '@material-ui/icons/Archive';
+import CloudDownload from '@material-ui/icons/CloudDownload';
 
 function actionFormatter(cell, row) {
   return row.actions;
@@ -130,11 +134,11 @@ class AnnotationTable extends React.Component {
     );
   }
   render() {
-    // if ( this.state.error ) {
-    //     return (
-    //         <div> {this.state.error} </div>
-    //     );
-    // }
+    if ( this.state.error ) {
+      return (
+        <div> {this.state.error} </div>
+      );
+    }
 
     const { classes } = this.props;
     let rows = [];
@@ -149,35 +153,50 @@ class AnnotationTable extends React.Component {
             actions = (
               <div className="text-center">
                 <span>
-                  <a
-                    className="button glyphicon glyphicon-folder-close"
-                    onClick={e => this.handleArchive(row)}
-                    title="Archive"
-                  />
-                  <a
-                    className="button glyphicon glyphicon-download-alt"
-                    onClick={()=>{
-                      RequestClient.getBinaryAsURL(row.archive_url, (url) => {
-                        let a = document.createElement('a');
-                        a.download = row.file_name;
-                        a.href = url;
-                        a.click();
-                      }, () => {});
-                    }}
-                    title="Download"
-                  />
+                  <Tooltip title="Archive">
+                    <Button
+                      classes={{root: classes.tableActionButton}}
+                      onClick={e => this.handleArchive(row)}
+                      className={classes.button}>
+                      <Archive fontSize="small"/>
+                    </Button>
+                  </Tooltip>
+                  <Tooltip title="Download">
+                    <Button
+                      classes={{root: classes.tableActionButton}}
+                      onClick={()=>{
+                        RequestClient.getBinaryAsURL(row.archive_url, (url) => {
+                          let a = document.createElement('a');
+                          a.download = row.file_name;
+                          a.href = url;
+                          a.click();
+                        }, () => {});
+                      }}
+                      className={classes.button}>
+                      <CloudDownload fontSize="small"/>
+                    </Button>
+                  </Tooltip>
                 </span>
               </div>
             );
           } else {
             actions = (
               <div className="text-center">
                 <span>
-                  <a
-                    className="button glyphicon glyphicon-folder-close"
-                    onClick={e => this.handleArchive(row)}
-                    title="Archive"
-                  />
+                  <Tooltip title="Archive">
+                    <Button
+                      classes={{root: classes.tableActionButton}}
+                      onClick={e => this.handleArchive(row)}
+                      className={classes.button}>
+                      <Archive fontSize="small"/>
+                    </Button>
+                  </Tooltip>
+                  <Button
+                    disabled
+                    classes={{root: classes.tableActionButton}}
+                    className={classes.button}>
+                    <CloudDownload fontSize="small"/>
+                  </Button>
                 </span>
               </div>
             );
@@ -248,7 +267,7 @@ class AnnotationTable extends React.Component {
           <TableHeaderColumn width="10%" dataField="dataset_id">
             Dataset ID
           </TableHeaderColumn>
-          <TableHeaderColumn width="10%" dataField="actions" dataFormat={actionFormatter}>
+          <TableHeaderColumn width="15%" dataField="actions" dataFormat={actionFormatter}>
             Actions
           </TableHeaderColumn>
         </ResizableTable>

front/app/dashboard/components/job/entry.jsx

@@ -3,6 +3,7 @@ import { withStyles } from '@material-ui/core/styles';
 import Fab from '@material-ui/core/Fab';
 import Paper from '@material-ui/core/Paper';
 import Add from '@material-ui/icons/Add';
+import Grid from '@material-ui/core/Grid';
 
 import JobTable from 'automan/dashboard/components/job/table';
 import JobForm from 'automan/dashboard/components/job/form';
@@ -27,20 +28,22 @@ class JobPage extends React.Component {
   render() {
     const { classes } = this.props;
     return (
-      <div className="container">
-        <JobForm formOpen={this.state.formOpen} hide={this.hide} />
-        <Paper className={classes.root}>
-          <JobTable onClickJob={this.handleClickJob} target={-1} />
-        </Paper>
-        <Fab
-          color="primary"
-          aria-label="Add"
-          className={classes.fab}
-          onClick={this.show}
-        >
-          <Add />
-        </Fab>
-      </div>
+      <Grid container spacing={24}>
+        <Grid item xs={12}>
+          <JobForm formOpen={this.state.formOpen} hide={this.hide} />
+          <Paper className={classes.root}>
+            <JobTable onClickJob={this.handleClickJob} target={-1} />
+          </Paper>
+          <Fab
+            color="primary"
+            aria-label="Add"
+            className={classes.fab}
+            onClick={this.show}
+          >
+            <Add />
+          </Fab>
+        </Grid>
+      </Grid>
     );
   }
 }