Add --apps option for aztk spark job get to list apps

Author: emlyn

cli/spark/endpoints/job/get.py

@@ -10,9 +10,12 @@ def setup_parser(parser: argparse.ArgumentParser):
                         dest='job_id',
                         required=True,
                         help='The unique id of your AZTK job')
+    parser.add_argument('--apps',
+                        action='store_true',
+                        help='Also list applications')
 
 
 def execute(args: typing.NamedTuple):
     spark_client = aztk.spark.Client(config.load_aztk_screts())
 
-    utils.print_job(spark_client, spark_client.get_job(args.job_id))
+    utils.print_job(spark_client, spark_client.get_job(args.job_id), args.apps)

cli/utils.py

@@ -4,7 +4,7 @@
 import threading
 import time
 from subprocess import call
-from typing import List
+from typing import Dict, List
 import azure.batch.models as batch_models
 import aztk.spark
 from aztk import error
@@ -247,7 +247,14 @@ def print_jobs(jobs: List[aztk.spark.models.Job]):
         )
 
 
-def print_job(client, job: aztk.spark.models.Job):
+def get_applications(client, job: aztk.spark.models.Job):
+    if job.applications:
+        return {j.name: j for j in job.applications}
+    else:
+        return client.list_applications(job.id)
+
+
+def print_job(client, job: aztk.spark.models.Job, apps: bool):
     print_format = '{:<36}| {:<15}'
 
     log.info("")
@@ -267,11 +274,12 @@ def print_job(client, job: aztk.spark.models.Job):
             log.info(print_format.format("Cluster", "Provisioning"))
             log.info("")
 
-    if job.applications:
-        application_summary(job.applications)
-    else:
-        application_summary(client.list_applications(job.id))
+    applications = get_applications(client, job)
+    application_summary(applications)
     log.info("")
+    if apps:
+        print_applications(applications)
+        log.info("")
 
 
 def node_state_count(cluster: aztk.spark.models.Cluster):
@@ -296,22 +304,22 @@ def print_cluster_summary(cluster: aztk.spark.models.Cluster):
     log.info("")
 
 
-def application_summary(applications):
+def application_summary(applications: Dict[str, aztk.spark.models.Application]):
     states = {"scheduling": 0}
     for state in batch_models.TaskState:
         states[state.name] = 0
 
     warn_scheduling = False
 
-    for application in applications:
-        if type(application) == str:
+    for name, application in applications.items():
+        if application is None:
             states["scheduling"] += 1
             warn_scheduling = True
         else:
             states[application.state] += 1
 
     print_format = '{:<17} {:<14}'
-    log.info("Applications")
+    log.info("Application States")
     log.info("-"*42)
     for state in states:
         if states[state] > 0:
@@ -320,15 +328,15 @@ def application_summary(applications):
     if warn_scheduling:
         log.warning("\nNo Spark applications will be scheduled until the master is selected.")
 
-def print_applications(applications):
+def print_applications(applications: Dict[str, aztk.spark.models.Application]):
     print_format = '{:<36}| {:<15}| {:<16} | {:^9} |'
     print_format_underline = '{:-<36}|{:-<16}|{:-<18}|{:-<11}|'
     log.info(print_format.format("Applications", "State", "Transition Time", "Exit Code"))
     log.info(print_format_underline.format('', '', '', ''))
 
     warn_scheduling = False
-    for name in applications:
-        if applications[name] is None:
+    for name, application in applications.items():
+        if application is None:
             log.info(
                 print_format.format(
                     name,
@@ -339,10 +347,9 @@ def print_applications(applications):
             )
             warn_scheduling = True
         else:
-            application = applications[name]
             log.info(
                 print_format.format(
-                    application.name,
+                    name,
                     application.state,
                     utc_to_local(application.state_transition_time),
                     application.exit_code if application.exit_code is not None else "-"