Example of a job with tasks that run other jobs (#22)

Note: this requires databricks/cli#1219.

Author: pietern

knowledge_base/job_with_run_job_tasks/.gitignore

@@ -0,0 +1 @@
+.databricks

knowledge_base/job_with_run_job_tasks/README.md

@@ -0,0 +1,16 @@
+# Job with `run_job` tasks
+
+This example demonstrates how to compose multiple jobs with `run_job` tasks.
+
+## Prerequisites
+
+* Databricks CLI v0.230.0 or above
+* Serverless enabled on the Databricks workspace
+
+## Usage
+
+Update the `host` field under `workspace` in `databricks.yml` to the Databricks workspace you wish to deploy to.
+
+Run `databricks bundle deploy` to deploy the bundle.
+
+Run `databricks bundle run primary` to run the primary job that triggers the leaf jobs.

knowledge_base/job_with_run_job_tasks/argv.py

@@ -0,0 +1,5 @@
+import sys
+
+# Display the task's arguments to confirm parameter propagation.
+print(f"argv[0]: {sys.argv[0]}")
+print(f"argv[1:]: {', '.join(sys.argv[1:])}")

knowledge_base/job_with_run_job_tasks/databricks.yml

@@ -0,0 +1,81 @@
+bundle:
+  name: job_with_run_job_tasks
+
+workspace:
+  host: https://e2-dogfood.staging.cloud.databricks.com
+
+permissions:
+  - level: CAN_VIEW
+    group_name: users
+
+resources:
+  jobs:
+    primary:
+      name: primary job
+
+      tasks:
+        - task_key: run_leaf1
+          run_job_task:
+            job_id: ${resources.jobs.leaf1.id}
+
+        - task_key: run_leaf2
+          depends_on:
+            - task_key: run_leaf1
+
+          run_job_task:
+            job_id: ${resources.jobs.leaf2.id}
+
+      parameters:
+        - name: input
+          default: test_propagation
+
+
+    leaf1:
+      name: leaf job 1
+
+      tasks:
+        - task_key: leaf
+          environment_key: default
+          spark_python_task:
+            python_file: ./argv.py
+            parameters:
+              - "leaf1"
+              - "{{ job.parameters.[input] }}"
+      parameters:
+        - # This parameter is passed by the primary job.
+          name: input
+          default: ""
+
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+
+
+    leaf2:
+      name: leaf job 2
+
+      tasks:
+        - task_key: leaf
+          environment_key: default
+          spark_python_task:
+            python_file: ./argv.py
+            parameters:
+              - "leaf2"
+              - "{{ job.parameters.[input] }}"
+
+      parameters:
+        - # This parameter is passed by the primary job.
+          name: input
+          default: ""
+
+      environments:
+        - environment_key: default
+          spec:
+            client: "1"
+
+
+targets:
+  dev:
+    default: true
+    mode: development