Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 12 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/databricks.yml.tmpl
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
bundle:
name: alert-task-$UNIQUE_NAME

resources:
jobs:
my_job:
name: alert-task-$UNIQUE_NAME
tasks:
- task_key: alert_task
alert_task:
workspace_path: ./my_alert.dbalert.json
warehouse_id: $TEST_DEFAULT_WAREHOUSE_ID
19 changes: 19 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/my_alert.dbalert.json
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
{
"query_lines": ["SELECT 1"],
"schedule": {
"quartz_cron_schedule": "0 0 * * * ?",
"timezone_id": "UTC"
},
"evaluation": {
"comparison_operator": "EQUAL",
"source": {
"name": "1",
"aggregation": "MAX"
},
"threshold": {
"value": {
"double_value": 1
}
}
}
}
5 changes: 5 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/out.test.toml

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

21 changes: 21 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/output.txt
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@

>>> [CLI] bundle deploy
Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/alert-task-[UNIQUE_NAME]/default/files...
Deploying resources...
Updating deployment state...
Deployment complete!

>>> [CLI] jobs get [JOB_ID]
{
"warehouse_id": "[TEST_DEFAULT_WAREHOUSE_ID]",
"workspace_path": "/Workspace/Users/[USERNAME]/.bundle/alert-task-[UNIQUE_NAME]/default/files/my_alert.dbalert.json"
}

>>> [CLI] bundle destroy --auto-approve
The following resources will be deleted:
delete resources.jobs.my_job

All files and directories at the following location will be deleted: /Workspace/Users/[USERNAME]/.bundle/alert-task-[UNIQUE_NAME]/default

Deleting files...
Destroy complete!
13 changes: 13 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/script
Original file line number Diff line number Diff line change
@@ -0,0 +1,13 @@
envsubst < databricks.yml.tmpl > databricks.yml

cleanup() {
trace $CLI bundle destroy --auto-approve
}
trap cleanup EXIT

trace $CLI bundle deploy

job_id=$($CLI bundle summary -o json | jq -r '.resources.jobs.my_job.id')
echo "$job_id:JOB_ID" >> ACC_REPLS

trace $CLI jobs get $job_id | jq '.settings.tasks[0].alert_task'
7 changes: 7 additions & 0 deletions acceptance/bundle/resources/jobs/alert-task/test.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,7 @@
Local = true
Cloud = true
RecordRequests = false
Ignore = ["databricks.yml", ".databricks"]

[Env]
MSYS_NO_PATHCONV = "1"
5 changes: 5 additions & 0 deletions bundle/config/mutator/paths/job_paths_visitor.go
Original file line number Diff line number Diff line change
Expand Up @@ -36,6 +36,11 @@ func jobTaskRewritePatterns(base dyn.Pattern) []jobRewritePattern {
TranslateModeFile,
noSkipRewrite,
},
{
base.Append(dyn.Key("alert_task"), dyn.Key("workspace_path")),
TranslateModeFile,
noSkipRewrite,
},
{
base.Append(dyn.Key("libraries"), dyn.AnyIndex(), dyn.Key("requirements")),
TranslateModeFile,
Expand Down
18 changes: 18 additions & 0 deletions bundle/config/mutator/paths/job_paths_visitor_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -49,6 +49,11 @@ func TestVisitJobPaths(t *testing.T) {
{Requirements: "requirements.txt"},
},
}
task7 := jobs.Task{
AlertTask: &jobs.AlertTask{
WorkspacePath: "abc",
},
}

job0 := &resources.Job{
JobSettings: jobs.JobSettings{
Expand All @@ -60,6 +65,7 @@ func TestVisitJobPaths(t *testing.T) {
task4,
task5,
task6,
task7,
},
},
}
Expand All @@ -79,6 +85,7 @@ func TestVisitJobPaths(t *testing.T) {
dyn.MustPathFromString("resources.jobs.job0.tasks[2].dbt_task.project_directory"),
dyn.MustPathFromString("resources.jobs.job0.tasks[3].sql_task.file.path"),
dyn.MustPathFromString("resources.jobs.job0.tasks[6].libraries[0].requirements"),
dyn.MustPathFromString("resources.jobs.job0.tasks[7].alert_task.workspace_path"),
}

assert.ElementsMatch(t, expected, actual)
Expand Down Expand Up @@ -125,10 +132,20 @@ func TestVisitJobPaths_foreach(t *testing.T) {
},
},
}
task1 := jobs.Task{
ForEachTask: &jobs.ForEachTask{
Task: jobs.Task{
AlertTask: &jobs.AlertTask{
WorkspacePath: "abc",
},
},
},
}
job0 := &resources.Job{
JobSettings: jobs.JobSettings{
Tasks: []jobs.Task{
task0,
task1,
},
},
}
Expand All @@ -144,6 +161,7 @@ func TestVisitJobPaths_foreach(t *testing.T) {
actual := collectVisitedPaths(t, root, VisitJobPaths)
expected := []dyn.Path{
dyn.MustPathFromString("resources.jobs.job0.tasks[0].for_each_task.task.notebook_task.notebook_path"),
dyn.MustPathFromString("resources.jobs.job0.tasks[1].for_each_task.task.alert_task.workspace_path"),
}

assert.ElementsMatch(t, expected, actual)
Expand Down
11 changes: 11 additions & 0 deletions bundle/config/mutator/translate_paths_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -290,6 +290,7 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
touchEmptyFile(t, filepath.Join(dir, "pipeline", "my_python_file.py"))
touchEmptyFile(t, filepath.Join(dir, "job", "my_sql_file.sql"))
touchEmptyFile(t, filepath.Join(dir, "job", "my_dbt_project", "dbt_project.yml"))
touchEmptyFile(t, filepath.Join(dir, "job", "my_alert.dbalert.json"))

b := &bundle.Bundle{
SyncRootPath: dir,
Expand Down Expand Up @@ -329,6 +330,11 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
ProjectDirectory: "./my_dbt_project",
},
},
{
AlertTask: &jobs.AlertTask{
WorkspacePath: "./my_alert.dbalert.json",
},
},
},
},
},
Expand Down Expand Up @@ -376,6 +382,11 @@ func TestTranslatePathsInSubdirectories(t *testing.T) {
"/bundle/job/my_dbt_project",
b.Config.Resources.Jobs["job"].Tasks[3].DbtTask.ProjectDirectory,
)
assert.Equal(
t,
"/bundle/job/my_alert.dbalert.json",
b.Config.Resources.Jobs["job"].Tasks[4].AlertTask.WorkspacePath,
)

assert.Equal(
t,
Expand Down