Skip to content

Commit 80670ec

Browse files
andrewnesterandrewnester
authored
Added bundle deployment bind and unbind command (#1131)
## Changes Added `bundle deployment bind` and `unbind` command. This command allows to bind bundle-defined resources to existing resources in Databricks workspace so they become DABs-managed. ## Tests Manually + added E2E test
1 parent e8b0698 commit 80670ec

25 files changed

Lines changed: 643 additions & 34 deletions

File tree

‎bundle/config/resources.go‎

Lines changed: 35 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,11 @@
11
package config
22

33
import (
4+
"context"
45
"fmt"
56

67
"github.com/databricks/cli/bundle/config/resources"
8+
"github.com/databricks/databricks-sdk-go"
79
)
810

911
// Resources defines Databricks resources associated with the bundle.
@@ -168,3 +170,36 @@ func (r *Resources) Merge() error {
168170
}
169171
return nil
170172
}
173+
174+
type ConfigResource interface {
175+
Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error)
176+
TerraformResourceName() string
177+
}
178+
179+
func (r *Resources) FindResourceByConfigKey(key string) (ConfigResource, error) {
180+
found := make([]ConfigResource, 0)
181+
for k := range r.Jobs {
182+
if k == key {
183+
found = append(found, r.Jobs[k])
184+
}
185+
}
186+
for k := range r.Pipelines {
187+
if k == key {
188+
found = append(found, r.Pipelines[k])
189+
}
190+
}
191+
192+
if len(found) == 0 {
193+
return nil, fmt.Errorf("no such resource: %s", key)
194+
}
195+
196+
if len(found) > 1 {
197+
keys := make([]string, 0, len(found))
198+
for _, r := range found {
199+
keys = append(keys, fmt.Sprintf("%s:%s", r.TerraformResourceName(), key))
200+
}
201+
return nil, fmt.Errorf("ambiguous: %s (can resolve to all of %s)", key, keys)
202+
}
203+
204+
return found[0], nil
205+
}

‎bundle/config/resources/job.go‎

Lines changed: 24 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,7 +1,12 @@
11
package resources
22

33
import (
4+
"context"
5+
"strconv"
6+
47
"github.com/databricks/cli/bundle/config/paths"
8+
"github.com/databricks/cli/libs/log"
9+
"github.com/databricks/databricks-sdk-go"
510
"github.com/databricks/databricks-sdk-go/marshal"
611
"github.com/databricks/databricks-sdk-go/service/jobs"
712
"github.com/imdario/mergo"
@@ -90,3 +95,22 @@ func (j *Job) MergeTasks() error {
9095
j.Tasks = tasks
9196
return nil
9297
}
98+
99+
func (j *Job) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
100+
jobId, err := strconv.Atoi(id)
101+
if err != nil {
102+
return false, err
103+
}
104+
_, err = w.Jobs.Get(ctx, jobs.GetJobRequest{
105+
JobId: int64(jobId),
106+
})
107+
if err != nil {
108+
log.Debugf(ctx, "job %s does not exist", id)
109+
return false, err
110+
}
111+
return true, nil
112+
}
113+
114+
func (j *Job) TerraformResourceName() string {
115+
return "databricks_job"
116+
}

‎bundle/config/resources/pipeline.go‎

Lines changed: 18 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,9 +1,12 @@
11
package resources
22

33
import (
4+
"context"
45
"strings"
56

67
"github.com/databricks/cli/bundle/config/paths"
8+
"github.com/databricks/cli/libs/log"
9+
"github.com/databricks/databricks-sdk-go"
710
"github.com/databricks/databricks-sdk-go/marshal"
811
"github.com/databricks/databricks-sdk-go/service/pipelines"
912
"github.com/imdario/mergo"
@@ -73,3 +76,18 @@ func (p *Pipeline) MergeClusters() error {
7376
p.Clusters = output
7477
return nil
7578
}
79+
80+
func (p *Pipeline) Exists(ctx context.Context, w *databricks.WorkspaceClient, id string) (bool, error) {
81+
_, err := w.Pipelines.Get(ctx, pipelines.GetPipelineRequest{
82+
PipelineId: id,
83+
})
84+
if err != nil {
85+
log.Debugf(ctx, "pipeline %s does not exist", id)
86+
return false, err
87+
}
88+
return true, nil
89+
}
90+
91+
func (p *Pipeline) TerraformResourceName() string {
92+
return "databricks_pipeline"
93+
}

‎bundle/deploy/lock/release.go‎

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -12,6 +12,8 @@ import (
1212
type Goal string
1313

1414
const (
15+
GoalBind = Goal("bind")
16+
GoalUnbind = Goal("unbind")
1517
GoalDeploy = Goal("deploy")
1618
GoalDestroy = Goal("destroy")
1719
)
@@ -46,6 +48,8 @@ func (m *release) Apply(ctx context.Context, b *bundle.Bundle) error {
4648
switch m.goal {
4749
case GoalDeploy:
4850
return b.Locker.Unlock(ctx)
51+
case GoalBind, GoalUnbind:
52+
return b.Locker.Unlock(ctx)
4953
case GoalDestroy:
5054
return b.Locker.Unlock(ctx, locker.AllowLockFileNotExist)
5155
default:

‎bundle/deploy/terraform/import.go‎

Lines changed: 108 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,108 @@
1+
package terraform
2+
3+
import (
4+
"bytes"
5+
"context"
6+
"fmt"
7+
"io"
8+
"os"
9+
"path/filepath"
10+
11+
"github.com/databricks/cli/bundle"
12+
"github.com/databricks/cli/libs/cmdio"
13+
"github.com/hashicorp/terraform-exec/tfexec"
14+
)
15+
16+
type BindOptions struct {
17+
AutoApprove bool
18+
ResourceType string
19+
ResourceKey string
20+
ResourceId string
21+
}
22+
23+
type importResource struct {
24+
opts *BindOptions
25+
}
26+
27+
// Apply implements bundle.Mutator.
28+
func (m *importResource) Apply(ctx context.Context, b *bundle.Bundle) error {
29+
dir, err := Dir(ctx, b)
30+
if err != nil {
31+
return err
32+
}
33+
34+
tf := b.Terraform
35+
if tf == nil {
36+
return fmt.Errorf("terraform not initialized")
37+
}
38+
39+
err = tf.Init(ctx, tfexec.Upgrade(true))
40+
if err != nil {
41+
return fmt.Errorf("terraform init: %w", err)
42+
}
43+
tmpDir, err := os.MkdirTemp("", "state-*")
44+
if err != nil {
45+
return fmt.Errorf("terraform init: %w", err)
46+
}
47+
tmpState := filepath.Join(tmpDir, TerraformStateFileName)
48+
49+
importAddress := fmt.Sprintf("%s.%s", m.opts.ResourceType, m.opts.ResourceKey)
50+
err = tf.Import(ctx, importAddress, m.opts.ResourceId, tfexec.StateOut(tmpState))
51+
if err != nil {
52+
return fmt.Errorf("terraform import: %w", err)
53+
}
54+
55+
buf := bytes.NewBuffer(nil)
56+
tf.SetStdout(buf)
57+
58+
//lint:ignore SA1019 We use legacy -state flag for now to plan the import changes based on temporary state file
59+
changed, err := tf.Plan(ctx, tfexec.State(tmpState), tfexec.Target(importAddress))
60+
if err != nil {
61+
return fmt.Errorf("terraform plan: %w", err)
62+
}
63+
64+
defer os.RemoveAll(tmpDir)
65+
66+
if changed && !m.opts.AutoApprove {
67+
output := buf.String()
68+
// Remove output starting from Warning until end of output
69+
output = output[:bytes.Index([]byte(output), []byte("Warning:"))]
70+
cmdio.LogString(ctx, output)
71+
ans, err := cmdio.AskYesOrNo(ctx, "Confirm import changes? Changes will be remotely applied only after running 'bundle deploy'.")
72+
if err != nil {
73+
return err
74+
}
75+
if !ans {
76+
return fmt.Errorf("import aborted")
77+
}
78+
}
79+
80+
// If user confirmed changes, move the state file from temp dir to state location
81+
f, err := os.Create(filepath.Join(dir, TerraformStateFileName))
82+
if err != nil {
83+
return err
84+
}
85+
defer f.Close()
86+
87+
tmpF, err := os.Open(tmpState)
88+
if err != nil {
89+
return err
90+
}
91+
defer tmpF.Close()
92+
93+
_, err = io.Copy(f, tmpF)
94+
if err != nil {
95+
return err
96+
}
97+
98+
return nil
99+
}
100+
101+
// Name implements bundle.Mutator.
102+
func (*importResource) Name() string {
103+
return "terraform.Import"
104+
}
105+
106+
func Import(opts *BindOptions) bundle.Mutator {
107+
return &importResource{opts: opts}
108+
}

‎bundle/deploy/terraform/unbind.go‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
package terraform
2+
3+
import (
4+
"context"
5+
"fmt"
6+
7+
"github.com/databricks/cli/bundle"
8+
"github.com/hashicorp/terraform-exec/tfexec"
9+
)
10+
11+
type unbind struct {
12+
resourceType string
13+
resourceKey string
14+
}
15+
16+
func (m *unbind) Apply(ctx context.Context, b *bundle.Bundle) error {
17+
tf := b.Terraform
18+
if tf == nil {
19+
return fmt.Errorf("terraform not initialized")
20+
}
21+
22+
err := tf.Init(ctx, tfexec.Upgrade(true))
23+
if err != nil {
24+
return fmt.Errorf("terraform init: %w", err)
25+
}
26+
27+
err = tf.StateRm(ctx, fmt.Sprintf("%s.%s", m.resourceType, m.resourceKey))
28+
if err != nil {
29+
return fmt.Errorf("terraform state rm: %w", err)
30+
}
31+
32+
return nil
33+
}
34+
35+
func (*unbind) Name() string {
36+
return "terraform.Unbind"
37+
}
38+
39+
func Unbind(resourceType string, resourceKey string) bundle.Mutator {
40+
return &unbind{resourceType: resourceType, resourceKey: resourceKey}
41+
}

‎bundle/phases/bind.go‎

Lines changed: 45 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,45 @@
1+
package phases
2+
3+
import (
4+
"github.com/databricks/cli/bundle"
5+
"github.com/databricks/cli/bundle/deploy/lock"
6+
"github.com/databricks/cli/bundle/deploy/terraform"
7+
)
8+
9+
func Bind(opts *terraform.BindOptions) bundle.Mutator {
10+
return newPhase(
11+
"bind",
12+
[]bundle.Mutator{
13+
lock.Acquire(),
14+
bundle.Defer(
15+
bundle.Seq(
16+
terraform.StatePull(),
17+
terraform.Interpolate(),
18+
terraform.Write(),
19+
terraform.Import(opts),
20+
terraform.StatePush(),
21+
),
22+
lock.Release(lock.GoalBind),
23+
),
24+
},
25+
)
26+
}
27+
28+
func Unbind(resourceType string, resourceKey string) bundle.Mutator {
29+
return newPhase(
30+
"unbind",
31+
[]bundle.Mutator{
32+
lock.Acquire(),
33+
bundle.Defer(
34+
bundle.Seq(
35+
terraform.StatePull(),
36+
terraform.Interpolate(),
37+
terraform.Write(),
38+
terraform.Unbind(resourceType, resourceKey),
39+
terraform.StatePush(),
40+
),
41+
lock.Release(lock.GoalUnbind),
42+
),
43+
},
44+
)
45+
}

‎bundle/phases/destroy.go‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -14,9 +14,9 @@ func Destroy() bundle.Mutator {
1414
lock.Acquire(),
1515
bundle.Defer(
1616
bundle.Seq(
17+
terraform.StatePull(),
1718
terraform.Interpolate(),
1819
terraform.Write(),
19-
terraform.StatePull(),
2020
terraform.Plan(terraform.PlanGoal("destroy")),
2121
terraform.Destroy(),
2222
terraform.StatePush(),

‎cmd/bundle/bundle.go‎

Lines changed: 2 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,6 +1,7 @@
11
package bundle
22

33
import (
4+
"github.com/databricks/cli/cmd/bundle/deployment"
45
"github.com/spf13/cobra"
56
)
67

@@ -24,5 +25,6 @@ func New() *cobra.Command {
2425
cmd.AddCommand(newInitCommand())
2526
cmd.AddCommand(newSummaryCommand())
2627
cmd.AddCommand(newGenerateCommand())
28+
cmd.AddCommand(deployment.NewDeploymentCommand())
2729
return cmd
2830
}

‎cmd/bundle/deploy.go‎

Lines changed: 2 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -3,14 +3,15 @@ package bundle
33
import (
44
"github.com/databricks/cli/bundle"
55
"github.com/databricks/cli/bundle/phases"
6+
"github.com/databricks/cli/cmd/bundle/utils"
67
"github.com/spf13/cobra"
78
)
89

910
func newDeployCommand() *cobra.Command {
1011
cmd := &cobra.Command{
1112
Use: "deploy",
1213
Short: "Deploy bundle",
13-
PreRunE: ConfigureBundleWithVariables,
14+
PreRunE: utils.ConfigureBundleWithVariables,
1415
}
1516

1617
var force bool

0 commit comments

Comments
 (0)