pipelines: unhide new commands

NEXT_CHANGELOG.md

@@ -7,6 +7,7 @@
 To disable this, set the environment variable DATABRICKS_CACHE_ENABLED to false.
 
 ### CLI
+* Add commands to pipelines command group ([#4275](https://github.com/databricks/cli/pull/4275))
 
 ### Bundles
 * Enable caching user identity by default ([#4202](https://github.com/databricks/cli/pull/4202))

acceptance/acceptance_test.go

@@ -216,7 +216,11 @@ func testAccept(t *testing.T, inprocessMode bool, singleTest string) int {
 	repls.SetPath(execPath, "[CLI]")
 
 	pipelinesPath := filepath.Join(buildDir, "pipelines") + exeSuffix
-	err = copyFile(execPath, pipelinesPath)
+	if _, err := os.Stat(pipelinesPath); err == nil {
+		err := os.Remove(pipelinesPath)
+		require.NoError(t, err)
+	}
+	err = os.Symlink(execPath, pipelinesPath)
 	require.NoError(t, err)
 	t.Setenv("PIPELINES", pipelinesPath)
 	repls.SetPath(pipelinesPath, "[PIPELINES]")

acceptance/pipelines/databricks-cli-help/script

@@ -1 +1 @@
-ENABLE_PIPELINES_CLI=1 $CLI pipelines
+$CLI pipelines

acceptance/pipelines/deploy/auto-approve/output.txt

@@ -1,5 +1,5 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files...
 Deploying resources...
 Updating deployment state...
@@ -10,7 +10,7 @@ View your pipeline foo here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]
 >>> rm resources.yml
 
 === Try to redeploy without --auto-approve - should fail
->>> errcode [PIPELINES] deploy
+>>> errcode [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files...
 
 This action will result in the deletion or recreation of the following Lakeflow Declarative Pipelines along with the
@@ -24,7 +24,7 @@ Error: the deployment requires destructive actions, but current console does not
 Exit code: 1
 
 === Redeploy with --auto-approve - should succeed
->>> [PIPELINES] deploy --auto-approve
+>>> [ENTRYPOINT] deploy --auto-approve
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-auto-approve/default/files...
 
 This action will result in the deletion or recreation of the following Lakeflow Declarative Pipelines along with the

acceptance/pipelines/deploy/auto-approve/script

@@ -1,10 +1,10 @@
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 
 title "Remove resources from configuration to test auto-approve"
 trace rm resources.yml
 
 title "Try to redeploy without --auto-approve - should fail"
-trace errcode $PIPELINES deploy
+trace errcode $ENTRYPOINT deploy
 
 title "Redeploy with --auto-approve - should succeed"
-trace $PIPELINES deploy --auto-approve
+trace $ENTRYPOINT deploy --auto-approve

acceptance/pipelines/deploy/create-pipeline/script

@@ -1,3 +1,3 @@
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id')
 trace $CLI pipelines get "${PIPELINE_ID}" | jq "{spec}"

acceptance/pipelines/deploy/create-pipeline/test.toml

@@ -0,0 +1,2 @@
+[EnvMatrix]
+ENTRYPOINT = ["$PIPELINES"]

acceptance/pipelines/deploy/fail-on-active-runs/output.txt

@@ -1,12 +1,12 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/pipeline-fail-on-active-runs/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]
 
->>> errcode [PIPELINES] deploy --fail-on-active-runs
+>>> [ENTRYPOINT] deploy --fail-on-active-runs
 Error: pipeline [UUID] is running
 
 

acceptance/pipelines/deploy/fail-on-active-runs/script

@@ -1,4 +1,4 @@
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 
 # We deploy the bundle again to check that the deploy is failing if the job is running
-trace errcode $PIPELINES deploy --fail-on-active-runs
+errcode trace $ENTRYPOINT deploy --fail-on-active-runs

acceptance/pipelines/deploy/force-lock/output.txt

@@ -3,15 +3,15 @@
 >>> [CLI] workspace import /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock
 
 === test deployment without force-lock (should fail)
->>> errcode [PIPELINES] deploy
+>>> errcode [ENTRYPOINT] deploy
 Error: Failed to acquire deployment lock: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Use --force-lock to override
 Error: deploy lock acquired by user-with-lock@databricks.com at [TIMESTAMP] +0000 UTC. Use --force-lock to override
 
 
 Exit code: 1
 
 === test deployment with force-lock (should succeed)
->>> [PIPELINES] deploy --force-lock
+>>> [ENTRYPOINT] deploy --force-lock
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-force-lock/default/files...
 Deploying resources...
 Updating deployment state...

acceptance/pipelines/deploy/force-lock/script

@@ -4,7 +4,7 @@ title "upload lock file"
 trace $CLI workspace import /Workspace/Users/$username/.bundle/test-pipeline-force-lock/default/state/deploy.lock --format AUTO --file ./deploy.lock
 
 title "test deployment without force-lock (should fail)"
-trace errcode $PIPELINES deploy
+trace errcode $ENTRYPOINT deploy
 
 title "test deployment with force-lock (should succeed)"
-trace $PIPELINES deploy --force-lock
+trace $ENTRYPOINT deploy --force-lock

acceptance/pipelines/deploy/oss-spark-error/output.txt

@@ -1,6 +1,6 @@
 
 === Fail to deploy OSS Spark pipelines-formatted YAML file
->>> musterr [PIPELINES] deploy
+>>> musterr [ENTRYPOINT] deploy
 Error: [TEST_TMP_DIR]/oss-pipeline.yml seems to be formatted for open-source Spark Declarative Pipelines.
 Pipelines CLI currently only supports Lakeflow Declarative Pipelines development.
 To see an example of a supported pipelines template, create a new Pipelines CLI project with "pipelines init".

acceptance/pipelines/deploy/oss-spark-error/script

@@ -1,2 +1,2 @@
 title "Fail to deploy OSS Spark pipelines-formatted YAML file"
-trace musterr $PIPELINES deploy
+trace musterr $ENTRYPOINT deploy

acceptance/pipelines/deploy/render-diagnostics-warning/output.txt

@@ -1,5 +1,5 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Warning: unknown field: unknown_property
   at resources.pipelines.test-pipeline
   in databricks.yml:8:7

acceptance/pipelines/deploy/render-diagnostics-warning/script

@@ -1,3 +1,3 @@
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 # print newline to comply with whitespace linter
 printf "<EOL>\n"

acceptance/pipelines/deploy/test.toml

@@ -0,0 +1,2 @@
+[EnvMatrix]
+ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"]

acceptance/pipelines/deploy/var-flag/script

@@ -1,5 +1,5 @@
 title "Test --var flag usage"
-trace $PIPELINES deploy --var="catalog=custom_catalog" --var="schema=custom_schema" --auto-approve
+trace $ENTRYPOINT deploy --var="catalog=custom_catalog" --var="schema=custom_schema" --auto-approve
 
 title "Verify: Check that variables were substituted correctly"
 PIPELINE_ID=$($CLI bundle summary -o json | jq -r '.resources.pipelines.foo.id')

acceptance/pipelines/deploy/var-flag/test.toml

@@ -0,0 +1,2 @@
+[EnvMatrix]
+ENTRYPOINT = ["$PIPELINES"]

acceptance/pipelines/dry-run/dry-run-pipeline/output.txt

@@ -1,13 +1,13 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]
 
 === Dry running pipeline, should have validate_only set to true
->>> [PIPELINES] dry-run
+>>> [ENTRYPOINT] dry-run
 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
 
 Update ID: [UUID]
@@ -20,7 +20,7 @@ Update ID: [UUID]
 }
 
 === Dry running pipeline with KEY, expect same output as without KEY
->>> [PIPELINES] dry-run my_pipeline
+>>> [ENTRYPOINT] dry-run my_pipeline
 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
 
 Update ID: [UUID]

acceptance/pipelines/dry-run/dry-run-pipeline/script

@@ -3,12 +3,12 @@ print_requests() {
     rm out.requests.txt
 }
 
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 rm out.requests.txt
 title "Dry running pipeline, should have validate_only set to true"
-trace $PIPELINES dry-run
+trace $ENTRYPOINT dry-run
 print_requests
 
 title "Dry running pipeline with KEY, expect same output as without KEY"
-trace $PIPELINES dry-run my_pipeline
+trace $ENTRYPOINT dry-run my_pipeline
 print_requests

acceptance/pipelines/dry-run/no-wait/output.txt

@@ -1,13 +1,13 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]
 
 === Dry running pipeline with --no-wait flag
->>> [PIPELINES] dry-run --no-wait
+>>> [ENTRYPOINT] dry-run --no-wait
 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
 
 Update ID: [UUID]

acceptance/pipelines/dry-run/no-wait/script

@@ -1,3 +1,3 @@
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 title "Dry running pipeline with --no-wait flag"
-trace $PIPELINES dry-run --no-wait
+trace $ENTRYPOINT dry-run --no-wait

acceptance/pipelines/dry-run/restart/output.txt

@@ -1,13 +1,13 @@
 
->>> [PIPELINES] deploy
+>>> [ENTRYPOINT] deploy
 Uploading bundle files to /Workspace/Users/[USERNAME]/.bundle/test-pipeline-run-flags/default/files...
 Deploying resources...
 Updating deployment state...
 Deployment complete!
 View your pipeline my_pipeline here: [DATABRICKS_URL]/pipelines/[UUID]?o=[NUMID]
 
 === Dry running pipeline with --restart flag, should stop the current pipeline and start a new run
->>> [PIPELINES] dry-run --restart
+>>> [ENTRYPOINT] dry-run --restart
 Update URL: [DATABRICKS_URL]/#joblist/pipelines/[UUID]/updates/[UUID]
 
 Update ID: [UUID]

acceptance/pipelines/dry-run/restart/script

@@ -3,9 +3,9 @@ print_requests() {
     rm out.requests.txt
 }
 
-trace $PIPELINES deploy
+trace $ENTRYPOINT deploy
 rm out.requests.txt
 
 title "Dry running pipeline with --restart flag, should stop the current pipeline and start a new run"
-trace $PIPELINES dry-run --restart
+trace $ENTRYPOINT dry-run --restart
 trace print_requests

acceptance/pipelines/dry-run/test.toml

@@ -0,0 +1,2 @@
+[EnvMatrix]
+ENTRYPOINT = ["$CLI pipelines", "$PIPELINES"]

acceptance/pipelines/e2e/test.toml

@@ -1,3 +1,3 @@
 Ignore = [
     '.databricks',
-]
+]

acceptance/pipelines/init/error-cases/output.txt

@@ -1,12 +1,12 @@
 
 === Test with invalid project name (contains uppercase letters)
->>> errcode [PIPELINES] init --config-file ./invalid_input.json --output-dir invalid-output
+>>> errcode [ENTRYPOINT] init --config-file ./invalid_input.json --output-dir invalid-output
 Error: failed to load config from file ./invalid_input.json: invalid value for project_name: "InvalidProjectName". Name must consist of lower case letters, numbers, and underscores.
 
 Exit code: 1
 
 === Test with non-existent config file
->>> errcode [PIPELINES] init --config-file ./nonexistent.json --output-dir invalid-output-2
+>>> errcode [ENTRYPOINT] init --config-file ./nonexistent.json --output-dir invalid-output-2
 Error: failed to load config from file ./nonexistent.json: open ./nonexistent.json: no such file or directory
 
 Exit code: 1

acceptance/pipelines/init/error-cases/script

@@ -1,5 +1,5 @@
 title "Test with invalid project name (contains uppercase letters)"
-trace errcode $PIPELINES init --config-file ./invalid_input.json --output-dir invalid-output
+trace errcode $ENTRYPOINT init --config-file ./invalid_input.json --output-dir invalid-output
 
 title "Test with non-existent config file"
-trace errcode $PIPELINES init --config-file ./nonexistent.json --output-dir invalid-output-2
+trace errcode $ENTRYPOINT init --config-file ./nonexistent.json --output-dir invalid-output-2