Skip to content

Commit d0c983a

Browse files
lennartkats-dbpieternPaulCornellDB
authored
Add an experimental default-sql template (#1051)
## Changes This adds a `default-sql` template! In this latest revision, I've hidden the new template from the list so we can merge it, iterate over it, and properly release the template at the right time. - [x] WorkspaceFS support for .sql files is in prod - [x] SQL extension is preconfigured based on extension settings (if possible) - [ ] Streaming tables support is either ungated or the template provides instructions about signup - _Mitigation for now: this template is hidden from the list of templates._ - [x] Support non-UC workspaces ## Tests - [x] Unit tests - [x] Manual testing - [x] More manual testing - [x] Reviewer testing --------- Co-authored-by: Pieter Noordhuis <pieter.noordhuis@databricks.com> Co-authored-by: PaulCornellDB <paul.cornell@databricks.com>
1 parent 2faa16c commit d0c983a

16 files changed

Lines changed: 349 additions & 4 deletions

File tree

‎cmd/bundle/init.go‎

Lines changed: 5 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -35,6 +35,11 @@ var nativeTemplates = []nativeTemplate{
3535
name: "default-python",
3636
description: "The default Python template for Notebooks / Delta Live Tables / Workflows",
3737
},
38+
{
39+
name: "default-sql",
40+
description: "The default SQL template for .sql files that run with Databricks SQL",
41+
hidden: true,
42+
},
3843
{
3944
name: "dbt-sql",
4045
description: "The dbt SQL template (https://www.databricks.com/blog/delivering-cost-effective-data-real-time-dbt-and-databricks)",

‎libs/template/renderer_test.go‎

Lines changed: 19 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -109,9 +109,9 @@ func TestBuiltinPythonTemplateValid(t *testing.T) {
109109
// Test option combinations
110110
options := []string{"yes", "no"}
111111
isServicePrincipal := false
112-
build := false
113112
catalog := "hive_metastore"
114113
cachedCatalog = &catalog
114+
build := false
115115
for _, includeNotebook := range options {
116116
for _, includeDlt := range options {
117117
for _, includePython := range options {
@@ -149,6 +149,24 @@ func TestBuiltinPythonTemplateValid(t *testing.T) {
149149
defer os.RemoveAll(tempDir)
150150
}
151151

152+
func TestBuiltinSQLTemplateValid(t *testing.T) {
153+
for _, personal_schemas := range []string{"yes", "no"} {
154+
for _, target := range []string{"dev", "prod"} {
155+
for _, isServicePrincipal := range []bool{true, false} {
156+
config := map[string]any{
157+
"project_name": "my_project",
158+
"http_path": "/sql/1.0/warehouses/123abc",
159+
"default_catalog": "users",
160+
"shared_schema": "lennart",
161+
"personal_schemas": personal_schemas,
162+
}
163+
build := false
164+
assertBuiltinTemplateValid(t, "default-sql", config, target, isServicePrincipal, build, t.TempDir())
165+
}
166+
}
167+
}
168+
}
169+
152170
func TestBuiltinDbtTemplateValid(t *testing.T) {
153171
for _, personal_schemas := range []string{"yes", "no"} {
154172
for _, target := range []string{"dev", "prod"} {

‎libs/template/templates/dbt-sql/databricks_template_schema.json‎

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -3,8 +3,8 @@
33
"properties": {
44
"project_name": {
55
"type": "string",
6-
"pattern": "^[A-Za-z_][A-Za-z0-9_]+$",
7-
"pattern_match_failure_message": "Name must consist of letters, numbers, and underscores.",
6+
"pattern": "^[A-Za-z_][A-Za-z0-9-_]+$",
7+
"pattern_match_failure_message": "Name must consist of letters, numbers, dashes, and underscores.",
88
"default": "dbt_project",
99
"description": "\nPlease provide a unique name for this project.\nproject_name",
1010
"order": 1

‎libs/template/templates/default-python/template/{{.project_name}}/databricks.yml.tmpl‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ targets:
1919
host: {{workspace_host}}
2020
2121
## Optionally, there could be a 'staging' target here.
22-
## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/index.html.)
22+
## (See Databricks docs on CI/CD at https://docs.databricks.com/dev-tools/bundles/ci-cd.html.)
2323
#
2424
# staging:
2525
# workspace:

‎libs/template/templates/default-sql/README.md‎

Lines changed: 3 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,3 @@
1+
# sql template
2+
3+
This folder provides a template for using SQL with Databricks Asset Bundles.

‎libs/template/templates/default-sql/databricks_template_schema.json‎

Lines changed: 53 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,53 @@
1+
{
2+
"welcome_message": "\nWelcome to the (EXPERIMENTAL) default SQL template for Databricks Asset Bundles!",
3+
"properties": {
4+
"project_name": {
5+
"type": "string",
6+
"default": "sql_project",
7+
"description": "\nPlease provide a unique name for this project.\nproject_name",
8+
"order": 1,
9+
"pattern": "^[A-Za-z_][A-Za-z0-9-_]+$",
10+
"pattern_match_failure_message": "Name must consist of letters, numbers, dashes, and underscores."
11+
},
12+
"http_path": {
13+
"type": "string",
14+
"pattern": "^/sql/.\\../warehouses/[a-z0-9]+$",
15+
"pattern_match_failure_message": "Path must be of the form /sql/1.0/warehouses/<warehouse id>",
16+
"description": "\nPlease provide the HTTP Path of the SQL warehouse you would like to use with dbt during development.\nYou can find this path by clicking on \"Connection details\" for your SQL warehouse.\nhttp_path [example: /sql/1.0/warehouses/abcdef1234567890]",
17+
"order": 2
18+
},
19+
"default_catalog": {
20+
"type": "string",
21+
"default": "{{if eq (default_catalog) \"\"}}hive_metastore{{else}}{{default_catalog}}{{end}}",
22+
"pattern": "^\\w*$",
23+
"pattern_match_failure_message": "Invalid catalog name.",
24+
"description": "\nPlease provide an initial catalog{{if eq (default_catalog) \"\"}} or metastore{{end}}.\ndefault_catalog",
25+
"order": 3
26+
},
27+
"personal_schemas": {
28+
"type": "string",
29+
"description": "\nWould you like to use a personal schema for each user working on this project? (e.g., 'catalog.{{short_name}}')\npersonal_schemas",
30+
"enum": [
31+
"yes, automatically use a schema based on the current user name during development",
32+
"no, use a single schema for all users during development"
33+
],
34+
"order": 4
35+
},
36+
"shared_schema": {
37+
"skip_prompt_if": {
38+
"properties": {
39+
"personal_schemas": {
40+
"const": "yes, automatically use a schema based on the current user name during development"
41+
}
42+
}
43+
},
44+
"type": "string",
45+
"default": "default",
46+
"pattern": "^\\w+$",
47+
"pattern_match_failure_message": "Invalid schema name.",
48+
"description": "\nPlease provide an initial schema during development.\ndefault_schema",
49+
"order": 5
50+
}
51+
},
52+
"success_message": "\n✨ Your new project has been created in the '{{.project_name}}' directory!\n\nPlease refer to the README.md file for \"getting started\" instructions.\nSee also the documentation at https://docs.databricks.com/dev-tools/bundles/index.html."
53+
}

‎libs/template/templates/default-sql/library/versions.tmpl‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{{define "latest_lts_dbr_version" -}}
2+
13.3.x-scala2.12
3+
{{- end}}
4+
5+
{{define "latest_lts_db_connect_version_spec" -}}
6+
>=13.3,<13.4
7+
{{- end}}

‎libs/template/templates/default-sql/template/{{.project_name}}/.vscode/extensions.json‎

Lines changed: 7 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,7 @@
1+
{
2+
"recommendations": [
3+
"databricks.databricks",
4+
"redhat.vscode-yaml",
5+
"databricks.sqltools-databricks-driver",
6+
]
7+
}

‎libs/template/templates/default-sql/template/{{.project_name}}/.vscode/settings.json.tmpl‎

Lines changed: 28 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,28 @@
1+
{
2+
"python.analysis.stubPath": ".vscode",
3+
"databricks.python.envFile": "${workspaceFolder}/.env",
4+
"jupyter.interactiveWindow.cellMarker.codeRegex": "^# COMMAND ----------|^# Databricks notebook source|^(#\\s*%%|#\\s*\\<codecell\\>|#\\s*In\\[\\d*?\\]|#\\s*In\\[ \\])",
5+
"jupyter.interactiveWindow.cellMarker.default": "# COMMAND ----------",
6+
"python.testing.pytestArgs": [
7+
"."
8+
],
9+
"python.testing.unittestEnabled": false,
10+
"python.testing.pytestEnabled": true,
11+
"python.analysis.extraPaths": ["src"],
12+
"files.exclude": {
13+
"**/*.egg-info": true,
14+
"**/__pycache__": true,
15+
".pytest_cache": true,
16+
},
17+
"sqltools.connections": [
18+
{
19+
"connectionMethod": "VS Code Extension (beta)",
20+
"catalog": "{{.default_catalog}}",
21+
"previewLimit": 50,
22+
"driver": "Databricks",
23+
"name": "databricks",
24+
"path": "{{.http_path}}"
25+
}
26+
],
27+
"sqltools.autoConnectTo": "",
28+
}

‎libs/template/templates/default-sql/template/{{.project_name}}/README.md.tmpl‎

Lines changed: 41 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,41 @@
1+
# {{.project_name}}
2+
3+
The '{{.project_name}}' project was generated by using the default-sql template.
4+
5+
## Getting started
6+
7+
1. Install the Databricks CLI from https://docs.databricks.com/dev-tools/cli/install.html
8+
9+
2. Authenticate to your Databricks workspace (if you have not done so already):
10+
```
11+
$ databricks configure
12+
```
13+
14+
3. To deploy a development copy of this project, type:
15+
```
16+
$ databricks bundle deploy --target dev
17+
```
18+
(Note that "dev" is the default target, so the `--target` parameter
19+
is optional here.)
20+
21+
This deploys everything that's defined for this project.
22+
For example, the default template would deploy a job called
23+
`[dev yourname] {{.project_name}}_job` to your workspace.
24+
You can find that job by opening your workpace and clicking on **Workflows**.
25+
26+
4. Similarly, to deploy a production copy, type:
27+
```
28+
$ databricks bundle deploy --target prod
29+
```
30+
31+
5. To run a job, use the "run" command:
32+
```
33+
$ databricks bundle run
34+
```
35+
36+
6. Optionally, install developer tools such as the Databricks extension for Visual Studio Code from
37+
https://docs.databricks.com/dev-tools/vscode-ext.html.
38+
39+
7. For documentation on the Databricks Asset Bundles format used
40+
for this project, and for CI/CD configuration, see
41+
https://docs.databricks.com/dev-tools/bundles/index.html.

0 commit comments

Comments
 (0)