From 1dbe894cf09582cd6e7b2e826deb4120a3a6d49a Mon Sep 17 00:00:00 2001 From: Vinay Roy Thykkutathil Date: Fri, 19 Dec 2025 22:08:58 -0500 Subject: [PATCH 1/3] DOCS: Added validated testing steps using Sbt against ADLS Gen2 Signed-off-by: Vinay Roy Thykkutathil --- README.md | 47 +++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 47 insertions(+) diff --git a/README.md b/README.md index e87e9ee4e..889807252 100644 --- a/README.md +++ b/README.md @@ -669,6 +669,53 @@ docker run -p : \ Note that `` should be the same as the port defined inside the config file. +### Local testing with SBT (Azure) + +For local Azure Data Lake Storage Gen2 testing with SBT, add your config directory (with `core-site.xml`) to the server's resource path and run the server: + +``` +build/sbt \ + "set server / Compile / unmanagedResourceDirectories += file(\"/path/to/server-configs\")" \ + "server/run --config=/path/to/server-configs/delta-sharing-server.yaml" +``` + +Example `delta-sharing-server.yaml`: + +```yaml +version: 1 +shares: +- name: "example" + schemas: + - name: "default" + tables: + - name: "example-table" + location: "abfss://@.dfs.core.windows.net/" + id: "00000000-0000-0000-0000-000000000001" +host: "localhost" +port: 8080 +endpoint: "/delta-sharing" +authorization: + bearerToken: "change-me" +``` + +Example `core-site.xml` (Shared Key auth): + +```xml + + + + + fs.azure.account.auth.type..dfs.core.windows.net + SharedKey + + + fs.azure.account.key..dfs.core.windows.net + YOUR-ACCOUNT-KEY + + +``` + +Make sure the account name in the table `location` matches the `` name in `core-site.xml`. Refer to [SBT docs](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html) for more commands. From 867df3cc48188c49d0320d01f499335878bf04f5 Mon Sep 17 00:00:00 2001 From: Vinay Roy Thykkutathil Date: Mon, 22 Dec 2025 15:47:13 -0500 Subject: [PATCH 2/3] Add profile loading from environment variables and reasonable defaults Signed-off-by: Vinay Roy Thykkutathil --- README.md | 6 +++ python/README.md | 3 ++ python/delta_sharing/protocol.py | 39 ++++++++++++-- python/delta_sharing/tests/test_protocol.py | 58 ++++++++++++++++++--- 4 files changed, 95 insertions(+), 11 deletions(-) diff --git a/README.md b/README.md index 889807252..ba25bc852 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ profile_file = "" # Create a SharingClient. client = delta_sharing.SharingClient(profile_file) +# You can also build a profile from environment variables (useful in CI). +# Required: DSHARING_VERSION, DSHARING_TOKEN, DSHARING_ENDPOINT +# Optional: DSHARING_EXPTIME +profile = delta_sharing.protocol.DeltaSharingProfile.from_env() +client = delta_sharing.SharingClient(profile) + # List all shared tables. client.list_all_tables() diff --git a/python/README.md b/python/README.md index 58bdb651a..987170334 100644 --- a/python/README.md +++ b/python/README.md @@ -9,6 +9,9 @@ This is the Python client library for Delta Sharing, which lets you load shared 1. Install using `pip install delta-sharing`. a. On some environments, you may also need to [install Rust](https://www.rust-lang.org/tools/install). This is because the `delta-sharing` package depends on the `delta-kernel-rust-sharing-wrapper` package, which does not have a pre-built Python wheel for all environments. As a result, pip will have to build `delta-kernel-rust-sharing-wrapper` from source. 2. To use the Python Connector, see [the project docs](https://github.com/delta-io/delta-sharing) for details. + If you need to load credentials from environment variables (e.g., CI), you can build a profile + with `delta_sharing.protocol.DeltaSharingProfile.from_env()` using + `DSHARING_VERSION`, `DSHARING_TOKEN`, `DSHARING_ENDPOINT`, and optional `DSHARING_EXPTIME`. ## Documentation diff --git a/python/delta_sharing/protocol.py b/python/delta_sharing/protocol.py index 7f643bd16..6a0f06cd0 100644 --- a/python/delta_sharing/protocol.py +++ b/python/delta_sharing/protocol.py @@ -15,6 +15,7 @@ # from dataclasses import dataclass, field from json import loads +import os from pathlib import Path from typing import ClassVar, Dict, IO, List, Optional, Sequence, Union, TypedDict @@ -44,7 +45,9 @@ class DeltaSharingProfile: scope: Optional[str] = None issuer: Optional[str] = None audience: Optional[str] = None - private_key: Optional[Dict[str, str]] = field(default=None, hash=False, compare=False) + private_key: Optional[Dict[str, str]] = field( + default=None, hash=False, compare=False + ) def __post_init__(self): if self.share_credentials_version > DeltaSharingProfile.CURRENT: @@ -144,7 +147,8 @@ def from_json(json) -> "DeltaSharingProfile": ) else: raise ValueError( - f"The current release does not supports {type} type. " "Please check type." + f"The current release does not supports {type} type. " + "Please check type." ) else: raise ValueError( @@ -154,6 +158,33 @@ def from_json(json) -> "DeltaSharingProfile": "Please upgrade to a newer release." ) + @staticmethod + def from_env( + version_env: str = "DSHARING_VERSION", + token_env: str = "DSHARING_TOKEN", + endpoint_env: str = "DSHARING_ENDPOINT", + expiration_env: str = "DSHARING_EXPTIME", + ) -> "DeltaSharingProfile": + version = os.environ.get(version_env) + token = os.environ.get(token_env) + endpoint = os.environ.get(endpoint_env) + expiration = os.environ.get(expiration_env) + + if version is None or token is None or endpoint is None: + raise ValueError( + "Missing required environment variables for Delta Sharing profile." + ) + + if endpoint.endswith("/"): + endpoint = endpoint[:-1] + + return DeltaSharingProfile( + share_credentials_version=int(version), + endpoint=endpoint, + bearer_token=token, + expiration_time=expiration, + ) + @dataclass(frozen=True) class Share: @@ -233,7 +264,9 @@ class Format: def from_json(json) -> "Format": if isinstance(json, (str, bytes, bytearray)): json = loads(json) - return Format(provider=json.get("provider", "parquet"), options=json.get("options", {})) + return Format( + provider=json.get("provider", "parquet"), options=json.get("options", {}) + ) @dataclass(frozen=True) diff --git a/python/delta_sharing/tests/test_protocol.py b/python/delta_sharing/tests/test_protocol.py index e413d34e4..3ad375f6b 100644 --- a/python/delta_sharing/tests/test_protocol.py +++ b/python/delta_sharing/tests/test_protocol.py @@ -93,7 +93,8 @@ def test_share_profile(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -191,7 +192,8 @@ def test_share_profile_bearer(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -294,7 +296,8 @@ def test_profile_share_oauth_client_credentials(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -374,7 +377,8 @@ def test_share_profile_oauth_jwt_bearer_private_key_jwt(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -487,7 +491,8 @@ def test_share_profile_basic(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -539,7 +544,9 @@ def test_protocol(): "minReaderVersion" : 100 } """ - with pytest.raises(ValueError, match="The table requires a newer version 100 to read."): + with pytest.raises( + ValueError, match="The table requires a newer version 100 to read." + ): Protocol.from_json(json) @@ -555,7 +562,9 @@ def test_protocol_delta(): } """ protocol = Protocol.from_json(json) - assert protocol == Protocol(3, 7, ["columnMapping"], ["columnMapping", "deletionVectors"]) + assert protocol == Protocol( + 3, 7, ["columnMapping"], ["columnMapping", "deletionVectors"] + ) json = """ { "deltaProtocol": { @@ -564,7 +573,9 @@ def test_protocol_delta(): } } """ - with pytest.raises(ValueError, match="The table requires a newer version 100 to read."): + with pytest.raises( + ValueError, match="The table requires a newer version 100 to read." + ): Protocol.from_json(json) @@ -914,3 +925,34 @@ def test_add_cdc_file(json: str, expected: AddCdcFile): ) def test_remove_file(json: str, expected: RemoveFile): assert RemoveFile.from_json(json) == expected + + +def test_share_profile_from_env_defaults(monkeypatch): + monkeypatch.setenv("DSHARING_VERSION", "1") + monkeypatch.setenv("DSHARING_TOKEN", "token") + monkeypatch.setenv("DSHARING_ENDPOINT", "https://localhost/delta-sharing/") + monkeypatch.setenv("DSHARING_EXPTIME", "2021-11-12T00:12:29.0Z") + + profile = DeltaSharingProfile.from_env() + + assert profile == DeltaSharingProfile( + 1, "https://localhost/delta-sharing", "token", "2021-11-12T00:12:29.0Z" + ) + + +def test_share_profile_from_env_custom_names(monkeypatch): + monkeypatch.setenv("CUSTOM_VERSION", "1") + monkeypatch.setenv("CUSTOM_TOKEN", "token") + monkeypatch.setenv("CUSTOM_ENDPOINT", "https://localhost/delta-sharing/") + monkeypatch.setenv("CUSTOM_EXPTIME", "2021-11-12T00:12:29.0Z") + + profile = DeltaSharingProfile.from_env( + version_env="CUSTOM_VERSION", + token_env="CUSTOM_TOKEN", + endpoint_env="CUSTOM_ENDPOINT", + expiration_env="CUSTOM_EXPTIME", + ) + + assert profile == DeltaSharingProfile( + 1, "https://localhost/delta-sharing", "token", "2021-11-12T00:12:29.0Z" + ) From c6cc5cf5dc7857cc2b84e5a7e2fc4f93287a092f Mon Sep 17 00:00:00 2001 From: Vinay Roy Thykkutathil Date: Mon, 22 Dec 2025 18:23:22 -0500 Subject: [PATCH 3/3] Fixed formatting issues Signed-off-by: Vinay Roy Thykkutathil --- python/delta_sharing/protocol.py | 15 ++++----------- python/delta_sharing/tests/test_protocol.py | 12 +++--------- 2 files changed, 7 insertions(+), 20 deletions(-) diff --git a/python/delta_sharing/protocol.py b/python/delta_sharing/protocol.py index 6a0f06cd0..1f1b38310 100644 --- a/python/delta_sharing/protocol.py +++ b/python/delta_sharing/protocol.py @@ -45,9 +45,7 @@ class DeltaSharingProfile: scope: Optional[str] = None issuer: Optional[str] = None audience: Optional[str] = None - private_key: Optional[Dict[str, str]] = field( - default=None, hash=False, compare=False - ) + private_key: Optional[Dict[str, str]] = field(default=None, hash=False, compare=False) def __post_init__(self): if self.share_credentials_version > DeltaSharingProfile.CURRENT: @@ -147,8 +145,7 @@ def from_json(json) -> "DeltaSharingProfile": ) else: raise ValueError( - f"The current release does not supports {type} type. " - "Please check type." + f"The current release does not supports {type} type. " "Please check type." ) else: raise ValueError( @@ -171,9 +168,7 @@ def from_env( expiration = os.environ.get(expiration_env) if version is None or token is None or endpoint is None: - raise ValueError( - "Missing required environment variables for Delta Sharing profile." - ) + raise ValueError("Missing required environment variables for Delta Sharing profile.") if endpoint.endswith("/"): endpoint = endpoint[:-1] @@ -264,9 +259,7 @@ class Format: def from_json(json) -> "Format": if isinstance(json, (str, bytes, bytearray)): json = loads(json) - return Format( - provider=json.get("provider", "parquet"), options=json.get("options", {}) - ) + return Format(provider=json.get("provider", "parquet"), options=json.get("options", {})) @dataclass(frozen=True) diff --git a/python/delta_sharing/tests/test_protocol.py b/python/delta_sharing/tests/test_protocol.py index 3ad375f6b..01a35e945 100644 --- a/python/delta_sharing/tests/test_protocol.py +++ b/python/delta_sharing/tests/test_protocol.py @@ -544,9 +544,7 @@ def test_protocol(): "minReaderVersion" : 100 } """ - with pytest.raises( - ValueError, match="The table requires a newer version 100 to read." - ): + with pytest.raises(ValueError, match="The table requires a newer version 100 to read."): Protocol.from_json(json) @@ -562,9 +560,7 @@ def test_protocol_delta(): } """ protocol = Protocol.from_json(json) - assert protocol == Protocol( - 3, 7, ["columnMapping"], ["columnMapping", "deletionVectors"] - ) + assert protocol == Protocol(3, 7, ["columnMapping"], ["columnMapping", "deletionVectors"]) json = """ { "deltaProtocol": { @@ -573,9 +569,7 @@ def test_protocol_delta(): } } """ - with pytest.raises( - ValueError, match="The table requires a newer version 100 to read." - ): + with pytest.raises(ValueError, match="The table requires a newer version 100 to read."): Protocol.from_json(json)