diff --git a/README.md b/README.md index e87e9ee4e..ba25bc852 100644 --- a/README.md +++ b/README.md @@ -84,6 +84,12 @@ profile_file = "" # Create a SharingClient. client = delta_sharing.SharingClient(profile_file) +# You can also build a profile from environment variables (useful in CI). +# Required: DSHARING_VERSION, DSHARING_TOKEN, DSHARING_ENDPOINT +# Optional: DSHARING_EXPTIME +profile = delta_sharing.protocol.DeltaSharingProfile.from_env() +client = delta_sharing.SharingClient(profile) + # List all shared tables. client.list_all_tables() @@ -669,6 +675,53 @@ docker run -p : \ Note that `` should be the same as the port defined inside the config file. +### Local testing with SBT (Azure) + +For local Azure Data Lake Storage Gen2 testing with SBT, add your config directory (with `core-site.xml`) to the server's resource path and run the server: + +``` +build/sbt \ + "set server / Compile / unmanagedResourceDirectories += file(\"/path/to/server-configs\")" \ + "server/run --config=/path/to/server-configs/delta-sharing-server.yaml" +``` + +Example `delta-sharing-server.yaml`: + +```yaml +version: 1 +shares: +- name: "example" + schemas: + - name: "default" + tables: + - name: "example-table" + location: "abfss://@.dfs.core.windows.net/" + id: "00000000-0000-0000-0000-000000000001" +host: "localhost" +port: 8080 +endpoint: "/delta-sharing" +authorization: + bearerToken: "change-me" +``` + +Example `core-site.xml` (Shared Key auth): + +```xml + + + + + fs.azure.account.auth.type..dfs.core.windows.net + SharedKey + + + fs.azure.account.key..dfs.core.windows.net + YOUR-ACCOUNT-KEY + + +``` + +Make sure the account name in the table `location` matches the `` name in `core-site.xml`. Refer to [SBT docs](https://www.scala-sbt.org/1.x/docs/Command-Line-Reference.html) for more commands. diff --git a/python/README.md b/python/README.md index 58bdb651a..987170334 100644 --- a/python/README.md +++ b/python/README.md @@ -9,6 +9,9 @@ This is the Python client library for Delta Sharing, which lets you load shared 1. Install using `pip install delta-sharing`. a. On some environments, you may also need to [install Rust](https://www.rust-lang.org/tools/install). This is because the `delta-sharing` package depends on the `delta-kernel-rust-sharing-wrapper` package, which does not have a pre-built Python wheel for all environments. As a result, pip will have to build `delta-kernel-rust-sharing-wrapper` from source. 2. To use the Python Connector, see [the project docs](https://github.com/delta-io/delta-sharing) for details. + If you need to load credentials from environment variables (e.g., CI), you can build a profile + with `delta_sharing.protocol.DeltaSharingProfile.from_env()` using + `DSHARING_VERSION`, `DSHARING_TOKEN`, `DSHARING_ENDPOINT`, and optional `DSHARING_EXPTIME`. ## Documentation diff --git a/python/delta_sharing/protocol.py b/python/delta_sharing/protocol.py index 7f643bd16..1f1b38310 100644 --- a/python/delta_sharing/protocol.py +++ b/python/delta_sharing/protocol.py @@ -15,6 +15,7 @@ # from dataclasses import dataclass, field from json import loads +import os from pathlib import Path from typing import ClassVar, Dict, IO, List, Optional, Sequence, Union, TypedDict @@ -154,6 +155,31 @@ def from_json(json) -> "DeltaSharingProfile": "Please upgrade to a newer release." ) + @staticmethod + def from_env( + version_env: str = "DSHARING_VERSION", + token_env: str = "DSHARING_TOKEN", + endpoint_env: str = "DSHARING_ENDPOINT", + expiration_env: str = "DSHARING_EXPTIME", + ) -> "DeltaSharingProfile": + version = os.environ.get(version_env) + token = os.environ.get(token_env) + endpoint = os.environ.get(endpoint_env) + expiration = os.environ.get(expiration_env) + + if version is None or token is None or endpoint is None: + raise ValueError("Missing required environment variables for Delta Sharing profile.") + + if endpoint.endswith("/"): + endpoint = endpoint[:-1] + + return DeltaSharingProfile( + share_credentials_version=int(version), + endpoint=endpoint, + bearer_token=token, + expiration_time=expiration, + ) + @dataclass(frozen=True) class Share: diff --git a/python/delta_sharing/tests/test_protocol.py b/python/delta_sharing/tests/test_protocol.py index e413d34e4..01a35e945 100644 --- a/python/delta_sharing/tests/test_protocol.py +++ b/python/delta_sharing/tests/test_protocol.py @@ -93,7 +93,8 @@ def test_share_profile(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -191,7 +192,8 @@ def test_share_profile_bearer(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -294,7 +296,8 @@ def test_profile_share_oauth_client_credentials(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -374,7 +377,8 @@ def test_share_profile_oauth_jwt_bearer_private_key_jwt(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -487,7 +491,8 @@ def test_share_profile_basic(tmp_path): } """ with pytest.raises( - ValueError, match="'shareCredentialsVersion' in the profile is 100 which is too new." + ValueError, + match="'shareCredentialsVersion' in the profile is 100 which is too new.", ): DeltaSharingProfile.read_from_file(io.StringIO(json)) @@ -914,3 +919,34 @@ def test_add_cdc_file(json: str, expected: AddCdcFile): ) def test_remove_file(json: str, expected: RemoveFile): assert RemoveFile.from_json(json) == expected + + +def test_share_profile_from_env_defaults(monkeypatch): + monkeypatch.setenv("DSHARING_VERSION", "1") + monkeypatch.setenv("DSHARING_TOKEN", "token") + monkeypatch.setenv("DSHARING_ENDPOINT", "https://localhost/delta-sharing/") + monkeypatch.setenv("DSHARING_EXPTIME", "2021-11-12T00:12:29.0Z") + + profile = DeltaSharingProfile.from_env() + + assert profile == DeltaSharingProfile( + 1, "https://localhost/delta-sharing", "token", "2021-11-12T00:12:29.0Z" + ) + + +def test_share_profile_from_env_custom_names(monkeypatch): + monkeypatch.setenv("CUSTOM_VERSION", "1") + monkeypatch.setenv("CUSTOM_TOKEN", "token") + monkeypatch.setenv("CUSTOM_ENDPOINT", "https://localhost/delta-sharing/") + monkeypatch.setenv("CUSTOM_EXPTIME", "2021-11-12T00:12:29.0Z") + + profile = DeltaSharingProfile.from_env( + version_env="CUSTOM_VERSION", + token_env="CUSTOM_TOKEN", + endpoint_env="CUSTOM_ENDPOINT", + expiration_env="CUSTOM_EXPTIME", + ) + + assert profile == DeltaSharingProfile( + 1, "https://localhost/delta-sharing", "token", "2021-11-12T00:12:29.0Z" + )