Add environment variable substitution support for YAML files

- Support ${ENV_VAR}, ${ENV_VAR:-default}, and $ENV_VAR syntax
- Environment variables substituted when YAML files are loaded
- Works in connection strings, table names, and any YAML value
- Update SemanticLayer.from_yaml() to read connection from YAML file
- Add 11 comprehensive tests for env var substitution
- Document env var usage in config.qmd and connections.qmd
- Compatible with Docker Compose and common YAML env var patterns

Author: nicosuave

docs/config.qmd

@@ -12,6 +12,7 @@ Basic semantic layer YAML file:
 # semantic_layer.yml
 
 # Database connection (optional, defaults to in-memory DuckDB)
+# Supports environment variable substitution
 connection: duckdb:///data/analytics.duckdb
 
 # Models define your tables and metrics
@@ -77,6 +78,47 @@ connection: databricks://token@server/http-path?catalog=main
 connection: spark://host:10000/database
 ```
 
+### Environment Variables in YAML
+
+Use environment variables for sensitive credentials:
+
+```yaml
+# Use ${ENV_VAR} syntax
+connection: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_NAME}
+
+# With default values
+connection: duckdb:///${DB_FILE:-/tmp/default.duckdb}
+
+# Simple form (uppercase vars only)
+connection: $DATABASE_URL
+```
+
+**Supported syntax:**
+
+- `${ENV_VAR}` - Substituted with environment variable value
+- `${ENV_VAR:-default}` - Use default if variable not set
+- `$ENV_VAR` - Simple form (uppercase variables only)
+
+**Example:**
+
+```bash
+# Set environment variables
+export DB_USER=analyst
+export DB_PASSWORD=secret
+export DB_HOST=localhost
+export DB_NAME=analytics
+```
+
+```yaml
+# semantic_layer.yml
+connection: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_NAME}
+
+models:
+  - name: orders
+    table: ${SCHEMA_NAME:-public}.orders
+    primary_key: order_id
+```
+
 See **[Database Connections](connections.qmd)** for complete connection string reference.
 
 ### Override via CLI

docs/connections.qmd

@@ -290,6 +290,57 @@ sidemantic query models/ \
 
 Store credentials in environment variables for security. Never commit credentials to version control.
 
+Environment variables can be used in:
+- **YAML configuration files** - Using `${VAR}` syntax
+- **CLI flags** - Using shell variables like `$VAR`
+
+### In YAML Files
+
+Sidemantic supports environment variable substitution in YAML files using standard syntax:
+
+**Syntax:**
+- `${ENV_VAR}` - Substituted with environment variable value
+- `${ENV_VAR:-default}` - Use default if variable not set
+- `$ENV_VAR` - Simple form (uppercase variables only)
+
+**Example YAML:**
+```yaml
+# semantic_layer.yml
+connection: postgres://${DB_USER}:${DB_PASSWORD}@${DB_HOST}:5432/${DB_NAME}
+
+models:
+  - name: orders
+    table: ${SCHEMA_NAME:-public}.orders
+    primary_key: order_id
+```
+
+**Set environment variables:**
+```bash
+export DB_USER=analyst
+export DB_PASSWORD=secret
+export DB_HOST=localhost
+export DB_NAME=analytics
+export SCHEMA_NAME=prod
+```
+
+**Load configuration:**
+```bash
+sidemantic query semantic_layer.yml --sql "SELECT revenue FROM orders"
+```
+
+The environment variables are substituted when the YAML file is loaded.
+
+### In CLI Flags
+
+Use shell environment variables in CLI commands:
+
+```bash
+export DATABASE_URL=postgres://user:pass@localhost:5432/analytics
+
+sidemantic query models/ --connection "$DATABASE_URL" \
+  --sql "SELECT revenue FROM orders"
+```
+
 ### Full Connection Strings
 
 Store complete connection string in an environment variable:

sidemantic/adapters/sidemantic.py

@@ -1,5 +1,7 @@
 """Sidemantic native YAML adapter with SQL syntax support."""
 
+import os
+import re
 from pathlib import Path
 
 import yaml
@@ -14,6 +16,59 @@
 from sidemantic.core.sql_definitions import parse_sql_definitions, parse_sql_file_with_frontmatter, parse_sql_model
 
 
+def substitute_env_vars(content: str) -> str:
+    """Substitute environment variables in YAML content.
+
+    Supports:
+    - ${ENV_VAR} - replaced with environment variable value
+    - ${ENV_VAR:-default} - replaced with value or default if not set
+    - $ENV_VAR - simple form without braces
+
+    Args:
+        content: YAML content string
+
+    Returns:
+        Content with environment variables substituted
+
+    Examples:
+        >>> os.environ['DB_HOST'] = 'localhost'
+        >>> substitute_env_vars('host: ${DB_HOST}')
+        'host: localhost'
+        >>> substitute_env_vars('host: ${MISSING:-default}')
+        'host: default'
+    """
+    # Pattern for ${ENV_VAR} or ${ENV_VAR:-default}
+    def replace_var(match):
+        var_expr = match.group(1)
+        # Check for default value syntax: VAR_NAME:-default
+        if ":-" in var_expr:
+            var_name, default = var_expr.split(":-", 1)
+            return os.environ.get(var_name, default)
+        else:
+            var_name = var_expr
+            value = os.environ.get(var_name)
+            if value is None:
+                # Keep original if not found (don't fail, let user handle missing vars)
+                return match.group(0)
+            return value
+
+    # Replace ${VAR} and ${VAR:-default}
+    content = re.sub(r"\$\{([^}]+)\}", replace_var, content)
+
+    # Replace $VAR (simple form, no braces)
+    # Only match valid environment variable names (alphanumeric + underscore)
+    def replace_simple_var(match):
+        var_name = match.group(1)
+        value = os.environ.get(var_name)
+        if value is None:
+            return match.group(0)
+        return value
+
+    content = re.sub(r"\$([A-Z_][A-Z0-9_]*)", replace_simple_var, content)
+
+    return content
+
+
 class SidemanticAdapter(BaseAdapter):
     """Adapter for Sidemantic native YAML format.
 
@@ -83,7 +138,12 @@ def parse(self, source: str | Path) -> SemanticGraph:
 
         # Handle YAML files
         with open(source_path) as f:
-            data = yaml.safe_load(f)
+            content = f.read()
+
+        # Substitute environment variables
+        content = substitute_env_vars(content)
+
+        data = yaml.safe_load(content)
 
         if not data:
             return graph

sidemantic/core/semantic_layer.py

@@ -365,22 +365,38 @@ def get_catalog_metadata(self, schema: str = "public") -> dict:
         return get_catalog_metadata(self.graph, schema=schema)
 
     @classmethod
-    def from_yaml(cls, path: str | Path, connection: str = "duckdb:///:memory:") -> SemanticLayer:
+    def from_yaml(cls, path: str | Path, connection: str | None = None) -> SemanticLayer:
         """Load semantic layer from native YAML file.
 
         Args:
             path: Path to YAML file
-            connection: Database connection string
+            connection: Database connection string (overrides connection in YAML file)
 
         Returns:
             SemanticLayer instance
         """
-        from sidemantic.adapters.sidemantic import SidemanticAdapter
+        import yaml
+
+        from sidemantic.adapters.sidemantic import SidemanticAdapter, substitute_env_vars
 
         adapter = SidemanticAdapter()
         graph = adapter.parse(path)
 
-        layer = cls(connection=connection)
+        # If connection not provided as parameter, try to read from YAML file
+        if connection is None:
+            with open(path) as f:
+                content = f.read()
+            # Substitute environment variables
+            content = substitute_env_vars(content)
+            data = yaml.safe_load(content)
+            if data and "connection" in data:
+                connection = data["connection"]
+
+        # Create layer with connection (or use default if still None)
+        if connection:
+            layer = cls(connection=connection)
+        else:
+            layer = cls()
         layer.graph = graph
 
         return layer