create_insightflow_structure.bat

@echo off
setlocal enabledelayedexpansion

REM InsightFlow - Project Structure Generator
REM Author: Saviour Henry
REM Date: January 2026

set PROJECT_NAME=InsightFlow
set INNER_PACKAGE=signaliq

echo.
echo ========================================
echo Creating InsightFlow project structure
echo ========================================
echo.

REM Create main project directory
if not exist %PROJECT_NAME% mkdir %PROJECT_NAME%
cd %PROJECT_NAME%

echo Creating directory structure...

REM Create all directories
mkdir api 2>nul
mkdir src\%INNER_PACKAGE% 2>nul
mkdir scripts 2>nul
mkdir sql 2>nul
mkdir tests\unit 2>nul
mkdir tests\api 2>nul
mkdir tests\integration 2>nul
mkdir tests\fixtures 2>nul
mkdir n8n\workflows 2>nul
mkdir docker 2>nul
mkdir systemd 2>nul
mkdir logs 2>nul
mkdir docs 2>nul
mkdir config 2>nul

echo Directory structure created!
echo.
echo Creating configuration files...

REM Create requirements.txt
(
echo fastapi==0.109.0
echo uvicorn[standard]==0.27.0
echo python-multipart==0.0.6
echo sqlalchemy==2.0.25
echo psycopg2-binary==2.9.9
echo pgvector==0.2.4
echo pydantic==2.5.3
echo pydantic-settings==2.1.0
echo sentence-transformers==2.2.2
echo hdbscan==0.8.33
echo numpy==1.26.3
echo scikit-learn==1.4.0
echo google-generativeai==0.3.2
echo slack-sdk==3.26.1
echo python-dotenv==1.0.0
echo requests==2.31.0
echo pytest==7.4.4
echo pytest-asyncio==0.23.3
echo pytest-cov==4.1.0
echo httpx==0.26.0
) > requirements.txt

REM Create .env.example
(
echo DATABASE_URL=postgresql://insight_user:insight_pass@localhost:5432/insightflow_db
echo GEMINI_API_KEY=your_gemini_api_key_here
echo LLM_PROVIDER=gemini
echo SLACK_WEBHOOK_URL=https://hooks.slack.com/services/YOUR/WEBHOOK/URL
echo SLACK_BOT_TOKEN=xoxb-your-token
echo SLACK_CHANNEL=#strategic-insights
echo N8N_WEBHOOK_URL=http://localhost:5678/webhook/insightflow
echo EMBEDDING_MODEL=all-MiniLM-L6-v2
echo EMBEDDING_BATCH_SIZE=32
echo CLUSTERING_MIN_SIZE=3
echo TOPIC_SIMILARITY_THRESHOLD=0.85
echo EMERGING_RISK_FREQ_PCT=30
echo EMERGING_RISK_URGENCY=7
echo AVOIDED_DECISION_COUNT=3
echo ATTENTION_SINK_PCT=0.3
echo SCOPE_CREEP_MESSAGES=5
echo WEEKLY_RUN_DAY=sunday
echo WEEKLY_RUN_HOUR=23
echo WEEKLY_RUN_TIMEZONE=UTC
) > .env.example

REM Create SQL schema
(
echo -- InsightFlow Database Schema
echo -- SignalIQ Core Engine
echo.
echo CREATE EXTENSION IF NOT EXISTS vector;
echo.
echo CREATE TABLE IF NOT EXISTS core_events ^(
echo   id TEXT PRIMARY KEY,
echo   source TEXT NOT NULL CHECK ^(source IN ^('email', 'meeting'^)^),
echo   timestamp TIMESTAMPTZ NOT NULL,
echo   actor TEXT NOT NULL,
echo   direction TEXT NOT NULL CHECK ^(direction IN ^('inbound', 'outbound', 'internal', 'unknown'^)^),
echo   subject TEXT,
echo   text TEXT NOT NULL,
echo   thread_id TEXT,
echo   decision TEXT NOT NULL CHECK ^(decision IN ^('made', 'deferred', 'none'^)^),
echo   action_owner TEXT,
echo   follow_up_required BOOLEAN NOT NULL,
echo   urgency_score INTEGER NOT NULL CHECK ^(urgency_score BETWEEN 0 AND 10^),
echo   sentiment TEXT NOT NULL DEFAULT 'unknown',
echo   raw_ref TEXT NOT NULL,
echo   embedding VECTOR^(384^),
echo   created_at TIMESTAMPTZ DEFAULT NOW^(^),
echo   updated_at TIMESTAMPTZ DEFAULT NOW^(^)
echo ^);
echo.
echo CREATE TABLE IF NOT EXISTS core_topics ^(
echo   topic_id UUID PRIMARY KEY DEFAULT gen_random_uuid^(^),
echo   centroid VECTOR^(384^) NOT NULL,
echo   created_at TIMESTAMPTZ DEFAULT NOW^(^),
echo   last_seen_at TIMESTAMPTZ DEFAULT NOW^(^),
echo   n_points INTEGER NOT NULL DEFAULT 0
echo ^);
echo.
echo CREATE TABLE IF NOT EXISTS core_event_topics ^(
echo   event_id TEXT REFERENCES core_events^(id^) ON DELETE CASCADE,
echo   topic_id UUID REFERENCES core_topics^(topic_id^) ON DELETE CASCADE,
echo   PRIMARY KEY ^(event_id, topic_id^)
echo ^);
echo.
echo CREATE TABLE IF NOT EXISTS out_weekly_briefs ^(
echo   week_start DATE PRIMARY KEY,
echo   week_end DATE NOT NULL,
echo   markdown TEXT NOT NULL,
echo   watchlist_json JSONB NOT NULL,
echo   audit_json JSONB NOT NULL,
echo   gemini_used BOOLEAN NOT NULL DEFAULT FALSE,
echo   gemini_model TEXT,
echo   gemini_response_id TEXT
echo ^);
echo.
echo CREATE INDEX IF NOT EXISTS idx_core_events_timestamp ON core_events^(timestamp^);
echo CREATE INDEX IF NOT EXISTS idx_core_events_thread_id ON core_events^(thread_id^);
echo CREATE INDEX IF NOT EXISTS idx_core_events_actor ON core_events^(actor^);
echo CREATE INDEX IF NOT EXISTS idx_core_topics_centroid ON core_topics USING hnsw ^(centroid vector_cosine_ops^);
echo CREATE INDEX IF NOT EXISTS idx_core_event_topics_topic ON core_event_topics^(topic_id^);
) > sql\schema.sql

REM Create Dockerfile
(
echo FROM python:3.10-slim
echo WORKDIR /app
echo RUN apt-get update ^&^& apt-get install -y gcc g++ postgresql-client ^&^& rm -rf /var/lib/apt/lists/*
echo COPY requirements.txt .
echo RUN pip install --no-cache-dir -r requirements.txt
echo COPY . .
echo EXPOSE 8000
echo CMD ["uvicorn", "api.main:app", "--host", "0.0.0.0", "--port", "8000"]
) > Dockerfile

REM Create docker-compose.yml
(
echo version: '3.8'
echo services:
echo   db:
echo     image: ankane/pgvector:latest
echo     environment:
echo       POSTGRES_USER: insight_user
echo       POSTGRES_PASSWORD: insight_pass
echo       POSTGRES_DB: insightflow_db
echo     ports:
echo       - "5432:5432"
echo     volumes:
echo       - postgres_data:/var/lib/postgresql/data
echo       - ./sql/schema.sql:/docker-entrypoint-initdb.d/schema.sql
echo   api:
echo     build: .
echo     ports:
echo       - "8000:8000"
echo     environment:
echo       - DATABASE_URL=postgresql://insight_user:insight_pass@db:5432/insightflow_db
echo     depends_on:
echo       - db
echo     volumes:
echo       - ./logs:/app/logs
echo volumes:
echo   postgres_data:
) > docker-compose.yml

REM Create .gitignore
(
echo __pycache__/
echo *.py[cod]
echo *$py.class
echo *.so
echo .Python
echo build/
echo dist/
echo *.egg-info/
echo .venv/
echo venv/
echo ENV/
echo env/
echo .env
echo .env.local
echo logs/*.log
echo *.db
echo *.sqlite
echo .pytest_cache/
echo .coverage
echo htmlcov/
echo .DS_Store
echo .vscode/
echo .idea/
echo *.swp
) > .gitignore

REM Create README.md
(
echo # InsightFlow
echo.
echo **SignalIQ Core Engine** - Transform scattered communications into strategic intelligence
echo.
echo ## Overview
echo.
echo InsightFlow is an automated system that ingests emails and meetings, normalizes them into a canonical schema, and uses ML-powered topic detection to surface strategic insights.
echo.
echo - **Embedding-based clustering**: Uses all-MiniLM-L6-v2 + HDBSCAN
echo - **Deterministic metrics**: 28-day baseline comparison
echo - **Rules-first intelligence**: Hard-coded business rules with optional Gemini enhancement
echo - **Evidence-backed delivery**: Weekly Slack briefs with full audit trail
echo.
echo ## Quick Start
echo.
echo ```bash
echo # 1. Create virtual environment
echo python -m venv .venv
echo .venv\Scripts\activate
echo.
echo # 2. Install dependencies
echo pip install -r requirements.txt
echo.
echo # 3. Configure environment
echo copy .env.example .env
echo # Edit .env with your credentials
echo.
echo # 4. Start with Docker
echo docker-compose up -d
echo.
echo # 5. Access API docs
echo # http://localhost:8000/docs
echo ```
echo.
echo ## Project Structure
echo.
echo ```
echo InsightFlow/
echo ├── api/              # FastAPI ingestion endpoints
echo ├── src/signaliq/     # Core SignalIQ engine
echo ├── scripts/          # Weekly job orchestration
echo ├── sql/              # PostgreSQL + pgvector schema
echo ├── tests/            # Unit, API, integration tests
echo ├── n8n/              # Workflow automation configs
echo └── docs/             # Documentation
echo ```
echo.
echo ## Architecture
echo.
echo **Data Flow**: External events → n8n normalization → FastAPI ingestion → PostgreSQL/pgvector → Weekly batch processing → Gemini enhancement → Slack delivery
echo.
echo **Core Components**:
echo - Ingestion layer with strict Pydantic validation
echo - Embedding generation using sentence-transformers
echo - HDBSCAN clustering for topic detection
echo - Deterministic metrics engine with baseline comparison
echo - Rules-based finding generation
echo - Optional Gemini API enhancement
echo - Markdown + JSON output rendering
echo.
echo ## Development
echo.
echo ```bash
echo # Run tests
echo pytest
echo.
echo # Run with coverage
echo pytest --cov=src
echo.
echo # Start API server
echo uvicorn api.main:app --reload
echo.
echo # Run weekly job ^(dry-run^)
echo python scripts/weekly_run.py --dry-run
echo ```
echo.
echo ## License
echo.
echo MIT License
echo.
echo ---
echo.
echo **Built by Saviour Henry** ^| Portfolio Project 2026
) > README.md

REM Create Python __init__.py files
type nul > api\__init__.py
type nul > src\__init__.py
type nul > src\%INNER_PACKAGE%\__init__.py
type nul > tests\__init__.py
type nul > tests\unit\__init__.py
type nul > tests\api\__init__.py
type nul > tests\integration\__init__.py

REM Create placeholder files
type nul > logs\.gitkeep

REM Create n8n workflows README
(
echo # n8n Workflows
echo.
echo Export your n8n workflows as JSON and place them here:
echo.
echo - `gmail_to_api.json` - Gmail integration workflow
echo - `readai_to_api.json` - Read.ai webhook workflow
echo.
echo ## Setup
echo.
echo 1. Import workflows in n8n
echo 2. Configure credentials ^(Gmail, HTTP endpoints^)
echo 3. Activate workflows
echo 4. Point webhooks to: http://localhost:8000/api/v1/ingest/*
) > n8n\workflows\README.md

REM Create systemd files
(
echo [Unit]
echo Description=InsightFlow Weekly Job
echo After=docker.service
echo.
echo [Service]
echo Type=oneshot
echo WorkingDirectory=C:\path\to\InsightFlow
echo ExecStart=docker-compose exec -T api python scripts/weekly_run.py
echo StandardOutput=journal
) > systemd\insightflow-weekly.service

(
echo [Unit]
echo Description=Run InsightFlow every Sunday at 23:00 UTC
echo.
echo [Timer]
echo OnCalendar=Sun *-*-* 23:00:00
echo Persistent=true
echo.
echo [Install]
echo WantedBy=timers.target
) > systemd\insightflow-weekly.timer

echo.
echo ========================================
echo Project structure created successfully!
echo ========================================
echo.
echo Project: %PROJECT_NAME%
echo Core package: src\%INNER_PACKAGE%
echo.
echo Created:
echo   - 15+ directories
echo   - requirements.txt ^(20 dependencies^)
echo   - .env.example ^(all config variables^)
echo   - SQL schema with pgvector
echo   - Docker configuration
echo   - README with quick start
echo   - .gitignore
echo.
echo Next steps:
echo   1. Open in VS Code: code .
echo   2. Create virtual environment: python -m venv .venv
echo   3. Activate: .venv\Scripts\activate
echo   4. Install: pip install -r requirements.txt
echo   5. Add Python source files to api\ and src\signaliq\
echo   6. Configure: copy .env.example .env
echo.
echo Python source files need to be created in:
echo   - api\main.py
echo   - src\signaliq\db.py, models.py, schemas.py, services.py
echo   - src\signaliq\clustering.py, rules.py, llm_client.py
echo   - scripts\weekly_run.py, ingest_demo.py
echo.
echo Ready for development!
echo.
pause