claude-code-docs/test-skills.sh at main · costiash/claude-code-docs · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
#!/usr/bin/env bash
set -euo pipefail

# test-skills.sh — Mechanical test harness for plugin skill quality
# Runs content-search and fuzzy-search with known queries, checks expected results.
# Output: PASS: <number> (total passed tests out of total)

# DOCS_DIR is the search scripts' grep-fallback location for fetched pages.
# v2 has no docs/ mirror — the local page cache lives at <repo>/cache.
DOCS_DIR="${DOCS_DIR:-$(cd "$(dirname "$0")" && pwd)/cache}"
SCRIPTS_DIR="$(cd "$(dirname "$0")" && pwd)/plugin/skills/claude-docs/scripts"

export DOCS_DIR

pass=0
fail=0
total=0

check() {
    local description="$1"
    local script="$2"
    shift 2
    local expected="$1"
    shift
    local args=("$@")

    total=$((total + 1))
    local output
    output=$("$SCRIPTS_DIR/$script" "${args[@]}" 2>/dev/null) || output=""

    if echo "$output" | grep -q "$expected"; then
        pass=$((pass + 1))
    else
        fail=$((fail + 1))
        echo "FAIL: $description — expected '$expected' in output" >&2
    fi
}

# === Content Search Tests ===
check "content: hooks"             content-search.sh claude-code__hooks.md          "hooks"
check "content: mcp"               content-search.sh claude-code__mcp.md            "mcp"
check "content: extended thinking"  content-search.sh docs__en__build-with-claude__extended-thinking.md "extended" "thinking"
check "content: streaming"         content-search.sh docs__en__build-with-claude__streaming.md         "streaming"
check "content: prompt caching"    content-search.sh docs__en__build-with-claude__prompt-caching.md    "prompt" "caching"
check "content: rate limits"       content-search.sh docs__en__api__rate-limits.md                     "rate" "limits"
check "content: agent sdk python"  content-search.sh claude-code__agent-sdk__python.md                 "agent" "sdk" "python"
check "content: vision"            content-search.sh docs__en__build-with-claude__vision.md            "vision"

# === Fuzzy Search Tests ===
check "fuzzy: hooks"       fuzzy-search.sh claude-code__hooks.md       "hooks"
check "fuzzy: mcp"         fuzzy-search.sh claude-code__mcp.md         "mcp"
check "fuzzy: plugins"     fuzzy-search.sh claude-code__plugins.md     "plugins"
check "fuzzy: agent sdk"   fuzzy-search.sh claude-code__agent-sdk      "agent sdk"
check "fuzzy: streaming"   fuzzy-search.sh streaming                   "streaming"
check "fuzzy: skills"      fuzzy-search.sh skills                      "skills"
check "fuzzy: vision"      fuzzy-search.sh vision                      "vision"

# === Edge Case Tests ===
check "content: computer use"      content-search.sh docs__en__agents-and-tools__tool-use__computer-use-tool.md  "computer"
check "content: batch processing"  content-search.sh docs__en__build-with-claude__batch-processing.md    "batch" "processing"
check "content: prompt engineering" content-search.sh docs__en__build-with-claude__prompt-engineering     "prompt" "engineering"
check "fuzzy: tool use overview"   fuzzy-search.sh   docs__en__agents-and-tools__tool-use__overview.md   "tool use overview"
check "fuzzy: batch"               fuzzy-search.sh   batch                                               "batch"

# === Robustness Tests ===
check "content: citations"         content-search.sh docs__en__build-with-claude__citations.md         "citations"
check "content: embeddings"        content-search.sh docs__en__build-with-claude__embeddings.md        "embeddings"
check "fuzzy: quickstart"          fuzzy-search.sh   claude-code__quickstart.md                        "quickstart"
check "fuzzy: settings"            fuzzy-search.sh   claude-code__settings.md                          "settings"
check "fuzzy: desktop"             fuzzy-search.sh   claude-code__desktop.md                           "desktop"

# === Additional Search Coverage ===
check "content: sandboxing"        content-search.sh claude-code__sandboxing.md                        "sandboxing"
check "content: context windows"   content-search.sh docs__en__build-with-claude__context-windows.md   "context" "windows"
check "fuzzy: headless"            fuzzy-search.sh   claude-code__headless.md                          "headless"
check "fuzzy: tool use"            fuzzy-search.sh   docs__en__agents-and-tools__tool-use              "tool use"
check "fuzzy: hooks guide"        fuzzy-search.sh   claude-code__hooks-guide.md                       "hooks guide"
check "content: devcontainer"      content-search.sh claude-code__devcontainer.md                      "devcontainer"
check "content: jetbrains"         content-search.sh claude-code__jetbrains.md                         "jetbrains"

# === Synonym-Adjacent Tests ===
check "content: structured outputs" content-search.sh docs__en__build-with-claude__structured-outputs.md  "structured" "outputs"
check "content: github actions"    content-search.sh claude-code__github-actions.md                       "github" "actions"
check "fuzzy: sub-agents"          fuzzy-search.sh   claude-code__sub-agents.md                           "sub-agents"

# === SDK Disambiguation Tests ===
check "content: python sdk"        content-search.sh docs__en__cli-sdks-libraries__sdks__python.md     "python" "sdk"
check "fuzzy: typescript sdk"      fuzzy-search.sh   docs__en__cli-sdks-libraries__sdks__typescript.md "typescript sdk"
check "fuzzy: go sdk"              fuzzy-search.sh   docs__en__cli-sdks-libraries__sdks__go.md         "go sdk"

# === Additional Coverage ===
check "content: model config"      content-search.sh claude-code__model-config.md                      "model" "config"
check "fuzzy: security"            fuzzy-search.sh   claude-code__security.md                          "security"
check "fuzzy: network config"      fuzzy-search.sh   claude-code__network-config.md                    "network config"
check "content: pricing"           content-search.sh docs__en__about-claude__pricing.md                 "pricing"
check "fuzzy: troubleshooting"     fuzzy-search.sh   claude-code__troubleshooting.md                    "troubleshooting"
check "content: pdf support"       content-search.sh docs__en__build-with-claude__pdf-support.md        "pdf"
check "content: data residency"    content-search.sh docs__en__manage-claude__data-residency.md         "residency"
check "content: glossary"          content-search.sh docs__en__about-claude__glossary.md                "glossary"
check "fuzzy: analytics"           fuzzy-search.sh   claude-code__analytics.md                          "analytics"
check "fuzzy: costs"               fuzzy-search.sh   claude-code__costs.md                              "costs"
check "content: model overview"    content-search.sh docs__en__about-claude__models__overview.md        "models" "overview"
check "content: error handling"    content-search.sh docs__en__api__errors.md                           "errors"
check "fuzzy: output styles"       fuzzy-search.sh   claude-code__output-styles.md                     "output styles"
check "fuzzy: checkpointing"       fuzzy-search.sh   claude-code__checkpointing.md                     "checkpointing"
check "fuzzy: monitoring usage"    fuzzy-search.sh   claude-code__monitoring-usage.md                   "monitoring usage"
check "fuzzy: interactive"         fuzzy-search.sh   claude-code__interactive-mode.md                   "interactive"
check "content: data usage"        content-search.sh claude-code__data-usage.md                        "data" "usage"
check "fuzzy: vs code"             fuzzy-search.sh   claude-code__vs-code.md                           "vs code"
check "fuzzy: terminal"            fuzzy-search.sh   claude-code__terminal-config.md                   "terminal"
check "content: fast mode"         content-search.sh docs__en__build-with-claude__fast-mode.md          "fast" "mode"
check "fuzzy: legal compliance"    fuzzy-search.sh   claude-code__legal-and-compliance.md               "legal"
check "content: compaction"        content-search.sh docs__en__build-with-claude__compaction.md         "compaction"
check "content: service tiers"     content-search.sh docs__en__api__service-tiers.md                    "service" "tiers"
check "fuzzy: statusline"          fuzzy-search.sh   claude-code__statusline.md                         "statusline"
check "fuzzy: amazon bedrock"      fuzzy-search.sh   claude-code__amazon-bedrock.md                     "amazon bedrock"
check "content: openai sdk compat" content-search.sh docs__en__cli-sdks-libraries__libraries__openai-sdk.md  "openai"

echo "PASS: $pass/$total"
echo "FAIL: $fail/$total"

# Non-zero exit when any check failed, so CI and callers can gate on this harness.
[ "$fail" -eq 0 ] || exit 1
exit 0