diff --git a/.changeset/pretty-sloths-switch.md b/.changeset/pretty-sloths-switch.md new file mode 100644 index 0000000..e5499bd --- /dev/null +++ b/.changeset/pretty-sloths-switch.md @@ -0,0 +1,5 @@ +--- +'@tanstack/intent': patch +--- + +Remove the unused feedback APIs, bundled feedback meta-skill, tests, and docs references. diff --git a/docs/getting-started/quick-start-consumers.md b/docs/getting-started/quick-start-consumers.md index 9a265a6..5e4fa0f 100644 --- a/docs/getting-started/quick-start-consumers.md +++ b/docs/getting-started/quick-start-consumers.md @@ -107,12 +107,3 @@ You can also check if any skills reference outdated source documentation: npx @tanstack/intent@latest stale ``` -## 5. Submit feedback (optional) - -After using a skill, you can submit feedback to help maintainers improve it: - -```bash -npx @tanstack/intent@latest meta feedback-collection -``` - -This prints a skill that guides your agent to collect structured feedback about gaps, errors, and improvements. diff --git a/packages/intent/README.md b/packages/intent/README.md index 2375425..fc4a9da 100644 --- a/packages/intent/README.md +++ b/packages/intent/README.md @@ -117,8 +117,6 @@ npx @tanstack/intent@latest setup The real risk with any derived artifact is staleness. `npx @tanstack/intent@latest stale` flags skills whose source docs have changed, generated skills that drift from `_artifacts`, and public workspace packages missing coverage. CI templates catch drift before it ships. -The feedback loop runs both directions. `npx @tanstack/intent@latest feedback` lets users submit structured reports when a skill produces wrong output — which skill, which version, what broke. That context flows back to the maintainer, and the fix ships to everyone on the next package update. Every support interaction produces an artifact that prevents the same class of problem for all future users — not just the one who reported it. - ## CLI Commands | Command | Description | @@ -131,7 +129,6 @@ The feedback loop runs both directions. `npx @tanstack/intent@latest feedback` l | `npx @tanstack/intent@latest validate [dir]` | Validate SKILL.md files | | `npx @tanstack/intent@latest setup` | Copy CI templates into your repo | | `npx @tanstack/intent@latest stale [dir] [--json]` | Check skills for version drift | -| `npx @tanstack/intent@latest feedback` | Submit skill feedback | ## License diff --git a/packages/intent/meta/domain-discovery/SKILL.md b/packages/intent/meta/domain-discovery/SKILL.md index fdfee28..34a03b0 100644 --- a/packages/intent/meta/domain-discovery/SKILL.md +++ b/packages/intent/meta/domain-discovery/SKILL.md @@ -572,7 +572,7 @@ Include the full draft domain_map.yaml in your message so the maintainer can review it. Also include a checklist of all docs files you read. **── STOP ── Do not proceed to Phase 4 until the maintainer has -reviewed the draft and responded. Their feedback on the draft informs +reviewed the draft and responded. Their response to the draft informs the detail interview questions.** --- diff --git a/packages/intent/meta/feedback-collection/SKILL.md b/packages/intent/meta/feedback-collection/SKILL.md deleted file mode 100644 index 080b84e..0000000 --- a/packages/intent/meta/feedback-collection/SKILL.md +++ /dev/null @@ -1,234 +0,0 @@ ---- -name: skill-feedback-collection -description: > - Collect structured feedback about skill usage after completing a coding task. - Activate at the end of any session where one or more SKILL.md files were - loaded. Captures agent signals (gaps, errors, corrections, human interventions) - and brief human input, then submits directly via gh CLI or provides manual - submission instructions. -metadata: - version: '2.0' - category: meta-tooling ---- - -# Skill Feedback Collection - -Run this at the end of any session where you loaded one or more SKILL.md files. -The goal is to capture what worked, what didn't, and what was missing — so skill -maintainers can improve future versions. - -This skill also covers **meta-skill feedback** — feedback about the scaffolding -process itself. When invoked after running domain-discovery, tree-generator, and -generate-skill, treat those three meta skills as the "skills" being evaluated. -Capture what worked and what didn't in each scaffolding phase so the meta skills -can be improved. - ---- - -## Phase 1 — Automated Signal Collection - -Review your own session transcript. No human interaction needed yet. - -### 1a: Skills inventory - -Before analyzing gaps and errors, inventory all skills that were available -during the session: - -- **Loaded and used:** Skills you read and actively followed. -- **Available but not loaded:** Skills that were installed (discoverable via - `npx @tanstack/intent@latest list`) but you never read. This is important — many issues stem from - the agent not loading the right skill, not from the skill itself being wrong. - -### 1b: Gap detection - -Identify moments where the skill was silent and you had to bridge the gap -yourself — via code reading, search, trial-and-error, or general knowledge. - -For each gap, note: - -- What you needed to do -- What the skill should have told you -- How you solved it (code reading, web search, guessing) - -### 1c: Error/correction tracking - -Identify moments where the skill prescribed an approach that produced an error. - -For each error, note: - -- What the skill said to do -- The error or incorrect behavior that resulted -- The fix you applied - -### 1d: Human intervention events - -Identify moments where the human clarified, corrected, or overrode your approach. - -For each intervention, note: - -- What you were doing when the human intervened -- What the human said or changed -- Whether the skill could have prevented this - -### 1e: Step duration anomalies - -Identify steps that consumed disproportionate effort compared to their apparent -complexity. These signal that the skill should provide a template, snippet, or -more detailed guidance. - ---- - -## Phase 2 — Human Interview - -Ask the human up to 4 questions. Keep it brief — skip questions if the session -already provided clear answers. Respect if they decline. - -1. "Was anything unclear about what was happening during the task?" -2. "Did anything feel frustrating or take longer than expected?" -3. "Were you uncertain about the output quality at any point?" -4. "Anything you'd want done differently next time?" - -Derive `userRating` from overall sentiment: - -- Mostly positive → `good` -- Mixed signals → `mixed` -- Mostly negative → `bad` - -If the human gives an explicit rating, use that instead. - ---- - -## Phase 3 — Build the Feedback - -Write one Markdown feedback file per skill used. Only include skills that were -actually used during the session — skip any that were loaded but never -referenced. - -### Template - -```markdown -# Skill Feedback: [skill name from SKILL.md frontmatter] - -**Package:** [npm package name that contains the skill] -**Skill version:** [metadata.version or library_version from frontmatter] -**Rating:** [good | mixed | bad] - -## Task - -[one-sentence summary of what the human asked you to do] - -## Skills Inventory - -**Loaded and used:** - -- [list each skill the agent read and actively followed during the session] - -**Available but not loaded:** - -- [list skills that were installed/available but the agent never read] - -## What Worked - -[patterns/instructions from the skill that were accurate and helpful] - -## What Failed - -[from 1c — skill instructions that produced errors] - -## Missing - -[from 1b — gaps where the skill should have covered] - -## Self-Corrections - -[from 1c fixes + 1d human interventions, combined] - -## User Comments - -[optional — direct quotes or paraphrased human input from Phase 2] -``` - -### Field derivation guide - -| Field | Source | -| ---------------- | ------------------------------------------------------------------ | -| Skill name | Frontmatter `name` field of the SKILL.md you loaded | -| Package | The npm package the skill lives in (e.g. `@tanstack/query-intent`) | -| Skill version | Frontmatter `metadata.version` or `library_version` | -| Task | Summarize the human's original request in one sentence | -| Skills Inventory | Which skills were loaded vs. available but not loaded (see below) | -| What Worked | List skill sections/patterns that were correct and useful | -| What Failed | From 1c — skill instructions that produced errors | -| Missing | From 1b — gaps where the skill was silent | -| Self-Corrections | From 1c fixes + 1d human interventions, combined | -| Rating | From Phase 2 sentiment analysis or explicit rating | -| User Comments | From Phase 2 answers, keep brief | - ---- - -## Phase 4 — Submit - -Determine the target repo from the skill's package. The repo is typically -derivable from the `repository` field in the package's `package.json`, or -from the `sources` field in the SKILL.md frontmatter. - -### Link to existing issues/discussions - -Before creating a new issue, search the target repo for existing issues or -discussions that match the feedback. Use `gh search issues` or the GitHub -web search with keywords from the "What Failed" and "Missing" sections. - -- If an **open issue** already describes the same problem, comment on it - with the feedback instead of creating a duplicate. Reference the skill - name and version in your comment. -- If a **closed issue** describes a problem the skill still gets wrong - (regression or stale skill content), reference the closed issue in the - new feedback issue body: `Related to #[number] — this was fixed in the -library but the skill still describes the old behavior.` -- If a **discussion thread** covers the same topic, link to it in the - feedback issue body so maintainers can see the community context. - -This prevents duplicate issues and gives maintainers richer context for -improving skills. - -### Privacy check - -Before submitting, determine whether the user's project is public or private. -Check with `gh repo view --json visibility` or look for a `private` field in -the project's `package.json`. If you can't determine visibility, assume private. - -**Private repos:** Feedback is submitted to a public issue tracker, so it must -not contain project-specific details. Before submission: - -1. Strip any project-specific code, file paths, internal API names, service - URLs, or business logic from all fields -2. Rewrite the "Task" field to describe the _type_ of task generically - (e.g. "set up authenticated data fetching" not "set up auth for our - internal billing API at api.acme.corp/billing") -3. Rewrite "What Failed" and "Missing" entries to reference only the - skill's own APIs and patterns, not the user's code -4. Show the sanitized feedback to the user and ask them to confirm it's - safe to submit before proceeding - -**Public repos:** No sanitization needed. Proceed directly to submission. - -### If `gh` CLI is available - -Submit directly as a GitHub issue: - -```bash -gh issue create --repo [owner/repo] --title "Skill Feedback: [skill-name] ([rating])" --label "skill:[skill-name]" --body-file intent-feedback.md -``` - -If the label doesn't exist, omit the `--label` flag — don't let a missing -label block submission. - -If submission succeeds, delete the feedback file. - -### If `gh` CLI is not available - -Tell the human: - -> "I've written skill feedback to `intent-feedback.md`. To submit it, -> open an issue at https://github.com/[owner/repo]/issues and paste the -> contents." diff --git a/packages/intent/meta/generate-skill/SKILL.md b/packages/intent/meta/generate-skill/SKILL.md index d0d7146..6056082 100644 --- a/packages/intent/meta/generate-skill/SKILL.md +++ b/packages/intent/meta/generate-skill/SKILL.md @@ -118,7 +118,7 @@ cannot already know: Before writing the skill body, search the library's GitHub repo for issues and discussions relevant to THIS skill's topic. This step is important for -both initial generation and regeneration — community feedback reveals +both initial generation and regeneration — issue discussions reveal failure modes that docs miss. **Search strategy:** @@ -443,11 +443,4 @@ Output is consumed by all major AI coding agents. To ensure consistency: - Critical info at start or end of sections (not buried in middle) - Each SKILL.md is self-contained except for declared `requires` ---- - -## Meta-skill feedback - -After generating all skills, run the `skill-feedback-collection` skill to -capture feedback about the scaffolding process (domain-discovery, -tree-generator, and generate-skill). ``` diff --git a/packages/intent/meta/tree-generator/SKILL.md b/packages/intent/meta/tree-generator/SKILL.md index 26b83a2..d948298 100644 --- a/packages/intent/meta/tree-generator/SKILL.md +++ b/packages/intent/meta/tree-generator/SKILL.md @@ -702,7 +702,7 @@ Run when: - The library has released a new version - A maintainer reports skills produce outdated code - A changelog or migration guide has been published since skill creation -- Feedback reports indicate skill content is inaccurate +- Issue reports indicate skill content is inaccurate ### Step 1 — Detect staleness diff --git a/packages/intent/src/commands/list.ts b/packages/intent/src/commands/list.ts index c5136b5..ea84286 100644 --- a/packages/intent/src/commands/list.ts +++ b/packages/intent/src/commands/list.ts @@ -162,15 +162,6 @@ export async function runListCommand( console.log() } - console.log('Feedback:') - console.log( - ' Submit feedback on skill usage to help maintainers improve the skills.', - ) - console.log( - ' Load: node_modules/@tanstack/intent/meta/feedback-collection/SKILL.md', - ) - console.log() - printWarnings(result.warnings) printNotices(result.notices, noticeOptions) } diff --git a/packages/intent/src/feedback.ts b/packages/intent/src/feedback.ts deleted file mode 100644 index d1561e3..0000000 --- a/packages/intent/src/feedback.ts +++ /dev/null @@ -1,406 +0,0 @@ -import { execFileSync, execSync } from 'node:child_process' -import { readFileSync, writeFileSync } from 'node:fs' -import { join } from 'node:path' -import type { - FeedbackFrequency, - FeedbackPayload, - IntentProjectConfig, - MetaFeedbackPayload, -} from './types.js' - -const META_FEEDBACK_REPO = 'TanStack/intent' - -// --------------------------------------------------------------------------- -// Secret detection -// --------------------------------------------------------------------------- - -const SECRET_PATTERNS = [ - /(?:ghp|gho|ghu|ghs|ghr)_[A-Za-z0-9_]{36,}/, // GitHub tokens - /(?:sk|pk)[-_](?:live|test)[-_][A-Za-z0-9]{24,}/, // Stripe keys - /AKIA[0-9A-Z]{16}/, // AWS access keys - /-----BEGIN (?:RSA |EC )?PRIVATE KEY-----/, // PEM private keys - /eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}/, // JWT-like tokens - /(?:Bearer|token)\s+[A-Za-z0-9_\-.~+/]{20,}/i, // Bearer tokens - /[A-Za-z0-9]{32,}(?=.*(?:key|secret|token|password))/i, // Generic secrets near keywords -] - -export function containsSecrets(text: string): boolean { - return SECRET_PATTERNS.some((pattern) => pattern.test(text)) -} - -// --------------------------------------------------------------------------- -// `gh` CLI detection -// --------------------------------------------------------------------------- - -export function hasGhCli(): boolean { - try { - execSync('gh --version', { stdio: 'ignore' }) - return true - } catch { - return false - } -} - -// --------------------------------------------------------------------------- -// Config resolution -// --------------------------------------------------------------------------- - -function getHomeConfigDir(): string { - return ( - process.env.XDG_CONFIG_HOME ?? - join(process.env.HOME ?? process.env.USERPROFILE ?? '', '.config') - ) -} - -function parseFrequency(value: unknown): FeedbackFrequency | null { - if (value === 'always' || value === 'never') return value - if (typeof value !== 'string') return null - - const match = /^every-(\d+)$/.exec(value) - if (!match) return null - - const count = Number(match[1]) - return Number.isInteger(count) && count > 0 - ? (`every-${count}` as FeedbackFrequency) - : null -} - -function readFrequency(filePath: string): FeedbackFrequency | null { - try { - const config = JSON.parse( - readFileSync(filePath, 'utf8'), - ) as Partial - return parseFrequency(config.feedback?.frequency) - } catch { - return null - } -} - -export function resolveFrequency(root: string): string { - // 1. User override (~/.config/intent/config.json) - const userConfigPath = join(getHomeConfigDir(), 'intent', 'config.json') - const userFrequency = readFrequency(userConfigPath) - if (userFrequency) return userFrequency - - // 2. Project config - const projectConfigPath = join(root, 'intent.config.json') - const projectFrequency = readFrequency(projectConfigPath) - if (projectFrequency) return projectFrequency - - // 3. Default - return 'every-5' -} - -// --------------------------------------------------------------------------- -// Feedback payload validation -// --------------------------------------------------------------------------- - -const REQUIRED_FIELDS: Array = [ - 'skill', - 'package', - 'skillVersion', - 'task', - 'whatWorked', - 'whatFailed', - 'missing', - 'selfCorrections', - 'userRating', -] - -export function validatePayload(payload: unknown): { - valid: boolean - errors: Array -} { - const errors: Array = [] - if (!payload || typeof payload !== 'object') { - return { valid: false, errors: ['Payload must be a JSON object'] } - } - const obj = payload as Record - - for (const field of REQUIRED_FIELDS) { - if (typeof obj[field] !== 'string' || obj[field].trim() === '') { - errors.push(`Missing or empty required field: ${field}`) - } - } - - if ( - obj.userRating && - !['good', 'mixed', 'bad'].includes(obj.userRating as string) - ) { - errors.push('userRating must be one of: good, mixed, bad') - } - - // Secret scan across all string values - const allText = Object.values(obj) - .filter((v) => typeof v === 'string') - .join('\n') - - if (containsSecrets(allText)) { - errors.push( - 'Payload appears to contain secrets or tokens — submission rejected', - ) - } - - return { valid: errors.length === 0, errors } -} - -// --------------------------------------------------------------------------- -// Meta-feedback payload validation -// --------------------------------------------------------------------------- - -const META_REQUIRED_FIELDS: Array = [ - 'metaSkill', - 'library', - 'agentUsed', - 'artifactQuality', - 'whatWorked', - 'whatFailed', - 'suggestions', - 'userRating', -] - -const VALID_META_SKILLS = [ - 'domain-discovery', - 'tree-generator', - 'generate-skill', - 'skill-staleness-check', -] - -const VALID_AGENTS = ['claude-code', 'cursor', 'copilot', 'codex', 'other'] - -const VALID_QUALITY_RATINGS = ['good', 'mixed', 'bad'] -const VALID_INTERVIEW_QUALITY_RATINGS = ['good', 'mixed', 'bad', 'skipped'] -const VALID_FAILURE_MODE_QUALITY_RATINGS = [ - 'good', - 'mixed', - 'bad', - 'not-applicable', -] - -export function validateMetaPayload(payload: unknown): { - valid: boolean - errors: Array -} { - const errors: Array = [] - if (!payload || typeof payload !== 'object') { - return { valid: false, errors: ['Payload must be a JSON object'] } - } - const obj = payload as Record - - for (const field of META_REQUIRED_FIELDS) { - if (typeof obj[field] !== 'string' || obj[field].trim() === '') { - errors.push(`Missing or empty required field: ${field}`) - } - } - - if (obj.metaSkill && !VALID_META_SKILLS.includes(obj.metaSkill as string)) { - errors.push(`metaSkill must be one of: ${VALID_META_SKILLS.join(', ')}`) - } - - if (obj.agentUsed && !VALID_AGENTS.includes(obj.agentUsed as string)) { - errors.push(`agentUsed must be one of: ${VALID_AGENTS.join(', ')}`) - } - - if ( - obj.artifactQuality && - !VALID_QUALITY_RATINGS.includes(obj.artifactQuality as string) - ) { - errors.push('artifactQuality must be one of: good, mixed, bad') - } - - if ( - obj.userRating && - !VALID_QUALITY_RATINGS.includes(obj.userRating as string) - ) { - errors.push('userRating must be one of: good, mixed, bad') - } - - if ( - obj.interviewQuality && - !VALID_INTERVIEW_QUALITY_RATINGS.includes(obj.interviewQuality as string) - ) { - errors.push('interviewQuality must be one of: good, mixed, bad, skipped') - } - - if ( - obj.failureModeQuality && - !VALID_FAILURE_MODE_QUALITY_RATINGS.includes( - obj.failureModeQuality as string, - ) - ) { - errors.push( - 'failureModeQuality must be one of: good, mixed, bad, not-applicable', - ) - } - - // Secret scan - const allText = Object.values(obj) - .filter((v) => typeof v === 'string') - .join('\n') - - if (containsSecrets(allText)) { - errors.push( - 'Payload appears to contain secrets or tokens — submission rejected', - ) - } - - return { valid: errors.length === 0, errors } -} - -// --------------------------------------------------------------------------- -// Markdown conversion -// --------------------------------------------------------------------------- - -export function metaToMarkdown(payload: MetaFeedbackPayload): string { - const lines = [ - `# Meta-Skill Feedback: ${payload.metaSkill}`, - '', - `**Library:** ${payload.library}`, - `**Agent:** ${payload.agentUsed}`, - `**Artifact quality:** ${payload.artifactQuality}`, - `**Rating:** ${payload.userRating}`, - ] - - if (payload.interviewQuality) { - lines.push(`**Interview quality:** ${payload.interviewQuality}`) - } - if (payload.failureModeQuality) { - lines.push(`**Failure mode quality:** ${payload.failureModeQuality}`) - } - - lines.push( - '', - '## What Worked', - payload.whatWorked, - '', - '## What Failed', - payload.whatFailed, - '', - '## Suggestions', - payload.suggestions, - ) - - return lines.join('\n') + '\n' -} - -export function toMarkdown(payload: FeedbackPayload): string { - const lines = [ - `# Skill Feedback: ${payload.skill}`, - '', - `**Package:** ${payload.package}`, - `**Skill version:** ${payload.skillVersion}`, - `**Rating:** ${payload.userRating}`, - '', - '## Task', - payload.task, - '', - '## What Worked', - payload.whatWorked, - '', - '## What Failed', - payload.whatFailed, - '', - '## Missing', - payload.missing, - '', - '## Self-Corrections', - payload.selfCorrections, - ] - - if (payload.userComments) { - lines.push('', '## User Comments', payload.userComments) - } - - return lines.join('\n') + '\n' -} - -// --------------------------------------------------------------------------- -// Submission -// --------------------------------------------------------------------------- - -export interface SubmitResult { - method: 'gh' | 'file' | 'stdout' - detail: string -} - -export function submitFeedback( - payload: FeedbackPayload, - repo: string, - opts: { ghAvailable: boolean; outputPath?: string }, -): SubmitResult { - const md = toMarkdown(payload) - - // Try gh - if (opts.ghAvailable) { - try { - const title = `Skill Feedback: ${payload.skill} (${payload.userRating})` - execFileSync( - 'gh', - ['issue', 'create', '--repo', repo, '--title', title, '--body', '-'], - { input: md, stdio: ['pipe', 'pipe', 'pipe'] }, - ) - return { method: 'gh', detail: `Submitted issue to ${repo}` } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err) - console.error(`GitHub submission failed: ${msg}`) - console.error('Falling back to file output.') - } - } - - // File fallback - if (opts.outputPath) { - writeFileSync(opts.outputPath, md, 'utf8') - return { method: 'file', detail: `Saved to ${opts.outputPath}` } - } - - // Stdout fallback - return { method: 'stdout', detail: md } -} - -// --------------------------------------------------------------------------- -// Meta-feedback submission -// --------------------------------------------------------------------------- - -export function submitMetaFeedback( - payload: MetaFeedbackPayload, - opts: { ghAvailable: boolean; outputPath?: string }, -): SubmitResult { - const md = metaToMarkdown(payload) - - if (opts.ghAvailable) { - try { - const title = `Meta-Skill Feedback: ${payload.metaSkill} (${payload.userRating})` - execFileSync( - 'gh', - [ - 'issue', - 'create', - '--repo', - META_FEEDBACK_REPO, - '--title', - title, - '--label', - `skill:${payload.metaSkill}`, - '--body', - '-', - ], - { input: md, stdio: ['pipe', 'pipe', 'pipe'] }, - ) - return { - method: 'gh', - detail: `Submitted issue to ${META_FEEDBACK_REPO}`, - } - } catch (err) { - const msg = err instanceof Error ? err.message : String(err) - console.error(`GitHub submission failed: ${msg}`) - console.error('Falling back to file output.') - } - } - - if (opts.outputPath) { - writeFileSync(opts.outputPath, md, 'utf8') - return { method: 'file', detail: `Saved to ${opts.outputPath}` } - } - - return { method: 'stdout', detail: md } -} diff --git a/packages/intent/src/index.ts b/packages/intent/src/index.ts index 880eb6d..75b49bd 100644 --- a/packages/intent/src/index.ts +++ b/packages/intent/src/index.ts @@ -7,17 +7,6 @@ export { createFailedStaleReviewItem, type StaleReviewItem, } from './workflow-review.js' -export { - containsSecrets, - hasGhCli, - metaToMarkdown, - resolveFrequency, - submitFeedback, - submitMetaFeedback, - toMarkdown, - validateMetaPayload, - validatePayload, -} from './feedback.js' export { findSkillFiles, getDeps, @@ -45,8 +34,6 @@ export type { SetupGithubActionsResult, } from './setup.js' export type { - AgentName, - FeedbackPayload, IntentConfig, IntentArtifactCoverageIgnore, IntentArtifactFile, @@ -54,9 +41,6 @@ export type { IntentArtifactSkill, IntentArtifactWarning, IntentPackage, - IntentProjectConfig, - MetaFeedbackPayload, - MetaSkillName, ScanOptions, ScanResult, SkillEntry, diff --git a/packages/intent/src/types.ts b/packages/intent/src/types.ts index 1abc747..00e5228 100644 --- a/packages/intent/src/types.ts +++ b/packages/intent/src/types.ts @@ -146,57 +146,3 @@ export interface IntentArtifactWarning { artifactPath: string message: string } - -// --------------------------------------------------------------------------- -// Feedback types -// --------------------------------------------------------------------------- - -export interface FeedbackPayload { - skill: string - package: string - skillVersion: string - task: string - whatWorked: string - whatFailed: string - missing: string - selfCorrections: string - userRating: 'good' | 'mixed' | 'bad' - userComments?: string -} - -// --------------------------------------------------------------------------- -// Meta-skill feedback types -// --------------------------------------------------------------------------- - -export type MetaSkillName = - | 'domain-discovery' - | 'tree-generator' - | 'generate-skill' - | 'skill-staleness-check' - -export type AgentName = 'claude-code' | 'cursor' | 'copilot' | 'codex' | 'other' - -export interface MetaFeedbackPayload { - metaSkill: MetaSkillName - library: string - agentUsed: AgentName - artifactQuality: 'good' | 'mixed' | 'bad' - interviewQuality?: 'good' | 'mixed' | 'bad' | 'skipped' - failureModeQuality?: 'good' | 'mixed' | 'bad' | 'not-applicable' - whatWorked: string - whatFailed: string - suggestions: string - userRating: 'good' | 'mixed' | 'bad' -} - -export type FeedbackFrequency = 'always' | 'never' | `every-${number}` - -// --------------------------------------------------------------------------- -// Config types -// --------------------------------------------------------------------------- - -export interface IntentProjectConfig { - feedback: { - frequency: FeedbackFrequency - } -} diff --git a/packages/intent/tests/feedback.test.ts b/packages/intent/tests/feedback.test.ts deleted file mode 100644 index cbeb231..0000000 --- a/packages/intent/tests/feedback.test.ts +++ /dev/null @@ -1,293 +0,0 @@ -import { - existsSync, - mkdirSync, - readFileSync, - rmSync, - writeFileSync, -} from 'node:fs' -import { join } from 'node:path' -import { tmpdir } from 'node:os' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - containsSecrets, - resolveFrequency, - submitFeedback, - toMarkdown, - validatePayload, -} from '../src/feedback.js' -import type { FeedbackPayload } from '../src/types.js' - -// --------------------------------------------------------------------------- -// Fixture helpers -// --------------------------------------------------------------------------- - -let tmpDir: string - -function setupDir(): string { - const dir = join( - tmpdir(), - `intent-fb-test-${Date.now()}-${Math.random().toString(36).slice(2)}`, - ) - mkdirSync(dir, { recursive: true }) - return dir -} - -function validPayload( - overrides: Partial = {}, -): FeedbackPayload { - return { - skill: 'db-core/live-queries', - package: '@tanstack/db', - skillVersion: '0.5.0', - task: 'Set up a live query subscription', - whatWorked: 'Query builder syntax was great', - whatFailed: 'Collection creation missed an import', - missing: 'No examples for nested joins', - selfCorrections: 'Had to add missing import manually', - userRating: 'good', - ...overrides, - } -} - -beforeEach(() => { - tmpDir = setupDir() -}) - -afterEach(() => { - if (existsSync(tmpDir)) { - rmSync(tmpDir, { recursive: true, force: true }) - } -}) - -// --------------------------------------------------------------------------- -// containsSecrets -// --------------------------------------------------------------------------- - -describe('containsSecrets', () => { - it('detects GitHub tokens', () => { - expect(containsSecrets('token: ghp_' + 'A'.repeat(36))).toBe(true) - }) - - it('detects AWS access keys', () => { - expect(containsSecrets('key: AKIA' + '0'.repeat(16))).toBe(true) - }) - - it('detects PEM private keys', () => { - expect(containsSecrets('-----BEGIN RSA PRIVATE KEY-----')).toBe(true) - }) - - it('detects Stripe keys', () => { - expect(containsSecrets('sk-live-' + 'a'.repeat(24))).toBe(true) - }) - - it('detects Bearer tokens', () => { - expect(containsSecrets('Bearer ' + 'a'.repeat(30))).toBe(true) - }) - - it('does not flag normal text', () => { - expect( - containsSecrets( - 'This is a perfectly normal feedback message about queries.', - ), - ).toBe(false) - }) - - it('does not flag short strings', () => { - expect(containsSecrets('key=abc123')).toBe(false) - }) -}) - -// --------------------------------------------------------------------------- -// validatePayload -// --------------------------------------------------------------------------- - -describe('validatePayload', () => { - it('accepts a valid payload', () => { - const result = validatePayload(validPayload()) - expect(result.valid).toBe(true) - expect(result.errors).toHaveLength(0) - }) - - it('rejects non-object input', () => { - const result = validatePayload('not an object') - expect(result.valid).toBe(false) - expect(result.errors[0]).toContain('JSON object') - }) - - it('rejects null', () => { - const result = validatePayload(null) - expect(result.valid).toBe(false) - }) - - it('reports missing required fields', () => { - const result = validatePayload({ skill: 'test' }) - expect(result.valid).toBe(false) - expect(result.errors.length).toBeGreaterThan(1) - expect(result.errors.some((e) => e.includes('package'))).toBe(true) - }) - - it('rejects empty string fields', () => { - const result = validatePayload(validPayload({ task: '' })) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('task'))).toBe(true) - }) - - it('rejects invalid userRating', () => { - const result = validatePayload( - validPayload({ userRating: 'excellent' as 'good' }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('userRating'))).toBe(true) - }) - - it('rejects payloads containing secrets', () => { - const result = validatePayload( - validPayload({ - whatFailed: 'Used token ghp_' + 'A'.repeat(36) + ' and it failed', - }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('secrets'))).toBe(true) - }) -}) - -// --------------------------------------------------------------------------- -// toMarkdown -// --------------------------------------------------------------------------- - -describe('toMarkdown', () => { - it('converts payload to markdown', () => { - const md = toMarkdown(validPayload()) - expect(md).toContain('# Skill Feedback: db-core/live-queries') - expect(md).toContain('**Package:** @tanstack/db') - expect(md).toContain('**Rating:** good') - expect(md).toContain('## Task') - expect(md).toContain('## What Worked') - expect(md).toContain('## What Failed') - expect(md).toContain('## Missing') - expect(md).toContain('## Self-Corrections') - }) - - it('includes user comments when present', () => { - const md = toMarkdown(validPayload({ userComments: 'Great overall!' })) - expect(md).toContain('## User Comments') - expect(md).toContain('Great overall!') - }) - - it('omits user comments section when not present', () => { - const md = toMarkdown(validPayload()) - expect(md).not.toContain('## User Comments') - }) -}) - -// --------------------------------------------------------------------------- -// submitFeedback -// --------------------------------------------------------------------------- - -describe('submitFeedback', () => { - it('saves to file when gh not available and outputPath given', () => { - const outPath = join(tmpDir, 'feedback.md') - const result = submitFeedback(validPayload(), 'tanstack/db', { - ghAvailable: false, - outputPath: outPath, - }) - expect(result.method).toBe('file') - expect(result.detail).toContain(outPath) - expect(existsSync(outPath)).toBe(true) - const content = readFileSync(outPath, 'utf8') - expect(content).toContain('# Skill Feedback') - }) - - it('returns stdout when no gh and no outputPath', () => { - const result = submitFeedback(validPayload(), 'tanstack/db', { - ghAvailable: false, - }) - expect(result.method).toBe('stdout') - expect(result.detail).toContain('# Skill Feedback') - }) -}) - -// --------------------------------------------------------------------------- -// resolveFrequency -// --------------------------------------------------------------------------- - -describe('resolveFrequency', () => { - it('returns project config frequency when set', () => { - writeFileSync( - join(tmpDir, 'intent.config.json'), - JSON.stringify({ feedback: { frequency: 'always' } }), - ) - expect(resolveFrequency(tmpDir)).toBe('always') - }) - - it('returns default when no config exists', () => { - expect(resolveFrequency(tmpDir)).toBe('every-5') - }) - - it('ignores invalid project config values and falls back to default', () => { - writeFileSync( - join(tmpDir, 'intent.config.json'), - JSON.stringify({ feedback: { frequency: 'sometimes' } }), - ) - - expect(resolveFrequency(tmpDir)).toBe('every-5') - }) - - it('accepts validated every-N frequencies', () => { - writeFileSync( - join(tmpDir, 'intent.config.json'), - JSON.stringify({ feedback: { frequency: 'every-12' } }), - ) - - expect(resolveFrequency(tmpDir)).toBe('every-12') - }) - - it('reads user override via XDG_CONFIG_HOME', () => { - const configDir = join(tmpDir, 'xdg') - mkdirSync(join(configDir, 'intent'), { recursive: true }) - writeFileSync( - join(configDir, 'intent', 'config.json'), - JSON.stringify({ feedback: { frequency: 'never' } }), - ) - - const originalXdg = process.env.XDG_CONFIG_HOME - process.env.XDG_CONFIG_HOME = configDir - try { - expect(resolveFrequency(tmpDir)).toBe('never') - } finally { - if (originalXdg !== undefined) { - process.env.XDG_CONFIG_HOME = originalXdg - } else { - delete process.env.XDG_CONFIG_HOME - } - } - }) - - it('user override takes precedence over project config', () => { - // Project says "always" - writeFileSync( - join(tmpDir, 'intent.config.json'), - JSON.stringify({ feedback: { frequency: 'always' } }), - ) - - // User override says "never" - const configDir = join(tmpDir, 'xdg2') - mkdirSync(join(configDir, 'intent'), { recursive: true }) - writeFileSync( - join(configDir, 'intent', 'config.json'), - JSON.stringify({ feedback: { frequency: 'never' } }), - ) - - const originalXdg = process.env.XDG_CONFIG_HOME - process.env.XDG_CONFIG_HOME = configDir - try { - expect(resolveFrequency(tmpDir)).toBe('never') - } finally { - if (originalXdg !== undefined) { - process.env.XDG_CONFIG_HOME = originalXdg - } else { - delete process.env.XDG_CONFIG_HOME - } - } - }) -}) diff --git a/packages/intent/tests/meta-feedback.test.ts b/packages/intent/tests/meta-feedback.test.ts deleted file mode 100644 index 2217c87..0000000 --- a/packages/intent/tests/meta-feedback.test.ts +++ /dev/null @@ -1,208 +0,0 @@ -import { existsSync, mkdirSync, readFileSync, rmSync } from 'node:fs' -import { join } from 'node:path' -import { tmpdir } from 'node:os' -import { afterEach, beforeEach, describe, expect, it } from 'vitest' -import { - metaToMarkdown, - submitMetaFeedback, - validateMetaPayload, -} from '../src/feedback.js' -import type { MetaFeedbackPayload } from '../src/types.js' - -// --------------------------------------------------------------------------- -// Fixtures -// --------------------------------------------------------------------------- - -let tmpDir: string - -function setupDir(): string { - const dir = join( - tmpdir(), - `intent-meta-fb-test-${Date.now()}-${Math.random().toString(36).slice(2)}`, - ) - mkdirSync(dir, { recursive: true }) - return dir -} - -function validMetaPayload( - overrides: Partial = {}, -): MetaFeedbackPayload { - return { - metaSkill: 'domain-discovery', - library: '@tanstack/query', - agentUsed: 'claude-code', - artifactQuality: 'good', - whatWorked: 'Interview questions surfaced real failure modes', - whatFailed: 'Missed some SSR-specific gotchas', - suggestions: 'Add more framework-specific probing', - userRating: 'mixed', - ...overrides, - } -} - -beforeEach(() => { - tmpDir = setupDir() -}) - -afterEach(() => { - if (existsSync(tmpDir)) { - rmSync(tmpDir, { recursive: true, force: true }) - } -}) - -// --------------------------------------------------------------------------- -// validateMetaPayload -// --------------------------------------------------------------------------- - -describe('validateMetaPayload', () => { - it('accepts a valid payload', () => { - const result = validateMetaPayload(validMetaPayload()) - expect(result.valid).toBe(true) - expect(result.errors).toHaveLength(0) - }) - - it('rejects non-object input', () => { - const result = validateMetaPayload('not an object') - expect(result.valid).toBe(false) - expect(result.errors[0]).toContain('JSON object') - }) - - it('rejects null', () => { - const result = validateMetaPayload(null) - expect(result.valid).toBe(false) - }) - - it('reports missing required fields', () => { - const result = validateMetaPayload({ metaSkill: 'domain-discovery' }) - expect(result.valid).toBe(false) - expect(result.errors.length).toBeGreaterThan(1) - expect(result.errors.some((e) => e.includes('library'))).toBe(true) - }) - - it('rejects empty string fields', () => { - const result = validateMetaPayload(validMetaPayload({ whatWorked: '' })) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('whatWorked'))).toBe(true) - }) - - it('rejects invalid metaSkill', () => { - const result = validateMetaPayload( - validMetaPayload({ metaSkill: 'not-a-skill' as any }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('metaSkill'))).toBe(true) - }) - - it('rejects invalid agentUsed', () => { - const result = validateMetaPayload( - validMetaPayload({ agentUsed: 'chatgpt' as any }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('agentUsed'))).toBe(true) - }) - - it('rejects invalid userRating', () => { - const result = validateMetaPayload( - validMetaPayload({ userRating: 'excellent' as any }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('userRating'))).toBe(true) - }) - - it('rejects payloads containing secrets', () => { - const result = validateMetaPayload( - validMetaPayload({ - whatFailed: 'Used token ghp_' + 'A'.repeat(36) + ' and it failed', - }), - ) - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('secrets'))).toBe(true) - }) - - it('accepts optional fields', () => { - const result = validateMetaPayload( - validMetaPayload({ - interviewQuality: 'good', - failureModeQuality: 'mixed', - }), - ) - expect(result.valid).toBe(true) - }) - - it('rejects invalid optional quality fields', () => { - const result = validateMetaPayload( - validMetaPayload({ - interviewQuality: 'excellent' as any, - failureModeQuality: 'unknown' as any, - }), - ) - - expect(result.valid).toBe(false) - expect(result.errors.some((e) => e.includes('interviewQuality'))).toBe(true) - expect(result.errors.some((e) => e.includes('failureModeQuality'))).toBe( - true, - ) - }) -}) - -// --------------------------------------------------------------------------- -// metaToMarkdown -// --------------------------------------------------------------------------- - -describe('metaToMarkdown', () => { - it('converts payload to markdown', () => { - const md = metaToMarkdown(validMetaPayload()) - expect(md).toContain('# Meta-Skill Feedback: domain-discovery') - expect(md).toContain('**Library:** @tanstack/query') - expect(md).toContain('**Agent:** claude-code') - expect(md).toContain('**Artifact quality:** good') - expect(md).toContain('**Rating:** mixed') - expect(md).toContain('## What Worked') - expect(md).toContain('## What Failed') - expect(md).toContain('## Suggestions') - }) - - it('includes optional quality fields when present', () => { - const md = metaToMarkdown( - validMetaPayload({ - interviewQuality: 'good', - failureModeQuality: 'bad', - }), - ) - expect(md).toContain('**Interview quality:** good') - expect(md).toContain('**Failure mode quality:** bad') - }) - - it('omits optional quality fields when not present', () => { - const md = metaToMarkdown(validMetaPayload()) - expect(md).not.toContain('Interview quality') - expect(md).not.toContain('Failure mode quality') - }) -}) - -// --------------------------------------------------------------------------- -// submitMetaFeedback -// --------------------------------------------------------------------------- - -describe('submitMetaFeedback', () => { - it('saves to file when gh not available and outputPath given', () => { - const outPath = join(tmpDir, 'meta-feedback.md') - const result = submitMetaFeedback(validMetaPayload(), { - ghAvailable: false, - outputPath: outPath, - }) - expect(result.method).toBe('file') - expect(result.detail).toContain(outPath) - expect(existsSync(outPath)).toBe(true) - const content = readFileSync(outPath, 'utf8') - expect(content).toContain('# Meta-Skill Feedback') - }) - - it('returns stdout when no gh and no outputPath', () => { - const result = submitMetaFeedback(validMetaPayload(), { - ghAvailable: false, - }) - expect(result.method).toBe('stdout') - expect(result.detail).toContain('# Meta-Skill Feedback') - }) -}) diff --git a/scripts/validate-skills.ts b/scripts/validate-skills.ts index 422f7d2..68885c1 100644 --- a/scripts/validate-skills.ts +++ b/scripts/validate-skills.ts @@ -41,7 +41,6 @@ const PROHIBITED_PATTERNS: Array<{ pattern: RegExp; description: string }> = [ const ALLOWED_SHELL_COMMANDS = [ 'intent list', - 'intent feedback', 'npm install @tanstack/', 'npx @tanstack/intent', ]