feat: implement LLM output sanitization, refine Diataxis prompts with explicit formatting rules, and update GitHub docs path to content/docs while escaping frontmatter quotes.

Author: jenilsavani9

apps/web/app/api/projects/route.ts

@@ -34,13 +34,15 @@ export async function POST(req: NextRequest) {
     // Use the access token (ghu_*) to authenticate
     const octokit = new Octokit({ auth: tokenResult.accessToken });
 
-    // Create Repo in user's account
-    const repoName = sanitizeRepoName(name);
-    const { data: createdRepo } = await octokit.request("POST /user/repos", {
+    // Create Repo from template in user's account
+    const repoName = `${sanitizeRepoName(name)}-docs`;
+    const { data: createdRepo } = await octokit.request("POST /repos/{template_owner}/{template_repo}/generate", {
+      template_owner: "SecrinLabs",
+      template_repo: "secrin-docs-template",
       name: repoName,
-      description: description || `Project: ${name}`,
+      description: description || `Documentation for ${name} - powered by Secrin`,
       private: true,
-      auto_init: true,
+      include_all_branches: false,
       headers: {
         "X-GitHub-Api-Version": "2022-11-28",
       },

packages/arc42gen/diataxis/generator.py

@@ -12,6 +12,7 @@
 from ..providers.base import BaseLLMProvider
 from ..providers.factory import create_llm_provider
 from ..templates import load_prompt
+from ..utils.sanitizer import sanitize_llm_content, sanitize_list_item, is_placeholder_content
 from .models import (
     Tutorial,
     TutorialStep,
@@ -84,15 +85,15 @@ def _build_tutorial_prompt(self, analysis: AnalysisResult) -> str:
         )
 
     def _parse_tutorial_response(self, response: str, default_title: str) -> Tutorial:
-        goal = self._extract_section(response, "GOAL") or "Complete the tutorial"
-        time = self._extract_section(response, "TIME") or ""
+        goal = sanitize_llm_content(self._extract_section(response, "GOAL") or "Complete the tutorial")
+        time = sanitize_llm_content(self._extract_section(response, "TIME") or "")
 
         prerequisites = []
         prereq_text = self._extract_section(response, "PREREQUISITES")
         if prereq_text:
             for line in prereq_text.split('\n'):
-                line = line.strip().lstrip('-').strip()
-                if line:
+                line = sanitize_list_item(line.strip().lstrip('-').strip())
+                if line and not is_placeholder_content(line):
                     prerequisites.append(line)
 
         steps = []
@@ -106,21 +107,21 @@ def _parse_tutorial_response(self, response: str, default_title: str) -> Tutoria
 
                 title_match = re.search(r"TITLE:\s*(.+?)(?=\n|INSTRUCTIONS)", step_text)
                 if title_match:
-                    title = title_match.group(1).strip()
+                    title = sanitize_llm_content(title_match.group(1).strip())
 
                 instr_match = re.search(r"INSTRUCTIONS:\s*(.+?)(?=\nCODE|CHECKPOINT|$)", step_text, re.DOTALL)
                 if instr_match:
-                    instructions = instr_match.group(1).strip()
+                    instructions = sanitize_llm_content(instr_match.group(1).strip())
 
                 code_match = re.search(r"CODE:\s*(.+?)(?=\nCHECKPOINT|$)", step_text, re.DOTALL)
                 if code_match:
-                    code = code_match.group(1).strip()
+                    code = code_match.group(1).strip()  # Don't sanitize code
 
                 check_match = re.search(r"CHECKPOINT:\s*(.+?)$", step_text, re.DOTALL)
                 if check_match:
-                    checkpoint = check_match.group(1).strip()
+                    checkpoint = sanitize_llm_content(check_match.group(1).strip())
 
-                if title:
+                if title and not is_placeholder_content(title):
                     steps.append(TutorialStep(
                         title=title,
                         instructions=instructions,
@@ -132,8 +133,8 @@ def _parse_tutorial_response(self, response: str, default_title: str) -> Tutoria
         next_text = self._extract_section(response, "NEXT_STEPS")
         if next_text:
             for line in next_text.split('\n'):
-                line = line.strip().lstrip('-').strip()
-                if line:
+                line = sanitize_list_item(line.strip().lstrip('-').strip())
+                if line and not is_placeholder_content(line):
                     next_steps.append(line)
 
         return Tutorial(
@@ -177,34 +178,55 @@ def _build_troubleshooting_prompt(self, analysis: AnalysisResult) -> str:
         )
 
     def _parse_howto_response(self, response: str, default_title: str) -> HowToGuide:
-        problem = self._extract_section(response, "PROBLEM") or ""
-        time = self._extract_section(response, "TIME") or ""
+        problem = sanitize_llm_content(self._extract_section(response, "PROBLEM") or "")
+        time = sanitize_llm_content(self._extract_section(response, "TIME") or "")
 
         prerequisites = []
         prereq_text = self._extract_section(response, "PREREQUISITES")
         if prereq_text:
             for line in prereq_text.split('\n'):
-                line = line.strip().lstrip('-').strip()
-                if line:
+                line = sanitize_list_item(line.strip().lstrip('-').strip())
+                if line and not is_placeholder_content(line):
                     prerequisites.append(line)
 
         steps = []
         steps_text = self._extract_section(response, "STEPS")
         if steps_text:
             for line in steps_text.split('\n'):
-                line = line.strip().lstrip('-').lstrip('0123456789.').strip()
-                if line:
+                line = sanitize_list_item(line.strip().lstrip('-').lstrip('0123456789.').strip())
+                if line and not is_placeholder_content(line):
                     steps.append(line)
 
         troubleshooting = []
-        for i in range(1, 10):
-            prob = self._extract_section(response, f"PROBLEM_{i}")
-            sol = self._extract_section(response, f"SOLUTION_{i}")
-            if prob and sol:
-                troubleshooting.append(TroubleshootingItem(
-                    problem=prob,
-                    solution=sol,
-                ))
+        # Parse new ISSUE/FIX format
+        troubleshooting_text = self._extract_section(response, "TROUBLESHOOTING")
+        if troubleshooting_text:
+            issue_pattern = r'ISSUE:\s*(.+?)(?=\nFIX:|$)'
+            fix_pattern = r'FIX:\s*(.+?)(?=\nISSUE:|$)'
+            issues = re.findall(issue_pattern, troubleshooting_text, re.DOTALL)
+            fixes = re.findall(fix_pattern, troubleshooting_text, re.DOTALL)
+            for issue, fix in zip(issues, fixes):
+                issue_clean = sanitize_llm_content(issue)
+                fix_clean = sanitize_llm_content(fix)
+                if issue_clean and fix_clean and not is_placeholder_content(issue_clean):
+                    troubleshooting.append(TroubleshootingItem(
+                        problem=issue_clean,
+                        solution=fix_clean,
+                    ))
+        
+        # Fallback: try old PROBLEM_N/SOLUTION_N format for backward compatibility
+        if not troubleshooting:
+            for i in range(1, 10):
+                prob = self._extract_section(response, f"PROBLEM_{i}")
+                sol = self._extract_section(response, f"SOLUTION_{i}")
+                if prob and sol:
+                    prob_clean = sanitize_llm_content(prob)
+                    sol_clean = sanitize_llm_content(sol)
+                    if prob_clean and sol_clean and not is_placeholder_content(prob_clean):
+                        troubleshooting.append(TroubleshootingItem(
+                            problem=prob_clean,
+                            solution=sol_clean,
+                        ))
 
         return HowToGuide(
             title=default_title,

packages/arc42gen/jobs.py

@@ -45,12 +45,15 @@ def _update_progress(step: int, total: int, message: str, status: str = "running
 
 def add_mdx_frontmatter(content: str, title: str, description: str = "") -> str:
     """Add MDX frontmatter for Fumadocs compatibility."""
-    frontmatter = f"""---
-title: {title}
-description: {description or title}
+    # Escape double quotes in title and description for YAML safety
+    safe_title = title.replace('"', '\\"')
+    safe_description = (description or title).replace('"', '\\"')
+    frontmatter = f'''---
+title: "{safe_title}"
+description: "{safe_description}"
 ---
 
-"""
+'''
     return frontmatter + content
 
 
@@ -72,13 +75,13 @@ def _commit_to_github(
     ref = repo.get_git_ref(f"heads/{branch}")
     latest_commit = repo.get_git_commit(ref.object.sha)
 
-    # Create tree elements for all files (under docs/ folder)
+    # Create tree elements for all files (under content/docs/ folder)
     tree_elements = []
     for path, content in files.items():
         blob = repo.create_git_blob(content, "utf-8")
         tree_elements.append(
             InputGitTreeElement(
-                path=f"docs/{path}",
+                path=f"content/docs/{path}",
                 mode="100644",
                 type="blob",
                 sha=blob.sha,

packages/arc42gen/templates/prompts/diataxis/howto.txt

@@ -5,26 +5,32 @@ Create a How-To guide for "{topic}" for this codebase:
 REPOSITORY: {repo_name}
 LANGUAGE: {language}
 
+IMPORTANT FORMATTING RULES:
+- Write final, complete content - NOT placeholders or templates
+- Do NOT use brackets like [description] or [step here]
+- Do NOT include template markers in your output
+- Write clear, actionable text that can be used directly
+
 Generate a how-to guide with EXACTLY this format:
 
 PROBLEM:
-[What problem does this guide solve]
+A clear description of what problem this guide helps solve
 
 PREREQUISITES:
-- [Prerequisite 1]
-- [Prerequisite 2]
+- First prerequisite needed
+- Second prerequisite needed
 
 TIME:
-[Estimated time]
+Estimated time to complete
 
 STEPS:
-- [Step 1]
-- [Step 2]
-- [Step 3]
+- First step with clear instructions
+- Second step with clear instructions
+- Third step with clear instructions
 
 TROUBLESHOOTING:
-PROBLEM_1: [Common problem]
-SOLUTION_1: [How to fix it]
+ISSUE: Description of a common problem developers might face
+FIX: How to resolve this issue
 
-PROBLEM_2: [Another common problem]
-SOLUTION_2: [How to fix it]
+ISSUE: Description of another common problem
+FIX: How to resolve this issue

packages/arc42gen/templates/prompts/diataxis/troubleshooting.txt

@@ -5,6 +5,12 @@ Create a Troubleshooting guide for this codebase:
 REPOSITORY: {repo_name}
 LANGUAGE: {language}
 
+IMPORTANT FORMATTING RULES:
+- Write final, complete content - NOT placeholders or templates
+- Do NOT use brackets like [description] or [step here]
+- Do NOT include template markers in your output
+- Write clear, actionable text that can be used directly
+
 Generate a troubleshooting guide with EXACTLY this format:
 
 PROBLEM:
@@ -15,18 +21,18 @@ PREREQUISITES:
 - Project set up locally
 
 STEPS:
-- Identify the error message
-- Check the troubleshooting section below
-- Follow the solution steps
+- Identify the error message you are seeing
+- Find the matching issue in the troubleshooting section below
+- Follow the solution steps provided
 
 TROUBLESHOOTING:
-PROBLEM_1: [Common issue like "Dependencies fail to install"]
-SOLUTION_1: [How to fix it]
+ISSUE: Dependencies fail to install or version conflicts occur
+FIX: Delete node_modules or virtual environment, clear cache, and reinstall dependencies
 
-PROBLEM_2: [Common issue like "Tests fail"]
-SOLUTION_2: [How to fix it]
+ISSUE: Tests fail unexpectedly on a fresh clone
+FIX: Ensure all environment variables are set and database migrations are run
 
-PROBLEM_3: [Common issue]
-SOLUTION_3: [How to fix it]
+ISSUE: Build or compilation errors after pulling latest changes
+FIX: Check for breaking changes in dependencies and update configuration if needed
 
-List 3-5 common problems developers might face.
+List 3-5 common problems developers might face based on the codebase.

packages/arc42gen/templates/prompts/diataxis/tutorial.txt

@@ -7,38 +7,45 @@ LANGUAGE: {language}
 MODULES: {modules}
 LOC: {total_loc}
 
+IMPORTANT FORMATTING RULES:
+- Write final, complete content - NOT placeholders or templates
+- Do NOT use brackets like [description] or [step here]
+- Do NOT include template markers in your output
+- Write clear, actionable text that can be used directly
+
 Generate a tutorial with EXACTLY this format:
 
 GOAL:
-[What the developer will accomplish after completing this tutorial]
+What the developer will accomplish after completing this tutorial
 
 PREREQUISITES:
-- [Prerequisite 1]
-- [Prerequisite 2]
+- First prerequisite (e.g., Python 3.10+ installed)
+- Second prerequisite (e.g., Git installed)
 
 TIME:
-[Estimated time, e.g., "15 minutes"]
+15 minutes
 
 STEP_1:
-TITLE: [Step title]
-INSTRUCTIONS: [What to do in this step]
-CODE: [Shell commands if any]
-CHECKPOINT: [How to verify this step worked]
+TITLE: Clone the Repository
+INSTRUCTIONS: Clone the project repository to your local machine using git
+CODE: git clone https://github.com/example/repo.git && cd repo
+CHECKPOINT: You should see the project files in your current directory
 
 STEP_2:
-TITLE: [Step title]
-INSTRUCTIONS: [What to do]
-CODE: [Commands]
-CHECKPOINT: [Verification]
+TITLE: Install Dependencies
+INSTRUCTIONS: Install the required dependencies for the project
+CODE: pip install -r requirements.txt
+CHECKPOINT: Installation completes without errors
 
 STEP_3:
-TITLE: [Step title]
-INSTRUCTIONS: [What to do]
-CODE: [Commands]
-CHECKPOINT: [Verification]
+TITLE: Run the Application
+INSTRUCTIONS: Start the application to verify everything works
+CODE: python main.py
+CHECKPOINT: The application starts and displays output
 
 NEXT_STEPS:
-- [What to explore next]
-- [Related documentation]
+- Explore the API documentation
+- Read the architecture overview
+- Try modifying a simple feature
 
 Create 3-5 clear steps for getting the project running locally.