fix: recursive chunker chunk size

Author: hpbyte

package.json

@@ -1,6 +1,6 @@
 {
   "name": "h-codex",
-  "version": "0.1.6",
+  "version": "0.1.8",
   "description": "A semantic code search tool for intelligent, cross-repo context retrieval.",
   "author": "Htoo Pyae Lwin",
   "license": "MIT",

packages/core/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@hpbyte/h-codex-core",
-  "version": "0.1.7",
+  "version": "0.1.8",
   "description": "Core indexing and search functionality for h-codex",
   "author": "Htoo Pyae Lwin",
   "license": "MIT",

packages/core/src/ingestion/chunker/cst-chunker.ts

@@ -127,4 +127,3 @@ export class CSTChunker {
 }
 
 export const cstChunker = new CSTChunker()
-

packages/core/src/ingestion/chunker/index.ts

@@ -12,12 +12,7 @@ export class Chunker {
 
       return await cstChunker.chunk({ filePath, projectId, language })
     } catch (error) {
-      if (error instanceof Error && error.message.includes('Unsupported language')) {
-        return await recursiveChunker.chunk({ filePath, projectId })
-      }
-
-      console.error(`Error processing file ${filePath}:`, error)
-      return []
+      return await recursiveChunker.chunk({ filePath, projectId })
     }
   }
 
@@ -44,4 +39,3 @@ export class Chunker {
 }
 
 export const chunker = new Chunker()
-

packages/core/src/ingestion/chunker/recursive-chunker.ts

@@ -1,4 +1,5 @@
 import * as fs from 'node:fs/promises'
+import * as path from 'path'
 import crypto from 'crypto'
 
 import type { ChunkParams, CodeChunkInsert } from '../../types'
@@ -40,10 +41,9 @@ export class RecursiveChunker {
       }
     }
 
-    const chunkSize = Math.floor(maxChunkSize / 2)
     const chunks: string[] = []
-    for (let i = 0; i < content.length; i += chunkSize) {
-      chunks.push(content.slice(i, i + chunkSize))
+    for (let i = 0; i < content.length; i += maxChunkSize) {
+      chunks.push(content.slice(i, i + maxChunkSize))
     }
 
     return chunks
@@ -58,13 +58,15 @@ export class RecursiveChunker {
     for (let i = 0; i < chunks.length; i++) {
       let chunk = chunks[i]!
 
-      if (countLengthWithoutWhitespace(chunk) < coalesce && i < chunks.length - 1) {
+      while (countLengthWithoutWhitespace(chunk) < coalesce && i < chunks.length - 1) {
         const nextChunk = chunks[i + 1]!
         const combinedChunk = chunk + '\n' + nextChunk
 
         if (combinedChunk.length <= maxChunkSize) {
           chunk = combinedChunk
           i++
+        } else {
+          break
         }
       }
 
@@ -78,7 +80,7 @@ export class RecursiveChunker {
         startLine: currentLineNumber,
         endLine,
         nodeType: 'text',
-        language: null,
+        language: this.getLanguage(filePath),
         hash: crypto.createHash('sha256').update(chunk).digest('hex'),
         size: chunk.length,
       })
@@ -88,7 +90,10 @@ export class RecursiveChunker {
 
     return processedChunks
   }
+
+  private getLanguage(filePath: string) {
+    return path.extname(filePath).toLowerCase().replace('.', '')
+  }
 }
 
 export const recursiveChunker = new RecursiveChunker()
-

packages/core/src/script.ts

@@ -0,0 +1,47 @@
+#!/usr/bin/env node
+
+import { indexer } from './ingestion'
+import { semanticSearch } from './search'
+
+const helpMesg = `
+  Usage: h-codexx <command> <path>
+
+  Commands:
+    index <path>   Index a directory of code
+    search <query> Search for a query
+`
+
+async function main() {
+  const args = process.argv.slice(2)
+
+  if (args.length === 0) {
+    console.log('No arguments provided')
+    return
+  }
+
+  const [command, arg] = args
+
+  if (!command || !arg) {
+    console.log(helpMesg)
+    return
+  }
+
+  if (!['index', 'search', 'clear'].includes(command)) {
+    console.log('Invalid command')
+    return
+  }
+
+  if (command === 'index') {
+    return indexer.index(arg)
+  }
+
+  if (command === 'search') {
+    return semanticSearch.search(arg)
+  }
+
+  if (command === 'clear') {
+    return indexer.clear(arg)
+  }
+}
+
+main()

packages/core/src/storage/migrations/drizzle/0002_language-optional.sql

@@ -0,0 +1 @@
+ALTER TABLE "code_chunks" ALTER COLUMN "language" DROP NOT NULL;