Merge pull request #124 from CyberStrikeus/feat/spa-nav-discovery

feat(hackbrowser): declarative nav harvest (Yön A of #120)

Author: badchars

packages/hackbrowser/src/agent.ts

@@ -198,7 +198,7 @@ function waitForBrowserClose(browser: import("playwright").Browser, signal?: Abo
 /**
  * Mark that login was detected. The actual re-queue happens in the BFS loop
  * AFTER the current page finishes exploration (so new discoveries from
- * collectDOMLinks are enqueued first, before re-visit URLs).
+ * collectNavLinks are enqueued first, before re-visit URLs).
  */
 function triggerReDiscovery(globalState: ReturnType<typeof createGlobalState>): void {
   if (globalState.authPhase === "authenticated") return
@@ -1084,12 +1084,12 @@ async function explorePageWithAI(
 
   // 6. Collect same-host links from DOM (BFS supplement)
   try {
-    const domLinks = await collectDOMLinks(page, pageUrl, inScope)
+    const domLinks = await collectNavLinks(page, pageUrl, inScope)
     for (const url of domLinks) {
       if (!linksToEnqueue.includes(url)) linksToEnqueue.push(url)
     }
   } catch {
-    log.debug("collectDOMLinks failed (page may have navigated)")
+    log.debug("collectNavLinks failed (page may have navigated)")
   }
 
   return linksToEnqueue
@@ -1743,27 +1743,52 @@ function resolveUrl(href: string, baseUrl: string, inScope: ScopeMatcher): strin
   return null
 }
 
-/** Collect <a href> links from DOM as BFS supplement. */
-async function collectDOMLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise<string[]> {
-  const hrefs: string[] = await page.$$eval("a[href]", (els) =>
-    els.map((el) => (el as HTMLAnchorElement).href).filter(Boolean),
+// Attributes that DECLARATIVELY encode a navigation destination readable
+// without clicking (a real URL, path, or hash-route). Order = priority per
+// element. `routerLink`/`[to]`/`data-route` are intentionally excluded — their
+// values are router-relative and need scheme-guessing; those routes are
+// discovered imperatively by clicking (phase B, #120).
+const DECLARATIVE_NAV_ATTRS = ["href", "data-href", "data-url"] as const
+const NAV_TARGET_SELECTOR = "a[href], [role=link], [data-href], [data-url]"
+
+/** Collect declarative navigation links from DOM as a BFS supplement. */
+async function collectNavLinks(page: Page, pageUrl: string, inScope: ScopeMatcher): Promise<string[]> {
+  // Beyond <a href>: many SPAs navigate via non-anchor elements that still
+  // declare their destination in an attribute (data-href/data-url) or carry
+  // role=link. Harvest the first destination-bearing attribute per element; the
+  // raw value is resolved against the page URL below (so "/x" and "#/x" work).
+  const hrefs: string[] = await page.$$eval(
+    NAV_TARGET_SELECTOR,
+    (els, attrs) => {
+      const out: string[] = []
+      for (const el of els) {
+        for (const attr of attrs) {
+          const value = el.getAttribute(attr)
+          if (value) {
+            out.push(value)
+            break
+          }
+        }
+      }
+      return out
+    },
+    [...DECLARATIVE_NAV_ATTRS],
   )
 
-  const results: string[] = []
+  // Route every harvested href through resolveUrl (resolve-against-page +
+  // in-scope + normalizeUrl) rather than an ad-hoc inline normalization. This
+  // aligns DOM-link dedup with the rest of the crawl — e.g. bare "#section"
+  // scroll anchors collapse to the base page instead of spawning phantom
+  // targets, while "#/route" hash-router URLs are preserved.
   const seen = new Set<string>()
-
+  const results: string[] = []
   for (const href of hrefs) {
-    try {
-      const u = new URL(href)
-      if (!inScope(u.hostname)) continue
-      const normalized = u.origin + u.pathname + u.search + u.hash
-      if (!seen.has(normalized)) {
-        seen.add(normalized)
-        results.push(normalized)
-      }
-    } catch {}
+    const resolved = resolveUrl(href, pageUrl, inScope)
+    if (resolved && !seen.has(resolved)) {
+      seen.add(resolved)
+      results.push(resolved)
+    }
   }
-
   return results
 }
 
@@ -2145,7 +2170,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo
       continue
     }
 
-    // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectDOMLinks
+    // Mark URL as visited BEFORE exploration — prevents re-enqueue during explore/collectNavLinks
     const normalizedEntryUrl = normalizeUrl(entry.url)
     visitedPages.add(normalizedEntryUrl)
     globalState.visitedPages.add(normalizedEntryUrl) // sync for explorePageWithAI's filterVisitedLinks
@@ -2253,7 +2278,7 @@ async function runMultiCredential(config: AgentConfig, credentials: CredentialCo
 
     // Collect DOM links from each context — enqueue with context tag
     for (const ctx of visitableContexts) {
-      const domLinks = await collectDOMLinks(ctx.page, entry.url, inScope)
+      const domLinks = await collectNavLinks(ctx.page, entry.url, inScope)
       for (const url of domLinks) {
         enqueueWithContext(url, ctx.id, pageQueue, visitedPages, inScope, pathPatternCounts)
       }
@@ -2632,7 +2657,7 @@ export async function run(config: AgentConfig): Promise<CrawlResult> {
         if (newFingerprint === oldFingerprint) {
           log.info("page unchanged after auth, skipping exploration", { url: currentUrl })
           // Still collect DOM links — navbar may have new links after login
-          const domLinks = await collectDOMLinks(page, currentUrl, inScope)
+          const domLinks = await collectNavLinks(page, currentUrl, inScope)
           for (const url of domLinks) {
             enqueueUrl(url, globalState, inScope)
           }