simplicity: remove much of the web changes

Author: brendan-kellam

packages/backend/src/jobManager.ts

@@ -123,7 +123,6 @@ export class BullMQJobManager implements JobManager {
                     `${LOG_TAG}:${spec.name}:job:${job.id ?? 'unknown'}`,
                 );
                 const lifecycleContext = this.jobLifecycleContext<TName>(job);
-                jobLogger.debug(`Started workload "${spec.name}"`);
 
                 try {
                     await workload.onStarted?.(lifecycleContext);
@@ -134,7 +133,6 @@ export class BullMQJobManager implements JobManager {
                         updateProgress: (progress) => job.updateProgress(progress),
                         trigger: (target, data) => this.trigger(target, data),
                     });
-                    jobLogger.debug(`Completed workload "${spec.name}"`);
                     return result;
                 } catch (error) {
                     jobLogger.error(`Workload "${spec.name}" attempt failed`, error);

packages/db/prisma/migrations/20260728190236_remove_connection_sync_job_rows/migration.sql

@@ -10,6 +10,15 @@ datasource db {
   url      = env("DATABASE_URL")
 }
 
+enum ConnectionSyncStatus {
+  SYNC_NEEDED
+  IN_SYNC_QUEUE
+  SYNCING
+  SYNCED
+  SYNCED_WITH_WARNINGS
+  FAILED
+}
+
 enum ChatVisibility {
   PRIVATE
   PUBLIC
@@ -65,10 +74,10 @@ model Repo {
   permissionSyncJobs RepoPermissionSyncJob[]
   permissionSyncedAt    DateTime? /// When the permissions were last synced successfully.
 
+  jobs RepoIndexingJob[]
   indexedAt          DateTime? /// When the repo was last indexed successfully.
-  latestIndexingJobId   String?
-
   indexedCommitHash  String? /// The commit hash of the last indexed commit (on HEAD).
+  latestIndexingJobStatus RepoIndexingJobStatus? /// The status of the latest indexing job.
   pushedAt           DateTime? /// The timestamp of the most recent commit across all branches.
 
   external_id           String /// The id of the repo in the external service
@@ -86,6 +95,35 @@ model Repo {
   @@index([indexedAt])
 }
 
+enum RepoIndexingJobStatus {
+  PENDING
+  IN_PROGRESS
+  COMPLETED
+  FAILED
+}
+
+enum RepoIndexingJobType {
+  INDEX
+  CLEANUP
+}
+
+model RepoIndexingJob {
+  id          String                      @id @default(cuid())
+  type        RepoIndexingJobType
+  status      RepoIndexingJobStatus @default(PENDING)
+  createdAt   DateTime                    @default(now())
+  updatedAt   DateTime                    @updatedAt
+  completedAt DateTime?
+  metadata    Json? /// For schema see repoIndexingJobMetadataSchema in packages/shared/src/types.ts
+
+  errorMessage String?
+
+  repo   Repo @relation(fields: [repoId], references: [id], onDelete: Cascade)
+  repoId Int
+
+  @@index([repoId, type, status])
+}
+
 enum RepoPermissionSyncJobStatus {
   PENDING
   IN_PROGRESS
@@ -143,9 +181,9 @@ model Connection {
   // The type of connection (e.g., github, gitlab, etc.)
   connectionType ConnectionType
 
+  syncJobs ConnectionSyncJob[]
   /// When the connection was last synced successfully.
   syncedAt           DateTime?
-  latestSyncJobId    String?
 
   /// Controls whether repository permissions are enforced for this connection.
   /// When `PERMISSION_SYNC_ENABLED` is false, this setting has no effect.
@@ -170,6 +208,27 @@ model Connection {
   @@unique([name, orgId])
 }
 
+enum ConnectionSyncJobStatus {
+  PENDING
+  IN_PROGRESS
+  COMPLETED
+  FAILED
+}
+
+model ConnectionSyncJob {
+  id          String                      @id @default(cuid())
+  status      ConnectionSyncJobStatus @default(PENDING)
+  createdAt   DateTime                    @default(now())
+  updatedAt   DateTime                    @updatedAt
+  completedAt DateTime?
+
+  warningMessages String[]
+  errorMessage String?
+
+  connection   Connection @relation(fields: [connectionId], references: [id], onDelete: Cascade)
+  connectionId Int
+}
+
 model RepoToConnection {
   addedAt DateTime @default(now())
 

packages/db/prisma/migrations/20260728231844_remove_repo_index_job_rows/migration.sql

@@ -2,6 +2,7 @@ import { Script } from "../scriptRunner";
 import { PrismaClient } from "../../dist";
 
 const NUM_REPOS = 1000;
+const NUM_INDEXING_JOBS_PER_REPO = 10000;
 const NUM_PERMISSION_JOBS_PER_REPO = 10000;
 
 export const injectRepoData: Script = {
@@ -36,6 +37,8 @@ export const injectRepoData: Script = {
     console.log(`Creating ${NUM_REPOS} repos...`);
 
     const statuses = ['PENDING', 'IN_PROGRESS', 'COMPLETED', 'FAILED'] as const;
+    const indexingJobTypes = ['INDEX', 'CLEANUP'] as const;
+
     for (let i = 0; i < NUM_REPOS; i++) {
         const repo = await prisma.repo.create({
             data: {
@@ -68,8 +71,23 @@ export const injectRepoData: Script = {
                 }
             });
         }
+
+        for (let j = 0; j < NUM_INDEXING_JOBS_PER_REPO; j++) {
+            const status = statuses[Math.floor(Math.random() * statuses.length)];
+            const type = indexingJobTypes[Math.floor(Math.random() * indexingJobTypes.length)];
+            await prisma.repoIndexingJob.create({
+                data: {
+                    repoId: repo.id,
+                    type,
+                    status,
+                    completedAt: status === 'COMPLETED' || status === 'FAILED' ? new Date() : null,
+                    errorMessage: status === 'FAILED' ? 'Mock indexing error' : null,
+                    metadata: {}
+                }
+            });
+        }
     }
 
-    console.log(`Created ${NUM_REPOS} repos with associated permission jobs.`);
+    console.log(`Created ${NUM_REPOS} repos with associated jobs.`);
     }
-};
+}; 

packages/db/prisma/schema.prisma

@@ -26,20 +26,6 @@ export const CONNECTION_QUEUE: QueueSpec<'connection'> = {
         keep: { completed: 50, failed: 50 },
         keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES,
     },
-    onEnqueued: async ({
-        prisma,
-        data: { connectionId },
-        jobId
-    }) => {
-        await prisma.connection.update({
-            where: {
-                id: connectionId
-            },
-            data: {
-                latestSyncJobId: jobId
-            }
-        });
-    },
     dedupKey: (data) => `connection:${data.connectionId}`,
 }
 
@@ -62,20 +48,6 @@ export const REPO_INDEX_QUEUE: QueueSpec<'repo-index'> = {
         keep: { completed: 50, failed: 50 },
         keepLogs: DEFAULT_JOB_LOGS_MAX_ENTRIES,
     },
-    onEnqueued: async ({
-        prisma,
-        data: { repoId },
-        jobId,
-    }) => {
-        await prisma.repo.update({
-            where: {
-                id: repoId,
-            },
-            data: {
-                latestIndexingJobId: jobId,
-            },
-        });
-    },
     dedupKey: (data) => `repo:${data.repoId}`,
 };
 

packages/db/tools/scripts/inject-repo-data.ts

@@ -4,7 +4,7 @@ import { createAudit } from "@/ee/features/audit/audit";
 import { ErrorCode } from "@/lib/errorCodes";
 import { notFound, ServiceError } from "@/lib/serviceError";
 import { sew } from "@/middleware/sew";
-import { OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
+import { ConnectionSyncJobStatus, OrgRole, Prisma, RepoIndexingJobStatus, RepoIndexingJobType } from "@sourcebot/db";
 import { GiteaConnectionConfig } from "@sourcebot/schemas/v3/gitea.type";
 import { GithubConnectionConfig } from "@sourcebot/schemas/v3/github.type";
 import { GitlabConnectionConfig } from "@sourcebot/schemas/v3/gitlab.type";
@@ -274,6 +274,42 @@ export const getReposStats = async () => sew(() =>
     })
 )
 
+export const getConnectionStats = async () => sew(() =>
+    withAuth(async ({ org, prisma }) => {
+        const [
+            numberOfConnections,
+            numberOfConnectionsWithFirstTimeSyncJobsInProgress,
+        ] = await Promise.all([
+            prisma.connection.count({
+                where: {
+                    orgId: org.id,
+                }
+            }),
+            prisma.connection.count({
+                where: {
+                    orgId: org.id,
+                    syncedAt: null,
+                    syncJobs: {
+                        some: {
+                            status: {
+                                in: [
+                                    ConnectionSyncJobStatus.PENDING,
+                                    ConnectionSyncJobStatus.IN_PROGRESS,
+                                ]
+                            }
+                        }
+                    }
+                }
+            })
+        ]);
+
+        return {
+            numberOfConnections,
+            numberOfConnectionsWithFirstTimeSyncJobsInProgress,
+        };
+    })
+);
+
 export const getRepoInfoByName = async (repoName: string) => sew(() =>
     withOptionalAuth(async ({ org, prisma }) => {
         // @note: repo names are represented by their remote url

packages/shared/src/queue.ts

@@ -2,6 +2,7 @@ import { cookies } from "next/headers";
 import { auth } from "@/auth";
 import { HOME_VIEW_COOKIE_NAME } from "@/lib/constants";
 import { HomeView } from "@/hooks/useHomeView";
+import { getConnectionStats } from "@/actions";
 import { getOrgAccountRequests } from "@/features/membership/actions";
 import { isServiceError } from "@/lib/utils";
 import { ServiceErrorException } from "@/lib/serviceError";
@@ -45,9 +46,11 @@ export async function DefaultSidebar() {
         if (!isOwner) {
             return false;
         }
+        const connectionStats = await getConnectionStats();
         const joinRequests = await getOrgAccountRequests();
+        const hasConnectionNotification = !isServiceError(connectionStats) && connectionStats.numberOfConnectionsWithFirstTimeSyncJobsInProgress > 0;
         const hasJoinRequestNotification = !isServiceError(joinRequests) && joinRequests.length > 0;
-        return hasJoinRequestNotification;
+        return hasConnectionNotification || hasJoinRequestNotification;
     })();
 
     return (

packages/web/src/actions.ts

@@ -1,4 +1,4 @@
-import { getRepos, getSearchContexts } from "@/actions";
+import { getRepos, getReposStats, getSearchContexts } from "@/actions";
 import { SourcebotLogo } from "@/app/components/sourcebotLogo";
 import { getConfiguredLanguageModelsInfo } from "@/features/chat/utils.server";
 import { CustomSlateEditor } from "@/features/chat/customSlateEditor";
@@ -35,6 +35,8 @@ export async function ChatLandingPage() {
         take: 10,
     });
 
+    const repoStats = await getReposStats();
+
     if (isServiceError(allRepos)) {
         throw new ServiceErrorException(allRepos);
     }
@@ -47,6 +49,10 @@ export async function ChatLandingPage() {
         throw new ServiceErrorException(carouselRepos);
     }
 
+    if (isServiceError(repoStats)) {
+        throw new ServiceErrorException(repoStats);
+    }
+
     const demoExamples = env.SOURCEBOT_DEMO_EXAMPLES_PATH ? await (async () => {
         try {
             return (await measure(() => loadJsonFile<DemoExamples>(env.SOURCEBOT_DEMO_EXAMPLES_PATH!, demoExamplesSchema), 'loadExamplesJsonFile')).data;
@@ -78,8 +84,7 @@ export async function ChatLandingPage() {
 
                     <div className="mt-8">
                         <RepositoryCarousel
-                            // @nocheckin
-                            numberOfReposWithIndex={0}
+                            numberOfReposWithIndex={repoStats.numberOfReposWithIndex}
                             displayRepos={carouselRepos}
                         />
                     </div>

packages/web/src/app/(app)/@sidebar/components/defaultSidebar/index.tsx

@@ -1,3 +1,7 @@
+import { getReposStats } from "@/actions";
+import { ServiceErrorException } from "@/lib/serviceError";
+import { isServiceError } from "@/lib/utils";
+
 interface LayoutProps {
     children: React.ReactNode;
 }
@@ -7,6 +11,11 @@ export default async function Layout(
 ) {
     const { children } = props;
 
+    const repoStats = await getReposStats();
+    if (isServiceError(repoStats)) {
+        throw new ServiceErrorException(repoStats);
+    }
+
     return (
         <div className="flex flex-col">
             <main className="flex-grow flex justify-center p-4 relative">