Eliminate ingestion cache from AI Chat Web template

eng/Versions.props

@@ -165,9 +165,9 @@
     <CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion>9.4.1-beta.277</CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion>
     <CommunityToolkitAspireOllamaSharpVersion>9.4.1-beta.277</CommunityToolkitAspireOllamaSharpVersion>
     <MicrosoftExtensionsServiceDiscoveryVersion>9.2.0</MicrosoftExtensionsServiceDiscoveryVersion>
-    <MicrosoftSemanticKernelConnectorsAzureAISearchVersion>1.47.0-preview</MicrosoftSemanticKernelConnectorsAzureAISearchVersion>
-    <MicrosoftSemanticKernelConnectorsQdrantVersion>1.47.0-preview</MicrosoftSemanticKernelConnectorsQdrantVersion>
-    <MicrosoftSemanticKernelCoreVersion>1.47.0</MicrosoftSemanticKernelCoreVersion>
+    <MicrosoftSemanticKernelConnectorsAzureAISearchVersion>1.49.0-preview</MicrosoftSemanticKernelConnectorsAzureAISearchVersion>
+    <MicrosoftSemanticKernelConnectorsQdrantVersion>1.49.0-preview</MicrosoftSemanticKernelConnectorsQdrantVersion>
+    <MicrosoftSemanticKernelCoreVersion>1.49.0</MicrosoftSemanticKernelCoreVersion>
     <OllamaSharpVersion>5.1.13</OllamaSharpVersion>
     <OpenTelemetryVersion>1.9.0</OpenTelemetryVersion>
     <PdfPigVersion>0.1.9</PdfPigVersion>

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/ChatWithCustomData-CSharp.AppHost.csproj.in

@@ -16,7 +16,6 @@
 <!--#if (UseQdrant)
     <PackageReference Include="Aspire.Hosting.Qdrant" Version="${AspireVersion}" />
 #endif -->
-    <PackageReference Include="CommunityToolkit.Aspire.Hosting.Sqlite" Version="${CommunityToolkitAspireHostingSqliteVersion}" />
 <!--#if (IsOllama)
     <PackageReference Include="CommunityToolkit.Aspire.Hosting.Ollama" Version="${CommunityToolkitAspireHostingOllamaVersion}" />
 #endif -->

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.AppHost/Program.cs

@@ -38,8 +38,6 @@
 #else // UseLocalVectorStore
 #endif
 
-var ingestionCache = builder.AddSqlite("ingestionCache");
-
 var webApp = builder.AddProject<Projects.ChatWithCustomData_CSharp_Web>("aichatweb-app");
 #if (IsOllama) // AI SERVICE PROVIDER REFERENCES
 webApp
@@ -58,8 +56,5 @@
     .WaitFor(vectorDB);
 #else // UseLocalVectorStore
 #endif
-webApp
-    .WithReference(ingestionCache)
-    .WaitFor(ingestionCache);
 
 builder.Build().Run();

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/ChatWithCustomData-CSharp.Web.csproj.in

@@ -30,11 +30,6 @@
 <!--#endif -->
 <!--#if (UseManagedIdentity) -->
     <PackageReference Include="Azure.Identity" Version="${AzureIdentityVersion}" />
-<!--#endif -->
-<!--#if (IsAspire) -->
-    <PackageReference Include="CommunityToolkit.Aspire.Microsoft.EntityFrameworkCore.Sqlite" Version="${CommunityToolkitAspireMicrosoftEntityFrameworkCoreSqliteVersion}" />
-<!--#else -->
-    <PackageReference Include="Microsoft.EntityFrameworkCore.Sqlite" Version="${MicrosoftEntityFrameworkCoreSqliteVersion}" />
 <!--#endif -->
     <PackageReference Include="Microsoft.Extensions.AI" Version="${MicrosoftExtensionsAIVersion}" />
     <PackageReference Include="Microsoft.SemanticKernel.Core" Version="${MicrosoftSemanticKernelCoreVersion}" />

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Components/Pages/Chat/Chat.razor

@@ -118,7 +118,7 @@
         await InvokeAsync(StateHasChanged);
         var results = await Search.SearchAsync(searchPhrase, filenameFilter, maxResults: 5);
         return results.Select(result =>
-            $"<result filename=\"{result.FileName}\" page_number=\"{result.PageNumber}\">{result.Text}</result>");
+            $"<result filename=\"{result.DocumentId}\" page_number=\"{result.PageNumber}\">{result.Text}</result>");
     }
 
     public void Dispose()

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Program.Aspire.cs

@@ -53,10 +53,8 @@
 #endif
 builder.Services.AddScoped<DataIngestor>();
 builder.Services.AddSingleton<SemanticSearch>();
-builder.AddSqliteDbContext<IngestionCacheDbContext>("ingestionCache");
 
 var app = builder.Build();
-IngestionCacheDbContext.Initialize(app.Services);
 
 app.MapDefaultEndpoints();
 

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Program.cs

@@ -1,4 +1,3 @@
-using Microsoft.EntityFrameworkCore;
 using Microsoft.Extensions.AI;
 using Microsoft.Extensions.VectorData;
 using ChatWithCustomData_CSharp.Web.Components;
@@ -102,11 +101,7 @@
 builder.Services.AddChatClient(chatClient).UseFunctionInvocation().UseLogging();
 builder.Services.AddEmbeddingGenerator(embeddingGenerator);
 
-builder.Services.AddDbContext<IngestionCacheDbContext>(options =>
-    options.UseSqlite("Data Source=ingestioncache.db"));
-
 var app = builder.Build();
-IngestionCacheDbContext.Initialize(app.Services);
 
 // Configure the HTTP request pipeline.
 if (!app.Environment.IsDevelopment())

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/SemanticSearchRecord.cs → src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/IngestedChunk.cs

@@ -2,7 +2,7 @@
 
 namespace ChatWithCustomData_CSharp.Web.Services;
 
-public class SemanticSearchRecord
+public class IngestedChunk
 {
     [VectorStoreRecordKey]
 #if (UseQdrant)
@@ -11,8 +11,8 @@ public class SemanticSearchRecord
     public required string Key { get; set; }
 #endif
 
-    [VectorStoreRecordData(IsFilterable = true)]
-    public required string FileName { get; set; }
+    [VectorStoreRecordData(IsIndexed = true)]
+    public required string DocumentId { get; set; }
 
     [VectorStoreRecordData]
     public int PageNumber { get; set; }
@@ -21,9 +21,9 @@ public class SemanticSearchRecord
     public required string Text { get; set; }
 
 #if (IsOllama)
-    [VectorStoreRecordVector(384, DistanceFunction.CosineSimilarity)] // 384 is the default vector size for the all-minilm embedding model
+    [VectorStoreRecordVector(384, DistanceFunction = DistanceFunction.CosineSimilarity)] // 384 is the default vector size for the all-minilm embedding model
 #else
-    [VectorStoreRecordVector(1536, DistanceFunction.CosineSimilarity)] // 1536 is the default vector size for the OpenAI text-embedding-3-small model
+    [VectorStoreRecordVector(1536, DistanceFunction = DistanceFunction.CosineSimilarity)] // 1536 is the default vector size for the OpenAI text-embedding-3-small model
 #endif
     public ReadOnlyMemory<float> Vector { get; set; }
 }

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/IngestedDocument.cs

@@ -0,0 +1,26 @@
+using Microsoft.Extensions.VectorData;
+
+namespace ChatWithCustomData_CSharp.Web.Services;
+
+public class IngestedDocument
+{
+    [VectorStoreRecordKey]
+#if (UseQdrant)
+    public required Guid Key { get; set; }
+#else
+    public required string Key { get; set; }
+#endif
+
+    [VectorStoreRecordData(IsIndexed = true)]
+    public required string SourceId { get; set; }
+
+    [VectorStoreRecordData]
+    public required string DocumentId { get; set; }
+
+    [VectorStoreRecordData]
+    public required string DocumentVersion { get; set; }
+
+    // The vector is not used but required for some vector databases
+    [VectorStoreRecordVector(1, DistanceFunction = DistanceFunction.CosineSimilarity)]
+    public ReadOnlyMemory<float> Vector { get; set; } = new ReadOnlyMemory<float>([0]);
+}

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/Ingestion/DataIngestor.cs

@@ -1,14 +1,12 @@
-using Microsoft.EntityFrameworkCore;
-using Microsoft.Extensions.AI;
+using Microsoft.Extensions.AI;
 using Microsoft.Extensions.VectorData;
 
 namespace ChatWithCustomData_CSharp.Web.Services.Ingestion;
 
 public class DataIngestor(
     ILogger<DataIngestor> logger,
     IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator,
-    IVectorStore vectorStore,
-    IngestionCacheDbContext ingestionCacheDb)
+    IVectorStore vectorStore)
 {
     public static async Task IngestDataAsync(IServiceProvider services, IIngestionSource source)
     {
@@ -20,48 +18,48 @@ public static async Task IngestDataAsync(IServiceProvider services, IIngestionSo
     public async Task IngestDataAsync(IIngestionSource source)
     {
 #if (UseQdrant)
-        var vectorCollection = vectorStore.GetCollection<Guid, SemanticSearchRecord>("data-ChatWithCustomData-CSharp.Web-ingestion");
+        var chunksCollection = vectorStore.GetCollection<Guid, IngestedChunk>("data-ChatWithCustomData-CSharp.Web-chunks");
+        var documentsCollection = vectorStore.GetCollection<Guid, IngestedDocument>("data-ChatWithCustomData-CSharp.Web-documents");
 #else
-        var vectorCollection = vectorStore.GetCollection<string, SemanticSearchRecord>("data-ChatWithCustomData-CSharp.Web-ingestion");
+        var chunksCollection = vectorStore.GetCollection<string, IngestedChunk>("data-ChatWithCustomData-CSharp.Web-chunks");
+        var documentsCollection = vectorStore.GetCollection<string, IngestedDocument>("data-ChatWithCustomData-CSharp.Web-documents");
 #endif
-        await vectorCollection.CreateCollectionIfNotExistsAsync();
+        await chunksCollection.CreateCollectionIfNotExistsAsync();
+        await documentsCollection.CreateCollectionIfNotExistsAsync();
 
-        var documentsForSource = ingestionCacheDb.Documents
-            .Where(d => d.SourceId == source.SourceId)
-            .Include(d => d.Records);
+        var sourceId = source.SourceId;
+        var documentsForSource = await documentsCollection.GetAsync(doc => doc.SourceId == sourceId, top: int.MaxValue).ToListAsync();
 
-        var deletedFiles = await source.GetDeletedDocumentsAsync(documentsForSource);
-        foreach (var deletedFile in deletedFiles)
+        var deletedDocuments = await source.GetDeletedDocumentsAsync(documentsForSource);
+        foreach (var deletedDocument in deletedDocuments)
         {
-            logger.LogInformation("Removing ingested data for {file}", deletedFile.Id);
-            await vectorCollection.DeleteBatchAsync(deletedFile.Records.Select(r => r.Id));
-            ingestionCacheDb.Documents.Remove(deletedFile);
+            logger.LogInformation("Removing ingested data for {documentId}", deletedDocument.DocumentId);
+            await DeleteChunksForDocumentAsync(deletedDocument);
+            await documentsCollection.DeleteAsync(deletedDocument.Key);
         }
-        await ingestionCacheDb.SaveChangesAsync();
 
-        var modifiedDocs = await source.GetNewOrModifiedDocumentsAsync(documentsForSource);
-        foreach (var modifiedDoc in modifiedDocs)
+        var modifiedDocuments = await source.GetNewOrModifiedDocumentsAsync(documentsForSource);
+        foreach (var modifiedDocument in modifiedDocuments)
         {
-            logger.LogInformation("Processing {file}", modifiedDoc.Id);
+            logger.LogInformation("Processing {documentId}", modifiedDocument.DocumentId);
+            await DeleteChunksForDocumentAsync(modifiedDocument);
 
-            if (modifiedDoc.Records.Count > 0)
-            {
-                await vectorCollection.DeleteBatchAsync(modifiedDoc.Records.Select(r => r.Id));
-            }
+            await documentsCollection.UpsertAsync(modifiedDocument);
 
-            var newRecords = await source.CreateRecordsForDocumentAsync(embeddingGenerator, modifiedDoc.Id);
-            await foreach (var id in vectorCollection.UpsertBatchAsync(newRecords)) { }
+            var newRecords = await source.CreateChunksForDocumentAsync(embeddingGenerator, modifiedDocument);
+            await chunksCollection.UpsertAsync(newRecords);
+        }
 
-            modifiedDoc.Records.Clear();
-            modifiedDoc.Records.AddRange(newRecords.Select(r => new IngestedRecord { Id = r.Key, DocumentId = modifiedDoc.Id }));
+        logger.LogInformation("Ingestion is up-to-date");
 
-            if (ingestionCacheDb.Entry(modifiedDoc).State == EntityState.Detached)
+        async Task DeleteChunksForDocumentAsync(IngestedDocument document)
+        {
+            var documentId = document.DocumentId;
+            var chunksToDelete = await chunksCollection.GetAsync(record => record.DocumentId == documentId, int.MaxValue).ToListAsync();
+            if (chunksToDelete.Any())
             {
-                ingestionCacheDb.Documents.Add(modifiedDoc);
+                await chunksCollection.DeleteAsync(chunksToDelete.Select(r => r.Key));
             }
         }
-
-        await ingestionCacheDb.SaveChangesAsync();
-        logger.LogInformation("Ingestion is up-to-date");
     }
 }

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/Ingestion/IIngestionSource.cs

@@ -6,9 +6,9 @@ public interface IIngestionSource
 {
     string SourceId { get; }
 
-    Task<IEnumerable<IngestedDocument>> GetNewOrModifiedDocumentsAsync(IQueryable<IngestedDocument> existingDocuments);
+    Task<IEnumerable<IngestedDocument>> GetNewOrModifiedDocumentsAsync(IReadOnlyList<IngestedDocument> existingDocuments);
 
-    Task<IEnumerable<IngestedDocument>> GetDeletedDocumentsAsync(IQueryable<IngestedDocument> existingDocuments);
+    Task<IEnumerable<IngestedDocument>> GetDeletedDocumentsAsync(IReadOnlyList<IngestedDocument> existingDocuments);
 
-    Task<IEnumerable<SemanticSearchRecord>> CreateRecordsForDocumentAsync(IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, string documentId);
+    Task<IEnumerable<IngestedChunk>> CreateChunksForDocumentAsync(IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, IngestedDocument document);
 }

src/ProjectTemplates/Microsoft.Extensions.AI.Templates/src/ChatWithCustomData/ChatWithCustomData-CSharp.Web/Services/Ingestion/PDFDirectorySource.cs

@@ -1,68 +1,66 @@
-using Microsoft.EntityFrameworkCore;
+using Microsoft.Extensions.AI;
 using Microsoft.SemanticKernel.Text;
-using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter;
-using UglyToad.PdfPig.DocumentLayoutAnalysis.WordExtractor;
 using UglyToad.PdfPig;
-using Microsoft.Extensions.AI;
 using UglyToad.PdfPig.Content;
+using UglyToad.PdfPig.DocumentLayoutAnalysis.PageSegmenter;
+using UglyToad.PdfPig.DocumentLayoutAnalysis.WordExtractor;
 
 namespace ChatWithCustomData_CSharp.Web.Services.Ingestion;
 
 public class PDFDirectorySource(string sourceDirectory) : IIngestionSource
 {
     public static string SourceFileId(string path) => Path.GetFileName(path);
+    public static string SourceFileVersion(string path) => File.GetLastWriteTimeUtc(path).ToString("o");
 
     public string SourceId => $"{nameof(PDFDirectorySource)}:{sourceDirectory}";
 
-    public async Task<IEnumerable<IngestedDocument>> GetNewOrModifiedDocumentsAsync(IQueryable<IngestedDocument> existingDocuments)
+    public Task<IEnumerable<IngestedDocument>> GetNewOrModifiedDocumentsAsync(IReadOnlyList<IngestedDocument> existingDocuments)
     {
         var results = new List<IngestedDocument>();
         var sourceFiles = Directory.GetFiles(sourceDirectory, "*.pdf");
+        var existingDocumentsById = existingDocuments.ToDictionary(d => d.DocumentId);
 
         foreach (var sourceFile in sourceFiles)
         {
             var sourceFileId = SourceFileId(sourceFile);
-            var sourceFileVersion = File.GetLastWriteTimeUtc(sourceFile).ToString("o");
-
-            var existingDocument = await existingDocuments.Where(d => d.SourceId == SourceId && d.Id == sourceFileId).FirstOrDefaultAsync();
-            if (existingDocument is null)
+            var sourceFileVersion = SourceFileVersion(sourceFile);
+            var existingDocumentVersion = existingDocumentsById.TryGetValue(sourceFileId, out var existingDocument) ? existingDocument.DocumentVersion : null;
+            if (existingDocumentVersion != sourceFileVersion)
             {
-                results.Add(new() { Id = sourceFileId, Version = sourceFileVersion, SourceId = SourceId });
-            }
-            else if (existingDocument.Version != sourceFileVersion)
-            {
-                existingDocument.Version = sourceFileVersion;
-                results.Add(existingDocument);
+#if (UseQdrant)
+                results.Add(new() { Key = Guid.CreateVersion7(), SourceId = SourceId, DocumentId = sourceFileId, DocumentVersion = sourceFileVersion });
+#else
+                results.Add(new() { Key = $"{SourceId}_{sourceFileId}", SourceId = SourceId, DocumentId = sourceFileId, DocumentVersion = sourceFileVersion });
+#endif
             }
         }
 
-        return results;
+        return Task.FromResult((IEnumerable<IngestedDocument>)results);
     }
 
-    public async Task<IEnumerable<IngestedDocument>> GetDeletedDocumentsAsync(IQueryable<IngestedDocument> existingDocuments)
+    public Task<IEnumerable<IngestedDocument>> GetDeletedDocumentsAsync(IReadOnlyList<IngestedDocument> existingDocuments)
     {
-        var sourceFiles = Directory.GetFiles(sourceDirectory, "*.pdf");
-        var sourceFileIds = sourceFiles.Select(SourceFileId).ToList();
-        return await existingDocuments
-            .Where(d => !sourceFileIds.Contains(d.Id))
-            .ToListAsync();
+        var currentFiles = Directory.GetFiles(sourceDirectory, "*.pdf");
+        var currentFileIds = currentFiles.ToLookup(SourceFileId);
+        var deletedDocuments = existingDocuments.Where(d => !currentFileIds.Contains(d.DocumentId));
+        return Task.FromResult(deletedDocuments);
     }
 
-    public async Task<IEnumerable<SemanticSearchRecord>> CreateRecordsForDocumentAsync(IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, string documentId)
+    public async Task<IEnumerable<IngestedChunk>> CreateChunksForDocumentAsync(IEmbeddingGenerator<string, Embedding<float>> embeddingGenerator, IngestedDocument document)
     {
-        using var pdf = PdfDocument.Open(Path.Combine(sourceDirectory, documentId));
+        using var pdf = PdfDocument.Open(Path.Combine(sourceDirectory, document.DocumentId));
         var paragraphs = pdf.GetPages().SelectMany(GetPageParagraphs).ToList();
 
         var embeddings = await embeddingGenerator.GenerateAsync(paragraphs.Select(c => c.Text));
 
-        return paragraphs.Zip(embeddings).Select((pair, index) => new SemanticSearchRecord
+        return paragraphs.Zip(embeddings).Select((pair, index) => new IngestedChunk
         {
 #if (UseQdrant)
             Key = Guid.CreateVersion7(),
 #else
-            Key = $"{Path.GetFileNameWithoutExtension(documentId)}_{pair.First.PageNumber}_{pair.First.IndexOnPage}",
+            Key = $"{Path.GetFileNameWithoutExtension(document.DocumentId)}_{pair.First.PageNumber}_{pair.First.IndexOnPage}",
 #endif
-            FileName = documentId,
+            DocumentId = document.DocumentId,
             PageNumber = pair.First.PageNumber,
             Text = pair.First.Text,
             Vector = pair.Second.Vector,