Refactor token usage and enhance logging capabilities

- Updated `TokenUsage` class properties to `PromptTokens` and `CompletionTokens`.
- Modified `Ask.razor` to display new token counts.
- Added logger to `ChatService` constructor for improved logging.
- Implemented logging for token usage in `ChatService` methods.
- Changed logging level to `Debug` for paragraph storage in `VectorSearchService`.
- Updated logging configuration in `appsettings.Development.json` for better visibility.

Author: marcominerva

SqlDatabaseVectorSearch/Components/Pages/Ask.razor

@@ -235,9 +235,9 @@
                 return string.Empty;
             }
 
-            return $"Input Token Count: {tokenUsage.InputTokenCount}<br />" +
-                   $"Output Token Count: {tokenUsage.OutputTokenCount}<br />" +
-                   $"Total Token Count: {tokenUsage.TotalTokenCount}";
+            return $"Prompt tokens: {tokenUsage.PromptTokens}<br />" +
+                   $"Completion tokens: {tokenUsage.CompletionTokens}<br />" +
+                   $"Total tokens: {tokenUsage.TotalTokens}";
         }
     }
 

SqlDatabaseVectorSearch/Models/TokenUsage.cs

@@ -1,6 +1,6 @@
 namespace SqlDatabaseVectorSearch.Models;
 
-public record class TokenUsage(int InputTokenCount, int OutputTokenCount)
+public record class TokenUsage(int PromptTokens, int CompletionTokens)
 {
-    public int TotalTokenCount => InputTokenCount + OutputTokenCount;
+    public int TotalTokens => PromptTokens + CompletionTokens;
 }

SqlDatabaseVectorSearch/Services/ChatService.cs

@@ -10,7 +10,7 @@
 
 namespace SqlDatabaseVectorSearch.Services;
 
-public class ChatService(IChatCompletionService chatCompletionService, TokenizerService tokenizerService, HybridCache cache, IOptions<AppSettings> appSettingsOptions)
+public class ChatService(IChatCompletionService chatCompletionService, TokenizerService tokenizerService, HybridCache cache, IOptions<AppSettings> appSettingsOptions, ILogger<ChatService> logger)
 {
     private readonly AppSettings appSettings = appSettingsOptions.Value;
 
@@ -36,6 +36,7 @@ The reformulation must always explicitly contain the subject of the question.
         await UpdateCacheAsync(conversationId, chat, cancellationToken);
 
         var tokenUsage = GetTokenUsage(reformulatedQuestion);
+        logger.LogDebug("Reformulation: {TokenUsage}", tokenUsage);
 
         return new(reformulatedQuestion.Content!, tokenUsage);
     }
@@ -53,6 +54,7 @@ public async Task<ChatResponse> AskQuestionAsync(Guid conversationId, IEnumerabl
         await SetChatHistoryAsync(conversationId, question, answer.Content!, cancellationToken);
 
         var tokenUsage = GetTokenUsage(answer);
+        logger.LogDebug("Ask question: {TokenUsage}", tokenUsage);
 
         return new(answer.Content!, tokenUsage);
     }
@@ -78,6 +80,7 @@ public async IAsyncEnumerable<ChatResponse> AskStreamingAsync(Guid conversationI
                 var tokenUsage = GetTokenUsage(token);
                 if (tokenUsage is not null)
                 {
+                    logger.LogDebug("Ask streaming: {TokenUsage}", tokenUsage);
                     yield return new(null, tokenUsage);
                 }
             }

SqlDatabaseVectorSearch/Services/VectorSearchService.cs

@@ -47,7 +47,7 @@ public async Task<ImportDocumentResponse> ImportAsync(Stream stream, string name
             // Save the document chunks and the corresponding embedding in the database.
             foreach (var (index, paragraph) in paragraphs.Index())
             {
-                logger.LogInformation("Storing a paragraph of {TokenCount} tokens.", tokenizerService.CountChatCompletionTokens(paragraph));
+                logger.LogDebug("Storing a paragraph of {TokenCount} tokens.", tokenizerService.CountChatCompletionTokens(paragraph));
 
                 var documentChunk = new Entities.DocumentChunk { Document = document, Index = index, Content = paragraph!, Embedding = embeddings[index].ToArray() };
                 dbContext.DocumentChunks.Add(documentChunk);
@@ -103,7 +103,9 @@ public async IAsyncEnumerable<QuestionResponse> AskStreamingAsync(Question quest
     {
         // Reformulate the question taking into account the context of the chat to perform keyword search and embeddings.
         var reformulatedQuestion = reformulate ? await chatService.CreateQuestionAsync(question.ConversationId, question.Text, cancellationToken) : new(question.Text);
+
         var embeddingTokenCount = tokenizerService.CountEmbeddingTokens(reformulatedQuestion.Text!);
+        logger.LogDebug("Embedding Token Count: {EmbeddingTokenCount}", embeddingTokenCount);
 
         // Perform Vector Search on SQL Database.
         var questionEmbedding = await textEmbeddingGenerationService.GenerateEmbeddingAsync(reformulatedQuestion.Text!, cancellationToken: cancellationToken);

SqlDatabaseVectorSearch/appsettings.Development.json

@@ -3,7 +3,7 @@
         "LogLevel": {
             "Default": "Information",
             "Microsoft.AspNetCore": "Warning",
-            "Microsoft.KernelMemory": "Debug"
+            "SqlDatabaseVectorSearch": "Debug"
         }
     }
 }