Skip to content

Commit 273a79d

Browse files
MarkDaoustcopybara-github
MarkDaoust
authored and
copybara-github
committed
feat: voice consent signature types across all SDK languages.
Most languages capture the "setup complete" message as a property on the session object. Golang returns the setup complete to the user as the first message. So there's no need to capture it, and that would be a breaking change. PiperOrigin-RevId: 944545654
1 parent de24c0d commit 273a79d

9 files changed

Lines changed: 447 additions & 24 deletions

File tree

‎DemoApp/LiveAudioToAudioRealtimeInput/Program.cs‎

Lines changed: 64 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -18,18 +18,47 @@
1818
using System.Text;
1919
using System.Text.Json;
2020
using System.Text.Json.Serialization;
21+
using System.Linq;
2122

2223
using Google.GenAI;
2324
using Google.GenAI.Types;
2425

2526
var builder = WebApplication.CreateBuilder(args);
27+
28+
string portStr = args.FirstOrDefault(a => a.StartsWith("--port="))?.Substring("--port=".Length);
29+
if (!string.IsNullOrEmpty(portStr)) {
30+
builder.WebHost.UseUrls($"http://localhost:{portStr}");
31+
}
32+
2633
var app = builder.Build();
2734

2835
app.UseWebSockets();
2936
app.UseDefaultFiles();
3037
app.UseStaticFiles();
3138

3239
bool isVertex = args.Contains("--vertex", StringComparer.OrdinalIgnoreCase);
40+
41+
string voiceSamplePath = args.FirstOrDefault(a => a.StartsWith("--voice-sample="))?.Substring("--voice-sample=".Length);
42+
string voiceConsentPath = args.FirstOrDefault(a => a.StartsWith("--voice-consent="))?.Substring("--voice-consent=".Length);
43+
string voiceSignature = args.FirstOrDefault(a => a.StartsWith("--voice-signature="))?.Substring("--voice-signature=".Length);
44+
45+
byte[] voiceSampleAudio = null;
46+
byte[] consentAudio = null;
47+
48+
if (!string.IsNullOrEmpty(voiceSamplePath)) {
49+
voiceSampleAudio = System.IO.File.ReadAllBytes(voiceSamplePath);
50+
51+
if (string.IsNullOrEmpty(voiceConsentPath) && string.IsNullOrEmpty(voiceSignature)) {
52+
throw new ArgumentException("Either --voice-consent or --voice-signature must be provided when --voice-sample is used.");
53+
}
54+
}
55+
56+
if (!string.IsNullOrEmpty(voiceConsentPath)) {
57+
consentAudio = System.IO.File.ReadAllBytes(voiceConsentPath);
58+
}
59+
60+
string parsedModel = args.FirstOrDefault(a => a.StartsWith("--model="))?.Substring("--model=".Length);
61+
3362
string model;
3463
string mimeType;
3564
Client client;
@@ -40,14 +69,14 @@
4069
throw new ArgumentNullException("GOOGLE_CLOUD_PROJECT not set for Vertex AI.");
4170
string location = System.Environment.GetEnvironmentVariable("GOOGLE_CLOUD_LOCATION") ?? "us-central1";
4271
client = new Client(project: project, location: location, vertexAI: true);
43-
model = "gemini-2.0-flash-live-preview-04-09";
72+
model = parsedModel ?? "gemini-2.0-flash-live-preview-04-09";
4473
mimeType = "audio/l16;rate=24000";
4574
} else {
4675
Console.WriteLine("Running in Gemini API mode.");
4776
string apiKey = System.Environment.GetEnvironmentVariable("GEMINI_API_KEY") ??
4877
throw new ArgumentNullException("GEMINI_API_KEY not set for Gemini API.");
4978
client = new Client(apiKey: apiKey);
50-
model = "gemini-2.5-flash-native-audio-preview-12-2025";
79+
model = parsedModel ?? "gemini-2.5-flash-native-audio-preview-12-2025";
5180
mimeType = "audio/pcm";
5281
}
5382

@@ -58,12 +87,31 @@
5887
}
5988

6089
using var localServerWs = await context.WebSockets.AcceptWebSocketAsync();
61-
var config = new LiveConnectConfig { ResponseModalities = new List<Modality> { Modality.Audio },
62-
SpeechConfig = new SpeechConfig { LanguageCode = "en-US" } };
90+
var config = new LiveConnectConfig {
91+
ResponseModalities = new List<Modality> { Modality.Audio },
92+
SpeechConfig = new SpeechConfig {
93+
LanguageCode = "en-US"
94+
}
95+
};
96+
97+
if (voiceSampleAudio != null) {
98+
config.SpeechConfig.VoiceConfig = new VoiceConfig {
99+
ReplicatedVoiceConfig = new ReplicatedVoiceConfig {
100+
MimeType = "audio/wav",
101+
VoiceSampleAudio = voiceSampleAudio,
102+
ConsentAudio = consentAudio,
103+
VoiceConsentSignature = voiceSignature != null ? new VoiceConsentSignature { Signature = voiceSignature } : null
104+
}
105+
};
106+
}
63107

64108
var geminiLiveSession = await client.Live.ConnectAsync(model, config);
65109
var cts = new CancellationTokenSource();
66110

111+
if (geminiLiveSession.SetupComplete?.VoiceConsentSignature?.Signature != null) {
112+
Console.WriteLine($"\n=== Voice Consent Signature Received ===\n{geminiLiveSession.SetupComplete.VoiceConsentSignature.Signature}\n========================================\n");
113+
}
114+
67115
Console.CancelKeyPress += (sender, e) => {
68116
e.Cancel = true;
69117
cts.Cancel();
@@ -98,6 +146,7 @@
98146
Console.WriteLine($"Error processing client message: {ex.Message}");
99147
}
100148
} else if (result.MessageType == WebSocketMessageType.Close) {
149+
Console.WriteLine($"WebSocket closed by server. Status: {result.CloseStatus}, Description: {result.CloseStatusDescription}");
101150
cts.Cancel();
102151
break;
103152
}
@@ -108,12 +157,18 @@
108157
var receiveTask = Task.Run(async () => {
109158
while (!cts.Token.IsCancellationRequested) {
110159
var serverMsg = await geminiLiveSession.ReceiveAsync();
111-
if (serverMsg != null) {
112-
var jsonResponse = JsonSerializer.Serialize(serverMsg);
113-
var responseBytes = Encoding.UTF8.GetBytes(jsonResponse);
114-
await localServerWs.SendAsync(new ArraySegment<byte>(responseBytes),
115-
WebSocketMessageType.Text, true, cts.Token);
160+
if (serverMsg == null) {
161+
var wsField = geminiLiveSession.GetType().GetField("_webSocket", System.Reflection.BindingFlags.NonPublic | System.Reflection.BindingFlags.Instance);
162+
var ws = wsField != null ? (System.Net.WebSockets.WebSocket)wsField.GetValue(geminiLiveSession) : null;
163+
string closeInfo = ws != null ? $"Status: {ws.CloseStatus}, Description: {ws.CloseStatusDescription}" : "Unknown";
164+
Console.WriteLine($"Gemini session ended (server closed connection). {closeInfo}");
165+
cts.Cancel();
166+
break;
116167
}
168+
var jsonResponse = JsonSerializer.Serialize(serverMsg);
169+
var responseBytes = Encoding.UTF8.GetBytes(jsonResponse);
170+
await localServerWs.SendAsync(new ArraySegment<byte>(responseBytes),
171+
WebSocketMessageType.Text, true, cts.Token);
117172
}
118173
}, cts.Token);
119174

‎Google.GenAI.Tests/AsyncSessionTest.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -450,5 +450,26 @@ public async Task CloseAsync_WhenWebSocketThrowsException_ShouldNotPropagate() {
450450

451451
_mockWebSocket.Verify(ws => ws.Dispose(), Times.Once);
452452
}
453+
454+
[TestMethod]
455+
public async Task ReadSetupCompleteAsync_PopulatesVoiceConsentSignature() {
456+
var setupCompleteMessage = "{\"setupComplete\":{\"voiceConsentSignature\":{\"signature\":\"test_sig\"}}}";
457+
byte[] messageBytes = Encoding.UTF8.GetBytes(setupCompleteMessage);
458+
459+
_mockWebSocket.Setup(ws => ws.State).Returns(WebSocketState.Open);
460+
461+
_mockWebSocket
462+
.Setup(ws => ws.ReceiveAsync(It.IsAny<ArraySegment<byte>>(), It.IsAny<CancellationToken>()))
463+
.Callback<ArraySegment<byte>, CancellationToken>((buffer, token) => {
464+
Array.Copy(messageBytes, 0, buffer.Array!, buffer.Offset, messageBytes.Length);
465+
})
466+
.ReturnsAsync(new WebSocketReceiveResult(messageBytes.Length, WebSocketMessageType.Text, true));
467+
468+
await _asyncSession.ReadSetupCompleteAsync();
469+
470+
Assert.IsNotNull(_asyncSession.SetupComplete);
471+
Assert.IsNotNull(_asyncSession.SetupComplete.VoiceConsentSignature);
472+
Assert.AreEqual("test_sig", _asyncSession.SetupComplete.VoiceConsentSignature.Signature);
473+
}
453474
}
454475
}

‎Google.GenAI/GoogleGenAIRealtimeClient.cs‎

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -92,10 +92,9 @@ public async Task<IRealtimeClientSession> CreateSessionAsync(
9292

9393
try
9494
{
95-
// The Google SDK's ConnectAsync sends the setup message but does NOT wait
96-
// for the server's SetupComplete acknowledgment. We must drain it here so
97-
// the session is fully ready (tools configured, modalities set) before the
98-
// caller starts sending audio or text.
95+
// The Google SDK's ConnectAsync awaits the server's SetupComplete acknowledgment
96+
// and buffers it. We consume the buffered message here so that the session is
97+
// ready and subsequent receives only yield actual session messages.
9998
var setupResponse = await asyncSession.ReceiveAsync(cancellationToken).ConfigureAwait(false);
10099
if (setupResponse?.SetupComplete is null)
101100
{

‎Google.GenAI/Live.cs‎

Lines changed: 21 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -62,6 +62,7 @@ public async Task<AsyncSession> ConnectAsync(string model, LiveConnectConfig con
6262
await clientWebSocket.SendAsync(new ArraySegment<byte>(buffer), WebSocketMessageType.Text, true, cancellationToken);
6363

6464
var session = new AsyncSession(clientWebSocket, _apiClient);
65+
await session.ReadSetupCompleteAsync(cancellationToken);
6566
success = true;
6667
return session;
6768
}
@@ -214,13 +215,26 @@ public class AsyncSession : IAsyncDisposable
214215
private readonly WebSocket _webSocket;
215216
private readonly ApiClient _apiClient;
216217
private int _isDisposed = 0; // 0 = false, 1 = true. Used with Interlocked.
218+
private LiveServerMessage? _bufferedMessage;
217219

218220
public AsyncSession(WebSocket webSocket, ApiClient apiClient)
219221
{
220222
_webSocket = webSocket;
221223
_apiClient = apiClient;
222224
}
223225

226+
public LiveServerSetupComplete? SetupComplete { get; private set; }
227+
228+
internal async Task ReadSetupCompleteAsync(CancellationToken cancellationToken = default)
229+
{
230+
var message = await ReceiveAsync(cancellationToken);
231+
if (message?.SetupComplete != null)
232+
{
233+
SetupComplete = message.SetupComplete;
234+
}
235+
_bufferedMessage = message;
236+
}
237+
224238
/// <summary>
225239
/// Sends non-realtime, turn-based content to the model.
226240
/// <para>
@@ -311,6 +325,13 @@ public async Task SendToolResponseAsync(LiveSendToolResponseParameters toolRespo
311325
/// <exception cref="WebSocketException">Thrown for underlying WebSocket errors that are not a graceful close.</exception>
312326
public async Task<LiveServerMessage?> ReceiveAsync(CancellationToken cancellationToken = default)
313327
{
328+
if (_bufferedMessage != null)
329+
{
330+
var msg = _bufferedMessage;
331+
_bufferedMessage = null;
332+
return msg;
333+
}
334+
314335
if (_isDisposed == 1)
315336
{
316337
return null;

‎Google.GenAI/LiveConverters.cs‎

Lines changed: 121 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -384,8 +384,11 @@ internal JsonNode GenerationConfigToVertex(JsonNode fromObject, JsonObject paren
384384
}
385385

386386
if (Common.GetValueByPath(fromObject, new string[] { "speechConfig" }) != null) {
387-
Common.SetValueByPath(toObject, new string[] { "speechConfig" },
388-
Common.GetValueByPath(fromObject, new string[] { "speechConfig" }));
387+
Common.SetValueByPath(
388+
toObject, new string[] { "speechConfig" },
389+
SpeechConfigToVertex(Common.ParseToJsonNode(Common.GetValueByPath(
390+
fromObject, new string[] { "speechConfig" })),
391+
toObject));
389392
}
390393

391394
if (Common.GetValueByPath(fromObject, new string[] { "stopSequences" }) != null) {
@@ -1092,10 +1095,12 @@ internal JsonNode LiveConnectConfigToVertex(JsonNode fromObject, JsonObject pare
10921095
}
10931096

10941097
if (Common.GetValueByPath(fromObject, new string[] { "speechConfig" }) != null) {
1095-
Common.SetValueByPath(parentObject,
1096-
new string[] { "setup", "generationConfig", "speechConfig" },
1097-
Transformers.TLiveSpeechConfig(Common.GetValueByPath(
1098-
fromObject, new string[] { "speechConfig" })));
1098+
Common.SetValueByPath(
1099+
parentObject, new string[] { "setup", "generationConfig", "speechConfig" },
1100+
SpeechConfigToVertex(
1101+
Common.ParseToJsonNode(Transformers.TLiveSpeechConfig(
1102+
Common.GetValueByPath(fromObject, new string[] { "speechConfig" }))),
1103+
toObject));
10991104
}
11001105

11011106
if (Common.GetValueByPath(fromObject, new string[] { "thinkingConfig" }) != null) {
@@ -1466,6 +1471,24 @@ internal JsonNode McpServerToVertex(JsonNode fromObject, JsonObject parentObject
14661471
return toObject;
14671472
}
14681473

1474+
internal JsonNode MultiSpeakerVoiceConfigToVertex(JsonNode fromObject,
1475+
JsonObject parentObject) {
1476+
JsonObject toObject = new JsonObject();
1477+
1478+
if (Common.GetValueByPath(fromObject, new string[] { "speakerVoiceConfigs" }) != null) {
1479+
JsonArray keyArray =
1480+
(JsonArray)Common.GetValueByPath(fromObject, new string[] { "speakerVoiceConfigs" });
1481+
JsonArray result = new JsonArray();
1482+
1483+
foreach (var record in keyArray) {
1484+
result.Add(SpeakerVoiceConfigToVertex(Common.ParseToJsonNode(record), toObject));
1485+
}
1486+
Common.SetValueByPath(toObject, new string[] { "speakerVoiceConfigs" }, result);
1487+
}
1488+
1489+
return toObject;
1490+
}
1491+
14691492
internal JsonNode PartToMldev(JsonNode fromObject, JsonObject parentObject) {
14701493
JsonObject toObject = new JsonObject();
14711494

@@ -1637,6 +1660,34 @@ internal JsonNode PartToVertex(JsonNode fromObject, JsonObject parentObject) {
16371660
return toObject;
16381661
}
16391662

1663+
internal JsonNode ReplicatedVoiceConfigToVertex(JsonNode fromObject, JsonObject parentObject) {
1664+
JsonObject toObject = new JsonObject();
1665+
1666+
if (Common.GetValueByPath(fromObject, new string[] { "mimeType" }) != null) {
1667+
Common.SetValueByPath(toObject, new string[] { "mimeType" },
1668+
Common.GetValueByPath(fromObject, new string[] { "mimeType" }));
1669+
}
1670+
1671+
if (Common.GetValueByPath(fromObject, new string[] { "voiceSampleAudio" }) != null) {
1672+
Common.SetValueByPath(
1673+
toObject, new string[] { "voiceSampleAudio" },
1674+
Common.GetValueByPath(fromObject, new string[] { "voiceSampleAudio" }));
1675+
}
1676+
1677+
if (!Common.IsZero(Common.GetValueByPath(fromObject, new string[] { "consentAudio" }))) {
1678+
throw new NotSupportedException(
1679+
"consentAudio parameter is only supported in Gemini Developer API mode, not in Gemini Enterprise Agent Platform mode.");
1680+
}
1681+
1682+
if (!Common.IsZero(
1683+
Common.GetValueByPath(fromObject, new string[] { "voiceConsentSignature" }))) {
1684+
throw new NotSupportedException(
1685+
"voiceConsentSignature parameter is only supported in Gemini Developer API mode, not in Gemini Enterprise Agent Platform mode.");
1686+
}
1687+
1688+
return toObject;
1689+
}
1690+
16401691
internal JsonNode SafetySettingToMldev(JsonNode fromObject, JsonObject parentObject) {
16411692
JsonObject toObject = new JsonObject();
16421693

@@ -1674,6 +1725,50 @@ internal JsonNode SessionResumptionConfigToMldev(JsonNode fromObject, JsonObject
16741725
return toObject;
16751726
}
16761727

1728+
internal JsonNode SpeakerVoiceConfigToVertex(JsonNode fromObject, JsonObject parentObject) {
1729+
JsonObject toObject = new JsonObject();
1730+
1731+
if (Common.GetValueByPath(fromObject, new string[] { "speaker" }) != null) {
1732+
Common.SetValueByPath(toObject, new string[] { "speaker" },
1733+
Common.GetValueByPath(fromObject, new string[] { "speaker" }));
1734+
}
1735+
1736+
if (Common.GetValueByPath(fromObject, new string[] { "voiceConfig" }) != null) {
1737+
Common.SetValueByPath(toObject, new string[] { "voiceConfig" },
1738+
VoiceConfigToVertex(Common.ParseToJsonNode(Common.GetValueByPath(
1739+
fromObject, new string[] { "voiceConfig" })),
1740+
toObject));
1741+
}
1742+
1743+
return toObject;
1744+
}
1745+
1746+
internal JsonNode SpeechConfigToVertex(JsonNode fromObject, JsonObject parentObject) {
1747+
JsonObject toObject = new JsonObject();
1748+
1749+
if (Common.GetValueByPath(fromObject, new string[] { "voiceConfig" }) != null) {
1750+
Common.SetValueByPath(toObject, new string[] { "voiceConfig" },
1751+
VoiceConfigToVertex(Common.ParseToJsonNode(Common.GetValueByPath(
1752+
fromObject, new string[] { "voiceConfig" })),
1753+
toObject));
1754+
}
1755+
1756+
if (Common.GetValueByPath(fromObject, new string[] { "languageCode" }) != null) {
1757+
Common.SetValueByPath(toObject, new string[] { "languageCode" },
1758+
Common.GetValueByPath(fromObject, new string[] { "languageCode" }));
1759+
}
1760+
1761+
if (Common.GetValueByPath(fromObject, new string[] { "multiSpeakerVoiceConfig" }) != null) {
1762+
Common.SetValueByPath(toObject, new string[] { "multiSpeakerVoiceConfig" },
1763+
MultiSpeakerVoiceConfigToVertex(
1764+
Common.ParseToJsonNode(Common.GetValueByPath(
1765+
fromObject, new string[] { "multiSpeakerVoiceConfig" })),
1766+
toObject));
1767+
}
1768+
1769+
return toObject;
1770+
}
1771+
16771772
internal JsonNode ToolToMldev(JsonNode fromObject, JsonObject parentObject) {
16781773
JsonObject toObject = new JsonObject();
16791774

@@ -1938,5 +2033,25 @@ internal JsonNode VoiceActivityFromVertex(JsonNode fromObject, JsonObject parent
19382033

19392034
return toObject;
19402035
}
2036+
2037+
internal JsonNode VoiceConfigToVertex(JsonNode fromObject, JsonObject parentObject) {
2038+
JsonObject toObject = new JsonObject();
2039+
2040+
if (Common.GetValueByPath(fromObject, new string[] { "replicatedVoiceConfig" }) != null) {
2041+
Common.SetValueByPath(toObject, new string[] { "replicatedVoiceConfig" },
2042+
ReplicatedVoiceConfigToVertex(
2043+
Common.ParseToJsonNode(Common.GetValueByPath(
2044+
fromObject, new string[] { "replicatedVoiceConfig" })),
2045+
toObject));
2046+
}
2047+
2048+
if (Common.GetValueByPath(fromObject, new string[] { "prebuiltVoiceConfig" }) != null) {
2049+
Common.SetValueByPath(
2050+
toObject, new string[] { "prebuiltVoiceConfig" },
2051+
Common.GetValueByPath(fromObject, new string[] { "prebuiltVoiceConfig" }));
2052+
}
2053+
2054+
return toObject;
2055+
}
19412056
}
19422057
}

0 commit comments

Comments
 (0)