1(function () { 2 if (window.tcRcTraining) { 3 return; 4 } 5 6 const sessions = {}; 7 let nextSessionRegistrationId = 0; 8 const MAX_NAVIGATION_HANDSHAKES = 32; 9 const MAX_COMPLETED_NAVIGATION_CALLS = 64; 10 // Truncated persona items are remembered only to refuse a transcript that arrives 11 // after the truncation. Bounded like the navigation set: a long session should not 12 // grow this without limit. 13 const MAX_TRUNCATED_PERSONA_ITEMS = 64; 14 const MAX_NAVIGATION_ID_LENGTH = 256; 15 const MAX_NAVIGATION_OUTPUT_LENGTH = 16 * 1024; 16 const NAVIGATION_ACK_PREFIX = "[CORE-E NAVIGATION ACK] "; 17 const MAX_CONTROL_EVENT_LENGTH = 128 * 1024; 18 19 // Provider events forwarded to .NET as observed deltas. A semantic subset rather 20 // than the raw stream: audio-buffer deltas arrive many times per second and carry 21 // nothing the session engine or the shadow comparison can use. 22 const OBSERVED_DELTA_KINDS = new Map([ 23 ["input_audio_buffer.speech_started", "LearnerSpeechStarted"], 24 ["input_audio_buffer.speech_stopped", "LearnerSpeechStopped"], 25 // Each transcription event is accepted under both spellings the sideband accepted. 26 // Missing either one leaves the learner item pending forever, and the send gate 27 // never goes quiet again â the conversation stops answering anything at all. 28 ["conversation.item.input_audio_transcription.completed", "LearnerTranscriptCompleted"], 29 ["input_audio_transcription.completed", "LearnerTranscriptCompleted"], 30 // A failure resolves the turn just as a completion does. 31 ["conversation.item.input_audio_transcription.failed", "LearnerTranscriptFailed"], 32 ["input_audio_transcription.failed", "LearnerTranscriptFailed"], 33 ["response.created", "PersonaResponseCreated"], 34 // Both spellings again. Missing one is quieter than a wedge but not harmless: the 35 // persona turn simply never reaches the transcript the grade is built from. 36 ["response.output_audio_transcript.done", "PersonaTranscriptCompleted"], 37 ["response.audio_transcript.done", "PersonaTranscriptCompleted"], 38 ["response.done", "PersonaResponseDone"], 39 ["output_audio_buffer.started", "PersonaPlaybackStarted"], 40 ["output_audio_buffer.stopped", "PersonaPlaybackStopped"], 41 // A clear is how a hint or a barge-in stops the persona mid-sentence. The provider 42 // acknowledges it with this rather than a stop, so without it the client believes 43 // audio is still playing for the rest of the session. 44 ["output_audio_buffer.cleared", "PersonaPlaybackCleared"] 45 ]); 46 47 // The only provider events the client-directed conversation sends. An allowlist 48 // rather than a passthrough so a malformed or unintended type cannot reach the 49 // provider on the same channel that carries the learner's session. 50 const CLIENT_DIRECTED_EVENT_TYPES = new Set([ 51 "session.update", 52 "response.create", 53 "response.cancel", 54 "output_audio_buffer.clear", 55 "conversation.item.create", 56 "conversation.item.delete" 57 ]); 58 // A forwarded session.update can be silently ineffective, which leaves the persona on 59 // its opening instructions for the rest of the scenario. Re-send before giving up, but 60 // bound it: an unbounded loop against a provider that will not accept the configuration 61 // is worse than reporting the drift. 62 const MAX_SESSION_CONFIGURATION_REASSERTS = 2; 63 // A director report is a diagnostic, not conversation state. Bounded so a runaway 64 // description cannot turn the delta channel into the problem being diagnosed. 65 const MAX_DIRECTOR_DECISION_LENGTH = 512; 66 const MAX_PROVIDER_ERROR_LENGTH = 512; 67 const NARRATION_DEFAULT_MAX_OUTPUT_TOKENS = 1024; 68 const NARRATION_AUDIO_MILLISECONDS_PER_TOKEN = 50; 69 const NARRATION_COMPLETION_GRACE_MILLISECONDS = 30000; 70 const NARRATION_MINIMUM_TIMEOUT_MILLISECONDS = 60000; 71 const NARRATION_MAXIMUM_TIMEOUT_MILLISECONDS = 120000; 72 73 // Without headphones the persona's voice re-enters the microphone and the 74 // server's turn detection hears it as the learner interrupting. Browser echo 75 // cancellation removes most of it, but what survives on a loud speaker is still 76 // enough to trigger a barge-in, so the outgoing microphone is additionally gated 77 // shut while the persona is audible. 78 const MIC_GATE = { 79 // Remote level above which the persona counts as audible. Deliberately lower 80 // than the visualiser's "speaking" threshold so the gate closes before the 81 // first syllable is loud enough to leak. 82 remoteAudibleLevel: 0.03, 83 // How loud the leak has to get before the gate engages at all. Under this the 84 // microphone is not meaningfully hearing the persona - headphones, or a quiet 85 // speaker the browser's echo cancellation already handles - and gating would 86 // only add latency to a learner who was never causing the problem. 87 echoDetectFloor: 0.03, 88 // A learner has to stay above the measured echo floor for this long before 89 // the gate opens, so one leaked word cannot cut the persona off. 90 bargeInHoldMs: 260, 91 // Keep the outbound microphone closed at the start of every playback burst
92 // while measuring the actual speaker leakage. This bootstrap window must 93 // include loud echo as well as quiet echo; otherwise loud echo is mistaken 94 // for a learner before an echo floor can ever be learned. 95 calibrationMs: 150, 96 // Playback-stop events and the analyser can lead one another slightly. Keep 97 // the microphone closed across that tail instead of reopening on the same 98 // frame that the output buffer reports stopped or cleared. 99 releaseHangoverMs: 120, 100 // No amount of echo-floor learning can make a level this quiet count as 101 // speech, which keeps a silent room from opening the gate on noise. 102 bargeInFloor: 0.055, 103 // How far above the learned echo floor the learner has to be. 104 bargeInMargin: 2.1, 105 openSeconds: 0.01, 106 closeSeconds: 0.03 107 }; 108 109 function getSession(id) { 110 if (!sessions[id] || sessions[id].retired) { 111 sessions[id] = { 112 id, 113 registrationId: ++nextSessionRegistrationId, 114 offerGeneration: 0, 115 retired: false, 116 dotNetRef: null, 117 peerConnection: null, 118 negotiationDataChannel: null, 119 runtimeContextSequence: 0, 120 observedDeltaSequence: 0, 121 pendingSessionConfiguration: null, 122 // Persona items the learner cut off, and persona transcripts waiting for 123 // their response to terminate before they are forwarded. 124 truncatedPersonaItemIds: new Set(), 125 bufferedPersonaTranscripts: new Map(), 126 navigationAcknowledgementSequence: 0, 127 pendingRuntimeContextItemIds: new Set(), 128 navigationHandshakes: new Map(), 129 completedNavigationToolCallIds: new Set(), 130 localStream: null, 131 outboundStream: null, 132 micContext: null, 133 micSource: null, 134 micGain: null, 135 micDestination: null, 136 micAnalyser: null, 137 micAnalyserData: null, 138 micFrame: null, 139 micLevel: 0, 140 micMuted: false, 141 micGateOpen: true, 142 micGateActive: false, 143 echoFloor: 0, 144 bargeInSince: null, 145 bargeInApproved: false, 146 earlySpeechCandidateSince: null, 147 providerPlaybackActive: false, 148 protectedPlaybackActive: false, 149 remoteSignalAudible: false, 150 remoteOnsetCalibrated: false, 151 micGateCalibrationUntil: 0, 152 micGateReleaseUntil: 0, 153 remoteLevel: 0, 154 remoteAudio: null, 155 outputMuted: false, 156 audioContext: null, 157 analyser: null, 158 analyserData: null, 159 analyserSource: null, 160 visualizerFrame: null, 161 visualizerTargetId: null, 162 smoothedVoiceLevel: 0, 163 playbackActive: false, 164 playbackInactiveTimeout: null, 165 hiddenSampleTimer: null, 166 narrationPeerConnection: null, 167 narrationDataChannel: null, 168 narrationAudio: null, 169 narrationCompletionResolve: null, 170 narrationTimeout: null 171 }; 172 } 173 174 return sessions[id]; 175 } 176 177 function getActiveSession(id) { 178 const session = sessions[id]; 179 return session && !session.retired ? session : null; 180 } 181 182 function sampleMicGate(session) { 183 if (!session?.micAnalyser || !session.micAnalyserData) { 184 return; 185 } 186 187 const level = measureLevel(session.micAnalyser, session.micAnalyserData); 188 session.micLevel = Math.max(level, session.micLevel * 0.8); 189 evaluateMicGate(session); 190 } 191 192 function sampleRemoteLevel(session) { 193 if (!session?.analyser || !session.analyserData) { 194 return; 195 } 196 197 session.analyser.getByteTimeDomainData(session.analyserData); 198 199 let sumSquares = 0; 200 for (let i = 0; i < session.analyserData.length; i++) { 201 const normalized = (session.analyserData[i] - 128) / 128; 202 sumSquares += normalized * normalized; 203 } 204 205 const rms = Math.sqrt(sumSquares / session.analyserData.length); 206 const boostedLevel = Math.min(1, Math.max(0, (rms - 0.014) * 12)); 207 session.smoothedVoiceLevel = Math.max(boostedLevel, session.smoothedVoiceLevel * 0.86); 208 updateVisualizer(session, session.smoothedVoiceLevel); 209 updatePlaybackActivity(session, session.smoothedVoiceLevel); 210 } 211 212 function ensureHiddenSampler(session) { 213 if (!session 214 || session.hiddenSampleTimer !== null 215 || typeof window.setInterval !== "function") { 216 return; 217 } 218 219 session.hiddenSampleTimer = window.setInterval(() => { 220 if (session.retired || document.visibilityState !== "hidden") { 221 return; 222 } 223 224 sampleRemoteLevel(session); 225 sampleMicGate(session); 226 }, 100); 227 } 228 229 function stopHiddenSampler(session) { 230 if (!session || session.hiddenSampleTimer === null) { 231 return; 232 } 233 234 if (typeof window.clearInterval === "function") { 235 window.clearInterval(session.hiddenSampleTimer); 236 } 237 session.hiddenSampleTimer = null; 238 } 239 240 function isCurrentOffer(id, session, registrationId, offerGeneration) { 241 return sessions[id] === session 242 && !session.retired 243 && session.registrationId === registrationId 244 && session.offerGeneration === offerGeneration; 245 } 246 247 function stopStreamTracks(stream) { 248 try { 249 stream?.getTracks()?.forEach(track => track.stop()); 250 } catch { 251 } 252 } 253 254 function measureLevel(analyser, buffer) { 255 analyser.getByteTimeDomainData(buffer); 256 257 let sumSquares = 0; 258 for (let i = 0; i < buffer.length; i++) { 259 const normalized = (buffer[i] - 128) / 128; 260 sumSquares += normalized * normalized; 261 } 262 263 const rms = Math.sqrt(sumSquares / buffer.length); 264 return Math.min(1, Math.max(0, (rms - 0.014) * 12)); 265 } 266 267 function setMicGateOpen(session, open) { 268 if (!session.micGain || session.micGateOpen === open) { 269 return; 270 } 271 272 session.micGateOpen = open; 273 const gain = session.micGain.gain; 274 const currentTime = session.micContext?.currentTime ?? 0; 275 const target = open ? 1 : 0; 276 277 try { 278 // Ramped rather than switched: an abrupt gain step is an audible click 279 // that the server would transcribe as a stray sound. 280 gain.cancelScheduledValues(currentTime); 281 gain.setTargetAtTime( 282 target, 283 currentTime, 284 open ? MIC_GATE.openSeconds : MIC_GATE.closeSeconds); 285 } catch { 286 gain.value = target; 287 } 288 } 289 290 function micGateNow() { 291 return window.performance?.now() ?? Date.now(); 292 } 293 294 function beginProtectedPlayback(session, restartCalibration = false) { 295 if (!session) { 296 return; 297 } 298 299 const now = micGateNow(); 300 if (!session.protectedPlaybackActive || restartCalibration) { 301 session.protectedPlaybackActive = true; 302 session.micGateCalibrationUntil = now + MIC_GATE.calibrationMs; 303 session.bargeInSince = null; 304 session.bargeInApproved = false;
305 } 306 307 session.micGateReleaseUntil = 0; 308 session.micGateActive = true; 309 setMicGateOpen(session, false); 310 } 311 312 function disablePlaybackProtection(session) { 313 if (!session) { 314 return; 315 } 316 317 session.protectedPlaybackActive = false; 318 session.remoteSignalAudible = false; 319 session.remoteOnsetCalibrated = false; 320 session.micGateCalibrationUntil = 0; 321 session.micGateReleaseUntil = 0; 322 session.bargeInSince = null; 323 session.bargeInApproved = false; 324 session.earlySpeechCandidateSince = null; 325 session.micGateActive = false; 326 evaluateMicGate(session); 327 } 328 329 function endProtectedPlayback(session) { 330 if (!session?.protectedPlaybackActive) { 331 return; 332 } 333 334 session.protectedPlaybackActive = false; 335 session.micGateCalibrationUntil = 0; 336 session.micGateReleaseUntil = micGateNow() + MIC_GATE.releaseHangoverMs; 337 session.bargeInSince = null; 338 session.bargeInApproved = false; 339 session.earlySpeechCandidateSince = null; 340 session.micGateActive = true; 341 setMicGateOpen(session, false); 342 } 343 344 function updateProtectedPlaybackFromRemoteLevel(session) { 345 if (!session) { 346 return; 347 } 348 349 if (session.outputMuted) { 350 disablePlaybackProtection(session); 351 return; 352 } 353 354 const remoteAudible = (session.remoteLevel ?? 0) >= MIC_GATE.remoteAudibleLevel; 355 const remoteOnset = remoteAudible && !session.remoteSignalAudible; 356 session.remoteSignalAudible = remoteAudible; 357 358 if (remoteOnset && !session.remoteOnsetCalibrated) { 359 // The provider can announce playback before the first audio sample reaches 360 // the analyser. Restart calibration exactly once on that first audible 361 // crossing so delayed loud leakage is measured rather than admitted. 362 session.remoteOnsetCalibrated = true; 363 beginProtectedPlayback(session, true); 364 } 365 366 if (session.providerPlaybackActive || remoteAudible) { 367 beginProtectedPlayback(session); 368 } else { 369 endProtectedPlayback(session); 370 session.remoteOnsetCalibrated = false; 371 } 372 } 373 374 function evaluateMicGate(session) { 375 if (!session?.micGain) { 376 return; 377 } 378 379 if (session.micMuted) { 380 session.bargeInSince = null; 381 session.earlySpeechCandidateSince = null; 382 setMicGateOpen(session, false); 383 return; 384 } 385 386 const now = micGateNow(); 387 const level = session.micLevel ?? 0; 388 389 if (!session.protectedPlaybackActive) { 390 // Decay the learned floor so a moment of loud playback does not keep 391 // raising the bar for the rest of the conversation. 392 session.echoFloor *= 0.97; 393 session.bargeInSince = null; 394 session.bargeInApproved = false; 395 396 if (now < session.micGateReleaseUntil) { 397 session.micGateActive = true; 398 setMicGateOpen(session, false); 399 return; 400 } 401 402 session.micGateReleaseUntil = 0; 403 session.micGateActive = false; 404 setMicGateOpen(session, true); 405 return; 406 } 407 408 if (session.earlySpeechCandidateSince != null) { 409 if (level < MIC_GATE.bargeInFloor) { 410 // A pre-playback sound that falls away was an impulse, not a learner 411 // holding the floor. It may now safely contribute to normal echo 412 // calibration if the protected bootstrap is still running. 413 session.earlySpeechCandidateSince = null; 414 } else if (now - session.earlySpeechCandidateSince >= MIC_GATE.bargeInHoldMs) { 415 session.earlySpeechCandidateSince = null; 416 session.bargeInApproved = true; 417 setMicGateOpen(session, true); 418 return; 419 } else { 420 // This energy existed before any remote audio was audible. Preserve it 421 // as a candidate instead of absorbing it into the echo floor, but keep 422 // the gate closed until it satisfies the full speech hold. 423 session.micGateActive = true; 424 setMicGateOpen(session, false); 425 return; 426 } 427 } 428 429 if (now < session.micGateCalibrationUntil) { 430 // Calibrate from the pre-gate analyser while outbound audio is closed. 431 // Loud samples are deliberately included: excluding anything above the 432 // initial speech threshold creates a bootstrap loop in which loud echo 433 // can never become part of the learned echo floor. 434 session.echoFloor = session.echoFloor > 0 435 ? session.echoFloor * 0.65 + level * 0.35 436 : level; 437 session.bargeInSince = null; 438 session.bargeInApproved = false;
439 session.micGateActive = true; 440 setMicGateOpen(session, false); 441 return; 442 } 443 444 if (session.bargeInApproved) { 445 setMicGateOpen(session, true); 446 return; 447 } 448 449 const threshold = Math.max( 450 MIC_GATE.bargeInFloor, 451 session.echoFloor >= MIC_GATE.echoDetectFloor 452 ? session.echoFloor * MIC_GATE.bargeInMargin 453 : MIC_GATE.bargeInFloor); 454 const speaking = level >= threshold; 455 456 if (!speaking) { 457 session.bargeInSince = null; 458 // Whatever the microphone hears below the barge-in threshold while the 459 // persona is talking is that persona leaking back in. Tracking it is what 460 // makes this adaptive: on headphones the floor never rises and the gate 461 // stays out of the way entirely, while on a loud speaker it climbs until 462 // only genuine speech gets through. 463 session.echoFloor = session.echoFloor * 0.75 + level * 0.25; 464 session.micGateActive = true; 465 setMicGateOpen(session, false); 466 return; 467 } 468 469 session.micGateActive = true; 470 471 if (speaking) { 472 session.bargeInSince ??= now; 473 const heldMs = now - session.bargeInSince; 474 if (heldMs >= MIC_GATE.bargeInHoldMs) { 475 session.bargeInApproved = true; 476 setMicGateOpen(session, true); 477 return; 478 } 479 } 480 481 setMicGateOpen(session, false); 482 } 483 484 function stopMicPipeline(session) { 485 if (!session) { 486 return; 487 } 488 489 if (session.micFrame) { 490 window.cancelAnimationFrame(session.micFrame); 491 session.micFrame = null; 492 } 493 494 try { 495 session.micSource?.disconnect(); 496 } catch { 497 } 498 499 try { 500 session.micGain?.disconnect(); 501 } catch { 502 } 503 504 try { 505 session.micAnalyser?.disconnect(); 506 } catch { 507 } 508 509 if (session.micContext) { 510 try { 511 void session.micContext.close(); 512 } catch { 513 } 514 } 515 516 session.micContext = null; 517 session.micSource = null; 518 session.micGain = null; 519 session.micDestination = null; 520 session.micAnalyser = null; 521 session.micAnalyserData = null; 522 session.micLevel = 0; 523 session.micGateOpen = true; 524 session.micGateActive = false; 525 session.echoFloor = 0; 526 session.bargeInSince = null; 527 session.bargeInApproved = false; 528 session.earlySpeechCandidateSince = null; 529 session.providerPlaybackActive = false; 530 session.protectedPlaybackActive = false; 531 session.remoteSignalAudible = false; 532 session.remoteOnsetCalibrated = false; 533 session.micGateCalibrationUntil = 0; 534 session.micGateReleaseUntil = 0; 535 } 536 537 // Returns the stream to publish: the gated one when Web Audio is available, and 538 // otherwise the raw capture, so a browser without it still gets a working call 539 // with only the browser's own echo cancellation. 540 function createMicPipeline(session, stream) { 541 const AudioContextCtor = window.AudioContext || window.webkitAudioContext; 542 if (!AudioContextCtor) { 543 return stream; 544 } 545 546 try { 547 const micContext = new AudioContextCtor(); 548 const micSource = micContext.createMediaStreamSource(stream); 549 const micGain = micContext.createGain(); 550 const micDestination = micContext.createMediaStreamDestination(); 551 const micAnalyser = micContext.createAnalyser(); 552 micAnalyser.fftSize = 256; 553 micAnalyser.smoothingTimeConstant = 0.6; 554 555 // The analyser taps the capture ahead of the gate, so the learner's level 556 // is still measurable while the gate is shut. Reading it after the gate 557 // would make barge-in impossible to detect. 558 micSource.connect(micAnalyser); 559 micSource.connect(micGain); 560 micGain.connect(micDestination); 561 562 session.micContext = micContext; 563 session.micSource = micSource; 564 session.micGain = micGain; 565 session.micDestination = micDestination; 566 session.micAnalyser = micAnalyser; 567 session.micAnalyserData = new Uint8Array(micAnalyser.fftSize); 568 session.micGateOpen = true; 569 session.micGain.gain.value = 1; 570 571 try { 572 if (micContext.state === "suspended") { 573 void micContext.resume(); 574 } 575 } catch { 576 } 577 578 const sample = () => { 579 if (!session.micAnalyser || !session.micAnalyserData) { 580 return; 581 } 582 583 sampleMicGate(session); 584 session.micFrame = window.requestAnimationFrame(sample); 585 }; 586 587 ensureHiddenSampler(session); 588 sample(); 589 return micDestination.stream; 590 } catch { 591 stopMicPipeline(session); 592 return stream; 593 } 594 } 595 596 // True when every field the server asked for is present and equal in what the provider 597 // reports. A subset check rather than equality: the effective session carries provider 598 // defaults the server never sent. 599 function sessionConfigurationContains(actual, expected) { 600 if (expected === null || typeof expected !== "object") { 601 return actual === expected; 602 } 603 604 if (Array.isArray(expected)) { 605 return Array.isArray(actual) 606 && actual.length === expected.length 607 && expected.every((item, index) => sessionConfigurationContains(actual[index], item)); 608 } 609 610 if (actual === null || typeof actual !== "object" || Array.isArray(actual)) { 611 return false;
612 } 613 614 return Object.keys(expected).every(key => 615 Object.prototype.hasOwnProperty.call(actual, key) 616 && sessionConfigurationContains(actual[key], expected[key])); 617 } 618 619 // Returns the delta kind to report, or null while the outcome is still undecided â 620 // either nothing was asserted, or a re-send is in flight and the next session.updated 621 // will settle it. 622 function verifySessionConfiguration(session, providerEvent) { 623 const pending = session.pendingSessionConfiguration; 624 if (!pending) { 625 return null; 626 } 627 628 if (sessionConfigurationContains(providerEvent.session, pending.expected)) { 629 session.pendingSessionConfiguration = null; 630 return "SessionConfigurationApplied"; 631 } 632 633 if (pending.reasserts >= MAX_SESSION_CONFIGURATION_REASSERTS) { 634 session.pendingSessionConfiguration = null; 635 return "SessionConfigurationDrifted"; 636 } 637 638 pending.reasserts++; 639 const dataChannel = session.negotiationDataChannel; 640 if (dataChannel && dataChannel.readyState === "open") { 641 try { 642 dataChannel.send(pending.eventJson); 643 } catch { 644 // The next session.updated still decides the outcome; a failed re-send 645 // simply spends one attempt. 646 } 647 } 648 649 return null; 650 } 651 652 // True when a finished response carried audio, false when its output carried none, 653 // null when the event does not say. 654 function responseProducedAudio(response) { 655 const output = Array.isArray(response?.output) ? response.output : null; 656 if (!output) { 657 return null; 658 } 659 660 return output.some(item => Array.isArray(item?.content) 661 && item.content.some(part => part?.type === "output_audio" || part?.type === "audio")); 662 } 663 664 function asStringValue(value) { 665 return typeof value === "string" && value ? value : null; 666 } 667 668 // Bounded: a runaway provider message must not turn the delta channel into the problem 669 // being diagnosed. 670 function describeProviderError(error) { 671 const parts = [error?.type, error?.code, error?.message] 672 .filter(part => typeof part === "string" && part); 673 const described = parts.length > 0 ? parts.join(": ") : "The provider reported an error."; 674 return described.length > MAX_PROVIDER_ERROR_LENGTH 675 ? described.slice(0, MAX_PROVIDER_ERROR_LENGTH) 676 : described; 677 } 678 679 function forwardObservedDelta(session, providerEvent) { 680 // Not an observed delta: the learner cut this persona item off, so whatever 681 // transcript the model generated for it overstates what was actually heard. 682 // Recorded, it becomes conversation context â and evaluator context â for words 683 // the learner never got. The sideband dropped these the same way; it saw the same 684 // event on its own socket. 685 if (providerEvent.type === "conversation.item.truncated") { 686 const truncatedItemId = typeof providerEvent.item_id === "string" 687 ? providerEvent.item_id 688 : null; 689 if (truncatedItemId) { 690 if (session.truncatedPersonaItemIds.size >= MAX_TRUNCATED_PERSONA_ITEMS) { 691 session.truncatedPersonaItemIds.delete( 692 session.truncatedPersonaItemIds.values().next().value); 693 } 694 session.truncatedPersonaItemIds.add(truncatedItemId); 695 session.bufferedPersonaTranscripts.delete(truncatedItemId); 696 697 // Forwarded as well as acted on here. The transcript may already have been 698 // sent, and the server holds a reply's evidence until playback settles, so 699 // this is how it learns that what it is holding was cut off. 700 emitObservedDelta(session, { 701 eventId: asStringValue(providerEvent.event_id), 702 kind: "PersonaItemTruncated", 703 itemId: truncatedItemId, 704 responseId: null, 705 providerEventType: providerEvent.type 706 }); 707 } 708 return; 709 } 710 711 // A tool call. The provider holds the conversation until it has an output, and only 712 // this client can send one â unanswered, the dangling call is something later 713 // response.create requests can be refused over. Announced on more than one event, 714 // so both are forwarded and the director answers a given call id once. 715 if (providerEvent.type === "response.function_call_arguments.done" 716 || providerEvent.type === "response.output_item.done") { 717 const call = providerEvent.item ?? providerEvent; 718 const callId = asStringValue(call.call_id) ?? asStringValue(providerEvent.call_id); 719 const toolName = asStringValue(call.name) ?? asStringValue(providerEvent.name); 720 if (!callId 721 || (providerEvent.type === "response.output_item.done" 722 && call.type !== "function_call")) { 723 return; 724 } 725 726 emitObservedDelta(session, { 727 eventId: asStringValue(providerEvent.event_id), 728 kind: "PersonaToolCall", 729 itemId: asStringValue(call.id) ?? asStringValue(providerEvent.item_id), 730 responseId: asStringValue(providerEvent.response_id), 731 transcriptText: null, 732 responseStatus: null, 733 providerEventType: providerEvent.type, 734 toolCallId: callId, 735 toolName 736 }); 737 return; 738 } 739 740 // A provider refusal. It is not conversation state, but a refused response.create 741 // never becomes a response, so the director has to hear about it or the turn it was 742 // waiting on never resolves and the session goes silent. 743 if (providerEvent.type === "error") { 744 const failed = providerEvent.error ?? {}; 745 emitObservedDelta(session, { 746 eventId: asStringValue(providerEvent.event_id), 747 kind: "ProviderError", 748 itemId: null, 749 responseId: null, 750 transcriptText: null, 751 responseStatus: null, 752 providerEventType: providerEvent.type, 753 failedClientEventId: asStringValue(failed.event_id), 754 // Kept apart from the description so a transient refusal can be told
755 // from one retrying cannot fix. 756 errorType: asStringValue(failed.type), 757 errorCode: asStringValue(failed.code), 758 detail: describeProviderError(failed) 759 }); 760 return; 761 } 762 763 const kind = providerEvent.type === "session.updated" 764 ? verifySessionConfiguration(session, providerEvent) 765 : OBSERVED_DELTA_KINDS.get(providerEvent.type); 766 if (!kind) { 767 return; 768 } 769 770 const asString = value => typeof value === "string" && value ? value : null; 771 const responseId = asString(providerEvent.response_id) 772 ?? asString(providerEvent.response?.id); 773 const itemId = asString(providerEvent.item_id) 774 ?? asString(providerEvent.item?.id) 775 // response.done carries its transcript-bearing item nested in the response. 776 ?? asString(providerEvent.response?.output?.[0]?.id); 777 778 const delta = { 779 eventId: asString(providerEvent.event_id), 780 kind, 781 itemId, 782 responseId, 783 transcriptText: asString(providerEvent.transcript), 784 responseStatus: asString(providerEvent.response?.status), 785 providerEventType: providerEvent.type 786 }; 787 788 // Whether there is audio still to play. Generation finishing is not the reply 789 // finishing: the server holds a reply's evidence until its playback settles, and 790 // needs to know whether there is any playback to wait for. 791 if (kind === "PersonaResponseDone") { 792 delta.responseHasAudio = responseProducedAudio(providerEvent.response); 793 } 794 795 // A persona transcript is held until its response terminates, because truncation 796 // can be reported after the transcript is generated. The server holds it further, 797 // until playback settles, since a cut-off can still come after that. The sequence number is assigned on the way out so the server still sees a 798 // contiguous stream. 799 if (kind === "PersonaTranscriptCompleted") { 800 if (itemId && session.truncatedPersonaItemIds.has(itemId)) { 801 return; 802 } 803 804 if (itemId) { 805 session.bufferedPersonaTranscripts.set(itemId, delta); 806 return; 807 } 808 } 809 810 if (kind === "PersonaResponseDone") { 811 for (const [bufferedItemId, buffered] of [...session.bufferedPersonaTranscripts]) { 812 if (buffered.responseId && responseId && buffered.responseId !== responseId) { 813 continue; 814 } 815 816 session.bufferedPersonaTranscripts.delete(bufferedItemId); 817 if (!session.truncatedPersonaItemIds.has(bufferedItemId)) { 818 emitObservedDelta(session, buffered); 819 } 820 } 821 } 822 823 emitObservedDelta(session, delta); 824 } 825 826 function emitObservedDelta(session, delta) { 827 void notify(session, "providerdelta", { 828 clientSequenceNumber: session.observedDeltaSequence++, 829 observedUtc: new Date().toISOString(), 830 ...delta 831 }); 832 } 833 834 async function notify(session, type, payload) { 835 if (!session || !session.dotNetRef) { 836 return false; 837 } 838 839 const dotNetRef = session.dotNetRef; 840 const payloadJson = typeof payload === "string" ? payload : JSON.stringify(payload ?? {}); 841 try { 842 await dotNetRef.invokeMethodAsync("OnEvent", type, payloadJson); 843 return true; 844 } catch { 845 // The Blazor component can be disposed while a browser event is queued. 846 // Do not turn that expected teardown race into an unhandled rejection. 847 if (session.dotNetRef === dotNetRef) { 848 session.dotNetRef = null; 849 } 850 return false; 851 } 852 } 853 854 function rememberBounded(map, key, value, limit) { 855 if (map.has(key)) { 856 map.delete(key); 857 } else if (map.size >= limit) { 858 map.delete(map.keys().next().value); 859 } 860 map.set(key, value); 861 } 862 863 function rememberCompletedNavigationCall(session, callId) { 864 if (session.completedNavigationToolCallIds.has(callId)) { 865 return; 866 } 867 if (session.completedNavigationToolCallIds.size >= MAX_COMPLETED_NAVIGATION_CALLS) { 868 session.completedNavigationToolCallIds.delete( 869 session.completedNavigationToolCallIds.values().next().value); 870 } 871 session.completedNavigationToolCallIds.add(callId); 872 } 873 874 function sendNavigationAcknowledgement(session, callId, outputItemId, route) { 875 const dataChannel = session?.negotiationDataChannel; 876 if (!dataChannel || dataChannel.readyState !== "open") { 877 return false;
878 } 879 880 const acknowledgementItemId = 881 `coree_nav_ack_${session.registrationId}_${++session.navigationAcknowledgementSequence}`; 882 try { 883 dataChannel.send(JSON.stringify({ 884 type: "conversation.item.create", 885 item: { 886 id: acknowledgementItemId, 887 type: "message", 888 role: "user", 889 content: [ 890 { 891 type: "input_text", 892 text: NAVIGATION_ACK_PREFIX + JSON.stringify({ 893 callId, 894 outputItemId, 895 route 896 }) 897 } 898 ] 899 } 900 })); 901 return true; 902 } catch { 903 return false; 904 } 905 } 906 907 async function tryCompleteNavigationHandshake(session, callId) { 908 const handshake = session.navigationHandshakes.get(callId); 909 if (!handshake?.callSeen 910 || !handshake.itemId 911 || typeof handshake.output !== "string" 912 || session.completedNavigationToolCallIds.has(callId)) { 913 return; 914 } 915 916 session.navigationHandshakes.delete(callId); 917 rememberCompletedNavigationCall(session, callId); 918 await notify(session, "authorizednavigation", { 919 callId, 920 itemId: handshake.itemId, 921 output: handshake.output 922 }); 923 } 924 925 function rememberNavigationToolCall(session, callId) { 926 if (typeof callId !== "string" 927 || !callId 928 || callId.length > MAX_NAVIGATION_ID_LENGTH 929 || session.completedNavigationToolCallIds.has(callId)) { 930 return; 931 } 932 933 const handshake = session.navigationHandshakes.get(callId) ?? { 934 callSeen: false, 935 itemId: null, 936 output: null 937 }; 938 handshake.callSeen = true; 939 rememberBounded( 940 session.navigationHandshakes, 941 callId, 942 handshake, 943 MAX_NAVIGATION_HANDSHAKES); 944 void tryCompleteNavigationHandshake(session, callId); 945 } 946 947 // The tool result is tied to the call the model itself issued. The item id is only 948 // relayed back for the acknowledgement, because the provider may echo a server 949 // created item under an id of its own choosing. 950 function rememberNavigationToolOutput(session, callId, itemId, output) { 951 if (typeof callId !== "string" 952 || !callId 953 || callId.length > MAX_NAVIGATION_ID_LENGTH 954 || typeof itemId !== "string" 955 || !itemId 956 || itemId.length > MAX_NAVIGATION_ID_LENGTH 957 || typeof output !== "string" 958 || output.length > MAX_NAVIGATION_OUTPUT_LENGTH 959 || session.completedNavigationToolCallIds.has(callId)) { 960 return; 961 } 962 963 const handshake = session.navigationHandshakes.get(callId) ?? { 964 callSeen: false, 965 itemId: null, 966 output: null 967 }; 968 handshake.itemId = itemId; 969 handshake.output = output; 970 rememberBounded( 971 session.navigationHandshakes, 972 callId, 973 handshake, 974 MAX_NAVIGATION_HANDSHAKES); 975 void tryCompleteNavigationHandshake(session, callId); 976 } 977 978 async function ensureRemoteAudio(session) { 979 if (session.remoteAudio) { 980 return session.remoteAudio; 981 } 982 983 const audio = document.createElement("audio"); 984 audio.autoplay = true; 985 audio.playsInline = true; 986 audio.muted = session.outputMuted; 987 audio.style.display = "none"; 988 document.body.appendChild(audio); 989 session.remoteAudio = audio; 990 return audio; 991 } 992 993 function getVisualizerTarget(session) { 994 if (!session || !session.visualizerTargetId) { 995 return null; 996 } 997 998 return document.getElementById(session.visualizerTargetId); 999 } 1000 1001 function updateVisualizer(session, level) { 1002 const target = getVisualizerTarget(session); 1003 if (!target) { 1004 return; 1005 } 1006 1007 const clampedLevel = Math.max(0, Math.min(1, level)); 1008 target.style.setProperty("--rc-voice-level", clampedLevel.toFixed(3)); 1009 target.dataset.voiceActive = clampedLevel > 0.06 ? "true" : "false"; 1010 } 1011 1012 function clearPlaybackInactiveTimeout(session) { 1013 if (!session?.playbackInactiveTimeout) { 1014 return; 1015 } 1016 1017 window.clearTimeout(session.playbackInactiveTimeout); 1018 session.playbackInactiveTimeout = null; 1019 } 1020 1021 /// The provider lifecycle is authoritative for the speaking status; the analysed 1022 /// remote level in updatePlaybackActivity stays as the fallback for when these 1023 /// events are late or unavailable. 1024 function markPlaybackActiveFromProvider(session, active) { 1025 if (!session) { 1026 return; 1027 } 1028 1029 clearPlaybackInactiveTimeout(session); 1030 if (session.playbackActive === active) { 1031 return; 1032 } 1033 1034 session.playbackActive = active; 1035 void notify(session, active ? "remote_audio_playback_started" : "remote_audio_playback_stopped", { 1036 state: active ? "speaking" : "idle" 1037 }); 1038 } 1039 1040 function updatePlaybackActivity(session, level) { 1041 if (!session) { 1042 return; 1043 } 1044 1045 const activeThreshold = 0.06; 1046 const inactiveThreshold = 0.025; 1047 const inactiveDebounceMs = 450; 1048 1049 // The provider's playback lifecycle is the earliest protection signal, while 1050 // the analysed remote level is the fallback when those events arrive late or 1051 // are unavailable. Release is intentionally governed by the shorter gate 1052 // hangover rather than the UI's longer speaking-status debounce below. 1053 session.remoteLevel = level; 1054 updateProtectedPlaybackFromRemoteLevel(session); 1055 evaluateMicGate(session); 1056 1057 if (level >= activeThreshold) { 1058 clearPlaybackInactiveTimeout(session); 1059 1060 if (session.playbackActive) { 1061 return; 1062 } 1063 1064 session.playbackActive = true; 1065 void notify(session, "remote_audio_playback_started", { 1066 state: "speaking" 1067 }); 1068 return; 1069 } 1070 1071 if (!session.playbackActive || level > inactiveThreshold || session.playbackInactiveTimeout) { 1072 return; 1073 } 1074 1075 session.playbackInactiveTimeout = window.setTimeout(() => { 1076 session.playbackInactiveTimeout = null; 1077 if (!session.playbackActive) { 1078 return; 1079 } 1080 1081 session.playbackActive = false;
1082 void notify(session, "remote_audio_playback_stopped", { 1083 state: "idle" 1084 }); 1085 }, inactiveDebounceMs); 1086 } 1087 1088 async function stopVisualizer(session) { 1089 if (!session) { 1090 return; 1091 } 1092 1093 if (session.visualizerFrame) { 1094 window.cancelAnimationFrame(session.visualizerFrame); 1095 session.visualizerFrame = null; 1096 } 1097 1098 try { 1099 session.analyserSource?.disconnect(); 1100 } catch { 1101 } 1102 1103 try { 1104 session.analyser?.disconnect(); 1105 } catch { 1106 } 1107 1108 if (session.audioContext) { 1109 try { 1110 await session.audioContext.close(); 1111 } catch { 1112 } 1113 } 1114 1115 session.audioContext = null; 1116 session.analyser = null; 1117 session.analyserData = null; 1118 session.analyserSource = null; 1119 session.smoothedVoiceLevel = 0; 1120 session.playbackActive = false; 1121 session.remoteLevel = 0; 1122 clearPlaybackInactiveTimeout(session); 1123 evaluateMicGate(session); 1124 updateVisualizer(session, 0); 1125 } 1126 1127 async function startVisualizer(session, stream) { 1128 if (!session || !stream) { 1129 return; 1130 } 1131 1132 await stopVisualizer(session); 1133 1134 const AudioContextCtor = window.AudioContext || window.webkitAudioContext; 1135 if (!AudioContextCtor) { 1136 return; 1137 } 1138 1139 const audioContext = new AudioContextCtor(); 1140 const analyser = audioContext.createAnalyser(); 1141 analyser.fftSize = 256; 1142 analyser.smoothingTimeConstant = 0.85; 1143 1144 const analyserSource = audioContext.createMediaStreamSource(stream); 1145 analyserSource.connect(analyser); 1146 1147 session.audioContext = audioContext; 1148 session.analyser = analyser; 1149 session.analyserSource = analyserSource; 1150 session.analyserData = new Uint8Array(analyser.fftSize); 1151 session.smoothedVoiceLevel = 0; 1152 1153 try { 1154 if (audioContext.state === "suspended") { 1155 await audioContext.resume(); 1156 } 1157 } catch { 1158 } 1159 1160 const sample = () => { 1161 if (!session.analyser || !session.analyserData) { 1162 return; 1163 } 1164 1165 sampleRemoteLevel(session); 1166 session.visualizerFrame = window.requestAnimationFrame(sample); 1167 }; 1168 1169 ensureHiddenSampler(session); 1170 sample(); 1171 } 1172 1173 async function cleanupSession(session) { 1174 if (!session) { 1175 return; 1176 } 1177 1178 await cleanupNarration(session); 1179 await stopVisualizer(session); 1180 stopMicPipeline(session); 1181 stopHiddenSampler(session); 1182 1183 if (session.negotiationDataChannel) { 1184 try { 1185 // A local teardown must not be reported back to the component as a 1186 // provider-side connection failure. 1187 session.negotiationDataChannel.onopen = null; 1188 session.negotiationDataChannel.onclose = null; 1189 session.negotiationDataChannel.onerror = null; 1190 session.negotiationDataChannel.onmessage = null; 1191 session.negotiationDataChannel.close(); 1192 } catch { 1193 } 1194 } 1195 1196 if (session.peerConnection) { 1197 try { 1198 session.peerConnection.ontrack = null; 1199 session.peerConnection.onconnectionstatechange = null; 1200 session.peerConnection.oniceconnectionstatechange = null; 1201 session.peerConnection.close(); 1202 } catch { 1203 } 1204 } 1205 1206 if (session.localStream) { 1207 stopStreamTracks(session.localStream); 1208 } 1209 1210 if (session.outboundStream && session.outboundStream !== session.localStream) { 1211 stopStreamTracks(session.outboundStream); 1212 } 1213 1214 if (session.remoteAudio) { 1215 session.remoteAudio.muted = false; 1216 session.remoteAudio.pause(); 1217 session.remoteAudio.srcObject = null; 1218 session.remoteAudio.remove(); 1219 session.remoteAudio = null; 1220 } 1221 1222 session.peerConnection = null; 1223 session.negotiationDataChannel = null; 1224 session.localStream = null; 1225 session.outboundStream = null; 1226 session.micMuted = false;
1227 session.outputMuted = false; 1228 session.pendingRuntimeContextItemIds.clear(); 1229 session.navigationHandshakes.clear(); 1230 session.completedNavigationToolCallIds.clear(); 1231 } 1232 1233 async function cleanupNarration(session) { 1234 if (!session) { 1235 return; 1236 } 1237 1238 if (session.narrationTimeout) { 1239 window.clearTimeout(session.narrationTimeout); 1240 session.narrationTimeout = null; 1241 } 1242 1243 const resolve = session.narrationCompletionResolve; 1244 session.narrationCompletionResolve = null; 1245 resolve?.(); 1246 1247 try { 1248 if (session.narrationDataChannel) { 1249 session.narrationDataChannel.onopen = null; 1250 session.narrationDataChannel.onmessage = null; 1251 session.narrationDataChannel.onerror = null; 1252 session.narrationDataChannel.close(); 1253 } 1254 } catch { 1255 } 1256 1257 try { 1258 session.narrationPeerConnection?.close(); 1259 } catch { 1260 } 1261 1262 if (session.narrationAudio) { 1263 try { 1264 session.narrationAudio.pause(); 1265 session.narrationAudio.srcObject = null; 1266 session.narrationAudio.remove(); 1267 } catch { 1268 } 1269 } 1270 1271 session.narrationPeerConnection = null; 1272 session.narrationDataChannel = null; 1273 session.narrationAudio = null; 1274 } 1275 1276 function waitForIceComplete(peerConnection, timeoutMs) { 1277 if (peerConnection.iceGatheringState === "complete") { 1278 return Promise.resolve(true); 1279 } 1280 1281 return new Promise(resolve => { 1282 let timeoutHandle = null; 1283 const handler = () => { 1284 if (peerConnection.iceGatheringState === "complete") { 1285 if (timeoutHandle) { 1286 clearTimeout(timeoutHandle); 1287 } 1288 peerConnection.removeEventListener("icegatheringstatechange", handler); 1289 resolve(true); 1290 } 1291 }; 1292 1293 peerConnection.addEventListener("icegatheringstatechange", handler); 1294 1295 timeoutHandle = window.setTimeout(() => { 1296 peerConnection.removeEventListener("icegatheringstatechange", handler); 1297 resolve(false); 1298 }, timeoutMs); 1299 }); 1300 } 1301 1302 window.tcRcTraining = { 1303 register: function (id, dotNetRef) { 1304 const session = getSession(id); 1305 session.dotNetRef = dotNetRef; 1306 }, 1307 1308 createOffer: async function (id) { 1309 const session = getSession(id); 1310 const registrationId = session.registrationId; 1311 const offerGeneration = ++session.offerGeneration; 1312 await cleanupSession(session); 1313 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1314 return ""; 1315 } 1316 1317 try { 1318 const stream = await navigator.mediaDevices.getUserMedia({ 1319 audio: { 1320 echoCancellation: true, 1321 noiseSuppression: true, 1322 autoGainControl: true, 1323 // Newer Chromium builds only apply the stronger, model-based echo 1324 // canceller when it is asked for by name. Unknown keys are ignored, 1325 // so this is safe on browsers that do not implement it. 1326 echoCancellationType: "system", 1327 voiceIsolation: true 1328 } 1329 }); 1330 1331 // disconnect retires the registration and advances its generation 1332 // before awaiting cleanup. A late permission/capture result therefore 1333 // belongs to no live session and must never be attached to a peer. 1334 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1335 stopStreamTracks(stream); 1336 return ""; 1337 } 1338 session.localStream = stream; 1339 1340 const peerConnection = new RTCPeerConnection(); 1341 // OpenAI's WebRTC offer requires an oai-events data channel. The browser 1342 // uses it only to share non-authoritative CORE-E runtime context; private 1343 // response and tool control stays on the authenticated sideband, which 1344 // independently validates provenance because a browser peer can be modified. 1345 const negotiationDataChannel = peerConnection.createDataChannel
1345("oai-events"); 1346 1347 negotiationDataChannel.onopen = function () { 1348 void notify(session, "datachannelstatechange", { 1349 state: negotiationDataChannel.readyState 1350 }); 1351 }; 1352 negotiationDataChannel.onclose = function () { 1353 void notify(session, "datachannelstatechange", { 1354 state: "closed" 1355 }); 1356 }; 1357 negotiationDataChannel.onerror = function () { 1358 void notify(session, "datachannelerror", { 1359 state: negotiationDataChannel.readyState ?? "unknown" 1360 }); 1361 }; 1362 negotiationDataChannel.onmessage = function (event) { 1363 if (session.retired || typeof event?.data !== "string") { 1364 return; 1365 } 1366 1367 let providerEvent; 1368 try { 1369 providerEvent = JSON.parse(event.data); 1370 } catch { 1371 return; 1372 } 1373 1374 if (typeof providerEvent?.type === "string") { 1375 // Forward before the local-concern handlers below, several of 1376 // which return early and would otherwise drop the delta. 1377 forwardObservedDelta(session, providerEvent); 1378 } 1379 1380 if (providerEvent?.type === "output_audio_buffer.started") { 1381 const isNewPlayback = !session.providerPlaybackActive; 1382 session.providerPlaybackActive = true; 1383 // The provider knows its output has begun before the analysed level 1384 // crosses a threshold, so the speaking status follows this rather 1385 // than waiting for the fallback to notice. 1386 markPlaybackActiveFromProvider(session, true); 1387 if (isNewPlayback) { 1388 const remoteAlreadyAudible = session.remoteSignalAudible 1389 || (session.remoteLevel ?? 0) >= MIC_GATE.remoteAudibleLevel; 1390 session.earlySpeechCandidateSince = !session.outputMuted 1391 && !session.micMuted 1392 && !remoteAlreadyAudible 1393 && (session.micLevel ?? 0) >= MIC_GATE.bargeInFloor 1394 ? micGateNow() 1395 : null; 1396 } 1397 if (session.outputMuted) { 1398 disablePlaybackProtection(session); 1399 } else { 1400 beginProtectedPlayback(session); 1401 } 1402 evaluateMicGate(session); 1403 return; 1404 } 1405 1406 if (providerEvent?.type === "output_audio_buffer.stopped") { 1407 session.providerPlaybackActive = false; 1408 // And it knows when the output ended. The analysed level decays 1409 // through a smoothed analyser and a debounce, which leaves the 1410 // status reading "Responding" for seconds after she has stopped. 1411 markPlaybackActiveFromProvider(session, false); 1412 session.remoteSignalAudible = false; 1413 session.remoteOnsetCalibrated = false; 1414 if (session.outputMuted) { 1415 disablePlaybackProtection(session); 1416 } else { 1417 endProtectedPlayback(session); 1418 } 1419 evaluateMicGate(session); 1420 void notify(session, "assistantresponsecompleted", { 1421 responseId: typeof providerEvent.response_id === "string" 1422 ? providerEvent.response_id 1423 : null 1424 }); 1425 return; 1426 } 1427 1428 if (providerEvent?.type === "output_audio_buffer.cleared") { 1429 markPlaybackActiveFromProvider(session, false); 1430 session.providerPlaybackActive = false;
1431 session.remoteSignalAudible = false; 1432 session.remoteOnsetCalibrated = false; 1433 if (session.outputMuted) { 1434 disablePlaybackProtection(session); 1435 } else { 1436 endProtectedPlayback(session); 1437 } 1438 evaluateMicGate(session); 1439 return; 1440 } 1441 1442 const item = providerEvent?.item; 1443 if (!item || typeof item !== "object") { 1444 return; 1445 } 1446 1447 const itemId = typeof item.id === "string" ? item.id : ""; 1448 const isConversationItemLifecycleEvent = 1449 providerEvent.type === "conversation.item.added" 1450 || providerEvent.type === "conversation.item.done" 1451 // Retain compatibility with sessions that still emit the 1452 // earlier event name while GA Realtime uses added/done. 1453 || providerEvent.type === "conversation.item.created"; 1454 if (isConversationItemLifecycleEvent 1455 && session.pendingRuntimeContextItemIds.has(itemId)) { 1456 session.pendingRuntimeContextItemIds.delete(itemId); 1457 void notify(session, "runtimecontextack", { itemId }); 1458 return; 1459 } 1460 1461 const callId = typeof item.call_id === "string" ? item.call_id : ""; 1462 if ((providerEvent.type === "response.output_item.added" 1463 || providerEvent.type === "response.output_item.done" 1464 || isConversationItemLifecycleEvent) 1465 && item.type === "function_call" 1466 && item.name === "navigate_to_app_page" 1467 && callId) { 1468 // The call request alone is not authority to navigate. Remember 1469 // it only so a later server-created function output can be tied 1470 // to the exact navigation tool invocation. 1471 rememberNavigationToolCall(session, callId); 1472 return; 1473 } 1474 1475 if (isConversationItemLifecycleEvent 1476 && item.type === "function_call_output") { 1477 rememberNavigationToolOutput(session, callId, itemId, item.output); 1478 } 1479 }; 1480 1481 session.peerConnection = peerConnection; 1482 session.negotiationDataChannel = negotiationDataChannel; 1483 const audio = await ensureRemoteAudio(session); 1484 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1485 return ""; 1486 } 1487 1488 peerConnection.ontrack = async function (event) { 1489 audio.srcObject = event.streams[0]; 1490 try { 1491 await audio.play(); 1492 } catch { 1493 } 1494 1495 await startVisualizer(session, event.streams[0]); 1496 }; 1497 1498 peerConnection.onconnectionstatechange = function () { 1499 void notify(session, "connectionstatechange", { 1500 state: peerConnection.connectionState 1501 }); 1502 }; 1503 1504 peerConnection.oniceconnectionstatechange = function () { 1505 void notify(session, "iceconnectionstatechange", { 1506 state: peerConnection.iceConnectionState 1507 }); 1508 }; 1509 1510 const outboundStream = createMicPipeline(session, stream); 1511 session.outboundStream = outboundStream; 1512 outboundStream.getTracks().forEach(track => peerConnection.addTrack(track, outboundStream)); 1513 1514 const offer = await peerConnection.createOffer(); 1515 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1516 return ""; 1517 } 1518 await peerConnection.setLocalDescription(offer); 1519 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1520 return ""; 1521 } 1522 const iceComplete = await waitForIceComplete(peerConnection, 5000); 1523 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1524 return ""; 1525 } 1526 if (!iceComplete) { 1527 await notify(session, "icegatheringtimeout", { 1528 state: peerConnection.iceGatheringState 1529 }); 1530 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1531 return ""; 1532 } 1533 } 1534 1535 await notify(session, "offercreated", { 1536 type: peerConnection.localDescription?.type ?? "offer" 1537 }); 1538 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1539 return ""; 1540 } 1541 1542 return peerConnection.localDescription?.sdp ?? ""; 1543 } catch (error) { 1544 // Never let an obsolete offer clean up a newer offer that reused the 1545 // same browser registration. Its invalidating operation owns cleanup. 1546 if (!isCurrentOffer(id, session, registrationId, offerGeneration)) { 1547 return ""; 1548 } 1549 1550 await cleanupSession(session); 1551 throw error; 1552 } 1553 }, 1554 1555 acceptAnswer: async function (id, answerSdp) { 1556 const session = getActiveSession(id); 1557 if (!session?.peerConnection) { 1558 throw new Error("No active RC training session exists."); 1559 } 1560 1561 await session.peerConnection.setRemoteDescription({ 1562 type: "answer", 1563 sdp: answerSdp 1564 }); 1565 1566 await notify(session, "answeraccepted", { 1567 type: "answer" 1568 }); 1569 }, 1570 1571 createNarrationOffer: async function (id) { 1572 const session = getActiveSession(id); 1573 if (!session) { 1574 throw new Error("No active RC training session exists."); 1575 } 1576 1577 await cleanupNarration(session); 1578 const peerConnection = new RTCPeerConnection(); 1579 const dataChannel = peerConnection.createDataChannel
1579("oai-events"); 1580 const audio = document.createElement("audio"); 1581 audio.autoplay = true; 1582 audio.playsInline = true; 1583 audio.style.display = "none"; 1584 document.body.appendChild(audio); 1585 1586 peerConnection.ontrack = async event => { 1587 audio.srcObject = event.streams[0]; 1588 try { 1589 await audio.play(); 1590 } catch { 1591 } 1592 }; 1593 peerConnection.addTransceiver("audio", { direction: "recvonly" }); 1594 session.narrationPeerConnection = peerConnection; 1595 session.narrationDataChannel = dataChannel; 1596 session.narrationAudio = audio; 1597 1598 const offer = await peerConnection.createOffer(); 1599 await peerConnection.setLocalDescription(offer); 1600 await waitForIceComplete(peerConnection, 5000); 1601 return peerConnection.localDescription?.sdp ?? ""; 1602 }, 1603 1604 playNarration: async function (id, answerSdp, narrationText, maxOutputTokens) { 1605 const session = getActiveSession(id); 1606 const peerConnection = session?.narrationPeerConnection; 1607 const dataChannel = session?.narrationDataChannel; 1608 if (!peerConnection || !dataChannel || !narrationText) { 1609 throw new Error("The RC scene narration is not ready."); 1610 } 1611 1612 await peerConnection.setRemoteDescription({ type: "answer", sdp: answerSdp }); 1613 await new Promise((resolve, reject) => { 1614 const narrationTokenBudget = Number.isFinite(maxOutputTokens) && maxOutputTokens > 0 1615 ? Math.min(4096, Math.max(1, Math.trunc(maxOutputTokens))) 1616 : NARRATION_DEFAULT_MAX_OUTPUT_TOKENS; 1617 const narrationTimeoutMilliseconds = Math.min( 1618 NARRATION_MAXIMUM_TIMEOUT_MILLISECONDS, 1619 Math.max( 1620 NARRATION_MINIMUM_TIMEOUT_MILLISECONDS, 1621 narrationTokenBudget * NARRATION_AUDIO_MILLISECONDS_PER_TOKEN 1622 + NARRATION_COMPLETION_GRACE_MILLISECONDS)); 1623 let modelResponseDone = false; 1624 let audioPlaybackStopped = false; 1625 let settled = false; 1626 1627 const settle = error => { 1628 if (settled) { 1629 return; 1630 } 1631 1632 settled = true; 1633 if (session.narrationTimeout) { 1634 window.clearTimeout(session.narrationTimeout); 1635 session.narrationTimeout = null; 1636 } 1637 1638 if (error) { 1639 reject(error); 1640 } else { 1641 resolve(); 1642 } 1643 }; 1644 const completeWhenReady = () => { 1645 if (modelResponseDone && audioPlaybackStopped) { 1646 settle(); 1647 } 1648 }; 1649 const sendNarration = () => { 1650 try { 1651 dataChannel.send(JSON.stringify({ 1652 type: "response.create", 1653 response: { 1654 conversation: "none", 1655 output_modalities: ["audio"], 1656 tools: [], 1657 max_output_tokens: narrationTokenBudget, 1658 instructions: "Read the supplied scene narration exactly once. Do not add any other words.", 1659 input: [{ 1660 type: "message", 1661 role: "user", 1662 content: [{ type: "input_text", text: narrationText }] 1663 }] 1664 } 1665 })); 1666 } catch (error) { 1667 settle(error); 1668 } 1669 }; 1670 1671 session.narrationCompletionResolve = () => settle(); 1672 session.narrationTimeout = window.setTimeout( 1673 () => settle(new Error("Scene narration playback timed out before it completed.")), 1674 narrationTimeoutMilliseconds); 1675 dataChannel.onmessage = event => { 1676 if (typeof event?.data !== "string") { 1677 return; 1678 } 1679 try { 1680 const providerEvent = JSON.parse(event.data); 1681 if (providerEvent?.type === "output_audio_buffer.stopped") { 1682 audioPlaybackStopped = true; 1683 completeWhenReady(); 1684 } else if (providerEvent?.type === "response.done") { 1685 const status = providerEvent.response?.status; 1686 if (status && status !== "completed") { 1687 settle(new Error( 1688 providerEvent.response?.status_details?.error?.message 1689 ?? `Scene narration ended with status ${status}.`)); 1690 return; 1691 } 1692 1693 modelResponseDone = true; 1694 completeWhenReady(); 1695 } else if (providerEvent?.type === "error") { 1696 settle(new Error(providerEvent.error?.message ?? "Scene narration failed.")); 1697 } 1698 } catch { 1699 } 1700 }; 1701 dataChannel.onerror = () => settle(new Error("Scene narration connection failed.")); 1702 if (dataChannel.readyState === "open") { 1703 sendNarration(); 1704 } else { 1705 dataChannel.onopen = sendNarration; 1706 } 1707 }).finally(async () => { 1708 await cleanupNarration(session); 1709 }); 1710 }, 1711 1712 stopNarration: async function (id) { 1713 const session = sessions[id]; 1714 await cleanupNarration(session); 1715 }, 1716 1717 setVisualizerTarget: function (id, elementId) { 1718 const session = getActiveSession(id); 1719 if (!session) { 1720 return false;
1721 } 1722 session.visualizerTargetId = elementId || null; 1723 updateVisualizer(session, session.smoothedVoiceLevel || 0); 1724 return true; 1725 }, 1726 1727 disconnect: async function (id) { 1728 const session = sessions[id]; 1729 if (!session) { 1730 return; 1731 } 1732 1733 // Invalidate pending createOffer continuations synchronously, before the 1734 // first cleanup await gives getUserMedia a chance to resolve. 1735 session.retired = true; 1736 session.offerGeneration++; 1737 await cleanupSession(session); 1738 await notify(session, "disconnected", { 1739 state: "closed" 1740 }); 1741 session.dotNetRef = null; 1742 if (sessions[id] === session) { 1743 delete sessions[id]; 1744 } 1745 }, 1746 1747 setMuted: async function (id, muted) { 1748 const session = getActiveSession(id); 1749 if (!session?.localStream) { 1750 return false; 1751 } 1752 1753 // Muting has to reach the published tracks, which are the gated ones when 1754 // the pipeline is in use, and it also pins the gate shut so the barge-in 1755 // detector cannot reopen it behind the learner's back. 1756 session.micMuted = muted; 1757 if (!muted && session.protectedPlaybackActive) { 1758 // A manually muted interval contains no useful outbound-gate history. 1759 // Recalibrate before allowing barge-in if playback is still active. 1760 session.micGateCalibrationUntil = micGateNow() + MIC_GATE.calibrationMs; 1761 session.bargeInSince = null; 1762 session.bargeInApproved = false; 1763 } 1764 evaluateMicGate(session); 1765 1766 const publishedStream = session.outboundStream ?? session.localStream; 1767 publishedStream.getAudioTracks().forEach(track => { 1768 track.enabled = !muted; 1769 }); 1770 1771 await notify(session, "mutestatechange", { 1772 muted: muted 1773 }); 1774 return true; 1775 }, 1776 1777 setOutputMuted: async function (id, muted) { 1778 const session = sessions[id]; 1779 if (!session || session.retired || !session.remoteAudio) { 1780 return false; 1781 } 1782 1783 // Keep receiving and analysing the remote stream so the voice visualizer 1784 // and playback lifecycle remain accurate while only audible output is muted. 1785 session.outputMuted = Boolean(muted); 1786 session.remoteAudio.muted = session.outputMuted; 1787 1788 if (session.outputMuted) { 1789 // Muted output cannot leak into the microphone. Do not let the still- 1790 // running remote analyser unnecessarily block a learner from speaking. 1791 disablePlaybackProtection(session); 1792 } else { 1793 // Becoming audible mid-response is a new acoustic onset even though 1794 // the provider response itself may have started while output was muted. 1795 session.remoteSignalAudible = false; 1796 session.remoteOnsetCalibrated = false; 1797 updateProtectedPlaybackFromRemoteLevel(session); 1798 evaluateMicGate(session); 1799 } 1800 1801 await notify(session, "outputmutestatechange", { 1802 muted: session.outputMuted 1803 }); 1804 return true; 1805 }, 1806 1807 sendRuntimeContext: function (id, runtimeContext) { 1808 const session = sessions[id]; 1809 const dataChannel = session?.negotiationDataChannel; 1810 if (!dataChannel || dataChannel.readyState !== "open") { 1811 return null; 1812 } 1813 1814 const itemId = `coree_ctx_${session.registrationId}_${++session.runtimeContextSequence}`; 1815 try { 1816 dataChannel.send(JSON.stringify({ 1817 type: "conversation.item.create", 1818 item: { 1819 id: itemId, 1820 type: "message", 1821 role: "user", 1822 content: [ 1823 { 1824 type: "input_text", 1825 text: "[CORE-E RUNTIME CONTEXT] " + JSON.stringify(runtimeContext ?? {}) 1826 } 1827 ] 1828 } 1829 })); 1830 // Runtime context is latest-wins. Retaining superseded IDs would grow 1831 // forever if the provider stopped echoing acknowledgements. 1832 session.pendingRuntimeContextItemIds.clear(); 1833 session.pendingRuntimeContextItemIds.add(itemId); 1834 return itemId; 1835 } catch { 1836 // The channel can transition to closing between the state check and send. 1837 return null; 1838 } 1839 }, 1840 1841 // Sends one pre-serialized provider control event on the session's oai-events 1842 // channel. Returns false when the channel is not open or the event is not one 1843 // the client is allowed to send, so the caller can retry or surface a stall 1844 // rather than assuming delivery. 1845 // The client director runs in .NET but its reports share this session's observed 1846 // delta sequence, because the server rejects anything out of order. Assigning the 1847 // number anywhere else puts two writers on one counter. 1848 reportDirectedResponse: function (id, requestId, responseId) { 1849 const session = sessions[id]; 1850 if (!session || session.retired 1851 || typeof requestId !== "string" || !requestId 1852 || typeof responseId !== "string" || !responseId) { 1853 return false;
1854 } 1855 1856 emitObservedDelta(session, { 1857 kind: "DirectedResponseStarted", 1858 responseId, 1859 directedRequestId: requestId, 1860 providerEventType: "client.directed_response" 1861 }); 1862 return true; 1863 }, 1864 1865 reportDirectorDecision: function (id, decision) { 1866 const session = sessions[id]; 1867 if (!session || session.retired 1868 || typeof decision !== "string" || !decision) { 1869 return false; 1870 } 1871 1872 emitObservedDelta(session, { 1873 kind: "DirectorDecision", 1874 detail: decision.length > MAX_DIRECTOR_DECISION_LENGTH 1875 ? decision.slice(0, MAX_DIRECTOR_DECISION_LENGTH) 1876 : decision, 1877 providerEventType: "client.director_decision" 1878 }); 1879 return true; 1880 }, 1881 1882 sendControlEvent: function (id, eventJson) { 1883 const session = sessions[id]; 1884 if (!session 1885 || session.retired 1886 || typeof eventJson !== "string" 1887 || !eventJson 1888 || eventJson.length > MAX_CONTROL_EVENT_LENGTH) { 1889 return false; 1890 } 1891 1892 let parsed; 1893 try { 1894 parsed = JSON.parse(eventJson); 1895 } catch { 1896 return false; 1897 } 1898 1899 if (!parsed || typeof parsed.type !== "string" 1900 || !CLIENT_DIRECTED_EVENT_TYPES.has(parsed.type)) { 1901 return false; 1902 } 1903 1904 const dataChannel = session.negotiationDataChannel; 1905 if (!dataChannel || dataChannel.readyState !== "open") { 1906 return false; 1907 } 1908 1909 try { 1910 dataChannel.send(eventJson); 1911 if (parsed.type === "session.update" 1912 && parsed.session !== null 1913 && typeof parsed.session === "object") { 1914 // Hold it so the next session.updated can be checked against what was 1915 // asked for, and re-sent if the provider did not take it. 1916 session.pendingSessionConfiguration = { 1917 eventJson, 1918 expected: parsed.session, 1919 reasserts: 0 1920 }; 1921 } 1922 return true; 1923 } catch { 1924 // The channel can transition to closing between the state check and send. 1925 return false; 1926 } 1927 }, 1928 1929 acknowledgeNavigation: function (id, callId, outputItemId, route) { 1930 const session = sessions[id]; 1931 if (!session 1932 || session.retired 1933 || typeof callId !== "string" 1934 || !callId 1935 || callId.length > MAX_NAVIGATION_ID_LENGTH 1936 || typeof outputItemId !== "string" 1937 || !outputItemId 1938 || outputItemId.length > MAX_NAVIGATION_ID_LENGTH 1939 || typeof route !== "string" 1940 || !route.startsWith("/") 1941 || route.startsWith("//") 1942 || route.length > 2_048) { 1943 return false; 1944 } 1945 1946 return sendNavigationAcknowledgement( 1947 session, 1948 callId, 1949 outputItemId, 1950 route); 1951 }, 1952 1953 scrollDialogToTop: function (markerId) { 1954 return new Promise(resolve => { 1955 requestAnimationFrame(() => { 1956 requestAnimationFrame(() => { 1957 const marker = document.getElementById(markerId); 1958 if (!marker) { 1959 resolve(false); 1960 return; 1961 } 1962 1963 const scrollTargets = []; 1964 let current = marker.parentElement; 1965 1966 while (current) { 1967 const style = window.getComputedStyle(current); 1968 const overflowY = style.overflowY; 1969 const isScrollable = (overflowY === "auto" || overflowY === "scroll" || overflowY === "overlay") 1970 && current.scrollHeight > current.clientHeight; 1971 1972 if (isScrollable || current.classList.contains("show-scroll") || current.classList.contains("mud-dialog-content")) { 1973 scrollTargets.push(current); 1974 } 1975 1976 current = current.parentElement; 1977 } 1978 1979 if (scrollTargets.length === 0) { 1980 marker.scrollIntoView({ 1981 behavior: "smooth", 1982 block: "start" 1983 }); 1984 window.scrollTo({ 1985 top: 0, 1986 behavior: "smooth" 1987 }); 1988 resolve(true); 1989 return; 1990 } 1991 1992 for (const target of scrollTargets) { 1993 target.scrollTo({ 1994 top: 0, 1995 behavior: "smooth" 1996 }); 1997 } 1998 1999 marker.scrollIntoView({ 2000 behavior: "smooth", 2001 block: "start" 2002 }); 2003 2004 resolve(true); 2005 }); 2006 }); 2007 }); 2008 }, 2009 2010 scrollRoadmapToCurrent: function (roadmapId) { 2011 return new Promise(resolve => { 2012 requestAnimationFrame(() => { 2013 const roadmap = document.getElementById(roadmapId); 2014 const currentStep = roadmap?.querySelector(".rc-roadmap-step.active"); 2015 if (!roadmap || !currentStep) { 2016 resolve(false); 2017 return; 2018 } 2019 2020 const roadmapRect = roadmap.getBoundingClientRect(); 2021 const currentStepRect = currentStep.getBoundingClientRect(); 2022 const centeredTop = roadmap.scrollTop
2023 + (currentStepRect.top - roadmapRect.top) 2024 - ((roadmap.clientHeight - currentStepRect.height) / 2); 2025 const reducedMotion = window.matchMedia("(prefers-reduced-motion: reduce)").matches; 2026 roadmap.scrollTo({ 2027 top: Math.max(0, centeredTop), 2028 behavior: reducedMotion ? "auto" : "smooth" 2029 }); 2030 resolve(true); 2031 }); 2032 }); 2033 } 2034 }; 2035})();
Line numbers count LF bytes from the start of the resource, as the search results do. Vendor segments are library code the classifier recognised; they are stored but not indexed. Bytes are shown as Latin1 characters, one per byte.