Skip to content

Commit a38f959

Browse files
pblazejpblazej
authored
Agent state visualization (#20)
- Adds `AgentBarAudioVisualizer` - it performs sequence-based animations for different agent states - it does not use [`PhaseAnimator`](https://developer.apple.com/documentation/swiftui/phaseanimator) that would need a "step" animation curve to achieve the desired effects without `.delay`, also we cannot simply reset the animation - it's not a simple wrapper around `BarAudioVisualizer` anymore, as we need item-based opacity - there's some duplication, but I'm open for suggestions here - Splits `AudioProcessor` into a separate file ### Considerations - We need some form of `.id(track)` or explicit `if` around empty track state to reinstantiate `@StateObject` of audio processor (and make it update the waveform) which is sort of a leaky abstraction - a separate initializer with non-optional track would be probably better
1 parent 6cb7972 commit a38f959

File tree

4 files changed

+393
-187
lines changed

4 files changed

+393
-187
lines changed

‎Sources/LiveKitComponents/Audio/AudioProcessor.swift‎

Lines changed: 104 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,104 @@
1+
/*
2+
* Copyright 2025 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import AVFoundation
18+
import LiveKit
19+
20+
public class AudioProcessor: ObservableObject, AudioRenderer {
21+
public let isCentered: Bool
22+
public let smoothingFactor: Float
23+
24+
// Normalized to 0.0-1.0 range.
25+
@Published public var bands: [Float]
26+
27+
private let _processor: AudioVisualizeProcessor
28+
private weak var _track: AudioTrack?
29+
30+
public init(track: AudioTrack?,
31+
bandCount: Int,
32+
isCentered: Bool = true,
33+
smoothingFactor: Float = 0.3)
34+
{
35+
self.isCentered = isCentered
36+
self.smoothingFactor = smoothingFactor
37+
bands = Array(repeating: 0.0, count: bandCount)
38+
39+
_processor = AudioVisualizeProcessor(bandsCount: bandCount)
40+
_track = track
41+
_track?.add(audioRenderer: self)
42+
}
43+
44+
deinit {
45+
_track?.remove(audioRenderer: self)
46+
}
47+
48+
public func render(pcmBuffer: AVAudioPCMBuffer) {
49+
let newBands = _processor.process(pcmBuffer: pcmBuffer)
50+
guard var newBands else { return }
51+
52+
// If centering is enabled, rearrange the normalized bands
53+
if isCentered {
54+
newBands.sort(by: >)
55+
newBands = centerBands(newBands)
56+
}
57+
58+
DispatchQueue.main.async { [weak self] in
59+
guard let self else { return }
60+
61+
self.bands = zip(self.bands, newBands).map { old, new in
62+
self._smoothTransition(from: old, to: new, factor: self.smoothingFactor)
63+
}
64+
}
65+
}
66+
67+
// MARK: - Private
68+
69+
/// Centers the sorted bands by placing higher values in the middle.
70+
@inline(__always) private func centerBands(_ sortedBands: [Float]) -> [Float] {
71+
var centeredBands = [Float](repeating: 0, count: sortedBands.count)
72+
var leftIndex = sortedBands.count / 2
73+
var rightIndex = leftIndex
74+
75+
for (index, value) in sortedBands.enumerated() {
76+
if index % 2 == 0 {
77+
// Place value to the right
78+
centeredBands[rightIndex] = value
79+
rightIndex += 1
80+
} else {
81+
// Place value to the left
82+
leftIndex -= 1
83+
centeredBands[leftIndex] = value
84+
}
85+
}
86+
87+
return centeredBands
88+
}
89+
90+
/// Applies an easing function to smooth the transition.
91+
@inline(__always) private func _smoothTransition(from oldValue: Float, to newValue: Float, factor: Float) -> Float {
92+
// Calculate the delta change between the old and new value
93+
let delta = newValue - oldValue
94+
// Apply an ease-in-out cubic easing curve
95+
let easedFactor = _easeInOutCubic(t: factor)
96+
// Calculate and return the smoothed value
97+
return oldValue + delta * easedFactor
98+
}
99+
100+
/// Easing function: ease-in-out cubic
101+
@inline(__always) private func _easeInOutCubic(t: Float) -> Float {
102+
t < 0.5 ? 4 * t * t * t : 1 - pow(-2 * t + 2, 3) / 2
103+
}
104+
}

‎Sources/LiveKitComponents/UI/Visualizer/AgentBarAudioVisualizer.swift‎

Lines changed: 187 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,187 @@
1+
/*
2+
* Copyright 2025 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
/*
18+
* Copyright 2025 LiveKit
19+
*
20+
* Licensed under the Apache License, Version 2.0 (the "License");
21+
* you may not use this file except in compliance with the License.
22+
* You may obtain a copy of the License at
23+
*
24+
* http://www.apache.org/licenses/LICENSE-2.0
25+
*
26+
* Unless required by applicable law or agreed to in writing, software
27+
* distributed under the License is distributed on an "AS IS" BASIS,
28+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
29+
* See the License for the specific language governing permissions and
30+
* limitations under the License.
31+
*/
32+
33+
import LiveKit
34+
import SwiftUI
35+
36+
/// A SwiftUI view that visualizes audio levels and agent states as a series of animated vertical bars.
37+
/// This visualizer is specifically designed to provide visual feedback for different agent states
38+
/// (connecting, initializing, listening, thinking, speaking) while also responding to real-time
39+
/// audio data when available.
40+
///
41+
/// `AgentBarAudioVisualizer` displays bars whose heights and opacities dynamically
42+
/// reflect the magnitude of audio frequencies in real time, creating an
43+
/// interactive, visual representation of the audio track's spectrum. This
44+
/// visualizer can be customized in terms of bar count, color, corner radius,
45+
/// spacing, and whether the bars are centered based on frequency magnitude.
46+
///
47+
/// Usage:
48+
/// ```
49+
/// let audioTrack: AudioTrack = ...
50+
/// let agentState: AgentState = ...
51+
/// AgentBarAudioVisualizer(audioTrack: audioTrack, agentState: agentState)
52+
/// ```
53+
///
54+
/// - Parameters:
55+
/// - audioTrack: The `AudioTrack` providing audio data to be visualized.
56+
/// - agentState: Triggers transitions between visualizer animation states.
57+
/// - barColor: The color used to fill each bar, defaulting to white.
58+
/// - barCount: The number of bars displayed, defaulting to 7.
59+
/// - barCornerRadius: The corner radius applied to each bar, giving a
60+
/// rounded appearance. Defaults to 100.
61+
/// - barSpacingFactor: Determines the spacing between bars as a factor
62+
/// of view width. Defaults to 0.015.
63+
/// - isCentered: A Boolean indicating whether higher-decibel bars
64+
/// should be centered. Defaults to `true`.
65+
///
66+
/// Example:
67+
/// ```
68+
/// AgentBarAudioVisualizer(audioTrack: audioTrack, barColor: .blue, barCount: 10)
69+
/// ```
70+
public struct AgentBarAudioVisualizer: View {
71+
public let barCount: Int
72+
public let barColor: Color
73+
public let barCornerRadius: CGFloat
74+
public let barSpacingFactor: CGFloat
75+
public let barMinOpacity: Double
76+
public let isCentered: Bool
77+
78+
private let agentState: AgentState
79+
80+
@StateObject private var audioProcessor: AudioProcessor
81+
82+
@State private var animationProperties: PhaseAnimationProperties
83+
@State private var animationPhase: Int = 0
84+
@State private var animationTask: Task<Void, Never>?
85+
86+
public init(audioTrack: AudioTrack?,
87+
agentState: AgentState,
88+
barColor: Color = .primary,
89+
barCount: Int = 5,
90+
barCornerRadius: CGFloat = 100,
91+
barSpacingFactor: CGFloat = 0.015,
92+
barMinOpacity: CGFloat = 0.16,
93+
isCentered: Bool = true)
94+
{
95+
self.agentState = agentState
96+
97+
self.barColor = barColor
98+
self.barCount = barCount
99+
self.barCornerRadius = barCornerRadius
100+
self.barSpacingFactor = barSpacingFactor
101+
self.barMinOpacity = Double(barMinOpacity)
102+
self.isCentered = isCentered
103+
104+
_audioProcessor = StateObject(wrappedValue: AudioProcessor(track: audioTrack,
105+
bandCount: barCount,
106+
isCentered: isCentered))
107+
108+
animationProperties = PhaseAnimationProperties(barCount: barCount)
109+
}
110+
111+
public var body: some View {
112+
GeometryReader { geometry in
113+
let highlightingSequence = animationProperties.highlightingSequence(agentState: agentState)
114+
let highlighted = highlightingSequence[animationPhase % highlightingSequence.count]
115+
let duration = animationProperties.duration(agentState: agentState)
116+
117+
bars(geometry: geometry, highlighted: highlighted)
118+
.onAppear {
119+
animationTask?.cancel()
120+
animationTask = Task {
121+
while !Task.isCancelled {
122+
try? await Task.sleep(nanoseconds: UInt64(duration * Double(NSEC_PER_SEC)))
123+
withAnimation(.easeInOut) { animationPhase += 1 }
124+
}
125+
}
126+
}
127+
.onDisappear {
128+
animationTask?.cancel()
129+
}
130+
.animation(.easeOut, value: agentState)
131+
.onChange(of: agentState) { _ in
132+
animationPhase = 0
133+
}
134+
}
135+
}
136+
137+
@ViewBuilder
138+
private func bars(geometry: GeometryProxy, highlighted: PhaseAnimationProperties.HighlightedBars) -> some View {
139+
let barMinHeight = (geometry.size.width - geometry.size.width * barSpacingFactor * CGFloat(barCount + 2)) / CGFloat(barCount)
140+
HStack(alignment: .center, spacing: geometry.size.width * barSpacingFactor) {
141+
ForEach(0 ..< audioProcessor.bands.count, id: \.self) { index in
142+
VStack {
143+
Spacer()
144+
RoundedRectangle(cornerRadius: barMinHeight)
145+
.fill(barColor)
146+
.opacity(highlighted.contains(index) ? 1 : barMinOpacity)
147+
.frame(height: (geometry.size.height - barMinHeight) * CGFloat(audioProcessor.bands[index]) + barMinHeight)
148+
Spacer()
149+
}
150+
}
151+
}
152+
.padding(geometry.size.width * barSpacingFactor)
153+
}
154+
}
155+
156+
extension AgentBarAudioVisualizer {
157+
private struct PhaseAnimationProperties {
158+
typealias HighlightedBars = Set<Int>
159+
160+
private let barCount: Int
161+
private let veryLongDuration: TimeInterval = 1000
162+
163+
init(barCount: Int) {
164+
self.barCount = barCount
165+
}
166+
167+
func duration(agentState: AgentState) -> TimeInterval {
168+
switch agentState {
169+
case .connecting, .initializing: return 2 / Double(barCount)
170+
case .listening: return 0.5
171+
case .thinking: return 0.15
172+
case .speaking: return veryLongDuration
173+
default: return veryLongDuration
174+
}
175+
}
176+
177+
func highlightingSequence(agentState: AgentState) -> [HighlightedBars] {
178+
switch agentState {
179+
case .connecting, .initializing: return (0 ..< barCount).map { HighlightedBars([$0, barCount - 1 - $0]) }
180+
case .thinking: return Array((0 ..< barCount) + (0 ..< barCount).reversed()).map { HighlightedBars([$0]) }
181+
case .listening: return barCount % 2 == 0 ? [[(barCount / 2) - 1, barCount / 2], []] : [[barCount / 2], []]
182+
case .speaking: return [HighlightedBars(0 ..< barCount)]
183+
default: return [[]]
184+
}
185+
}
186+
}
187+
}

‎Sources/LiveKitComponents/UI/Visualizer/BarAudioVisualizer.swift‎

Lines changed: 102 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,102 @@
1+
/*
2+
* Copyright 2025 LiveKit
3+
*
4+
* Licensed under the Apache License, Version 2.0 (the "License");
5+
* you may not use this file except in compliance with the License.
6+
* You may obtain a copy of the License at
7+
*
8+
* http://www.apache.org/licenses/LICENSE-2.0
9+
*
10+
* Unless required by applicable law or agreed to in writing, software
11+
* distributed under the License is distributed on an "AS IS" BASIS,
12+
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13+
* See the License for the specific language governing permissions and
14+
* limitations under the License.
15+
*/
16+
17+
import LiveKit
18+
import SwiftUI
19+
20+
/// A SwiftUI view that visualizes audio levels as a series of vertical bars,
21+
/// responding to real-time audio data processed from an audio track.
22+
///
23+
/// `BarAudioVisualizer` displays bars whose heights and opacities dynamically
24+
/// reflect the magnitude of audio frequencies in real time, creating an
25+
/// interactive, visual representation of the audio track's spectrum. This
26+
/// visualizer can be customized in terms of bar count, color, corner radius,
27+
/// spacing, and whether the bars are centered based on frequency magnitude.
28+
///
29+
/// Usage:
30+
/// ```
31+
/// let audioTrack: AudioTrack = ...
32+
/// BarAudioVisualizer(audioTrack: audioTrack)
33+
/// ```
34+
///
35+
/// - Parameters:
36+
/// - audioTrack: The `AudioTrack` providing audio data to be visualized.
37+
/// - barColor: The color used to fill each bar, defaulting to white.
38+
/// - barCount: The number of bars displayed, defaulting to 7.
39+
/// - barCornerRadius: The corner radius applied to each bar, giving a
40+
/// rounded appearance. Defaults to 100.
41+
/// - barSpacingFactor: Determines the spacing between bars as a factor
42+
/// of view width. Defaults to 0.015.
43+
/// - isCentered: A Boolean indicating whether higher-decibel bars
44+
/// should be centered. Defaults to `true`.
45+
///
46+
/// Example:
47+
/// ```
48+
/// BarAudioVisualizer(audioTrack: audioTrack, barColor: .blue, barCount: 10)
49+
/// ```
50+
public struct BarAudioVisualizer: View {
51+
public let barColor: Color
52+
public let barCount: Int
53+
public let barCornerRadius: CGFloat
54+
public let barSpacingFactor: CGFloat
55+
public let barMinOpacity: Double
56+
public let isCentered: Bool
57+
58+
@StateObject private var audioProcessor: AudioProcessor
59+
60+
public init(audioTrack: AudioTrack?,
61+
barColor: Color = .primary,
62+
barCount: Int = 5,
63+
barCornerRadius: CGFloat = 100,
64+
barSpacingFactor: CGFloat = 0.015,
65+
barMinOpacity: CGFloat = 0.16,
66+
isCentered: Bool = true)
67+
{
68+
self.barColor = barColor
69+
self.barCount = barCount
70+
self.barCornerRadius = barCornerRadius
71+
self.barSpacingFactor = barSpacingFactor
72+
self.barMinOpacity = Double(barMinOpacity)
73+
self.isCentered = isCentered
74+
75+
_audioProcessor = StateObject(wrappedValue: AudioProcessor(track: audioTrack,
76+
bandCount: barCount,
77+
isCentered: isCentered))
78+
}
79+
80+
public var body: some View {
81+
GeometryReader { geometry in
82+
bars(geometry: geometry)
83+
}
84+
}
85+
86+
@ViewBuilder
87+
private func bars(geometry: GeometryProxy) -> some View {
88+
let barMinHeight = (geometry.size.width - geometry.size.width * barSpacingFactor * CGFloat(barCount + 2)) / CGFloat(barCount)
89+
HStack(alignment: .center, spacing: geometry.size.width * barSpacingFactor) {
90+
ForEach(0 ..< audioProcessor.bands.count, id: \.self) { index in
91+
VStack {
92+
Spacer()
93+
RoundedRectangle(cornerRadius: barMinHeight)
94+
.fill(barColor)
95+
.frame(height: (geometry.size.height - barMinHeight) * CGFloat(audioProcessor.bands[index]) + barMinHeight)
96+
Spacer()
97+
}
98+
}
99+
}
100+
.padding(geometry.size.width * barSpacingFactor)
101+
}
102+
}

0 commit comments

Comments
 (0)