Merging visionos branch into main (#87)

* Move code to extension

* Don't have MTKView on VisionOS

* Add encode function and draw(in:)

* Resize waveformTexture

* Use drawable size for width of waveformTexture

* Move code out

* FloatPlot shouldn't be an MTKView

* Make private

* Add MetalView

* Add createDisplayLink

* Logging

* Add logging

* visionOS support

* uncomment display lnk

* Update FloatPlot.swift

* Fix warning

* Fix warnings

* Fix macOS build

* visionOS support

Co-Authored-By: Aurelius Prochazka <[email protected]>

* FloatPloat now defaults to a clear background

---------

Co-authored-by: Taylor Holliday <[email protected]>
Co-authored-by: Aurelius Prochazka <[email protected]>

Sources/AudioKitUI/Controls/ADSRView.swift

@@ -502,7 +502,7 @@ extension CGPoint {
 #else
 
 import AVFoundation
-import Cocoa
+import AppKit
 
 /// Call back for values for attack, decay, sustain, and release parameters
 public typealias ADSRCallback = (AUValue, AUValue, AUValue, AUValue) -> Void

Sources/AudioKitUI/Visualizations/FloatPlot.swift

@@ -5,7 +5,7 @@ import Metal
 import MetalKit
 
 // This must be in sync with the definition in shaders.metal
-public struct FragmentConstants {
+struct FragmentConstants {
  public var foregroundColor: SIMD4<Float>
  public var backgroundColor: SIMD4<Float>
  public var isFFT: Bool
@@ -17,13 +17,15 @@ public struct FragmentConstants {
  public var padding: Int = 0
 }
 
-public class FloatPlot: MTKView, MTKViewDelegate {
- let waveformTexture: MTLTexture!
+class FloatPlot: NSObject {
+ var waveformTexture: MTLTexture?
  let commandQueue: MTLCommandQueue!
  let pipelineState: MTLRenderPipelineState!
- let bufferSampleCount: Int
+ var bufferSampleCount: Int
  var dataCallback: () -> [Float]
  var constants: FragmentConstants
+ let layerRenderPassDescriptor: MTLRenderPassDescriptor
+ let device = MTLCreateSystemDefaultDevice()
 
  public init(frame frameRect: CGRect,
  constants: FragmentConstants,
@@ -32,15 +34,6 @@ public class FloatPlot: MTKView, MTKViewDelegate {
  self.constants = constants
  bufferSampleCount = Int(frameRect.width)
 
- let desc = MTLTextureDescriptor()
- desc.textureType = .type1D
- desc.width = Int(frameRect.width)
- desc.pixelFormat = .r32Float
- assert(desc.height == 1)
- assert(desc.depth == 1)
-
- let device = MTLCreateSystemDefaultDevice()
- waveformTexture = device?.makeTexture(descriptor: desc)
  commandQueue = device!.makeCommandQueue()
 
  let library = try! device?.makeDefaultLibrary(bundle: Bundle.module)
@@ -51,7 +44,6 @@ public class FloatPlot: MTKView, MTKViewDelegate {
  let pipelineStateDescriptor = MTLRenderPipelineDescriptor()
  pipelineStateDescriptor.vertexFunction = vertexProgram
  pipelineStateDescriptor.fragmentFunction = fragmentProgram
- pipelineStateDescriptor.sampleCount = 1
 
  let colorAttachment = pipelineStateDescriptor.colorAttachments[0]!
  colorAttachment.pixelFormat = .bgra8Unorm
@@ -63,23 +55,45 @@ public class FloatPlot: MTKView, MTKViewDelegate {
 
  pipelineState = try! device!.makeRenderPipelineState(descriptor: pipelineStateDescriptor)
 
- super.init(frame: frameRect, device: device)
-
- clearColor = .init(red: 0.0, green: 0.0, blue: 0.0, alpha: 0)
-
- delegate = self
+ layerRenderPassDescriptor = MTLRenderPassDescriptor()
+ layerRenderPassDescriptor.colorAttachments[0].loadAction = .clear
+ layerRenderPassDescriptor.colorAttachments[0].storeAction = .store
+ layerRenderPassDescriptor.colorAttachments[0].clearColor = MTLClearColorMake(0, 0, 0, 0);
  }
 
  @available(*, unavailable)
  required init(coder _: NSCoder) {
  fatalError("init(coder:) has not been implemented")
  }
 
+ func resize(width: Int) {
+
+ if width == 0 {
+ return
+ }
+
+ let desc = MTLTextureDescriptor()
+ desc.textureType = .type1D
+ desc.width = width
+ desc.pixelFormat = .r32Float
+ assert(desc.height == 1)
+ assert(desc.depth == 1)
+
+ waveformTexture = device?.makeTexture(descriptor: desc)
+ bufferSampleCount = width
+
+ }
+
  func updateWaveform(samples: [Float]) {
  if samples.count == 0 {
  return
  }
 
+ guard let waveformTexture else {
+ print("⚠️ updateWaveform: waveformTexture is nil")
+ return
+ }
+
  var resampled = [Float](repeating: 0, count: bufferSampleCount)
 
  for i in 0 ..< bufferSampleCount {
@@ -97,24 +111,60 @@ public class FloatPlot: MTKView, MTKViewDelegate {
  }
  }
 
- public func mtkView(_: MTKView, drawableSizeWillChange _: CGSize) {
- // We may want to resize the texture.
+ func encode(to commandBuffer: MTLCommandBuffer, pass: MTLRenderPassDescriptor) {
+ guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: pass) else { return }
+
+ encoder.setRenderPipelineState(pipelineState)
+ encoder.setFragmentTexture(waveformTexture, index: 0)
+ assert(MemoryLayout<FragmentConstants>.size == 48)
+ encoder.setFragmentBytes(&constants, length: MemoryLayout<FragmentConstants>.size, index: 0)
+ encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
+ encoder.endEncoding()
  }
 
- public func draw(in view: MTKView) {
+ func draw(to layer: CAMetalLayer) {
+
  updateWaveform(samples: dataCallback())
+
+ let size = layer.drawableSize
+ let w = Float(size.width)
+ let h = Float(size.height)
+ // let scale = Float(view.contentScaleFactor)
 
- if let commandBuffer = commandQueue.makeCommandBuffer() {
- if let renderPassDescriptor = currentRenderPassDescriptor {
- guard let encoder = commandBuffer.makeRenderCommandEncoder(descriptor: renderPassDescriptor) else { return }
+ if w == 0 || h == 0 {
+ return
+ }
+
+ guard let commandBuffer = commandQueue.makeCommandBuffer() else {
+ return
+ }
+
+ if let currentDrawable = layer.nextDrawable() {
+
+ layerRenderPassDescriptor.colorAttachments[0].texture = currentDrawable.texture
 
- encoder.setRenderPipelineState(pipelineState)
- encoder.setFragmentTexture(waveformTexture, index: 0)
- assert(MemoryLayout<FragmentConstants>.size == 48)
- encoder.setFragmentBytes(&constants, length: MemoryLayout<FragmentConstants>.size, index: 0)
- encoder.drawPrimitives(type: .triangleStrip, vertexStart: 0, vertexCount: 4)
- encoder.endEncoding()
+ encode(to: commandBuffer, pass: layerRenderPassDescriptor)
 
+ commandBuffer.present(currentDrawable)
+ } else {
+ print("⚠️ couldn't get drawable")
+ }
+ commandBuffer.commit()
+ }
+}
+
+#if !os(visionOS)
+extension FloatPlot: MTKViewDelegate {
+ public func mtkView(_ view: MTKView, drawableSizeWillChange size: CGSize) {
+ resize(width: Int(size.width))
+ }
+
+ public func draw(in view: MTKView) {
+ updateWaveform(samples: dataCallback())
+
+ if let commandBuffer = commandQueue.makeCommandBuffer() {
+ if let renderPassDescriptor = view.currentRenderPassDescriptor {
+ encode(to: commandBuffer, pass: renderPassDescriptor)
  if let drawable = view.currentDrawable {
  commandBuffer.present(drawable)
  }
@@ -125,3 +175,38 @@ public class FloatPlot: MTKView, MTKViewDelegate {
  }
  }
 }
+#endif
+
+#if !os(visionOS)
+public class FloatPlotCoordinator {
+ var renderer: FloatPlot
+
+ init(renderer: FloatPlot) {
+ self.renderer = renderer
+ }
+
+ var view: MTKView {
+ let view = MTKView(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), device: renderer.device)
+ view.clearColor = .init(red: 0.0, green: 0.0, blue: 0.0, alpha: 0)
+ view.delegate = renderer
+ return view
+ }
+}
+#else
+public class FloatPlotCoordinator {
+ var renderer: FloatPlot
+
+ init(renderer: FloatPlot) {
+ self.renderer = renderer
+ }
+
+ var view: MetalView {
+ let view = MetalView(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024))
+ view.renderer = renderer
+ view.metalLayer.pixelFormat = .bgra8Unorm
+ view.metalLayer.isOpaque = false
+ view.createDisplayLink()
+ return view
+ }
+}
+#endif

Sources/AudioKitUI/Visualizations/MetalView.swift

@@ -0,0 +1,90 @@
+// Copyright AudioKit. All Rights Reserved. Revision History at http:https://github.com/AudioKit/Waveform/
+
+#if os(iOS) || os(visionOS)
+import UIKit
+
+class MetalView: UIView {
+
+ var renderer: FloatPlot?
+ var displayLink: CADisplayLink?
+
+ @objc static override var layerClass: AnyClass {
+ CAMetalLayer.self
+ }
+
+ var metalLayer: CAMetalLayer {
+ layer as! CAMetalLayer
+ }
+
+ func createDisplayLink() {
+ displayLink = CADisplayLink(target: self,
+ selector: #selector(render))
+
+ displayLink?.add(to: .current,
+ forMode: .default)
+ }
+
+ override func draw(_ rect: CGRect) {
+ render()
+ }
+
+ override func draw(_ layer: CALayer, in ctx: CGContext) {
+ render()
+ }
+
+ override func display(_ layer: CALayer) {
+ render()
+ }
+
+ @objc func render() {
+ guard let renderer else {
+ print("⚠️ no renderer")
+ return
+ }
+ renderer.draw(to: metalLayer)
+ }
+
+ func resizeDrawable() {
+
+ var newSize = bounds.size
+ newSize.width *= contentScaleFactor
+ newSize.height *= contentScaleFactor
+
+ if newSize.width <= 0 || newSize.height <= 0 {
+ return
+ }
+
+ if newSize.width == metalLayer.drawableSize.width &&
+ newSize.height == metalLayer.drawableSize.height {
+ return
+ }
+
+ metalLayer.drawableSize = newSize
+ renderer?.resize(width: Int(newSize.width))
+
+ setNeedsDisplay()
+ }
+
+ @objc override var frame: CGRect {
+ get { super.frame }
+ set {
+ super.frame = newValue
+ resizeDrawable()
+ }
+ }
+
+ @objc override func layoutSubviews() {
+ super.layoutSubviews()
+ resizeDrawable()
+ }
+
+ @objc override var bounds: CGRect {
+ get { super.bounds }
+ set {
+ super.bounds = newValue
+ resizeDrawable()
+ }
+ }
+
+}
+#endif

Sources/AudioKitUI/Visualizations/NodeFFTView.swift

@@ -4,6 +4,7 @@ import Accelerate
 import AudioKit
 import AVFoundation
 import SwiftUI
+import MetalKit
 
 public struct NodeFFTView: ViewRepresentable {
  var nodeTap: FFTTap
@@ -13,7 +14,7 @@ public struct NodeFFTView: ViewRepresentable {
  nodeTap = FFTTap(node, bufferSize: UInt32(bufferSampleCount), callbackQueue: .main) { _ in }
  }
 
- internal var plot: FloatPlot {
+ public func makeCoordinator() -> FloatPlotCoordinator {
  nodeTap.start()
 
  let constants = FragmentConstants(foregroundColor: Color.yellow.simd,
@@ -26,14 +27,14 @@ public struct NodeFFTView: ViewRepresentable {
  nodeTap.fftData
  }
 
- return plot
+ return .init(renderer: plot)
  }
 
  #if os(macOS)
- public func makeNSView(context: Context) -> FloatPlot { return plot }
- public func updateNSView(_ nsView: FloatPlot, context: Context) {}
+ public func makeNSView(context: Context) -> NSView { return context.coordinator.view }
+ public func updateNSView(_ nsView: NSView, context: Context) {}
  #else
- public func makeUIView(context: Context) -> FloatPlot { return plot }
- public func updateUIView(_ uiView: FloatPlot, context: Context) {}
+ public func makeUIView(context: Context) -> UIView { return context.coordinator.view }
+ public func updateUIView(_ uiView: UIView, context: Context) {}
  #endif
 }

Sources/AudioKitUI/Visualizations/NodeOutputView.swift

@@ -4,6 +4,7 @@ import Accelerate
 import AudioKit
 import AVFoundation
 import SwiftUI
+import MetalKit
 
 public struct NodeOutputView: ViewRepresentable {
  private var nodeTap: RawDataTap
@@ -18,19 +19,21 @@ public struct NodeOutputView: ViewRepresentable {
  nodeTap = RawDataTap(node, bufferSize: UInt32(bufferSize), callbackQueue: .main)
  }
 
- var plot: FloatPlot {
+ public func makeCoordinator() -> FloatPlotCoordinator {
  nodeTap.start()
 
- return FloatPlot(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), constants: constants) {
- return nodeTap.data
+ let plot = FloatPlot(frame: CGRect(x: 0, y: 0, width: 1024, height: 1024), constants: constants) {
+ nodeTap.data
  }
+
+ return .init(renderer: plot)
  }
 
  #if os(macOS)
- public func makeNSView(context: Context) -> FloatPlot { return plot }
- public func updateNSView(_ nsView: FloatPlot, context: Context) {}
+ public func makeNSView(context: Context) -> NSView { return context.coordinator.view }
+ public func updateNSView(_ nsView: NSView, context: Context) {}
  #else
- public func makeUIView(context: Context) -> FloatPlot { return plot }
- public func updateUIView(_ uiView: FloatPlot, context: Context) {}
+ public func makeUIView(context: Context) -> UIView { return context.coordinator.view }
+ public func updateUIView(_ uiView: UIView, context: Context) {}
  #endif
 }