Merge branch 'frame'

Cargo.toml

@@ -11,7 +11,7 @@ bark-core = { path = "bark-core" }
 bark-protocol = { path = "bark-protocol" }
 
 bitflags = { version = "2.4", features = ["bytemuck"] }
-bytemuck = { version = "1.13", features = ["derive"] }
+bytemuck = { version = "1.14", features = ["derive", "must_cast"] }
 derive_more = { version = "0.99" }
 heapless = "0.7"
 log = "0.4"

bark-core/src/audio.rs

@@ -0,0 +1,19 @@
+use bytemuck::{Pod, Zeroable};
+
+pub type Sample = f32;
+
+#[derive(Pod, Zeroable, Copy, Clone, Debug)]
+#[repr(C)]
+pub struct Frame(pub Sample, pub Sample);
+
+#[derive(Copy, Clone, Debug)]
+#[repr(transparent)]
+pub struct FrameCount(pub usize);
+
+pub fn as_interleaved(frames: &[Frame]) -> &[Sample] {
+ bytemuck::must_cast_slice(frames)
+}
+
+pub fn as_interleaved_mut(frames: &mut [Frame]) -> &mut [Sample] {
+ bytemuck::must_cast_slice_mut(frames)
+}

bark-core/src/decode/mod.rs

@@ -3,11 +3,13 @@ pub mod pcm;
 
 use core::fmt::Display;
 
-use bark_protocol::packet::Audio;
 use thiserror::Error;
 
+use bark_protocol::FRAMES_PER_PACKET;
+use bark_protocol::packet::Audio;
 use bark_protocol::types::{AudioPacketHeader, AudioPacketFormat};
-use bark_protocol::SAMPLES_PER_PACKET;
+
+use crate::audio::Frame;
 
 #[derive(Debug, Error)]
 pub enum NewDecoderError {
@@ -19,8 +21,10 @@ pub enum NewDecoderError {
 
 #[derive(Debug, Error)]
 pub enum DecodeError {
- #[error("wrong length: {length}, expected: {expected}")]
+ #[error("wrong byte length: {length}, expected: {expected}")]
  WrongLength { length: usize, expected: usize },
+ #[error("wrong frame count: {frames}, expected: {expected}")]
+ WrongFrameCount { frames: usize, expected: usize },
  #[error("opus codec error: {0}")]
  Opus(#[from] ::opus::Error),
 }
@@ -29,7 +33,7 @@ pub struct Decoder {
  decode: DecodeFormat,
 }
 
-pub type SampleBuffer = [f32; SAMPLES_PER_PACKET];
+pub type FrameBuffer = [Frame; FRAMES_PER_PACKET];
 
 impl Decoder {
  pub fn new(header: &AudioPacketHeader) -> Result<Self, NewDecoderError> {
@@ -47,14 +51,14 @@ impl Decoder {
  &self.decode as &dyn Display
  }
 
- pub fn decode(&mut self, packet: Option<&Audio>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
+ pub fn decode(&mut self, packet: Option<&Audio>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
  let bytes = packet.map(|packet| packet.buffer_bytes());
  self.decode.decode_packet(bytes, out)
  }
 }
 
 trait Decode: Display {
- fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError>;
+ fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError>;
 }
 
 enum DecodeFormat {
@@ -64,7 +68,7 @@ enum DecodeFormat {
 }
 
 impl Decode for DecodeFormat {
- fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
+ fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
  match self {
  DecodeFormat::S16LE(dec) => dec.decode_packet(bytes, out),
  DecodeFormat::F32LE(dec) => dec.decode_packet(bytes, out),

bark-core/src/decode/opus.rs

@@ -2,7 +2,9 @@ use core::fmt::{self, Display};
 
 use bark_protocol::SAMPLE_RATE;
 
-use super::{Decode, DecodeError, SampleBuffer};
+use crate::audio;
+
+use super::{Decode, DecodeError, FrameBuffer};
 
 pub struct OpusDecoder {
  opus: opus::Decoder,
@@ -26,16 +28,16 @@ impl Display for OpusDecoder {
 }
 
 impl Decode for OpusDecoder {
- fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
- let expected = out.len() / 2;
+ fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
+ let expected = out.len();
 
- let length = match bytes {
- Some(bytes) => self.opus.decode_float(bytes, out, false)?,
- None => self.opus.decode_float(&[], out, true)?,
+ let frames = match bytes {
+ Some(bytes) => self.opus.decode_float(bytes, audio::as_interleaved_mut(out), false)?,
+ None => self.opus.decode_float(&[], audio::as_interleaved_mut(out), true)?,
  };
 
- if expected != length {
- return Err(DecodeError::WrongLength { length, expected });
+ if expected != frames {
+ return Err(DecodeError::WrongFrameCount { frames, expected });
  }
 
  Ok(())

bark-core/src/decode/pcm.rs

@@ -1,6 +1,8 @@
 use core::fmt::{self, Display};
 
-use super::{Decode, DecodeError, SampleBuffer};
+use crate::audio;
+
+use super::{Decode, DecodeError, FrameBuffer};
 
 pub struct S16LEDecoder;
 
@@ -11,7 +13,7 @@ impl Display for S16LEDecoder {
 }
 
 impl Decode for S16LEDecoder {
- fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
+ fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
  decode_packed(bytes, out, |bytes| {
  let input = i16::from_le_bytes(bytes);
  let scale = i16::MAX as f32;
@@ -29,26 +31,28 @@ impl Display for F32LEDecoder {
 }
 
 impl Decode for F32LEDecoder {
- fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut SampleBuffer) -> Result<(), DecodeError> {
+ fn decode_packet(&mut self, bytes: Option<&[u8]>, out: &mut FrameBuffer) -> Result<(), DecodeError> {
  decode_packed(bytes, out, f32::from_le_bytes)
  }
 }
 
 fn decode_packed<const N: usize>(
  bytes: Option<&[u8]>,
- out: &mut SampleBuffer,
+ out: &mut FrameBuffer,
  func: impl Fn([u8; N]) -> f32,
 ) -> Result<(), DecodeError> {
+ let out_samples = audio::as_interleaved_mut(out);
+
  let Some(bytes) = bytes else {
  // PCM codecs have no packet loss correction
  // just zero fill and return
- out.fill(0.0);
+ out_samples.fill(0.0);
  return Ok(());
  };
 
- check_length(bytes, out.len() * N)?;
+ check_length(bytes, out_samples.len() * N)?;
 
- for (input, output) in bytes.chunks_exact(N).zip(out) {
+ for (input, output) in bytes.chunks_exact(N).zip(out_samples) {
  // when array_chunks stabilises we can use that instead
  // but for now use try_into to turn a &[u8] (guaranteed len == width)
  // into a [u8; width]

bark-core/src/encode/mod.rs

@@ -6,6 +6,8 @@ use core::fmt::Display;
 use bark_protocol::types::AudioPacketFormat;
 use thiserror::Error;
 
+use crate::audio::Frame;
+
 #[derive(Debug, Error)]
 pub enum NewEncoderError {
  #[error("opus codec error: {0}")]
@@ -22,5 +24,5 @@ pub enum EncodeError {
 
 pub trait Encode: Display + Send {
  fn header_format(&self) -> AudioPacketFormat;
- fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError>;
+ fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError>;
 }

bark-core/src/encode/opus.rs

@@ -2,6 +2,8 @@ use core::fmt::{self, Display};
 
 use bark_protocol::{types::AudioPacketFormat, SAMPLE_RATE};
 
+use crate::audio::{Frame, self};
+
 use super::{Encode, EncodeError, NewEncoderError};
 
 pub struct OpusEncoder {
@@ -35,7 +37,7 @@ impl Encode for OpusEncoder {
  AudioPacketFormat::OPUS
  }
 
- fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
- Ok(self.opus.encode_float(samples, out)?)
+ fn encode_packet(&mut self, samples: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
+ Ok(self.opus.encode_float(audio::as_interleaved(samples), out)?)
  }
 }

bark-core/src/encode/pcm.rs

@@ -2,6 +2,8 @@ use core::fmt::{self, Display};
 
 use bark_protocol::types::AudioPacketFormat;
 
+use crate::audio::{Frame, self};
+
 use super::{Encode, EncodeError};
 
 pub struct S16LEEncoder;
@@ -17,8 +19,8 @@ impl Encode for S16LEEncoder {
  AudioPacketFormat::S16LE
  }
 
- fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
- encode_packed(samples, out, |sample| {
+ fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
+ encode_packed(frames, out, |sample| {
  let scale = i16::MAX as f32;
  let sample = sample.clamp(-1.0, 1.0) * scale;
  i16::to_le_bytes(sample as i16)
@@ -39,16 +41,17 @@ impl Encode for F32LEEncoder {
  AudioPacketFormat::F32LE
  }
 
- fn encode_packet(&mut self, samples: &[f32], out: &mut [u8]) -> Result<usize, EncodeError> {
- encode_packed(samples, out, f32::to_le_bytes)
+ fn encode_packet(&mut self, frames: &[Frame], out: &mut [u8]) -> Result<usize, EncodeError> {
+ encode_packed(frames, out, f32::to_le_bytes)
  }
 }
 
 fn encode_packed<const N: usize>(
- samples: &[f32],
+ frames: &[Frame],
  out: &mut [u8],
  func: impl Fn(f32) -> [u8; N],
 ) -> Result<usize, EncodeError> {
+ let samples = audio::as_interleaved(frames);
  let out = check_length(out, samples.len() * N)?;
 
  for (output, input) in out.chunks_exact_mut(N).zip(samples) {

bark-core/src/lib.rs

@@ -1,3 +1,4 @@
+pub mod audio;
 pub mod consts;
 pub mod decode;
 pub mod encode;

bark-core/src/receive/resample.rs

@@ -2,7 +2,7 @@ use std::ffi::{c_void, c_int, CStr};
 use std::fmt::Debug;
 use std::ptr;
 
-use bark_protocol::time::SampleDuration;
+use crate::audio::{Frame, FrameCount};
 
 use self::ffi::speex_resampler_strerror;
 
@@ -46,10 +46,8 @@ pub struct Resampler {
 unsafe impl Send for Resampler {}
 
 pub struct ProcessResult {
- /// per-channel
- pub input_read: SampleDuration,
- /// per-channel
- pub output_written: SampleDuration,
+ pub input_read: FrameCount,
+ pub output_written: FrameCount,
 }
 
 impl Resampler {
@@ -93,30 +91,27 @@ impl Resampler {
  Ok(())
  }
 
- pub fn process_interleaved(&mut self, input: &[f32], output: &mut [f32])
+ pub fn process(&mut self, input: &[Frame], output: &mut [Frame])
  -> Result<ProcessResult, SpeexError>
  {
- // speex API takes frame count:
- let input_len = input.len() / usize::from(bark_protocol::CHANNELS);
- let output_len = output.len() / usize::from(bark_protocol::CHANNELS);
-
  // usize could technically be 64 bit, speex only takes u32 sizes,
  // we don't want to panic or truncate, so let's just pick a reasonable
  // length and cap input and output since the API allows us to.
  // i'm going to say a reasonable length for a single call is 1<<20.
  let max_reasonable_len = 1 << 20;
- let input_len = std::cmp::min(input_len, max_reasonable_len);
- let output_len = std::cmp::min(output_len, max_reasonable_len);
+ let input_len = std::cmp::min(input.len(), max_reasonable_len);
+ let output_len = std::cmp::min(output.len(), max_reasonable_len);
 
  let mut input_len = u32::try_from(input_len).unwrap();
  let mut output_len = u32::try_from(output_len).unwrap();
 
  let err = unsafe {
  ffi::speex_resampler_process_interleaved_float(
  self.ptr.0,
- input.as_ptr(),
+ input.as_ptr().cast(),
+ // speex API takes frame count already:
  &mut input_len,
- output.as_mut_ptr(),
+ output.as_mut_ptr().cast(),
  &mut output_len,
  )
  };
@@ -126,8 +121,8 @@ impl Resampler {
  }
 
  Ok(ProcessResult {
- input_read: SampleDuration::from_frame_count(u64::from(input_len)),
- output_written: SampleDuration::from_frame_count(u64::from(output_len)),
+ input_read: FrameCount(usize::try_from(input_len).unwrap()),
+ output_written: FrameCount(usize::try_from(output_len).unwrap()),
  })
  }
 }

bark/src/audio/input.rs

@@ -1,7 +1,7 @@
 use alsa::Direction;
 use alsa::pcm::PCM;
-use bark_protocol::{CHANNELS, time::Timestamp};
-use bark_protocol::time::SampleDuration;
+use bark_core::audio::{Frame, self};
+use bark_protocol::time::{Timestamp, SampleDuration};
 use nix::errno::Errno;
 use thiserror::Error;
 
@@ -24,7 +24,7 @@ impl Input {
  Ok(Input { pcm })
  }
 
- pub fn read(&self, mut audio: &mut [f32]) -> Result<Timestamp, ReadAudioError> {
+ pub fn read(&self, mut audio: &mut [Frame]) -> Result<Timestamp, ReadAudioError> {
  let now = Timestamp::from_micros_lossy(time::now());
  let timestamp = now.saturating_sub(self.delay()?);
 
@@ -36,7 +36,7 @@ impl Input {
  Ok(timestamp)
  }
 
- fn read_partial(&self, audio: &mut [f32]) -> Result<usize, ReadAudioError> {
+ fn read_partial(&self, audio: &mut [Frame]) -> Result<usize, ReadAudioError> {
  let io = unsafe {
  // the checked versions of this function call
  // snd_pcm_hw_params_current which mallocs under the hood
@@ -45,10 +45,8 @@ impl Input {
 
  loop {
  // try to write audio
- let err = match io.readi(audio) {
- Ok(n) => {
- return Ok(n * CHANNELS.0 as usize);
- }
+ let err = match io.readi(audio::as_interleaved_mut(audio)) {
+ Ok(n) => { return Ok(n) }
  Err(e) => e,
  };