[performance] Add aggressive systematic inlining in ByteBuffer and Fl…

…atBufferBuilder (google#7253)

Co-authored-by: Joakim Hassila <[email protected]>

swift/Sources/FlatBuffers/ByteBuffer.swift

@@ -199,6 +199,7 @@ public struct ByteBuffer {
 
  /// Fills the buffer with padding by adding to the writersize
  /// - Parameter padding: Amount of padding between two to be serialized objects
+ @inline(__always)
  @usableFromInline
  mutating func fill(padding: Int) {
  assert(padding >= 0, "Fill should be larger than or equal to zero")
@@ -208,6 +209,7 @@ public struct ByteBuffer {
 
  /// Adds an array of type Scalar to the buffer memory
  /// - Parameter elements: An array of Scalars
+ @inline(__always)
  @usableFromInline
  mutating func push<T: Scalar>(elements: [T]) {
  let size = elements.count &* MemoryLayout<T>.size
@@ -221,6 +223,7 @@ public struct ByteBuffer {
  /// - Parameters:
  /// - value: Object that will be written to the buffer
  /// - size: size to subtract from the WriterIndex
+ @usableFromInline
  @inline(__always)
  mutating func push<T: NativeStruct>(struct value: T, size: Int) {
  ensureSpace(size: size)
@@ -233,6 +236,7 @@ public struct ByteBuffer {
  /// - Parameters:
  /// - value: Object that will be written to the buffer
  /// - len: Offset to subtract from the WriterIndex
+ @inline(__always)
  @usableFromInline
  mutating func push<T: Scalar>(value: T, len: Int) {
  ensureSpace(size: len)
@@ -244,6 +248,7 @@ public struct ByteBuffer {
  /// Adds a string to the buffer using swift.utf8 object
  /// - Parameter str: String that will be added to the buffer
  /// - Parameter len: length of the string
+ @inline(__always)
  @usableFromInline
  mutating func push(string str: String, len: Int) {
  ensureSpace(size: len)
@@ -263,6 +268,7 @@ public struct ByteBuffer {
  /// - Parameters:
  /// - bytes: Pointer to the view
  /// - len: Size of string
+ @usableFromInline
  @inline(__always)
  mutating func push(
  bytes: UnsafeBufferPointer<String.UTF8View.Element>,
@@ -284,6 +290,7 @@ public struct ByteBuffer {
  /// - value: Value that needs to be written to the buffer
  /// - index: index to write to
  /// - direct: Should take into consideration the capacity of the buffer
+ @inline(__always)
  func write<T>(value: T, index: Int, direct: Bool = false) {
  var index = index
  if !direct {
@@ -297,6 +304,7 @@ public struct ByteBuffer {
  /// Makes sure that buffer has enouch space for each of the objects that will be written into it
  /// - Parameter size: size of object
  @discardableResult
+ @usableFromInline
  @inline(__always)
  mutating func ensureSpace(size: Int) -> Int {
  if size &+ _writerSize > _storage.capacity {
@@ -308,6 +316,7 @@ public struct ByteBuffer {
 
  /// pops the written VTable if it's already written into the buffer
  /// - Parameter size: size of the `VTable`
+ @usableFromInline
  @inline(__always)
  mutating func pop(_ size: Int) {
  assert(
@@ -318,11 +327,13 @@ public struct ByteBuffer {
  }
 
  /// Clears the current size of the buffer
+ @inline(__always)
  mutating public func clearSize() {
  _writerSize = 0
  }
 
  /// Clears the current instance of the buffer, replacing it with new memory
+ @inline(__always)
  mutating public func clear() {
  _writerSize = 0
  alignment = 1
@@ -333,6 +344,7 @@ public struct ByteBuffer {
  /// - Parameters:
  /// - def: Type of the object
  /// - position: the index of the object in the buffer
+ @inline(__always)
  public func read<T>(def: T.Type, position: Int) -> T {
  _storage.memory.advanced(by: position).load(as: T.self)
  }
@@ -360,6 +372,7 @@ public struct ByteBuffer {
  /// - index: index of the string in the buffer
  /// - count: length of the string
  /// - type: Encoding of the string
+ @inline(__always)
  public func readString(
  at index: Int,
  count: Int,
@@ -376,6 +389,7 @@ public struct ByteBuffer {
 
  /// Creates a new Flatbuffer object that's duplicated from the current one
  /// - Parameter removeBytes: the amount of bytes to remove from the current Size
+ @inline(__always)
  public func duplicate(removing removeBytes: Int = 0) -> ByteBuffer {
  assert(removeBytes > 0, "Can NOT remove negative bytes")
  assert(
@@ -402,6 +416,7 @@ public struct ByteBuffer {
  /// which allows us to skip the first 4 bytes instead of recreating the buffer
  @discardableResult
  @usableFromInline
+ @inline(__always)
  mutating func skipPrefix() -> Int32 {
  _writerSize = _writerSize &- MemoryLayout<Int32>.size
  return read(def: Int32.self, position: 0)

swift/Sources/FlatBuffers/FlatBufferBuilder.swift

@@ -142,6 +142,7 @@ public struct FlatBufferBuilder {
  ///
  /// *NOTE: Never call this function, this is only supposed to be called
  /// by the generated code*
+ @inline(__always)
  mutating public func require(table: Offset, fields: [Int32]) {
  for field in fields {
  let start = _bb.capacity &- Int(table.o)
@@ -228,6 +229,7 @@ public struct FlatBufferBuilder {
  /// ```
  /// - Parameter numOfFields: Number of elements to be written to the buffer
  /// - Returns: Offset of the newly started table
+ @inline(__always)
  mutating public func startTable(with numOfFields: Int) -> UOffset {
  notNested()
  isNested = true
@@ -307,6 +309,7 @@ public struct FlatBufferBuilder {
  // MARK: - Builds Buffer
 
  /// Asserts to see if the object is not nested
+ @inline(__always)
  @usableFromInline
  mutating internal func notNested() {
  assert(!isNested, "Object serialization must not be nested")
@@ -315,6 +318,7 @@ public struct FlatBufferBuilder {
  /// Changes the minimuim alignment of the buffer
  /// - Parameter size: size of the current alignment
  @inline(__always)
+ @usableFromInline
  mutating internal func minAlignment(size: Int) {
  if size > _minAlignment {
  _minAlignment = size
@@ -326,6 +330,7 @@ public struct FlatBufferBuilder {
  /// - bufSize: Current size of the buffer + the offset of the object to be written
  /// - elementSize: Element size
  @inline(__always)
+ @usableFromInline
  mutating internal func padding(
  bufSize: UInt32,
  elementSize: UInt32) -> UInt32
@@ -337,6 +342,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - len:Length of the object
  /// - alignment: Alignment type
+ @inline(__always)
  @usableFromInline
  mutating internal func preAlign(len: Int, alignment: Int) {
  minAlignment(size: alignment)
@@ -349,13 +355,15 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - len: Length of the object
  /// - type: Type of the object to be written
+ @inline(__always)
  @usableFromInline
  mutating internal func preAlign<T: Scalar>(len: Int, type: T.Type) {
  preAlign(len: len, alignment: MemoryLayout<T>.size)
  }
 
  /// Refers to an object that's written in the buffer
  /// - Parameter off: the objects index value
+ @inline(__always)
  @usableFromInline
  mutating internal func refer(to off: UOffset) -> UOffset {
  let size = MemoryLayout<UOffset>.size
@@ -367,6 +375,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - offset: The offset of the element witten
  /// - position: The position of the element
+ @inline(__always)
  @usableFromInline
  mutating internal func track(offset: UOffset, at position: VOffset) {
  _vtableStorage.add(loc: FieldLoc(offset: offset, position: position))
@@ -386,6 +395,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - len: Length of vector to be created
  /// - elementSize: Size of object type to be written
+ @inline(__always)
  mutating public func startVector(_ len: Int, elementSize: Int) {
  notNested()
  isNested = true
@@ -407,6 +417,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter len: Length of the buffer
  /// - Returns: Returns the current ``Offset`` in the ``ByteBuffer``
+ @inline(__always)
  mutating public func endVector(len: Int) -> Offset {
  assert(isNested, "Calling endVector without calling startVector")
  isNested = false
@@ -427,6 +438,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter elements: elements to be written into the buffer
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector<T: Scalar>(_ elements: [T]) -> Offset {
  createVector(elements, size: elements.count)
  }
@@ -444,6 +456,7 @@ public struct FlatBufferBuilder {
  /// - Parameter elements: Elements to be written into the buffer
  /// - Parameter size: Count of elements
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector<T: Scalar>(
  _ elements: [T],
  size: Int) -> Offset
@@ -468,6 +481,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter elements: elements to be written into the buffer
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector<T: Enum>(_ elements: [T]) -> Offset {
  createVector(elements, size: elements.count)
  }
@@ -485,6 +499,7 @@ public struct FlatBufferBuilder {
  /// - Parameter elements: Elements to be written into the buffer
  /// - Parameter size: Count of elements
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector<T: Enum>(
  _ elements: [T],
  size: Int) -> Offset
@@ -511,6 +526,7 @@ public struct FlatBufferBuilder {
  /// ```
  /// - Parameter offsets: Array of offsets of type ``Offset``
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector(ofOffsets offsets: [Offset]) -> Offset {
  createVector(ofOffsets: offsets, len: offsets.count)
  }
@@ -529,6 +545,7 @@ public struct FlatBufferBuilder {
  /// - Parameter offsets: Array of offsets of type ``Offset``
  /// - Parameter size: Count of elements
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector(
  ofOffsets offsets: [Offset],
  len: Int) -> Offset
@@ -555,6 +572,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter str: Array of string
  /// - returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector(ofStrings str: [String]) -> Offset {
  var offsets: [Offset] = []
  for s in str {
@@ -576,6 +594,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter structs: A vector of ``NativeStruct``
  /// - Returns: ``Offset`` of the vector
+ @inline(__always)
  mutating public func createVector<T: NativeStruct>(ofStructs structs: [T])
  -> Offset
  {
@@ -605,6 +624,7 @@ public struct FlatBufferBuilder {
  /// - s: ``NativeStruct`` to be inserted into the ``ByteBuffer``
  /// - position: The predefined position of the object
  /// - Returns: ``Offset`` of written struct
+ @inline(__always)
  @discardableResult
  mutating public func create<T: NativeStruct>(
  struct s: T, position: VOffset) -> Offset
@@ -630,6 +650,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - s: ``NativeStruct`` to be inserted into the ``ByteBuffer``
  /// - Returns: ``Offset`` of written struct
+ @inline(__always)
  @discardableResult
  mutating public func create<T: NativeStruct>(
  struct s: T) -> Offset
@@ -654,6 +675,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter str: String to be serialized
  /// - returns: ``Offset`` of inserted string
+ @inline(__always)
  mutating public func create(string str: String?) -> Offset {
  guard let str = str else { return Offset() }
  let len = str.utf8.count
@@ -684,6 +706,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter str: String to be serialized
  /// - returns: ``Offset`` of inserted string
+ @inline(__always)
  mutating public func createShared(string str: String?) -> Offset {
  guard let str = str else { return Offset() }
  if let offset = stringOffsetMap[str] {
@@ -704,6 +727,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - offset: ``Offset`` of another object to be written
  /// - position: The predefined position of the object
+ @inline(__always)
  mutating public func add(offset: Offset, at position: VOffset) {
  if offset.isEmpty { return }
  add(element: refer(to: offset.o), def: 0, at: position)
@@ -712,6 +736,7 @@ public struct FlatBufferBuilder {
  /// Pushes a value of type ``Offset`` into the ``ByteBuffer``
  /// - Parameter o: ``Offset``
  /// - returns: Current position of the ``Offset``
+ @inline(__always)
  @discardableResult
  mutating public func push(element o: Offset) -> UOffset {
  push(element: refer(to: o.o))
@@ -740,6 +765,7 @@ public struct FlatBufferBuilder {
  /// - element: Element to insert
  /// - def: Default value for that element
  /// - position: The predefined position of the element
+ @inline(__always)
  mutating public func add<T: Scalar>(
  element: T,
  def: T,
@@ -758,6 +784,7 @@ public struct FlatBufferBuilder {
  /// - Parameters:
  /// - element: Optional element of type scalar
  /// - position: The predefined position of the element
+ @inline(__always)
  mutating public func add<T: Scalar>(element: T?, at position: VOffset) {
  guard let element = element else { return }
  track(offset: push(element: element), at: position)
@@ -769,6 +796,7 @@ public struct FlatBufferBuilder {
  ///
  /// - Parameter element: Element to insert
  /// - returns: Postion of the Element
+ @inline(__always)
  @discardableResult
  mutating public func push<T: Scalar>(element: T) -> UOffset {
  let size = MemoryLayout<T>.size
@@ -814,12 +842,14 @@ extension FlatBufferBuilder: CustomDebugStringConvertible {
 
  /// Creates the memory to store the buffer in
  @usableFromInline
+ @inline(__always)
  init() {
  memory = UnsafeMutableRawBufferPointer.allocate(
  byteCount: 0,
  alignment: 0)
  }
 
+ @inline(__always)
  deinit {
  memory.deallocate()
  }
@@ -836,6 +866,7 @@ extension FlatBufferBuilder: CustomDebugStringConvertible {
  /// Adds a FieldLoc into the buffer, which would track how many have been written,
  /// and max offset
  /// - Parameter loc: Location of encoded element
+ @inline(__always)
  func add(loc: FieldLoc) {
  memory.baseAddress?.advanced(by: writtenIndex).storeBytes(
  of: loc,
@@ -846,6 +877,7 @@ extension FlatBufferBuilder: CustomDebugStringConvertible {
  }
 
  /// Clears the data stored related to the encoded buffer
+ @inline(__always)
  func clear() {
  maxOffset = 0
  numOfFields = 0