metal: matrix-matrix multiplication kernel

This commit removes MPS and uses custom matrix-matrix multiplication kernels for all quantization types. This commit also adds grouped-query attention to support llama2 70B.
ggerganov · Aug 15, 2023 · 5f6de2a · 5f6de2a
1 parent 25d43e0
commit 5f6de2a
Show file tree

Hide file tree

Showing 6 changed files with 497 additions and 632 deletions.
diff --git a/CMakeLists.txt b/CMakeLists.txt
@@ -298,7 +298,6 @@ if (LLAMA_METAL)
  find_library(FOUNDATION_LIBRARY Foundation REQUIRED)
  find_library(METAL_FRAMEWORK Metal REQUIRED)
  find_library(METALKIT_FRAMEWORK MetalKit REQUIRED)
- find_library(METALPERFORMANCE_FRAMEWORK MetalPerformanceShaders REQUIRED)
 
  set(GGML_SOURCES_METAL ggml-metal.m ggml-metal.h)
 
@@ -315,7 +314,6 @@ if (LLAMA_METAL)
  ${FOUNDATION_LIBRARY}
  ${METAL_FRAMEWORK}
  ${METALKIT_FRAMEWORK}
- ${METALPERFORMANCE_FRAMEWORK}
  )
 endif()
 

diff --git a/Makefile b/Makefile
@@ -283,7 +283,7 @@ endif # LLAMA_CLBLAST
 ifdef LLAMA_METAL
  CFLAGS += -DGGML_USE_METAL -DGGML_METAL_NDEBUG
  CXXFLAGS += -DGGML_USE_METAL
- LDFLAGS += -framework Foundation -framework Metal -framework MetalKit -framework MetalPerformanceShaders
+ LDFLAGS += -framework Foundation -framework Metal -framework MetalKit
  OBJS += ggml-metal.o
 endif # LLAMA_METAL
 

diff --git a/flake.nix b/flake.nix
@@ -14,8 +14,6 @@
  with pkgs.darwin.apple_sdk_11_0.frameworks; [
  Accelerate
  MetalKit
- MetalPerformanceShaders
- MetalPerformanceShadersGraph
  ]
  else if isAarch32 && isDarwin then
  with pkgs.darwin.apple_sdk.frameworks; [