Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

sync : whisper.cpp #823

Merged
merged 14 commits into from
May 14, 2024
Merged

sync : whisper.cpp #823

Changes from 1 commit
Commits
Show all changes
14 commits
Select commit Hold shift + click to select a range
8821421
whisper : update grammar-parser.cpp (whisper/2058)
eltociear Apr 15, 2024
6d3b994
main : pass nullptr when regex is empty (whisper/2070)
ggerganov Apr 17, 2024
85a1d20
whisper : more prominent log message for sub-1s audio (whisper/2065)
ggerganov Apr 24, 2024
623b291
Remove unnecessary memory reallocation in fft (whisper/2080)
goldwaving Apr 28, 2024
083d952
ggml : expose SSE3 and SSSE3 for MSVC when AVX is available (whisper/…
przemoc May 8, 2024
9466438
ggml : try fix ppc64 (whisper/0)
ggerganov May 12, 2024
14001e4
whisper : remove old flash attn code (whisper/0)
ggerganov May 13, 2024
b842efa
metal : tune soft_max number of threads (whisper/0)
ggerganov May 13, 2024
737407f
main : fix double quote escaping in csv output (whisper/2090)
mashizora May 13, 2024
71ce179
whisper : switch back to F32 mask (whisper/0)
ggerganov May 13, 2024
36caae3
main : add options for temperature control (whisper/2088)
ziegenberg May 13, 2024
42526d6
main : dont print timings with --no-prints (whisper/2108)
ziegenberg May 13, 2024
c7385b4
whisper : fix model path encoding in windows (whisper/2086)
thewh1teagle May 14, 2024
b2773a4
sync : whisper.cpp
ggerganov May 14, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Jump to
Jump to file
Failed to load files.
Diff view
Diff view
Prev Previous commit
Next Next commit
whisper : remove old flash attn code (whisper/0)
  • Loading branch information
@ggerganov
ggerganov committed May 14, 2024
commit 14001e43d3b8ff967ff20550baceb01e6658ff43
42 changes: 5 additions & 37 deletions examples/whisper/whisper.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -147,7 +147,6 @@ static void whisper_log_callback_default(ggml_log_level level, const char * text
} \
} while (0)

//#define WHISPER_USE_FLASH_ATTN
//#define WHISPER_USE_FLASH_FF
#define WHISPER_MAX_DECODERS 8
#define WHISPER_MAX_NODES 4096
Expand Down Expand Up @@ -1951,32 +1950,6 @@ static struct ggml_cgraph * whisper_build_graph_encoder(

// ------

#ifdef WHISPER_USE_FLASH_ATTN
struct ggml_tensor * Q =
ggml_permute(ctx0,
ggml_cpy(ctx0,
Qcur,
ggml_new_tensor_3d(ctx0, wctx.itype, n_state/n_head, n_head, n_ctx)),
0, 2, 1, 3);

struct ggml_tensor * K =
ggml_permute(ctx0,
ggml_cpy(ctx0,
Kcur,
ggml_new_tensor_3d(ctx0, wctx.itype, n_state/n_head, n_head, n_ctx)),
0, 2, 1, 3);

struct ggml_tensor * V =
ggml_cpy(ctx0,
ggml_permute(ctx0,
ggml_reshape_3d(ctx0,
Vcur,
n_state/n_head, n_head, n_ctx),
1, 2, 0, 3),
ggml_new_tensor_3d(ctx0, wctx.itype, n_ctx, n_state/n_head, n_head));

struct ggml_tensor * KQV = ggml_flash_attn(ctx0, Q, K, V, false);
#else
struct ggml_tensor * Q =
ggml_permute(ctx0,
ggml_cpy(ctx0,
Expand All @@ -1994,9 +1967,7 @@ static struct ggml_cgraph * whisper_build_graph_encoder(
// K * Q
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);

struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, KQscale);

struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_scaled);
struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, nullptr, KQscale, 0.0f);

struct ggml_tensor * V =
ggml_cpy(ctx0,
Expand All @@ -2009,7 +1980,7 @@ static struct ggml_cgraph * whisper_build_graph_encoder(
);

struct ggml_tensor * KQV = ggml_mul_mat(ctx0, V, KQ_soft_max);
#endif

struct ggml_tensor * KQV_merged = ggml_permute(ctx0, KQV, 0, 2, 1, 3);

cur = ggml_cpy(ctx0,
Expand Down Expand Up @@ -2323,6 +2294,8 @@ static struct ggml_cgraph * whisper_build_graph_decoder(
ggml_set_name(KQ_mask, "KQ_mask");
ggml_set_input(KQ_mask);

struct ggml_tensor * KQ_mask_f16 = ggml_cast(ctx0, KQ_mask, GGML_TYPE_F16);

// token encoding + position encoding
struct ggml_tensor * cur =
ggml_add(ctx0,
Expand Down Expand Up @@ -2406,12 +2379,7 @@ static struct ggml_cgraph * whisper_build_graph_decoder(
// K * Q
struct ggml_tensor * KQ = ggml_mul_mat(ctx0, K, Q);

//struct ggml_tensor * KQ_scaled = ggml_scale(ctx0, KQ, KQ_scale);

//struct ggml_tensor * KQ_masked = ggml_diag_mask_inf(ctx0, KQ, n_past);
struct ggml_tensor * KQ_masked = ggml_add(ctx0, KQ, KQ_mask);

struct ggml_tensor * KQ_soft_max = ggml_soft_max(ctx0, KQ_masked);
struct ggml_tensor * KQ_soft_max = ggml_soft_max_ext(ctx0, KQ, KQ_mask_f16, 1.0f, 0.0f);

struct ggml_tensor * V =
ggml_view_3d(ctx0, kv_self.v,
Expand Down