cuda : non-cont concat support

[ggerganov]

ref #7118

Should fix DeepSeek V2 on CUDA (#7519 (comment))

[github-actions]

📈 llama.cpp server for bench-server-baseline on Standard_NC4as_T4_v3 for phi-2-q4_0: 527 iterations 🚀

Expand details for performance related PR only

• Concurrent users: 8, duration: 10m
• HTTP request : avg=8899.04ms p(95)=20966.1ms fails=, finish reason: stop=468 truncated=59
• Prompt processing (pp): avg=99.57tk/s p(95)=400.16tk/s
• Token generation (tg): avg=31.63tk/s p(95)=45.04tk/s
• ggml-org/models/phi-2/ggml-model-q4_0.gguf parallel=8 ctx-size=16384 ngl=33 batch-size=2048 ubatch-size=256 pp=1024 pp+tg=2048 branch=gg/concat-non-cont commit=738008fbcccf5b5fd6cb93378e5f55cbc44b6b91

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 527 iterations"
    y-axis "llamacpp:prompt_tokens_seconds"
    x-axis "llamacpp:prompt_tokens_seconds" 1716972379 --> 1716973009
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 868.66, 868.66, 868.66, 868.66, 868.66, 826.98, 826.98, 826.98, 826.98, 826.98, 807.58, 807.58, 807.58, 807.58, 807.58, 814.51, 814.51, 814.51, 814.51, 814.51, 869.01, 869.01, 869.01, 869.01, 869.01, 883.89, 883.89, 883.89, 883.89, 883.89, 867.22, 867.22, 867.22, 867.22, 867.22, 863.4, 863.4, 863.4, 863.4, 863.4, 849.23, 849.23, 849.23, 849.23, 849.23, 860.87, 860.87, 860.87, 860.87, 860.87, 859.52, 859.52, 859.52, 859.52, 859.52, 879.96, 879.96, 879.96, 879.96, 879.96, 897.19, 897.19, 897.19, 897.19, 897.19, 901.43, 901.43, 901.43, 901.43, 901.43, 912.88, 912.88, 912.88, 912.88, 912.88, 912.52, 912.52, 912.52, 912.52, 912.52, 909.31, 909.31, 909.31, 909.31, 909.31, 912.16, 912.16, 912.16, 912.16, 912.16, 919.01, 919.01, 919.01, 919.01, 919.01, 907.69, 907.69, 907.69, 907.69, 907.69, 911.42, 911.42, 911.42, 911.42, 911.42, 913.77, 913.77, 913.77, 913.77, 913.77, 913.14, 913.14, 913.14, 913.14, 913.14, 873.86, 873.86, 873.86, 873.86, 873.86, 868.33, 868.33, 868.33, 868.33, 868.33, 869.76, 869.76, 869.76, 869.76, 869.76, 842.0, 842.0, 842.0, 842.0, 842.0, 841.62, 841.62, 841.62, 841.62, 841.62, 840.76, 840.76, 840.76, 840.76, 840.76, 842.61, 842.61, 842.61, 842.61, 842.61, 845.8, 845.8, 845.8, 845.8, 845.8, 844.93, 844.93, 844.93, 844.93, 844.93, 843.9, 843.9, 843.9, 843.9, 843.9, 843.49, 843.49, 843.49, 843.49, 843.49, 844.97, 844.97, 844.97, 844.97, 844.97, 844.07, 844.07, 844.07, 844.07, 844.07, 847.21, 847.21, 847.21, 847.21, 847.21, 850.4, 850.4, 850.4, 850.4, 850.4, 848.52, 848.52, 848.52, 848.52, 848.52, 845.78, 845.78, 845.78, 845.78, 845.78, 848.16, 848.16, 848.16, 848.16, 848.16, 849.9, 849.9, 849.9, 849.9, 849.9, 849.6, 849.6, 849.6, 849.6, 849.6, 852.7, 852.7, 852.7, 852.7, 852.7, 809.45, 809.45, 809.45, 809.45, 809.45, 800.75, 800.75, 800.75, 800.75, 800.75, 799.56, 799.56, 799.56, 799.56, 799.56, 803.93, 803.93, 803.93, 803.93, 803.93, 806.97, 806.97, 806.97, 806.97, 806.97, 806.23, 806.23, 806.23, 806.23, 806.23, 809.9, 809.9, 809.9, 809.9, 809.9, 812.47, 812.47, 812.47, 812.47, 812.47, 812.42, 812.42, 812.42, 812.42, 812.42, 816.64, 816.64, 816.64, 816.64, 816.64, 817.27, 817.27, 817.27, 817.27, 817.27, 825.28, 825.28, 825.28, 825.28, 825.28, 824.84, 824.84, 824.84, 824.84, 824.84, 826.52, 826.52, 826.52, 826.52, 826.52, 827.37, 827.37, 827.37, 827.37, 827.37, 827.69, 827.69, 827.69, 827.69, 827.69, 827.51]
                    

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 527 iterations"
    y-axis "llamacpp:predicted_tokens_seconds"
    x-axis "llamacpp:predicted_tokens_seconds" 1716972379 --> 1716973009
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 36.78, 36.78, 36.78, 36.78, 36.78, 36.77, 36.77, 36.77, 36.77, 36.77, 26.64, 26.64, 26.64, 26.64, 26.64, 27.41, 27.41, 27.41, 27.41, 27.41, 27.42, 27.42, 27.42, 27.42, 27.42, 28.58, 28.58, 28.58, 28.58, 28.58, 29.13, 29.13, 29.13, 29.13, 29.13, 29.9, 29.9, 29.9, 29.9, 29.9, 30.59, 30.59, 30.59, 30.59, 30.59, 30.79, 30.79, 30.79, 30.79, 30.79, 30.85, 30.85, 30.85, 30.85, 30.85, 31.05, 31.05, 31.05, 31.05, 31.05, 31.41, 31.41, 31.41, 31.41, 31.41, 30.96, 30.96, 30.96, 30.96, 30.96, 30.39, 30.39, 30.39, 30.39, 30.39, 30.25, 30.25, 30.25, 30.25, 30.25, 29.29, 29.29, 29.29, 29.29, 29.29, 29.62, 29.62, 29.62, 29.62, 29.62, 29.38, 29.38, 29.38, 29.38, 29.38, 29.27, 29.27, 29.27, 29.27, 29.27, 29.39, 29.39, 29.39, 29.39, 29.39, 29.51, 29.51, 29.51, 29.51, 29.51, 29.75, 29.75, 29.75, 29.75, 29.75, 29.81, 29.81, 29.81, 29.81, 29.81, 29.91, 29.91, 29.91, 29.91, 29.91, 29.99, 29.99, 29.99, 29.99, 29.99, 30.29, 30.29, 30.29, 30.29, 30.29, 30.25, 30.25, 30.25, 30.25, 30.25, 30.25, 30.25, 30.25, 30.25, 30.25, 30.46, 30.46, 30.46, 30.46, 30.46, 30.56, 30.56, 30.56, 30.56, 30.56, 30.7, 30.7, 30.7, 30.7, 30.7, 30.77, 30.77, 30.77, 30.77, 30.77, 30.84, 30.84, 30.84, 30.84, 30.84, 30.8, 30.8, 30.8, 30.8, 30.8, 30.63, 30.63, 30.63, 30.63, 30.63, 30.27, 30.27, 30.27, 30.27, 30.27, 30.12, 30.12, 30.12, 30.12, 30.12, 29.85, 29.85, 29.85, 29.85, 29.85, 29.8, 29.8, 29.8, 29.8, 29.8, 29.98, 29.98, 29.98, 29.98, 29.98, 30.08, 30.08, 30.08, 30.08, 30.08, 30.22, 30.22, 30.22, 30.22, 30.22, 30.3, 30.3, 30.3, 30.3, 30.3, 30.23, 30.23, 30.23, 30.23, 30.23, 29.9, 29.9, 29.9, 29.9, 29.9, 29.18, 29.18, 29.18, 29.18, 29.18, 29.02, 29.02, 29.02, 29.02, 29.02, 29.03, 29.03, 29.03, 29.03, 29.03, 28.99, 28.99, 28.99, 28.99, 28.99, 29.05, 29.05, 29.05, 29.05, 29.05, 29.08, 29.08, 29.08, 29.08, 29.08, 29.11, 29.11, 29.11, 29.11, 29.11, 29.1, 29.1, 29.1, 29.1, 29.1, 29.04, 29.04, 29.04, 29.04, 29.04, 29.06, 29.06, 29.06, 29.06, 29.06, 29.06, 29.06, 29.06, 29.06, 29.06, 29.21, 29.21, 29.21, 29.21, 29.21, 29.34, 29.34, 29.34, 29.34, 29.34, 29.35, 29.35, 29.35, 29.35, 29.35, 29.48]
                    

Details

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 527 iterations"
    y-axis "llamacpp:kv_cache_usage_ratio"
    x-axis "llamacpp:kv_cache_usage_ratio" 1716972379 --> 1716973009
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.27, 0.27, 0.27, 0.27, 0.27, 0.46, 0.46, 0.46, 0.46, 0.46, 0.33, 0.33, 0.33, 0.33, 0.33, 0.13, 0.13, 0.13, 0.13, 0.13, 0.2, 0.2, 0.2, 0.2, 0.2, 0.25, 0.25, 0.25, 0.25, 0.25, 0.08, 0.08, 0.08, 0.08, 0.08, 0.17, 0.17, 0.17, 0.17, 0.17, 0.19, 0.19, 0.19, 0.19, 0.19, 0.26, 0.26, 0.26, 0.26, 0.26, 0.21, 0.21, 0.21, 0.21, 0.21, 0.1, 0.1, 0.1, 0.1, 0.1, 0.31, 0.31, 0.31, 0.31, 0.31, 0.33, 0.33, 0.33, 0.33, 0.33, 0.3, 0.3, 0.3, 0.3, 0.3, 0.29, 0.29, 0.29, 0.29, 0.29, 0.14, 0.14, 0.14, 0.14, 0.14, 0.19, 0.19, 0.19, 0.19, 0.19, 0.18, 0.18, 0.18, 0.18, 0.18, 0.22, 0.22, 0.22, 0.22, 0.22, 0.2, 0.2, 0.2, 0.2, 0.2, 0.15, 0.15, 0.15, 0.15, 0.15, 0.18, 0.18, 0.18, 0.18, 0.18, 0.3, 0.3, 0.3, 0.3, 0.3, 0.12, 0.12, 0.12, 0.12, 0.12, 0.17, 0.17, 0.17, 0.17, 0.17, 0.26, 0.26, 0.26, 0.26, 0.26, 0.2, 0.2, 0.2, 0.2, 0.2, 0.15, 0.15, 0.15, 0.15, 0.15, 0.14, 0.14, 0.14, 0.14, 0.14, 0.17, 0.17, 0.17, 0.17, 0.17, 0.19, 0.19, 0.19, 0.19, 0.19, 0.21, 0.21, 0.21, 0.21, 0.21, 0.19, 0.19, 0.19, 0.19, 0.19, 0.32, 0.32, 0.32, 0.32, 0.32, 0.27, 0.27, 0.27, 0.27, 0.27, 0.34, 0.34, 0.34, 0.34, 0.34, 0.28, 0.28, 0.28, 0.28, 0.28, 0.18, 0.18, 0.18, 0.18, 0.18, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.13, 0.16, 0.16, 0.16, 0.16, 0.16, 0.12, 0.12, 0.12, 0.12, 0.12, 0.39, 0.39, 0.39, 0.39, 0.39, 0.5, 0.5, 0.5, 0.5, 0.5, 0.44, 0.44, 0.44, 0.44, 0.44, 0.39, 0.39, 0.39, 0.39, 0.39, 0.18, 0.18, 0.18, 0.18, 0.18, 0.27, 0.27, 0.27, 0.27, 0.27, 0.2, 0.2, 0.2, 0.2, 0.2, 0.17, 0.17, 0.17, 0.17, 0.17, 0.19, 0.19, 0.19, 0.19, 0.19, 0.24, 0.24, 0.24, 0.24, 0.24, 0.19, 0.19, 0.19, 0.19, 0.19, 0.11, 0.11, 0.11, 0.11, 0.11, 0.27, 0.27, 0.27, 0.27, 0.27, 0.09, 0.09, 0.09, 0.09, 0.09, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.15, 0.13, 0.13, 0.13, 0.13, 0.13, 0.23]
                    

More

---
config:
    xyChart:
        titleFontSize: 12
        width: 900
        height: 600
    themeVariables:
        xyChart:
            titleColor: "#000000"
---
xychart-beta
    title "llama.cpp bench-server-baseline on Standard_NC4as_T4_v3
 duration=10m 527 iterations"
    y-axis "llamacpp:requests_processing"
    x-axis "llamacpp:requests_processing" 1716972379 --> 1716973009
    line [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 5.0, 5.0, 5.0, 5.0, 5.0, 8.0, 8.0, 8.0, 8.0, 8.0, 3.0, 3.0, 3.0, 3.0, 3.0, 2.0, 2.0, 2.0, 2.0, 2.0, 8.0, 8.0, 8.0, 8.0, 8.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 4.0, 4.0, 4.0, 4.0, 4.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 5.0, 5.0, 5.0, 5.0, 5.0, 4.0, 4.0, 4.0, 4.0, 4.0, 5.0, 5.0, 5.0, 5.0, 5.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 3.0, 7.0, 7.0, 7.0, 7.0, 7.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 7.0, 3.0, 3.0, 3.0, 3.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 7.0, 7.0, 7.0, 7.0, 7.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 6.0, 6.0, 6.0, 6.0, 6.0, 8.0, 8.0, 8.0, 8.0, 8.0, 2.0, 2.0, 2.0, 2.0, 2.0, 3.0, 3.0, 3.0, 3.0, 3.0, 8.0, 8.0, 8.0, 8.0, 8.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 6.0, 3.0, 3.0, 3.0, 3.0, 3.0, 6.0, 6.0, 6.0, 6.0, 6.0, 4.0, 4.0, 4.0, 4.0, 4.0, 2.0, 2.0, 2.0, 2.0, 2.0, 1.0, 1.0, 1.0, 1.0, 1.0, 7.0, 7.0, 7.0, 7.0, 7.0, 2.0]