aboutsummaryrefslogtreecommitdiff
diff options
context:
space:
mode:
authorLostRuins <39025047+LostRuins@users.noreply.github.com>2023-07-11 22:01:08 +0800
committerGitHub <noreply@github.com>2023-07-11 22:01:08 +0800
commitbbef28218fe827265716b66977719b9ee2b21165 (patch)
treec38db93b20493f8c1e9c4daa67004e8fea262c42
parent5656d10599bd756dc0f17284e418e704200b43f3 (diff)
Possible solution to allow K-quants on models with n_vocab!=32000 (#2148)
* This allows LLAMA models that were previously incompatible with K quants to function mostly as normal. This happens when a model has a vocab != 32000, e.g 32001 which means it's not divisible by 256 or 64. Since the problematic dimensions only apply for `tok_embeddings.weight` and `output.weight` (dimentions 4096 x n_vocab), we can simply quantize these layers to Q8_0 whereas the majority of the hidden layers are still K-quanted since they have compatible dimensions. * Fix indentation Co-authored-by: Georgi Gerganov <ggerganov@gmail.com> * As an alternative, to avoid failing on Metal due to lack of Q8_0 support, instead quantize tok_embeddings.weight to Q4_0 and retain output.weight as F16. This results in a net gain of about 55mb for a 7B model compared to previous approach, but should minimize adverse impact to model quality. --------- Co-authored-by: Georgi Gerganov <ggerganov@gmail.com>
-rw-r--r--llama.cpp18
1 files changed, 14 insertions, 4 deletions
diff --git a/llama.cpp b/llama.cpp
index ad7283f..08ec21a 100644
--- a/llama.cpp
+++ b/llama.cpp
@@ -2454,15 +2454,14 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
} else {
new_type = quantized_type;
#ifdef GGML_USE_K_QUANTS
+ bool convert_incompatible_tensor = false;
if (quantized_type == GGML_TYPE_Q2_K || quantized_type == GGML_TYPE_Q3_K || quantized_type == GGML_TYPE_Q4_K ||
quantized_type == GGML_TYPE_Q5_K || quantized_type == GGML_TYPE_Q6_K) {
int nx = tensor.ne.at(0);
int ny = tensor.ne.at(1);
if (nx % QK_K != 0 || ny % QK_K != 0) {
- fprintf(stderr, "\n\n========================= Tensor sizes %d x %d are not divisible by %d\n",nx,ny,QK_K);
- fprintf(stderr, "This is required to be able to use k-quants for now!\n");
- fprintf(stderr, "========================================================================================\n\n");
- throw std::runtime_error("Unsupported tensor size encountered\n");
+ fprintf(stderr, "\n\nTensor sizes %d x %d are not divisible by %d, required for k-quants.\n",nx,ny,QK_K);
+ convert_incompatible_tensor = true;
}
}
if (tensor.name == "output.weight") {
@@ -2490,6 +2489,17 @@ static void llama_model_quantize_internal(const std::string & fname_inp, const s
if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_M || ftype == LLAMA_FTYPE_MOSTLY_Q2_K) new_type = GGML_TYPE_Q4_K;
else if (ftype == LLAMA_FTYPE_MOSTLY_Q3_K_L) new_type = GGML_TYPE_Q5_K;
}
+ if (convert_incompatible_tensor) {
+ if (tensor.name == "output.weight") {
+ new_type = GGML_TYPE_F16; //fall back to F16 instead of just failing.
+ fprintf(stderr, "F16 will be used for this tensor instead.\n");
+ } else if (tensor.name == "tok_embeddings.weight") {
+ new_type = GGML_TYPE_Q4_0; //fall back to Q4_0 instead of just failing.
+ fprintf(stderr, "Q4_0 will be used for this tensor instead.\n");
+ } else {
+ throw std::runtime_error("Unsupported tensor size encountered\n");
+ }
+ }
#endif
float * f32_data;