Skip to content

hrx: IQ1_S and IQ1_M weights on HRX0 (shared dequantizer, K-quant decode kernels) #53

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
bong-water-water-bong merged 1 commit into from
Oct 1, 2026
Merged

hrx: IQ1_S and IQ1_M weights on HRX0 (shared dequantizer, K-quant decode kernels) #53

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
34 changes: 25 additions & 9 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-kquant-decode.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,7 @@
// limitations under the License.

// Decode projections (1 token, or 2-8 for MTP / speculative verify batches) on Q2_K, Q3_K, Q4_K, Q5_K,
// Q6_K, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_NL, IQ4_XS and Q8_0 weights, read in their GGUF block layout
// Q6_K, IQ1_S, IQ1_M, IQ2_XXS, IQ2_XS, IQ2_S, IQ3_XXS, IQ3_S, IQ4_NL, IQ4_XS and Q8_0 weights, read in their GGUF block layout
// (ops/kquant_decode_f32.loom): FFN gate/up pairs fused with SwiGLU, and plain projections with an
// optional following residual ADD. Mixed-quant models (Unsloth UD-Q4_K_XL and similar) pair these
// types freely per layer; without this they take the generic dequantize-4-values-at-a-time
Expand Down Expand Up @@ -46,6 +46,8 @@ static constexpr KernelCatalogRef kKQuantMulMatDecodeTokensKernel =
bool kquant_format(CommonMulMatWeightFormat format) {
switch (format) {
case CommonMulMatWeightFormat::Q2K:
case CommonMulMatWeightFormat::IQ1_S:
case CommonMulMatWeightFormat::IQ1_M:
case CommonMulMatWeightFormat::IQ2_XXS:
case CommonMulMatWeightFormat::IQ2_XS:
case CommonMulMatWeightFormat::IQ3_XXS:
Expand All @@ -64,12 +66,26 @@ bool kquant_format(CommonMulMatWeightFormat format) {
}
}

// Formats whose lane functions read a codebook staged in workgroup memory. A gate/up pair shares
// that one buffer, so its two formats may not need different codebooks.
bool kquant_needs_grid(CommonMulMatWeightFormat format) {
return format == CommonMulMatWeightFormat::IQ3_S || format == CommonMulMatWeightFormat::IQ2_XXS ||
format == CommonMulMatWeightFormat::IQ2_XS || format == CommonMulMatWeightFormat::IQ3_XXS ||
format == CommonMulMatWeightFormat::IQ2_S;
// The codebook a format's lane functions read from workgroup memory (IQ1_S and IQ1_M share one),
// or 0. A gate/up pair shares that one buffer, so its two formats may not need different codebooks.
int kquant_grid(CommonMulMatWeightFormat format) {
switch (format) {
case CommonMulMatWeightFormat::IQ3_S:
return 21;
case CommonMulMatWeightFormat::IQ2_XXS:
return 24;
case CommonMulMatWeightFormat::IQ2_XS:
return 25;
case CommonMulMatWeightFormat::IQ2_S:
return 22;
case CommonMulMatWeightFormat::IQ3_XXS:
return 28;
case CommonMulMatWeightFormat::IQ1_S:
case CommonMulMatWeightFormat::IQ1_M:
return 26;
default:
return 0;
}
}

// The kernels' config ranges (ops/kquant_decode_f32.loom).
Expand Down Expand Up @@ -163,8 +179,8 @@ bool match_kquant_swiglu_decode(const DispatchMatchContext & context, DispatchMa
return false;
}
const CommonMulMatMatch other = match_few_token_mul_mat(graph, peer, kKQuantSwiGLUDecodeKernel);
if (other.matched() && kquant_needs_grid(other.weight_format) && kquant_needs_grid(root.weight_format) &&
other.weight_format != root.weight_format) {
if (other.matched() && kquant_grid(other.weight_format) != 0 && kquant_grid(root.weight_format) != 0 &&
kquant_grid(other.weight_format) != kquant_grid(root.weight_format)) {
return false;
}
if (!other.matched() || !kquant_format(other.weight_format) || other.input->id != root.input->id ||
Expand Down
4 changes: 4 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-common.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -187,6 +187,10 @@ inline ggml_type common_mul_mat_format_type(CommonMulMatWeightFormat format) {
return GGML_TYPE_Q5_1;
case CommonMulMatWeightFormat::IQ3_XXS:
return GGML_TYPE_IQ3_XXS;
case CommonMulMatWeightFormat::IQ1_S:
return GGML_TYPE_IQ1_S;
case CommonMulMatWeightFormat::IQ1_M:
return GGML_TYPE_IQ1_M;
case CommonMulMatWeightFormat::IQ2_XXS:
return GGML_TYPE_IQ2_XXS;
case CommonMulMatWeightFormat::IQ2_XS:
Expand Down
12 changes: 12 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-weight-format.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,8 @@ enum class CommonMulMatWeightFormat {
Q4_1,
Q5_0,
Q5_1,
IQ1_S,
IQ1_M,
IQ2_XXS,
IQ2_XS,
IQ3_XXS,
Expand Down Expand Up @@ -68,6 +70,12 @@ inline bool common_mul_mat_format_for_type(ggml_type type, CommonMulMatWeightFor
case GGML_TYPE_IQ3_XXS:
format = CommonMulMatWeightFormat::IQ3_XXS;
return true;
case GGML_TYPE_IQ1_S:
format = CommonMulMatWeightFormat::IQ1_S;
return true;
case GGML_TYPE_IQ1_M:
format = CommonMulMatWeightFormat::IQ1_M;
return true;
case GGML_TYPE_IQ2_XXS:
format = CommonMulMatWeightFormat::IQ2_XXS;
return true;
Expand Down Expand Up @@ -134,6 +142,10 @@ inline int64_t common_mul_mat_format_config_value(CommonMulMatWeightFormat forma
return 51;
case CommonMulMatWeightFormat::IQ3_XXS:
return 28;
case CommonMulMatWeightFormat::IQ1_S:
return 26;
case CommonMulMatWeightFormat::IQ1_M:
return 27;
case CommonMulMatWeightFormat::IQ2_XXS:
return 24;
case CommonMulMatWeightFormat::IQ2_XS:
Expand Down
Loading
Loading