Skip to content

ggml-hrx: support BF16 weights #95

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
AaronStGeorge merged 1 commit into from
Sep 2, 2026
Merged

ggml-hrx: support BF16 weights #95

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
3 changes: 3 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-get-rows.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -68,6 +68,9 @@ static bool format_config_for_type(ggml_type type, FormatConfig & config) {
case GGML_TYPE_F16:
config = { type, 16 };
return true;
case GGML_TYPE_BF16:
config = { type, 17 };
return true;
case GGML_TYPE_F32:
config = { type, 32 };
return true;
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat-weight-format.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ enum class CommonMulMatWeightFormat {
Q8_0,
Q8_1,
F16,
BF16,
F32,
};

Expand All @@ -32,6 +33,9 @@ inline bool common_mul_mat_format_for_type(ggml_type type, CommonMulMatWeightFor
case GGML_TYPE_F16:
format = CommonMulMatWeightFormat::F16;
return true;
case GGML_TYPE_BF16:
format = CommonMulMatWeightFormat::BF16;
return true;
case GGML_TYPE_F32:
format = CommonMulMatWeightFormat::F32;
return true;
Expand All @@ -52,6 +56,8 @@ inline int64_t common_mul_mat_format_config_value(CommonMulMatWeightFormat forma
return 81;
case CommonMulMatWeightFormat::F16:
return 16;
case CommonMulMatWeightFormat::BF16:
return 17;
case CommonMulMatWeightFormat::F32:
return 32;
}
Expand Down
9 changes: 8 additions & 1 deletion ggml/src/ggml-hrx/dispatch_registration/llm/dispatch-attention-qkv.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ enum class AttentionWeightFormat {
Q8_0,
Q8_1,
F16,
BF16,
F32,
};

Expand All @@ -93,6 +94,9 @@ static bool format_for_type(ggml_type type, AttentionWeightFormat & format) {
case GGML_TYPE_F16:
format = AttentionWeightFormat::F16;
return true;
case GGML_TYPE_BF16:
format = AttentionWeightFormat::BF16;
return true;
case GGML_TYPE_F32:
format = AttentionWeightFormat::F32;
return true;
Expand All @@ -113,14 +117,17 @@ static int64_t format_config_value(AttentionWeightFormat format) {
return 81;
case AttentionWeightFormat::F16:
return 16;
case AttentionWeightFormat::BF16:
return 17;
case AttentionWeightFormat::F32:
return 32;
}
return 0;
}

static bool is_dense_float_weight_format(AttentionWeightFormat format) {
return format == AttentionWeightFormat::F16 || format == AttentionWeightFormat::F32;
return format == AttentionWeightFormat::F16 || format == AttentionWeightFormat::BF16 ||
format == AttentionWeightFormat::F32;
}

static std::string to_config_value(int64_t value) {
Expand Down
37 changes: 37 additions & 0 deletions ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/manifest.json
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -143,6 +143,9 @@
{
"path": "motifs/f16_f16.loom"
},
{
"path": "motifs/bf16_f16.loom"
},
{
"path": "motifs/f32_f16.loom"
}
Expand Down Expand Up @@ -486,6 +489,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -498,6 +502,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -551,6 +556,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -565,6 +571,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -620,6 +627,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -634,6 +642,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -682,6 +691,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -695,6 +705,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -743,6 +754,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -754,6 +766,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -846,6 +859,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -856,6 +870,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -909,6 +924,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -921,6 +937,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -980,6 +997,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -992,6 +1010,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1155,6 +1174,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1167,6 +1187,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1214,6 +1235,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1226,6 +1248,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1273,6 +1296,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1285,6 +1309,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1334,6 +1359,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1346,6 +1372,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1401,6 +1428,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1413,6 +1441,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1472,6 +1501,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1482,6 +1512,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1527,6 +1558,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1537,6 +1569,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1698,6 +1731,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1710,6 +1744,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down Expand Up @@ -1773,6 +1808,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand All @@ -1785,6 +1821,7 @@
"motifs/q8_0_f16.loom",
"motifs/q8_1_f16.loom",
"motifs/f16_f16.loom",
"motifs/bf16_f16.loom",
"motifs/f32_f16.loom"
]
},
Expand Down
12 changes: 12 additions & 0 deletions ggml/src/ggml-hrx/kernel-corpus/kernels/loom-libs/motifs/bf16_f16.loom
View file
Open in desktop
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
// Copyright 2026 The HRX Authors
// SPDX-License-Identifier: Apache-2.0

// Loads four adjacent BF16 weights and truncates them for FP16 matrix staging.
func.def inline @ggml_bf16_f16_vector4(%weight: buffer, %row_byte_base: offset, %input_size: index, %k: index) -> (vector<4xf16>) {
%bounded_input_size = index.assume %input_size [range(%input_size, 256, 32768), mul(%input_size, 256)] : index
%row_view = buffer.view %weight[%row_byte_base] : buffer -> view<[%bounded_input_size]xbf16>
%values_bf16 = vector.load %row_view[%k] : view<[%bounded_input_size]xbf16> -> vector<4xbf16>
%values_f32 = vector.extf %values_bf16 : vector<4xbf16> to vector<4xf32>
%values = vector.fptrunc %values_f32 : vector<4xf32> to vector<4xf16>
func.return %values : vector<4xf16>
}
Loading