gemm block quantization infrastructure for llm decoder style

CMakeLists.txt

@@ -90,6 +90,7 @@ option(NCNN_ASAN "build for address sanitizer" OFF)
 option(NCNN_BUILD_BENCHMARK "build benchmark" ON)
 option(NCNN_PYTHON "build python api" OFF)
 option(NCNN_INT8 "int8 inference" ON)
+option(NCNN_WEIGHT_QUANT "weight quantized inference" ON)
 option(NCNN_BF16 "bf16 inference" ON)
 option(NCNN_FORCE_INLINE "force inline some function" ON)
 

docs/developer-guide/operation-param-weight-table.md

@@ -131,6 +131,25 @@
 ||2|expand_c|0|
 ||3|axes|[ ]|
 |Flatten|||
+|Gemm|0|alpha|1.f|A_data B_data C_data A_data_int8_scales B_data_int8_scales/B_data_quantize_scales B_data_input_scales|
+||1|beta|1.f|
+||2|transA|0|
+||3|transB|0|
+||4|constantA|0|
+||5|constantB|0|
+||6|constantC|0|
+||7|constantM|0|
+||8|constantN|0|
+||9|constantK|0|
+||10|constant_broadcast_type_C|0|
+||11|output_N1M|0|
+||12|output_elempack|0|
+||13|output_elemtype|0|
+||14|output_transpose|0|
+||18|quantize_term|0|
+||20|constant_TILE_M|0|
+||21|constant_TILE_N|0|
+||22|constant_TILE_K|0|
 |HardSigmoid|0|alpha|0.2f||
 ||1|beta|0.5f|
 |HardSwish|0|alpha|0.2f||
@@ -298,6 +317,12 @@
 |RNN|0|num_output|0|
 ||1|weight_data_size|0|
 ||2|direction|0|
-|MultiHeadAttention|0|embed_dim|0|
+|MultiHeadAttention|0|embed_dim|0|q_weight_data q_bias_data k_weight_data k_bias_data v_weight_data v_bias_data out_weight_data out_bias_data q_weight_data_int8_scales k_weight_data_int8_scales v_weight_data_int8_scales out_weight_data_int8_scales q_weight_data_quantize_scales k_weight_data_quantize_scales v_weight_data_quantize_scales out_weight_data_quantize_scales q_weight_data_input_scales k_weight_data_input_scales v_weight_data_input_scales out_weight_data_input_scales|
 ||1|num_head|1|
 ||2|weight_data_size|0|
+||3|kdim|embed_dim|
+||4|vdim|embed_dim|
+||5|attn_mask|0|
+||6|scale|1.f / sqrt(embed_dim / num_heads)|
+||7|kv_cache|0|
+||18|quantize_term|0|

docs/developer-guide/operators.md

@@ -1151,7 +1151,7 @@ y = (gemm(a, b) + c * beta) * alpha
 | 12        | output_elempack | int | 0         |                   |
 | 13        | output_elemtype | int | 0         |                   |
 | 14        | output_transpose | int| 0         |                   |
-| 18        | int8_scale_term | int | 0         |                   |
+| 18        | quantize_term | int | 0         | 0=no quant, nonzero below 400=legacy int8, 4xx/6xx/8xx=weight block quant |
 | 20        | constant_TILE_M | int | 0         |                   |
 | 21        | constant_TILE_N | int | 0         |                   |
 | 22        | constant_TILE_K | int | 0         |                   |
@@ -1163,6 +1163,16 @@ y = (gemm(a, b) + c * beta) * alpha
 | C_data        | float | [1], [M] or [N] or [1, M] or [N,1] or [N, M] |
 | A_data_int8_scales| float | [M]               |
 | B_data_int8_scales| float | [1]               |
+| B_data_quantize_scales| float | [ceil(K / block_size), N] for block quantized constant B |
+| B_data_input_scales| float | [K] for block quantized constant B with input scale |
+
+For weight-only block quantized Gemm:
+
+* `constantA=0`, `constantB=1`, `transA=0`, `transB=1`
+* output is fp32 pack1 with `output_N1M=0`, `output_elempack=0`, `output_transpose=0`
+* `B_data` is tagged int8 bytes with shape `[ceil(K * weight_bits / 8), N]`
+* `quantize_term = bits * 100 + input_scale * 10 + block_code`
+* `block_code`: 0=32, 1=64, 2=128
 
 # GridSample
 ```
@@ -1575,7 +1585,7 @@ y = affine(out)
 | 5         | attn_mask     | int   | 0         |                   |
 | 6         | scale         | float | 1.f / sqrt(embed_dim / num_heads) | |
 | 7         | kv_cache      | int   | 0         |                   |
-| 18        | int8_scale_term | int | 0         |                   |
+| 18        | quantize_term | int | 0         | 0=no quant, nonzero below 400=legacy int8, 4xx/6xx/8xx=weight block quant |
 
 | weight        | type  | shape                 |
 | ------------- | ----- | --------------------- |
@@ -1591,6 +1601,16 @@ y = affine(out)
 | k_weight_data_int8_scales| float | [embed_dim] |
 | v_weight_data_int8_scales| float | [embed_dim] |
 | out_weight_data_int8_scales| float | [1]      |
+| q_weight_data_quantize_scales| float | [ceil(qdim / block_size), embed_dim] for block quantized weight |
+| k_weight_data_quantize_scales| float | [ceil(kdim / block_size), embed_dim] for block quantized weight |
+| v_weight_data_quantize_scales| float | [ceil(vdim / block_size), embed_dim] for block quantized weight |
+| out_weight_data_quantize_scales| float | [ceil(embed_dim / block_size), qdim] for block quantized weight |
+| q_weight_data_input_scales| float | [qdim] for block quantized weight with input scale |
+| k_weight_data_input_scales| float | [kdim] for block quantized weight with input scale |
+| v_weight_data_input_scales| float | [vdim] for block quantized weight with input scale |
+| out_weight_data_input_scales| float | [embed_dim] for block quantized weight with input scale |
+
+Weight-only block quantized MultiHeadAttention stores q/k/v/out weights as tagged int8 bytes. The `quantize_term` rule is same as Gemm.
 
 # MVN
 ```

docs/how-to-use-and-FAQ/build-minimal-library.md

@@ -56,6 +56,13 @@ cmake -DNCNN_INT8=OFF ..
 
 * Cannot use quantized int8 inference.
 
+### disable NCNN_WEIGHT_QUANT
+
+```
+cmake -DNCNN_WEIGHT_QUANT=OFF ..
+```
+
+* Cannot use weight quantized inference.
 
 ### drop pixel drawing functions
 

docs/how-to-use-and-FAQ/quantized-int8-inference.md

@@ -101,6 +101,66 @@ ncnn2table can generate static weight scales without a calibration dataset for R
 ./ncnn2int8 mobilenet-opt.param mobilenet-opt.bin mobilenet-int8.param mobilenet-int8.bin mobilenet.table
 ```
 
+## Weight-only block quantized Gemm and MultiHeadAttention
+
+LLM-oriented `Gemm` and `MultiHeadAttention` weight-only block quantization is separate from the post training int8 flow above. It stores weight as signed int4/int6/int8 blocks and keeps activation/output in fp32.
+
+The workflow is similar to `ncnn2table` and `ncnn2int8`:
+
+```shell
+./ncnnllm2table in.param in.bin model.llm.table method=minmax bits=6 block=64
+./ncnnllm2int in.param in.bin out.param out.bin model.llm.table
+```
+
+method can be minmax,mseclip,awq,gptq. bits can be 4,6,8. block can be 32,64,128. thread is the CPU thread count.
+
+awq and gptq need calibration data, same as npy calibration in ncnn2table.
+
+```shell
+./ncnnllm2table in.param in.bin calib.list awq.llm.table method=awq bits=4 block=64 type=1 shape=[...]
+./ncnnllm2int in.param in.bin awq.param awq.bin awq.llm.table
+```
+
+```shell
+./ncnnllm2table in.param in.bin calib.list gptq.llm.table method=gptq bits=4 block=128 type=1 shape=[...]
+./ncnnllm2int in.param in.bin gptq.param gptq.bin gptq.llm.table
+```
+
+The calibration list format follows `ncnn2table`.
+
+```text
+gemm_name_param_1 bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_0  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_1  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_2  bits=4 block=64 method=mseclip scale0 scale1 ...
+mha_name_param_3  bits=4 block=64 method=mseclip scale0 scale1 ...
+```
+
+For `MultiHeadAttention`, `_param_0/_param_1/_param_2/_param_3` are q/k/v/out weights.
+
+```text
+gemm_name_param_1_input_scale method=awq scale0 scale1 ...
+mha_name_param_0_input_scale  method=awq scale0 scale1 ...
+```
+
+`method=gptq` uses fixed symmetric block scales and standard GPTQ error compensation. It writes packed qweight files and records them in the table.
+
+```text
+gemm_name_param_1 bits=4 block=128 method=gptq qweight=gemm.qweight scale0 scale1 ...
+```
+
+The table is text and may be edited before conversion. Missing `Gemm` rows are skipped. `MultiHeadAttention` q/k/v/out rows must exist together. Unused rows are rejected and at least one layer must be quantized.
+
+The generated layer `quantize_term` is `bits * 100 + input_scale * 10 + block_code`, and block_code 0/1/2 means block 32/64/128.
+
+For quick conversion without saving a table, `ncnnllm2int` can still compute scales directly:
+
+```shell
+./ncnnllm2int in.param in.bin out.param out.bin method=minmax bits=6 block=64
+```
+
+This format is signed symmetric scale-only. Zero point is not used.
+
 ## use ncnn int8 inference
 
 the ncnn library would use int8 inference automatically, nothing changed in your code

src/layer/arm/gemm_arm.cpp

@@ -4584,8 +4584,13 @@ static int gemm_AT_BT_arm(const Mat& AT, const Mat& BT, const Mat& C, Mat& top_b
 
 int Gemm_arm::create_pipeline(const Option& opt)
 {
+    if (weight_block_quantize)
+    {
+        return 0;
+    }
+
 #if NCNN_INT8
-    if (int8_scale_term)
+    if (quantize_term)
     {
         return create_pipeline_int8(opt);
     }
@@ -4742,8 +4747,13 @@ int Gemm_arm::create_pipeline(const Option& opt)
 
 int Gemm_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& opt) const
 {
+    if (weight_block_quantize)
+    {
+        return Gemm::forward(bottom_blobs, top_blobs, opt);
+    }
+
 #if NCNN_INT8
-    if (int8_scale_term)
+    if (quantize_term)
     {
         return forward_int8(bottom_blobs, top_blobs, opt);
     }

src/layer/arm/multiheadattention_arm.cpp

@@ -34,6 +34,9 @@ MultiHeadAttention_arm::MultiHeadAttention_arm()
 
 int MultiHeadAttention_arm::create_pipeline(const Option& _opt)
 {
+    if (weight_block_quantize)
+        return 0;
+
     Option opt = _opt;
     if (int8_scale_term)
     {
@@ -265,6 +268,9 @@ int MultiHeadAttention_arm::create_pipeline(const Option& _opt)
 
 int MultiHeadAttention_arm::destroy_pipeline(const Option& _opt)
 {
+    if (weight_block_quantize)
+        return 0;
+
     Option opt = _opt;
     if (int8_scale_term)
     {
@@ -330,6 +336,9 @@ int MultiHeadAttention_arm::destroy_pipeline(const Option& _opt)
 
 int MultiHeadAttention_arm::forward(const std::vector<Mat>& bottom_blobs, std::vector<Mat>& top_blobs, const Option& _opt) const
 {
+    if (weight_block_quantize)
+        return MultiHeadAttention::forward(bottom_blobs, top_blobs, _opt);
+
     int q_blob_i = 0;
     int k_blob_i = 0;
     int v_blob_i = 0;