Skip to content

Commit f81044d

Browse files
zonglinpengfacebook-github-bot
zonglinpeng
authored and
facebook-github-bot
committed
AI: [G3] dequantize_per_tensor asym SIMD: integer zero-point subtract
Summary: Optimizes the per-tensor `dequantize_per_tensor_out` SIMD fast path on Fusion-G3 for int8 / uint8 / int16 / uint16 -> float32. What is optimized: the zero-point subtract is kept in the int32 domain (`PDX_SUB_MX32`) and the int->float conversion is folded into the multiply by feeding the widened int32 codes straight into the mixed-type `PDX_MUL_MXF32(int32, scale)` (the idiom the vendor NNLib dequantize kernel uses). Per element the symmetric path is then a single fused multiply, and the asymmetric path is one integer subtract plus one fused multiply, with no separate int->float convert instruction on either path. Bit-identical to the previous code for 8/16-bit inputs, since `q - zp` is exact in int32 and converts exactly to float, matching `float(q) - float(zp)`. What is improved from 00b95cf2b4: that revision moved the subtract into the integer domain but kept an explicit `(xb_vecMxf32)` convert, so the asymmetric FP pipe still did convert + multiply (a `sub -> convert -> mul` dependency chain) and the symmetric path was left untouched (also convert + multiply). This diff removes that convert on both paths via the fused mixed-type multiply, shortening the asymmetric chain to `sub -> fused-mul` and the symmetric chain to a single fused multiply. That clears the residual large-tensor asymmetric regression 00b95cf2b4 still had -- e.g. `objv1_1x400x400x1_u8` (160000 elems) goes 0.75x -> 1.00x vs stock and `dpev26_4x60x4x4_u16` goes 0.89x -> 1.17x -- while keeping every symmetric case at parity or better, and the object file is ~144 B smaller. Differential Revision: D110529287
1 parent bb194ea commit f81044d

1 file changed

Lines changed: 84 additions & 96 deletions

File tree

‎backends/cadence/fusion_g3/operators/op_dequantize.cpp‎

Lines changed: 84 additions & 96 deletions
Original file line numberDiff line numberDiff line change
@@ -612,34 +612,37 @@ Tensor& dequantize_per_tensor_out(
612612
const int8_t* __restrict__ input_data = input.const_data_ptr<int8_t>();
613613
#if defined(__XTENSA__)
614614
// Direct inline PDX SIMD dequantization for per-tensor int8->float32.
615-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
616-
// independent loads, converts, and FP MACs across unrolled iterations,
617-
// hiding load-to-use latency and reducing loop overhead.
615+
// The zero-point subtract stays in the int32 domain (PDX_SUB_MX32) and
616+
// the int->float conversion is folded into PDX_MUL_MXF32 (mixed int32 x
617+
// float32 -> float32), so each element costs one fused mul (sym) or one
618+
// int sub + one fused mul (asym) with no separate convert. The int sub
619+
// issues on the integer unit, overlapping the float mul, so the asym path
620+
// stays store-bound. 4x-unrolled to hide load-to-use latency.
618621
if (dequant_simd_aligned(input_data, out_data)) {
619622
const int numel = inp_shape[0];
620623
auto vIn = reinterpret_cast<const xb_vecMx8*>(input_data);
621624
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
622625
const xb_vecMxf32 v_scale{
623626
scale_data, scale_data, scale_data, scale_data};
624627
int i = 0;
628+
// 4x unrolled main loop: 16 elements per iteration
629+
const int e16 = (numel >> 4) << 4;
625630
if (zero_point_data != 0) {
631+
const xb_vecMx32 v_zp{
632+
zero_point_data,
633+
zero_point_data,
634+
zero_point_data,
635+
zero_point_data};
626636
const float zp_f = static_cast<float>(zero_point_data);
627-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
628-
// 4x unrolled main loop: 16 elements per iteration
629-
const int e16 = (numel >> 4) << 4;
630637
for (; i < e16; i += 16) {
631638
xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0);
632639
xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0);
633640
xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0);
634641
xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0);
635-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
636-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
637-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
638-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
639-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
640-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
641-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
642-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
642+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
643+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
644+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
645+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
643646
vIn += 4;
644647
vOut += 4;
645648
}
@@ -649,21 +652,15 @@ Tensor& dequantize_per_tensor_out(
649652
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
650653
}
651654
} else {
652-
// 4x unrolled main loop: 16 elements per iteration
653-
const int e16 = (numel >> 4) << 4;
654655
for (; i < e16; i += 16) {
655656
xb_vecMx32 vP0 = PDX_LV32_MX8_I(vIn, 0);
656657
xb_vecMx32 vP1 = PDX_LV32_MX8_I(vIn + 1, 0);
657658
xb_vecMx32 vP2 = PDX_LV32_MX8_I(vIn + 2, 0);
658659
xb_vecMx32 vP3 = PDX_LV32_MX8_I(vIn + 3, 0);
659-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
660-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
661-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
662-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
663-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
664-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
665-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
666-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
660+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
661+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
662+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
663+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
667664
vIn += 4;
668665
vOut += 4;
669666
}
@@ -704,34 +701,37 @@ Tensor& dequantize_per_tensor_out(
704701
const uint8_t* __restrict__ input_data = input.const_data_ptr<uint8_t>();
705702
#if defined(__XTENSA__)
706703
// Direct inline PDX SIMD dequantization for per-tensor uint8->float32.
707-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
708-
// independent loads, converts, and FP MACs across unrolled iterations,
709-
// hiding load-to-use latency and reducing loop overhead.
704+
// The zero-point subtract stays in the int32 domain (PDX_SUB_MX32) and
705+
// the int->float conversion is folded into PDX_MUL_MXF32 (mixed int32 x
706+
// float32 -> float32), so each element costs one fused mul (sym) or one
707+
// int sub + one fused mul (asym) with no separate convert. The int sub
708+
// issues on the integer unit, overlapping the float mul, so the asym path
709+
// stays store-bound. 4x-unrolled to hide load-to-use latency.
710710
if (dequant_simd_aligned(input_data, out_data)) {
711711
const int numel = inp_shape[0];
712712
auto vIn = reinterpret_cast<const xb_vecMxu8*>(input_data);
713713
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
714714
const xb_vecMxf32 v_scale{
715715
scale_data, scale_data, scale_data, scale_data};
716716
int i = 0;
717+
// 4x unrolled main loop: 16 elements per iteration
718+
const int e16 = (numel >> 4) << 4;
717719
if (zero_point_data != 0) {
720+
const xb_vecMxu32 v_zp{
721+
static_cast<uint32_t>(zero_point_data),
722+
static_cast<uint32_t>(zero_point_data),
723+
static_cast<uint32_t>(zero_point_data),
724+
static_cast<uint32_t>(zero_point_data)};
718725
const float zp_f = static_cast<float>(zero_point_data);
719-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
720-
// 4x unrolled main loop: 16 elements per iteration
721-
const int e16 = (numel >> 4) << 4;
722726
for (; i < e16; i += 16) {
723727
xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0);
724728
xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0);
725729
xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0);
726730
xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0);
727-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
728-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
729-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
730-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
731-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
732-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
733-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
734-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
731+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
732+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
733+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
734+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
735735
vIn += 4;
736736
vOut += 4;
737737
}
@@ -741,21 +741,15 @@ Tensor& dequantize_per_tensor_out(
741741
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
742742
}
743743
} else {
744-
// 4x unrolled main loop: 16 elements per iteration
745-
const int e16 = (numel >> 4) << 4;
746744
for (; i < e16; i += 16) {
747745
xb_vecMxu32 vP0 = PDX_LVU32_MX8_I(vIn, 0);
748746
xb_vecMxu32 vP1 = PDX_LVU32_MX8_I(vIn + 1, 0);
749747
xb_vecMxu32 vP2 = PDX_LVU32_MX8_I(vIn + 2, 0);
750748
xb_vecMxu32 vP3 = PDX_LVU32_MX8_I(vIn + 3, 0);
751-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
752-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
753-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
754-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
755-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
756-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
757-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
758-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
749+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
750+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
751+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
752+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
759753
vIn += 4;
760754
vOut += 4;
761755
}
@@ -796,34 +790,37 @@ Tensor& dequantize_per_tensor_out(
796790
const int16_t* __restrict__ input_data = input.const_data_ptr<int16_t>();
797791
#if defined(__XTENSA__)
798792
// Direct inline PDX SIMD dequantization for per-tensor int16->float32.
799-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
800-
// independent loads, converts, and FP MACs across unrolled iterations,
801-
// hiding load-to-use latency and reducing loop overhead.
793+
// The zero-point subtract stays in the int32 domain (PDX_SUB_MX32) and
794+
// the int->float conversion is folded into PDX_MUL_MXF32 (mixed int32 x
795+
// float32 -> float32), so each element costs one fused mul (sym) or one
796+
// int sub + one fused mul (asym) with no separate convert. The int sub
797+
// issues on the integer unit, overlapping the float mul, so the asym path
798+
// stays store-bound. 4x-unrolled to hide load-to-use latency.
802799
if (dequant_simd_aligned(input_data, out_data)) {
803800
const int numel = inp_shape[0];
804801
auto vIn = reinterpret_cast<const xb_vecMx16*>(input_data);
805802
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
806803
const xb_vecMxf32 v_scale{
807804
scale_data, scale_data, scale_data, scale_data};
808805
int i = 0;
806+
// 4x unrolled main loop: 16 elements per iteration
807+
const int e16 = (numel >> 4) << 4;
809808
if (zero_point_data != 0) {
809+
const xb_vecMx32 v_zp{
810+
zero_point_data,
811+
zero_point_data,
812+
zero_point_data,
813+
zero_point_data};
810814
const float zp_f = static_cast<float>(zero_point_data);
811-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
812-
// 4x unrolled main loop: 16 elements per iteration
813-
const int e16 = (numel >> 4) << 4;
814815
for (; i < e16; i += 16) {
815816
xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0);
816817
xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0);
817818
xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0);
818819
xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0);
819-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
820-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
821-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
822-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
823-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
824-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
825-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
826-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
820+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
821+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
822+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
823+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
827824
vIn += 4;
828825
vOut += 4;
829826
}
@@ -833,21 +830,15 @@ Tensor& dequantize_per_tensor_out(
833830
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
834831
}
835832
} else {
836-
// 4x unrolled main loop: 16 elements per iteration
837-
const int e16 = (numel >> 4) << 4;
838833
for (; i < e16; i += 16) {
839834
xb_vecMx32 vP0 = PDX_LV32_MX16_I(vIn, 0);
840835
xb_vecMx32 vP1 = PDX_LV32_MX16_I(vIn + 1, 0);
841836
xb_vecMx32 vP2 = PDX_LV32_MX16_I(vIn + 2, 0);
842837
xb_vecMx32 vP3 = PDX_LV32_MX16_I(vIn + 3, 0);
843-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
844-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
845-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
846-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
847-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
848-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
849-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
850-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
838+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
839+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
840+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
841+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
851842
vIn += 4;
852843
vOut += 4;
853844
}
@@ -889,34 +880,37 @@ Tensor& dequantize_per_tensor_out(
889880
input.const_data_ptr<uint16_t>();
890881
#if defined(__XTENSA__)
891882
// Direct inline PDX SIMD dequantization for per-tensor uint16->float32.
892-
// 4x-unrolled to improve ILP: the Fusion G3 scheduler can interleave
893-
// independent loads, converts, and FP MACs across unrolled iterations,
894-
// hiding load-to-use latency and reducing loop overhead.
883+
// The zero-point subtract stays in the int32 domain (PDX_SUB_MX32) and
884+
// the int->float conversion is folded into PDX_MUL_MXF32 (mixed int32 x
885+
// float32 -> float32), so each element costs one fused mul (sym) or one
886+
// int sub + one fused mul (asym) with no separate convert. The int sub
887+
// issues on the integer unit, overlapping the float mul, so the asym path
888+
// stays store-bound. 4x-unrolled to hide load-to-use latency.
895889
if (dequant_simd_aligned(input_data, out_data)) {
896890
const int numel = inp_shape[0];
897891
auto vIn = reinterpret_cast<const xb_vecMxu16*>(input_data);
898892
auto vOut = reinterpret_cast<xb_vecMxf32*>(out_data);
899893
const xb_vecMxf32 v_scale{
900894
scale_data, scale_data, scale_data, scale_data};
901895
int i = 0;
896+
// 4x unrolled main loop: 16 elements per iteration
897+
const int e16 = (numel >> 4) << 4;
902898
if (zero_point_data != 0) {
899+
const xb_vecMxu32 v_zp{
900+
static_cast<uint32_t>(zero_point_data),
901+
static_cast<uint32_t>(zero_point_data),
902+
static_cast<uint32_t>(zero_point_data),
903+
static_cast<uint32_t>(zero_point_data)};
903904
const float zp_f = static_cast<float>(zero_point_data);
904-
const xb_vecMxf32 v_zp{zp_f, zp_f, zp_f, zp_f};
905-
// 4x unrolled main loop: 16 elements per iteration
906-
const int e16 = (numel >> 4) << 4;
907905
for (; i < e16; i += 16) {
908906
xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0);
909907
xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0);
910908
xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0);
911909
xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0);
912-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
913-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
914-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
915-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
916-
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF0, v_zp), v_scale);
917-
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF1, v_zp), v_scale);
918-
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF2, v_zp), v_scale);
919-
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MXF32(vF3, v_zp), v_scale);
910+
vOut[0] = PDX_MUL_MXF32(PDX_SUB_MX32(vP0, v_zp), v_scale);
911+
vOut[1] = PDX_MUL_MXF32(PDX_SUB_MX32(vP1, v_zp), v_scale);
912+
vOut[2] = PDX_MUL_MXF32(PDX_SUB_MX32(vP2, v_zp), v_scale);
913+
vOut[3] = PDX_MUL_MXF32(PDX_SUB_MX32(vP3, v_zp), v_scale);
920914
vIn += 4;
921915
vOut += 4;
922916
}
@@ -926,21 +920,15 @@ Tensor& dequantize_per_tensor_out(
926920
(static_cast<float>(input_data[i]) - zp_f) * scale_data;
927921
}
928922
} else {
929-
// 4x unrolled main loop: 16 elements per iteration
930-
const int e16 = (numel >> 4) << 4;
931923
for (; i < e16; i += 16) {
932924
xb_vecMxu32 vP0 = PDX_LVU32_MX16_I(vIn, 0);
933925
xb_vecMxu32 vP1 = PDX_LVU32_MX16_I(vIn + 1, 0);
934926
xb_vecMxu32 vP2 = PDX_LVU32_MX16_I(vIn + 2, 0);
935927
xb_vecMxu32 vP3 = PDX_LVU32_MX16_I(vIn + 3, 0);
936-
xb_vecMxf32 vF0 = (xb_vecMxf32)vP0;
937-
xb_vecMxf32 vF1 = (xb_vecMxf32)vP1;
938-
xb_vecMxf32 vF2 = (xb_vecMxf32)vP2;
939-
xb_vecMxf32 vF3 = (xb_vecMxf32)vP3;
940-
vOut[0] = PDX_MUL_MXF32(vF0, v_scale);
941-
vOut[1] = PDX_MUL_MXF32(vF1, v_scale);
942-
vOut[2] = PDX_MUL_MXF32(vF2, v_scale);
943-
vOut[3] = PDX_MUL_MXF32(vF3, v_scale);
928+
vOut[0] = PDX_MUL_MXF32(vP0, v_scale);
929+
vOut[1] = PDX_MUL_MXF32(vP1, v_scale);
930+
vOut[2] = PDX_MUL_MXF32(vP2, v_scale);
931+
vOut[3] = PDX_MUL_MXF32(vP3, v_scale);
944932
vIn += 4;
945933
vOut += 4;
946934
}

0 commit comments

Comments
 (0)