Add Optional Activation node to NodeUnit #22888
+317
−592
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change | ||||
|---|---|---|---|---|---|---|
|
|
@@ -215,4 +215,141 @@ | |||||
|
|
||||||
| #endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) | ||||||
|
|
||||||
| namespace { | ||||||
|
|
||||||
| bool GetDataTypeMinMax(int32_t data_type, int32_t& min, int32_t& max) { | ||||||
| switch (data_type) { | ||||||
| case ONNX_NAMESPACE::TensorProto::INT8: | ||||||
| min = static_cast<int32_t>(std::numeric_limits<int8_t>::min()); | ||||||
| max = static_cast<int32_t>(std::numeric_limits<int8_t>::max()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::UINT8: | ||||||
| min = static_cast<int32_t>(std::numeric_limits<uint8_t>::min()); | ||||||
| max = static_cast<int32_t>(std::numeric_limits<uint8_t>::max()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::INT16: | ||||||
| min = static_cast<int32_t>(std::numeric_limits<int16_t>::min()); | ||||||
| max = static_cast<int32_t>(std::numeric_limits<int16_t>::max()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::UINT16: | ||||||
| min = static_cast<int32_t>(std::numeric_limits<uint16_t>::min()); | ||||||
| max = static_cast<int32_t>(std::numeric_limits<uint16_t>::max()); | ||||||
| break; | ||||||
| default: | ||||||
| return false; | ||||||
| } | ||||||
| return true; | ||||||
| } | ||||||
| bool GetQSalarScaleZp(const GraphViewer& graph_viewer, const Node& q_node, float& scale, int32_t& zp, | ||||||
|
There was a problem hiding this comment.
Suggested change
Would be good to refactor some of the utils here as there seems to be a fair bit of duplication. e.g. maybe a general purpose helper that reads the scale and zp values (scalar or otherwise), and has a bool to indicate if they're scalar. that helper could be used by many of the utils here. |
||||||
| int32_t& data_type) { | ||||||
| assert(q_node.OpType() == QOpName); | ||||||
| const auto& q_input_defs = q_node.InputDefs(); | ||||||
| if (q_input_defs.size() != 3 || !q_input_defs[2]->Exists()) { | ||||||
|
There was a problem hiding this comment. zp is optional and defaults to zero, so do we need to require 3 inputs here? |
||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| const ONNX_NAMESPACE::TensorProto* scale_tensor_proto = | ||||||
| graph_viewer.GetConstantInitializer(q_input_defs[1]->Name(), true); | ||||||
| if (!scale_tensor_proto) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| // Support scalar float scale only for now. Need to extend to other float types if needed. | ||||||
| Initializer scale_initializer(*scale_tensor_proto, graph_viewer.ModelPath()); | ||||||
| if (scale_initializer.dims().size() != 0 || scale_initializer.data_type() != ONNX_NAMESPACE::TensorProto::FLOAT) { | ||||||
| return false; | ||||||
| } | ||||||
| scale = *scale_initializer.data<float>(); | ||||||
|
|
||||||
| const ONNX_NAMESPACE::TensorProto* zp_tensor_proto = | ||||||
| graph_viewer.GetConstantInitializer(q_input_defs[2]->Name(), true); | ||||||
| if (!zp_tensor_proto) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| Initializer zp_initializer(*zp_tensor_proto, graph_viewer.ModelPath()); | ||||||
| if (zp_initializer.dims().size() != 0) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| data_type = zp_initializer.data_type(); | ||||||
| switch (data_type) { | ||||||
| case ONNX_NAMESPACE::TensorProto::INT8: | ||||||
| zp = static_cast<int32_t>(*zp_initializer.data<int8_t>()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::UINT8: | ||||||
| zp = static_cast<int32_t>(*zp_initializer.data<uint8_t>()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::INT16: | ||||||
| zp = static_cast<int32_t>(*zp_initializer.data<int16_t>()); | ||||||
| break; | ||||||
| case ONNX_NAMESPACE::TensorProto::UINT16: | ||||||
| zp = static_cast<int32_t>(*zp_initializer.data<uint16_t>()); | ||||||
| break; | ||||||
| default: | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| return true; | ||||||
| } | ||||||
|
|
||||||
| bool CanRemoveRelu(const GraphViewer& graph_viewer, const Node& q_node) { | ||||||
| float scale = 0.0f; | ||||||
| int32_t zp = 0; | ||||||
| int32_t data_type = 0; | ||||||
| if (!GetQSalarScaleZp(graph_viewer, q_node, scale, zp, data_type)) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| int32_t data_type_min = 0; | ||||||
| int32_t data_type_max = 0; | ||||||
| if (!GetDataTypeMinMax(data_type, data_type_min, data_type_max)) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| // Relu can be removed if the zero-point is set to the smallest quantized value. | ||||||
| return zp == data_type_min; | ||||||
| } | ||||||
|
|
||||||
| bool CanRemoveClip(const GraphViewer& graph_viewer, const Node& clip_node, const Node& q_node) { | ||||||
| float scale = 0.0f; | ||||||
| int32_t zp = 0; | ||||||
| int32_t data_type = 0; | ||||||
| if (!GetQSalarScaleZp(graph_viewer, q_node, scale, zp, data_type)) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| float min = 0.0f; | ||||||
| float max = 0.0f; | ||||||
| if (!optimizer_utils::GetClipConstantMinMax(graph_viewer.GetGraph(), clip_node, min, max)) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| int32_t q_clip_min = static_cast<int32_t>(::rint(min / scale)) + zp; | ||||||
| int32_t q_clip_max = static_cast<int32_t>(::rint(max / scale)) + zp; | ||||||
|
|
||||||
| int32_t data_type_min = 0; | ||||||
| int32_t data_type_max = 0; | ||||||
| if (!GetDataTypeMinMax(data_type, data_type_min, data_type_max)) { | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| // The Clip can be removed if its range entirely overlaps the quantization range. | ||||||
| // QClip range: [------------------] | ||||||
| // Quant range: [-------------] | ||||||
| return q_clip_min <= data_type_min && q_clip_max >= data_type_max; | ||||||
| } | ||||||
|
|
||||||
| } // namespace | ||||||
|
|
||||||
| bool CanFuseActivationQ(const GraphViewer& graph_viewer, const Node& activation_node, const Node& q_node) { | ||||||
| const std::string& activation_op_type = activation_node.OpType(); | ||||||
|
Check warning on line 346 in onnxruntime/core/optimizer/qdq_transformer/qdq_util.cc
|
||||||
| if (activation_op_type == "Relu") { | ||||||
| return CanRemoveRelu(graph_viewer, q_node); | ||||||
| } else if (activation_op_type == "Clip") { | ||||||
| return CanRemoveClip(graph_viewer, activation_node, q_node); | ||||||
| } | ||||||
| return false; | ||||||
| } | ||||||
|
|
||||||
| } // namespace onnxruntime::QDQ | ||||||
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -15,6 +15,7 @@ namespace onnxruntime { | |
|
|
||
| class Node; | ||
| class Path; | ||
| class GraphViewer; | ||
|
|
||
| namespace QDQ { | ||
|
|
||
|
|
@@ -76,5 +77,9 @@ bool MatchQNode(const Node& node); | |
| // Check DQ node op type, version, and domain. | ||
| bool MatchDQNode(const Node& node); | ||
| #endif // !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) | ||
|
|
||
| // Check if an activation node can be fused with a Q node. | ||
| bool CanFuseActivationQ(const GraphViewer& graph_viewer, const Node& activation_node, const Node& q_node); | ||
|
|
||
| } // namespace QDQ | ||
| } // namespace onnxruntime | ||
Oops, something went wrong.
Oops, something went wrong.
Add this suggestion to a batch that can be applied as a single commit.
This suggestion is invalid because no changes were made to the code.
Suggestions cannot be applied while the pull request is closed.
Suggestions cannot be applied while viewing a subset of changes.
Only one suggestion per line can be applied in a batch.
Add this suggestion to a batch that can be applied as a single commit.
Applying suggestions on deleted lines is not supported.
You must change the existing code in this line in order to create a valid suggestion.
Outdated suggestions cannot be applied.
This suggestion has been applied or marked resolved.
Suggestions cannot be applied from pending reviews.
Suggestions cannot be applied on multi-line comments.
Suggestions cannot be applied while the pull request is queued to merge.
Suggestion cannot be applied right now. Please check back later.
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
We should have a comment explaining what the 'activation' node is and how it is expected to be used.
IIUC
If that's correct I'd almost be inclined to call it something like redundant_clip_node (given Relu is a form of Clip).
Also as the OpenVINO EP (IIRC) is doing the fallback to higher precision does it need an update to be aware of the activation node in the NodeUnit?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
How to combine values of Clip/Relu with Q? Is there a formular I can follow to adjust the scale/zp value in the Q?