Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[CPU] [ARM] [INT8] FullyConnected #25171

Merged
merged 35 commits into from
Dec 18, 2024
Merged
Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
35 commits
Select commit Hold shift + click to select a range
8212473
[CPU] [ARM] FullyConnected: int8 support
eshoguli Jun 26, 2024
7dd226e
[CPU] [ACL] FullyConnected fp32 executor refactoring
eshoguli Aug 13, 2024
d4c30c6
cleanup and refactoring
eshoguli Aug 16, 2024
5dbd319
address comments #1
alvoron Nov 14, 2024
03315bf
Merge branch 'master' into es/aarch64/int8
alvoron Nov 14, 2024
e6d4880
fixed build
alvoron Nov 14, 2024
8189730
fix merge
alvoron Nov 15, 2024
c743709
removed generate_values
alvoron Nov 15, 2024
c40e695
removed MatMulWithDequantizationTransformation
alvoron Nov 18, 2024
ff886c4
revert network_helper changes
alvoron Nov 19, 2024
39fcf09
delete empty DQScales check and moved test skip
alvoron Nov 19, 2024
2e9a2a0
return empty dequantizationScales check
alvoron Nov 19, 2024
1c639ef
added test case with bias
alvoron Nov 20, 2024
5df26e4
test with bias is added to x64 scope
alvoron Nov 22, 2024
b9df131
removed FusedWithMatMulI8 and MM transformation logic
alvoron Nov 22, 2024
b5f3487
fixed test and apply the last comment
alvoron Nov 25, 2024
ceff99e
simplified isSuitableChildForFC logic
alvoron Nov 25, 2024
459519b
update aclLowpFCTypeMapping
alvoron Nov 26, 2024
3688201
revert StaticMemory changes
alvoron Nov 27, 2024
1e95bb6
Merge branch 'master' into es/aarch64/int8
alvoron Nov 27, 2024
5d8c67d
Merge branch 'master' into es/aarch64/int8
alvoron Dec 11, 2024
7a07337
changes required after #26239
alvoron Dec 12, 2024
aeca18e
Merge branch 'master' into es/aarch64/int8
alvoron Dec 12, 2024
44e04cf
fix code style and warnings
alvoron Dec 12, 2024
fddd3f4
rollback dq scales fusing
alvoron Dec 13, 2024
55e0d0d
removed DQ check
alvoron Dec 13, 2024
9226262
stop wrapping FQ with Convert
alvoron Dec 16, 2024
aa64460
Revert "stop wrapping FQ with Convert"
alvoron Dec 16, 2024
0079f5f
mark getDeQuantizedScales as OV_CPU_MAYBE_UNUSED_FUNCTION
alvoron Dec 16, 2024
5455c1b
added missed code to prepareWeightMemory
alvoron Dec 16, 2024
118cfa8
fix fuse condition
alvoron Dec 16, 2024
f782508
Merge branch 'master' into es/aarch64/int8
alvoron Dec 17, 2024
5721c63
clang-format fix
alvoron Dec 17, 2024
bf9f2f6
Merge branch 'master' into es/aarch64/int8
alvoron Dec 17, 2024
83bc0c3
fix x64 expected nodes
alvoron Dec 17, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Prev Previous commit
Next Next commit
rollback dq scales fusing
  • Loading branch information
alvoron committed Dec 13, 2024
commit fddd3f462a4a1b65f9a3e5938a481b649a488364
Original file line number Diff line number Diff line change
Expand Up @@ -64,7 +64,7 @@ inline void ConvertToCPUSpecificOpset(std::shared_ptr<ov::Model>& model, const C
config.inferencePrecision);
});

CPU_REGISTER_PASS_COMMON(manager, pass::ConvertFCToFCQuantizedLegacy);
CPU_REGISTER_PASS_X64(manager, pass::ConvertFCToFCQuantizedLegacy);
CPU_REGISTER_PASS_X64(manager, MoveFCReshapeToWeights);
CPU_REGISTER_PASS_X64(manager, ov::pass::Validate);
CPU_REGISTER_PASS_COMMON(manager, AlignMatMulInputRanks);
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
#include "snippets/op/subgraph.hpp"
#include "snippets/utils/utils.hpp"

#include "low_precision/rt_info/bias_attribute.hpp"
#include "transformations/utils/utils.hpp"
#include "transformations/utils.hpp"
#include "utils/general_utils.h"
Expand Down Expand Up @@ -73,57 +72,15 @@ bool isFullyConnected(const std::shared_ptr<const ov::Node>& node) {
ov::op::util::is_on_constant_path(out_weights);
}

bool canBePerformedAsScaleShift(const std::shared_ptr<const Node> &node, const int channelAxis) {
size_t fusingPort = 0;
size_t numNonConstInputs = 0;
ov::PartialShape dataShape;
for (size_t i = 0; i < node->get_input_size(); i++) {
const auto parent = node->get_input_node_shared_ptr(i);
if (!ov::is_type<ov::op::v0::Constant>(parent)) {
fusingPort = i;
dataShape = node->get_input_partial_shape(i);
// only one non-const parent is allowed
if (++numNonConstInputs != 1)
return false;
} else {
// every const parent must have exactly one child
const auto out = parent->outputs();
const bool has_only_child = (out.size() == 1) && (out[0].get_target_inputs().size() == 1);
if (!has_only_child)
return false;
}
}

const auto isBroadcastableToDataInput = [&]() {
for (size_t i = 0; i < node->get_input_size(); i++) {
if (i == fusingPort)
continue;
const ov::PartialShape weightShape = node->get_input_partial_shape(i);
if (!isPerTensorOrPerChannelBroadcastable(dataShape.get_max_shape(), weightShape.get_max_shape(), channelAxis, true))
return false;
}
return true;
};

// Prelu and MulAdd are still ignored
// isConvertablePowerStatic() is ignored
return (ov::is_type<ov::opset1::Add>(node) ||
ov::is_type<ov::opset1::Multiply>(node) ||
ov::is_type<ov::opset1::Subtract>(node) ||
ov::is_type<ov::opset1::Divide>(node)) &&
isBroadcastableToDataInput();
}

bool SupportsFusingWithConvolution_Simple(const std::shared_ptr<const Node> &node, const int channelAxis = DEFAULT_AXIS) {
bool SupportsFusingWithConvolution_Simple(const std::shared_ptr<const Node> &node) {
// Note: some other operations support this fusing (SoftPlus, Sqrt).
// Skip them here, when they are supported by Snippets ARM. Ticket: 141170.
return ov::is_type<ov::op::v0::Abs>(node) ||
ov::is_type<ov::op::v0::Clamp>(node) ||
ov::is_type<ov::op::v0::Elu>(node) ||
ov::is_type<ov::op::v0::Relu>(node) ||
ov::is_type<ov::op::v0::Sigmoid>(node) ||
ov::is_type<ov::op::v0::Tanh>(node) ||
canBePerformedAsScaleShift(node, channelAxis);
ov::is_type<ov::op::v0::Tanh>(node);
}
// Convolution is a special case, since it supports peculiar fusings
bool isSuitableConvolutionParent(const std::shared_ptr<const Node> &node) {
Expand Down Expand Up @@ -262,27 +219,6 @@ auto is_skipped_op(const std::shared_ptr<ov::Node>& op) -> bool {
ov::is_type<ov::op::v0::Parameter>(op) ||
ov::is_type<ov::op::v0::Result>(op);
}

bool isSuitableChildForFC(const std::shared_ptr<const Node> &node, const bool canMatMulBeExecutedInI8, int& fusingAxis) {
// Invoke SupportsFusingWithConvolution_Simple directly instead of isSuitableChildForFusingSimple to
// eliminate getNumNonConstInputs() check
if (SupportsFusingWithConvolution_Simple(node, fusingAxis)) {
size_t num_non_const_inputs = 0;
size_t num_mm_inputs = 0;
for (const auto &parent_out : node->input_values()) {
if (one_of(GetNodeFusingType(parent_out.get_node_shared_ptr()), NodeFusingType::FusedWithFC, NodeFusingType::FusedWithFCI8))
num_mm_inputs++;
else if (!ov::op::util::is_on_constant_path(parent_out))
num_non_const_inputs++;
}
return (num_non_const_inputs + num_mm_inputs == 1);
}
return false;
}

inline bool canBeMatMulExecutedInInt8(const ov::element::Type& firstType, const ov::element::Type& secondType) {
return one_of(firstType, ov::element::i8, ov::element::u8) && secondType == ov::element::i8;
}
} // namespace

bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr<ov::Model> &m) {
Expand All @@ -291,7 +227,6 @@ bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr<ov::Model> &m) {
for (auto &node : m->get_ordered_ops()) {
if (is_skipped_op(node))
continue;

if (isSuitableConvolutionParent(node)) {
// Initiate fusing chain
SetNodeFusingType(node, NodeFusingType::FusedWithConvolution);
Expand All @@ -304,10 +239,9 @@ bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr<ov::Model> &m) {
SetNodeFusingType(node, NodeFusingType::FusedWithMisc);
} else if (isSuitableMatMulParent(node)) {
const bool is_fc = isFullyConnected(node);
const bool is_i8 = canBeMatMulExecutedInInt8(node->get_input_element_type(0), node->get_input_element_type(1));
const auto out_rank = node->get_output_partial_shape(0).rank();
if (is_fc) {
SetNodeFusingType(node, is_i8 ? NodeFusingType::FusedWithFCI8 : NodeFusingType::FusedWithFC);
SetNodeFusingType(node, NodeFusingType::FusedWithFC);
channelAxis = out_rank.is_static() ? (out_rank.get_length() == 3 ? 2 : 1) : DEFAULT_AXIS;
} else {
SetNodeFusingType(node, NodeFusingType::FusedWithMatMul);
Expand All @@ -332,10 +266,6 @@ bool SnippetsMarkSkipped::run_on_model(const std::shared_ptr<ov::Model> &m) {
if (isSuitablePoolChild(node)) {
PropagateIfHasOnlyChild(node, fusingChainType);
}
} else if (one_of(fusingChainType, NodeFusingType::FusedWithFC, NodeFusingType::FusedWithFCI8)) {
const bool isExecutedInINT8 = fusingChainType == NodeFusingType::FusedWithFCI8;
if (isSuitableChildForFC(node, isExecutedInINT8, channelAxis))
PropagateIfHasOnlyChild(node, fusingChainType);
}
}
}
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,7 +34,7 @@ enum class NodeFusingType : int64_t {
NotSet,
FusedTerminator,
FusedWithConvolution, FusedWithBinaryConvolution,
FusedWithMatMul, FusedWithFC, FusedWithFCI8, FusedWithMisc};
FusedWithMatMul, FusedWithFC, FusedWithMisc};

} // namespace intel_cpu
} // namespace ov
Original file line number Diff line number Diff line change
Expand Up @@ -82,7 +82,7 @@ const std::vector<FullyConnectedParams> activations = {
false, // per-channel
true, // FQ
false, // bias
"fullyConnected"
"fullyconnected_original"
},
{
true, // activation
Expand All @@ -96,7 +96,7 @@ const std::vector<FullyConnectedParams> activations = {
false, // per-channel
true, // FQ
true, // bias
"fullyConnected/DequantizationMultiply"
"fullyConnected"
},
};

Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ const std::vector<FullyConnectedParams> activations = {
false, // per-channel
true, // FQ
false, // bias
"fullyConnected"
"fullyconnected_original"
},
{
true, // activation
Expand All @@ -66,7 +66,7 @@ const std::vector<FullyConnectedParams> activations = {
false, // per-channel
true, // FQ
true, // bias
"fullyConnected/DequantizationMultiply"
"fullyConnected"
},
};

Expand Down
Loading