From 22e9847efa2a89344193f8fdc76677efcc1acfb3 Mon Sep 17 00:00:00 2001 From: Amy Zhuang Date: Thu, 10 Oct 2019 10:05:08 -0700 Subject: [PATCH] Migrate #3736 from master. (#3745) * Use Eigen kernel for REFLECT mode Pad. * Do not call is_optimized_et. --- src/ngraph/runtime/cpu/builder/pad.cpp | 32 +++++---- src/ngraph/runtime/cpu/cpu_emitter.cpp | 40 ++++++----- src/ngraph/runtime/cpu/cpu_kernels.hpp | 3 + src/ngraph/runtime/cpu/kernel/pad.cpp | 2 + src/ngraph/runtime/cpu/kernel/pad.hpp | 95 ++++++++++++++++++++++++-- 5 files changed, 137 insertions(+), 35 deletions(-) diff --git a/src/ngraph/runtime/cpu/builder/pad.cpp b/src/ngraph/runtime/cpu/builder/pad.cpp index 98e89039542..6e3ca732c6b 100644 --- a/src/ngraph/runtime/cpu/builder/pad.cpp +++ b/src/ngraph/runtime/cpu/builder/pad.cpp @@ -50,7 +50,8 @@ namespace ngraph auto padding_above = pad->get_padding_above(); auto pad_mode = pad->get_pad_mode(); - if (pad_mode == ngraph::op::PadMode::CONSTANT) + if (pad_mode == ngraph::op::PadMode::CONSTANT || + pad_mode == ngraph::op::PadMode::REFLECT) { std::function)> kernel; @@ -65,6 +66,7 @@ namespace ngraph out_shape, padding_below, padding_above, + pad_mode, arg_buffer_index, padding_value_index, out_buffer_index](CPURuntimeContext* ctx, @@ -76,6 +78,7 @@ namespace ngraph out_shape, CoordinateDiff(padding_below.begin(), padding_below.end()), CoordinateDiff(padding_above.begin(), padding_above.end()), + pad_mode, ectx->arena); }; functors.emplace_back(functor); @@ -123,7 +126,8 @@ namespace ngraph auto padding_above = pad->get_padding_above(); auto pad_mode = pad->get_pad_mode(); - if (pad_mode == ngraph::op::PadMode::CONSTANT) + if (pad_mode == ngraph::op::PadMode::CONSTANT || + pad_mode == ngraph::op::PadMode::REFLECT) { std::function)> kernel; @@ -132,17 +136,19 @@ namespace ngraph arg_shape.size(), runtime::cpu::kernel::pad_and_slice); - auto functor = [kernel, arg_shape, out_shape, padding_below, padding_above]( - const std::vector& inputs, std::vector& outputs) { - kernel(inputs[0], - outputs[0], - inputs[1], - arg_shape, - out_shape, - CoordinateDiff(padding_below.begin(), padding_below.end()), - CoordinateDiff(padding_above.begin(), padding_above.end()), - 0); - }; + auto functor = + [kernel, arg_shape, out_shape, padding_below, padding_above, pad_mode]( + const std::vector& inputs, std::vector& outputs) { + kernel(inputs[0], + outputs[0], + inputs[1], + arg_shape, + out_shape, + CoordinateDiff(padding_below.begin(), padding_below.end()), + CoordinateDiff(padding_above.begin(), padding_above.end()), + pad_mode, + 0); + }; return functor; } else diff --git a/src/ngraph/runtime/cpu/cpu_emitter.cpp b/src/ngraph/runtime/cpu/cpu_emitter.cpp index 3f1ba15783b..969253b060b 100644 --- a/src/ngraph/runtime/cpu/cpu_emitter.cpp +++ b/src/ngraph/runtime/cpu/cpu_emitter.cpp @@ -3107,8 +3107,26 @@ namespace ngraph auto arg0_shape = args[0].get_shape(); auto result_shape = out[0].get_shape(); + std::string pad_mode_string; + switch (pad->get_pad_mode()) + { + case ngraph::op::PadMode::CONSTANT: + pad_mode_string = "ngraph::op::PadMode::CONSTANT"; + break; + case ngraph::op::PadMode::EDGE: + pad_mode_string = "ngraph::op::PadMode::EDGE"; + break; + case ngraph::op::PadMode::REFLECT: + pad_mode_string = "ngraph::op::PadMode::REFLECT"; + break; + case ngraph::op::PadMode::SYMMETRIC: + pad_mode_string = "ngraph::op::PadMode::SYMMETRIC"; + break; + } + if (arg0_shape.size() == 4 && args[0].get_element_type() == element::f32 && - pad->get_pad_mode() == ngraph::op::PadMode::CONSTANT) + (pad->get_pad_mode() == ngraph::op::PadMode::CONSTANT || + pad->get_pad_mode() == ngraph::op::PadMode::REFLECT)) { writer << "cpu::kernel::pad_4d_float32(" << args[0].get_name() << ",\n" << " " << out[0].get_name() << ",\n" @@ -3118,26 +3136,12 @@ namespace ngraph << " {" << join(pad->get_padding_below()) << "},\n" << " {" << join(pad->get_padding_above()) - << "}, 0);\n"; + << "}, \n" + << " " << pad_mode_string << ",\n" + << " 0);\n"; } else { - std::string pad_mode_string; - switch (pad->get_pad_mode()) - { - case ngraph::op::PadMode::CONSTANT: - pad_mode_string = "ngraph::op::PadMode::CONSTANT"; - break; - case ngraph::op::PadMode::EDGE: - pad_mode_string = "ngraph::op::PadMode::EDGE"; - break; - case ngraph::op::PadMode::REFLECT: - pad_mode_string = "ngraph::op::PadMode::REFLECT"; - break; - case ngraph::op::PadMode::SYMMETRIC: - pad_mode_string = "ngraph::op::PadMode::SYMMETRIC"; - break; - } writer << "reference::pad<" << out[0].get_type() << ">(" << args[0].get_name() << ",\n"; writer << " " << args[1].get_name() << ",\n"; diff --git a/src/ngraph/runtime/cpu/cpu_kernels.hpp b/src/ngraph/runtime/cpu/cpu_kernels.hpp index 83d92f9c766..ef8ab8c68e6 100644 --- a/src/ngraph/runtime/cpu/cpu_kernels.hpp +++ b/src/ngraph/runtime/cpu/cpu_kernels.hpp @@ -21,6 +21,8 @@ #include #include +#include "ngraph/op/pad.hpp" + // CBLAS types and wrappers namespace cblas @@ -146,6 +148,7 @@ namespace ngraph const Shape& output_shape, const CoordinateDiff& padding_below, const CoordinateDiff& padding_above, + const ngraph::op::PadMode pad_mode, int arena); void reduce_sum_all_1d_float32(float* input, diff --git a/src/ngraph/runtime/cpu/kernel/pad.cpp b/src/ngraph/runtime/cpu/kernel/pad.cpp index ddeeb376434..57330dd8746 100644 --- a/src/ngraph/runtime/cpu/kernel/pad.cpp +++ b/src/ngraph/runtime/cpu/kernel/pad.cpp @@ -31,6 +31,7 @@ namespace ngraph const Shape& output_shape, const CoordinateDiff& padding_below, const CoordinateDiff& padding_above, + const ngraph::op::PadMode pad_mode, int arena) { pad_and_slice(input, @@ -40,6 +41,7 @@ namespace ngraph output_shape, padding_below, padding_above, + pad_mode, arena); } } diff --git a/src/ngraph/runtime/cpu/kernel/pad.hpp b/src/ngraph/runtime/cpu/kernel/pad.hpp index 7be8b78ccd7..2895bcb039e 100644 --- a/src/ngraph/runtime/cpu/kernel/pad.hpp +++ b/src/ngraph/runtime/cpu/kernel/pad.hpp @@ -67,15 +67,19 @@ namespace ngraph const Shape& output_shape, const CoordinateDiff& padding_below, const CoordinateDiff& padding_above, + const ngraph::op::PadMode pad_mode, int arena) { - Eigen::array out_dims, in_dims; + Eigen::array out_dims, in_dims, temp_dims; Eigen::array, Rank> padding; Eigen::array indices; + bool has_negative_below_padding = false; + for (int i = 0; i < Rank; i++) { out_dims[i] = output_shape[i]; + temp_dims[i] = output_shape[i]; in_dims[i] = input_shape[i]; padding[i] = { @@ -88,6 +92,8 @@ namespace ngraph { NGRAPH_CHECK(padding_below[i] > INT_MIN); indices[i] = -padding_below[i]; + temp_dims[i] -= padding_below[i]; + has_negative_below_padding = true; } else { @@ -97,12 +103,93 @@ namespace ngraph Eigen::TensorMap> out( static_cast(output), out_dims); + Eigen::TensorMap> temp( + static_cast(output), temp_dims); Eigen::TensorMap> in( static_cast(input), in_dims); - out.device(ngraph::runtime::cpu::executor::GetCPUExecutor().get_device(arena)) = - in.pad(padding, *static_cast(pad_value)) - .slice(indices, out_dims); + if (pad_mode == ngraph::op::PadMode::CONSTANT) + { + out.device(ngraph::runtime::cpu::executor::GetCPUExecutor().get_device( + arena)) = in.pad(padding, *static_cast(pad_value)) + .slice(indices, out_dims); + } + else + { + // clang-format off + // PadMode::REFLECT + // We should have dim >= 2 for each dim. + // Example: + // + // Input shape: [4] + // Padding: 6 below, 13 above + // Output shape: [23] + // + // Input: 1 2 3 4 + // Expected output: 1 2 3 4 3 2 1 2 3 4 3 2 1 2 3 4 3 2 1 2 3 4 3 + // Pattern: ... | original n elements | middle (n - 2) elements of original n in reverse order | + // original n elements | middle (n - 2) elements of original n in reverse order | ... + // | 1 2 3 4 | 3 2 | 1 2 3 4 | 3 2 | 1 2 3 4 | 3 2 | 1 2 3 4 | 3 + // clang-format on + auto generator = + [&](const Eigen::array& out_index) { + Eigen::array in_index; + for (size_t i = 0; i < Rank; i++) + { + auto origin_length = in_dims[i]; + auto p_below = padding_below[i] >= 0 ? padding_below[i] : 0; + if (out_index[i] < p_below) + { + // padding below + auto reverse = p_below - out_index[i]; + auto res = reverse % (origin_length * 2 - 2); + if (res <= origin_length - 2) + { + // copy one of the middle n-2 items + in_index[i] = res; + } + else + { + // copy one of the n items + in_index[i] = origin_length * 2 - 2 - res; + } + } + else if (out_index[i] < in_dims[i] + p_below) + { + // original + in_index[i] = out_index[i] - p_below; + } + else + { + // padding above + auto pos = out_index[i] - in_dims[i] - p_below; + auto res = pos % (origin_length * 2 - 2); + if (res < origin_length - 2) + { + // copy one of the middle n-2 items + in_index[i] = origin_length - 2 - res; + } + else + { + // copy one of the n items + in_index[i] = res - (origin_length - 2); + } + } + } + return in(in_index); + }; + + if (has_negative_below_padding) + { + out.device(ngraph::runtime::cpu::executor::GetCPUExecutor().get_device( + arena)) = temp.generate(generator).slice(indices, out_dims); + } + else + { + out.device(ngraph::runtime::cpu::executor::GetCPUExecutor().get_device( + arena)) = out.generate(generator); + } + } } template