diff options
Diffstat (limited to 'Drivers/CMSIS/NN/Source')
32 files changed, 7639 insertions, 7515 deletions
diff --git a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q15.c b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q15.c index cb8a08f..fd447e5 100644 --- a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q15.c +++ b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q15.c @@ -1,96 +1,101 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_activations_q15.c - * Description: Q15 neural network activation function using direct table look-up - * - * $Date: 09. October 2020 - * $Revision: V.1.0.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nn_tables.h" -#include "arm_nnfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Acti - * @{ - */ - -/** - * @brief neural network activation function using direct table look-up - * - * @note Refer header file for details. - * - */ - -void arm_nn_activations_direct_q15(q15_t *data, uint16_t size, uint16_t int_width, arm_nn_activation_type type) -{ - uint16_t i = size; - q15_t *pIn = data; - q15_t *pOut = data; - uint16_t shift_size = 8 + 3 - int_width; - uint32_t bit_mask = 0x7FF >> int_width; - uint32_t full_frac = bit_mask + 1; - const q15_t *lookup_table; - - switch (type) - { - case ARM_SIGMOID: - lookup_table = sigmoidTable_q15; - break; - case ARM_TANH: - default: - lookup_table = tanhTable_q15; - break; - } - - while (i) - { - q15_t out; - q15_t in = *pIn++; - q15_t frac = (uint32_t)in & bit_mask; - q15_t value = lookup_table[(uint8_t)(in >> shift_size)]; - if ((in >> shift_size) != 0x7f) - { - q15_t value2 = lookup_table[(uint8_t)(1 + ((uint8_t)(in >> shift_size)))]; - /* doing the interpolation here for better accuracy */ - out = ((q31_t)(full_frac - frac) * value + (q31_t)value2 * frac) >> shift_size; - } - else - { - /* the largest positive value does not have a right side for linear interpolation */ - out = value; - } - - *pOut++ = out; - i--; - } -} - -/** - * @} end of Acti group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_activations_q15.c
+ * Description: Q15 neural network activation function using direct table look-up
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_common_tables.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Acti
+ * @{
+ */
+
+ /**
+ * @brief Q15 neural network activation function using direct table look-up
+ * @param[in,out] data pointer to input
+ * @param[in] size number of elements
+ * @param[in] int_width bit-width of the integer part, assume to be smaller than 3
+ * @param[in] type type of activation functions
+ * @return none.
+ *
+ * @details
+ *
+ * This is the direct table look-up approach.
+ *
+ * Assume here the integer part of the fixed-point is <= 3.
+ * More than 3 just not making much sense, makes no difference with
+ * saturation followed by any of these activation functions.
+ */
+
+void arm_nn_activations_direct_q15(q15_t * data, uint16_t size, uint16_t int_width, arm_nn_activation_type type)
+{
+ uint16_t i = size;
+ q15_t *pIn = data;
+ q15_t *pOut = data;
+ uint16_t shift_size = 8 + 3 - int_width;
+ uint32_t bit_mask = 0x7FF >> int_width;
+ uint32_t full_frac = bit_mask + 1;
+ const q15_t *lookup_table;
+
+ switch (type)
+ {
+ case ARM_SIGMOID:
+ lookup_table = sigmoidTable_q15;
+ break;
+ case ARM_TANH:
+ default:
+ lookup_table = tanhTable_q15;
+ break;
+ }
+
+ while (i)
+ {
+ q15_t out;
+ q15_t in = *pIn++;
+ q15_t frac = (uint32_t) in & bit_mask;
+ q15_t value = lookup_table[__USAT(in >> shift_size, 8)];
+ q15_t value2 = lookup_table[__USAT(1 + (in >> shift_size), 8)];
+
+ /* doing the interpolation here for better accuracy */
+ out = ((q31_t) (full_frac - frac) * value + (q31_t) value2 * frac) >> shift_size;
+
+ *pOut++ = out;
+ i--;
+ }
+
+}
+
+/**
+ * @} end of Acti group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q7.c b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q7.c index 72a0b15..2953bd5 100644 --- a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q7.c +++ b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_nn_activations_q7.c @@ -1,89 +1,91 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_activations_q7.c - * Description: Q7 neural network activation function using direct table look-up - * - * $Date: 09. October 2020 - * $Revision: V.1.0.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nn_tables.h" -#include "arm_nnfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Acti - * @{ - */ - -/** - * @brief Q7 neural network activation function using direct table look-up - * @param[in,out] data pointer to input - * @param[in] size number of elements - * @param[in] int_width bit-width of the integer part, assume to be smaller than 3 - * @param[in] type type of activation functions - * - * @details - * - * This is the direct table look-up approach. - * - * Assume here the integer part of the fixed-point is <= 3. - * More than 3 just not making much sense, makes no difference with - * saturation followed by any of these activation functions. - */ - -void arm_nn_activations_direct_q7(q7_t *data, uint16_t size, uint16_t int_width, arm_nn_activation_type type) -{ - uint16_t i = size; - q7_t *pIn = data; - q7_t *pOut = data; - q7_t in; - q7_t out; - uint16_t shift_size = 3 - int_width; - const q7_t *lookup_table; - switch (type) - { - case ARM_SIGMOID: - lookup_table = sigmoidTable_q7; - break; - case ARM_TANH: - default: - lookup_table = tanhTable_q7; - break; - } - while (i) - { - in = *pIn++; - out = lookup_table[(uint8_t)(in >> shift_size)]; - *pOut++ = out; - i--; - } -} - -/** - * @} end of Acti group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_activations_q7.c
+ * Description: Q7 neural network activation function using direct table look-up
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_common_tables.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Acti
+ * @{
+ */
+
+ /**
+ * @brief Q7 neural network activation function using direct table look-up
+ * @param[in,out] data pointer to input
+ * @param[in] size number of elements
+ * @param[in] int_width bit-width of the integer part, assume to be smaller than 3
+ * @param[in] type type of activation functions
+ * @return none.
+ *
+ * @details
+ *
+ * This is the direct table look-up approach.
+ *
+ * Assume here the integer part of the fixed-point is <= 3.
+ * More than 3 just not making much sense, makes no difference with
+ * saturation followed by any of these activation functions.
+ */
+
+void arm_nn_activations_direct_q7(q7_t * data, uint16_t size, uint16_t int_width, arm_nn_activation_type type)
+{
+ uint16_t i = size;
+ q7_t *pIn = data;
+ q7_t *pOut = data;
+ q7_t in;
+ q7_t out;
+ uint16_t shift_size = 3 - int_width;
+ const q7_t *lookup_table;
+ switch (type)
+ {
+ case ARM_SIGMOID:
+ lookup_table = sigmoidTable_q7;
+ break;
+ case ARM_TANH:
+ default:
+ lookup_table = tanhTable_q7;
+ break;
+ }
+ while (i)
+ {
+ in = *pIn++;
+ out = lookup_table[(uint8_t) (in >> shift_size)];
+ *pOut++ = out;
+ i--;
+ }
+}
+
+/**
+ * @} end of Acti group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q15.c b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q15.c index 1d4ea4e..6a1b907 100644 --- a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q15.c +++ b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q15.c @@ -1,104 +1,106 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_relu_q15.c - * Description: Q15 version of ReLU - * - * $Date: 20. July 2021 - * $Revision: V.1.0.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Acti - * @{ - */ - -/** - * @brief Q15 RELU function - * @param[in,out] data pointer to input - * @param[in] size number of elements - * - * @details - * - * Optimized relu with QSUB instructions. - * - */ - -void arm_relu_q15(q15_t *data, uint16_t size) -{ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for M cores with DSP extension */ - - uint16_t i = size >> 1; - q15_t *input = data; - q15_t *output = data; - q31_t in; - q31_t buf; - q31_t mask; - - while (i) - { - in = arm_nn_read_q15x2_ia((const q15_t **)&input); - - /* extract the first bit */ - buf = __ROR(in & 0x80008000, 15); - - /* if MSB=1, mask will be 0xFF, 0x0 otherwise */ - mask = __QSUB16(0x00000000, buf); - - arm_nn_write_q15x2_ia(&output, in & (~mask)); - i--; - } - - if (size & 0x1) - { - if (*input < 0) - { - *input = 0; - } - input++; - } -#else - /* Run the following code as reference implementation for M cores without DSP extension */ - uint16_t i; - - for (i = 0; i < size; i++) - { - if (data[i] < 0) - data[i] = 0; - } - -#endif /* ARM_MATH_DSP */ -} - -/** - * @} end of Acti group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_relu_q15.c
+ * Description: Q15 version of ReLU
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Acti
+ * @{
+ */
+
+ /**
+ * @brief Q15 RELU function
+ * @param[in,out] data pointer to input
+ * @param[in] size number of elements
+ * @return none.
+ *
+ * @details
+ *
+ * Optimized relu with QSUB instructions.
+ *
+ */
+
+void arm_relu_q15(q15_t * data, uint16_t size)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ uint16_t i = size >> 1;
+ q15_t *pIn = data;
+ q15_t *pOut = data;
+ q31_t in;
+ q31_t buf;
+ q31_t mask;
+
+ while (i)
+ {
+ in = *__SIMD32(pIn)++;
+
+ /* extract the first bit */
+ buf = __ROR(in & 0x80008000, 15);
+
+ /* if MSB=1, mask will be 0xFF, 0x0 otherwise */
+ mask = __QSUB16(0x00000000, buf);
+
+ *__SIMD32(pOut)++ = in & (~mask);
+ i--;
+ }
+
+ if (size & 0x1)
+ {
+ if (*pIn < 0)
+ {
+ *pIn = 0;
+ }
+ pIn++;
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i;
+
+ for (i = 0; i < size; i++)
+ {
+ if (data[i] < 0)
+ data[i] = 0;
+ }
+
+#endif /* ARM_MATH_DSP */
+
+}
+
+/**
+ * @} end of Acti group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q7.c b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q7.c index a3163cd..caa027b 100644 --- a/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q7.c +++ b/Drivers/CMSIS/NN/Source/ActivationFunctions/arm_relu_q7.c @@ -1,109 +1,110 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_relu_q7.c - * Description: Q7 version of ReLU - * - * $Date: 20. July 2021 - * $Revision: V.1.1.3 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Acti - * @{ - */ - -/** - * @brief Q7 RELU function - * @param[in,out] data pointer to input - * @param[in] size number of elements - * - * @details - * - * Optimized relu with QSUB instructions. - * - */ - -void arm_relu_q7(q7_t *data, uint16_t size) -{ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for M cores with DSP extension */ - - uint16_t i = size >> 2; - q7_t *input = data; - q7_t *output = data; - q31_t in; - q31_t buf; - q31_t mask; - - while (i) - { - in = arm_nn_read_q7x4_ia((const q7_t **)&input); - - /* extract the first bit */ - buf = (int32_t)__ROR((uint32_t)in & 0x80808080, 7); - - /* if MSB=1, mask will be 0xFF, 0x0 otherwise */ - mask = __QSUB8(0x00000000, buf); - - arm_nn_write_q7x4_ia(&output, in & (~mask)); - - i--; - } - - i = size & 0x3; - while (i) - { - if (*input < 0) - { - *input = 0; - } - input++; - i--; - } - -#else - /* Run the following code as reference implementation for cores without DSP extension */ - - uint16_t i; - - for (i = 0; i < size; i++) - { - if (data[i] < 0) - data[i] = 0; - } - -#endif -} - -/** - * @} end of Acti group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_relu_q7.c
+ * Description: Q7 version of ReLU
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Acti
+ * @{
+ */
+
+ /**
+ * @brief Q7 RELU function
+ * @param[in,out] data pointer to input
+ * @param[in] size number of elements
+ * @return none.
+ *
+ * @details
+ *
+ * Optimized relu with QSUB instructions.
+ *
+ */
+
+void arm_relu_q7(q7_t * data, uint16_t size)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ uint16_t i = size >> 2;
+ q7_t *pIn = data;
+ q7_t *pOut = data;
+ q31_t in;
+ q31_t buf;
+ q31_t mask;
+
+ while (i)
+ {
+ in = *__SIMD32(pIn)++;
+
+ /* extract the first bit */
+ buf = __ROR(in & 0x80808080, 7);
+
+ /* if MSB=1, mask will be 0xFF, 0x0 otherwise */
+ mask = __QSUB8(0x00000000, buf);
+
+ *__SIMD32(pOut)++ = in & (~mask);
+ i--;
+ }
+
+ i = size & 0x3;
+ while (i)
+ {
+ if (*pIn < 0)
+ {
+ *pIn = 0;
+ }
+ pIn++;
+ i--;
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i;
+
+ for (i = 0; i < size; i++)
+ {
+ if (data[i] < 0)
+ data[i] = 0;
+ }
+
+#endif /* ARM_MATH_DSP */
+
+}
+
+/**
+ * @} end of Acti group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c index 3db3ba4..4c69e7c 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c @@ -1,235 +1,235 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_1x1_HWC_q7_fast_nonsquare.c - * Description: Fast Q7 version of 1x1 convolution (non-square shape) - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Fast Q7 version of 1x1 convolution (non-sqaure shape) - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in_x input tensor dimention x - * @param[in] dim_im_in_y input tensor dimention y - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel_x filter kernel size x - * @param[in] dim_kernel_y filter kernel size y - * @param[in] padding_x padding size x - * @param[in] padding_y padding size y - * @param[in] stride_x convolution stride x - * @param[in] stride_y convolution stride y - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out_x output tensor dimension x - * @param[in] dim_im_out_y output tensor dimension y - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * This function is optimized for convolution with 1x1 kernel size (i.e., dim_kernel_x=1 - * and dim_kernel_y=1). It can be used for the second half of MobileNets [1] after depthwise - * separable convolution. - * - * This function is the version with full list of optimization tricks, but with - * some constraints: - * ch_im_in is multiple of 4 - * ch_im_out is multiple of 2 - * - * [1] MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications - * https://arxiv.org/abs/1704.04861 - */ - -arm_status arm_convolve_1x1_HWC_q7_fast_nonsquare(const q7_t *Im_in, - const uint16_t dim_im_in_x, - const uint16_t dim_im_in_y, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel_x, - const uint16_t dim_kernel_y, - const uint16_t padding_x, - const uint16_t padding_y, - const uint16_t stride_x, - const uint16_t stride_y, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out_x, - const uint16_t dim_im_out_y, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - (void)dim_im_in_y; - int16_t i_out_y, i_out_x; - int16_t i_ch_out; - - /* ----------------------- - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1 || padding_x != 0 || - padding_y != 0 || stride_x != 1 || stride_y != 1) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - /* This part implements the im2col function */ - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_out_y * dim_im_in_x + i_out_x) * ch_im_in, pBuffer, ch_im_in); - pBuffer += ch_im_in; - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* check if there is left-over for compute */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) - { - q31_t sum = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); - const q15_t *pB = bufferA; - /* basically each time it process 4 entries */ - uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2; - - while (colCnt) - { - - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad_reordered(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia(&pB); - - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut = (q7_t)__SSAT((sum >> out_shift), 8); - pOut++; - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1 || padding_x != 0 || - padding_y != 0 || stride_x != 1 || stride_y != 1) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out_y; j++) - { - for (k = 0; k < dim_im_out_x; k++) - { - conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel_y; m++) - { - for (n = 0; n < dim_kernel_x; n++) - { - // if-for implementation - in_row = stride_y * j + m - padding_y; - in_col = stride_x * k + n - padding_x; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_y + n) * ch_im_in + - l]; - } - } - } - } - Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_1x1_HWC_q7_fast_nonsquare.c
+ * Description: Fast Q7 version of 1x1 convolution (non-square shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Fast Q7 version of 1x1 convolution (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is optimized for convolution with 1x1 kernel size (i.e., dim_kernel_x=1
+ * and dim_kernel_y=1). It can be used for the second half of MobileNets [1] after depthwise
+ * separable convolution.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 4
+ * ch_im_out is multiple of 2
+ *
+ * [1] MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications
+ * https://arxiv.org/abs/1704.04861
+ */
+
+arm_status arm_convolve_1x1_HWC_q7_fast_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ch_out;
+
+ /* -----------------------
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
+ || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_out_y * dim_im_in_x + i_out_x) * ch_im_in, pBuffer,
+ ch_im_in);
+ pBuffer += ch_im_in;
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ q31_t sum = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ int i, j, k, l, m, n;
+ int conv_out;
+ int in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
+ || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ // if-for implementation
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_y + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c index 0a6868a..ee08d74 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c @@ -1,209 +1,207 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q15_basic.c - * Description: Q15 version of convolution - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Basic Q15 convolution function - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * This basic version is designed to work for any input tensor and weight - * dimension. - */ - -arm_status arm_convolve_HWC_q15_basic(const q15_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q15_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q15_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q15_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - uint16_t im2col_out_pixel_index = 0; - q15_t *pBuffer = bufferA; - q15_t *pOut = Im_out; - q15_t *im_buffer = bufferA; - const q15_t *pA; - int i; - - /* This part implements the im2col function */ - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* Filling 0 for out-of-bound paddings */ - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, - * ch_im_in); */ - memcpy(pBuffer, - (q15_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, - sizeof(q15_t) * ch_im_in); - } - pBuffer += ch_im_in; - } - } - - pA = wt; - for (i = 0; i < ch_im_out; i++) - { - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - const q15_t *pB = im_buffer; - uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; - while (colCnt) - { - q31_t inA1 = arm_nn_read_q15x2_ia(&pA); - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inA2 = arm_nn_read_q15x2_ia(&pA); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB); - - sum = __SMLAD(inA1, inB1, sum); - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; - while (colCnt) - { - q15_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut = (q15_t)__SSAT((sum >> out_shift), 16); - pOut++; - } - - /* counter reset */ - pBuffer = im_buffer; - im2col_out_pixel_index++; - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out; j++) - { - for (k = 0; k < dim_im_out; k++) - { - conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel; m++) - { - for (n = 0; n < dim_kernel; n++) - { - in_row = stride * j + m - padding; - in_col = stride * k + n - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + n) * ch_im_in + l]; - } - } - } - } - Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t)__SSAT((conv_out >> out_shift), 16); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_basic.c
+ * Description: Q15 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q15 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * This basic version is designed to work for any input tensor and weight
+ * dimension.
+ */
+
+arm_status
+arm_convolve_HWC_q15_basic(const q15_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ uint16_t im2col_out_pixel_index = 0;
+ q15_t *pBuffer = bufferA;
+ q15_t *pOut = Im_out;
+ q15_t *im_buffer = bufferA;
+ const q15_t *pA;
+ int i;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ pA = wt;
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = im_buffer;
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA)++;
+ q31_t inB2 = *__SIMD32(pB)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q15_t) __SSAT((sum >> out_shift), 16);
+ pOut++;
+ }
+
+ /* counter reset */
+ pBuffer = im_buffer;
+ im2col_out_pixel_index++;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c index 66fbc00..a02aaa0 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c @@ -1,259 +1,255 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q15_fast.c - * Description: Fast Q15 version of convolution - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Fast Q15 convolution function - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * <b>Input dimension constraints:</b> - * - * ch_im_in is multiple of 2 - * - * ch_im_out is multiple of 2 - * - * dim_im_out is a multiple of 2 - * - */ - -arm_status arm_convolve_HWC_q15_fast(const q15_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q15_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q15_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q15_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - q15_t *pBuffer = bufferA; - q15_t *im_buffer = bufferA; - q15_t *pOut = Im_out; - - if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0 || dim_im_out & 0x1) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - /* This part implements the im2col function */ - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, - * ch_im_in); */ - memcpy(pBuffer, - (q15_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, - sizeof(q15_t) * ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (i_out_x & 0x1) - { - int i; - /* initialize the matrix pointers for A */ - const q15_t *pA = wt; - - /* set up the second output pointers */ - q15_t *pOut2 = pOut + ch_im_out; - - /* this loop over rows in A */ - for (i = 0; i < ch_im_out; i += 2) - { - /* setup pointers for B */ - const q15_t *pB = im_buffer; - const q15_t *pB2 = pB + ch_im_in * dim_kernel * dim_kernel; - - /* aling the second pointer for A */ - const q15_t *pA2 = pA + ch_im_in * dim_kernel * dim_kernel; - - /* init the sum with bias */ - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 1; - /* accumulate over the vector */ - while (colCnt) - { - q31_t inA1 = arm_nn_read_q15x2_ia(&pA); - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inA2 = arm_nn_read_q15x2_ia(&pA2); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB2); - - sum = __SMLAD(inA1, inB1, sum); - sum2 = __SMLAD(inA1, inB2, sum2); - sum3 = __SMLAD(inA2, inB1, sum3); - sum4 = __SMLAD(inA2, inB2, sum4); - - colCnt--; - } /* while over colCnt */ - colCnt = ch_im_in * dim_kernel * dim_kernel & 0x1; - while (colCnt) - { - q15_t inA1 = *pA++; - q15_t inB1 = *pB++; - q15_t inA2 = *pA2++; - q15_t inB2 = *pB2++; - - sum += inA1 * inB1; - sum2 += inA1 * inB2; - sum3 += inA2 * inB1; - sum4 += inA2 * inB2; - colCnt--; - } /* while over colCnt */ - *pOut++ = (q15_t)__SSAT(sum >> out_shift, 16); - *pOut++ = (q15_t)__SSAT(sum3 >> out_shift, 16); - *pOut2++ = (q15_t)__SSAT(sum2 >> out_shift, 16); - *pOut2++ = (q15_t)__SSAT(sum4 >> out_shift, 16); - - /* skip the row computed with A2 */ - pA += ch_im_in * dim_kernel * dim_kernel; - } /* for over ch_im_out */ - - pOut += ch_im_out; - /* counter reset */ - pBuffer = im_buffer; - } - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out; j++) - { - for (k = 0; k < dim_im_out; k++) - { - conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel; m++) - { - for (n = 0; n < dim_kernel; n++) - { - in_row = stride * j + m - padding; - in_col = stride * k + n - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + n) * ch_im_in + l]; - } - } - } - } - Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t)__SSAT((conv_out >> out_shift), 16); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_fast.c
+ * Description: Fast Q15 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q15 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 2
+ *
+ * ch_im_out is multipe of 2
+ *
+ */
+
+arm_status
+arm_convolve_HWC_q15_fast(const q15_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ q15_t *pBuffer = bufferA;
+ q15_t *im_buffer = bufferA;
+ q15_t *pOut = Im_out;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (i_out_x & 0x1)
+ {
+ int i;
+ /* initialize the matrix pointers for A */
+ const q15_t *pA = wt;
+
+ /* set up the second output pointers */
+ q15_t *pOut2 = pOut + ch_im_out;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ q15_t *pB = im_buffer;
+ const q15_t *pB2 = pB + ch_im_in * dim_kernel * dim_kernel;
+
+ /* aling the second pointer for A */
+ const q15_t *pA2 = pA + ch_im_in * dim_kernel * dim_kernel;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 1;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA2)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum2 = __SMLAD(inA1, inB2, sum2);
+ sum3 = __SMLAD(inA2, inB1, sum3);
+ sum4 = __SMLAD(inA2, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x1;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
+ *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
+
+ /* skip the row computed with A2 */
+ pA += ch_im_in * dim_kernel * dim_kernel;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+ /* counter reset */
+ pBuffer = im_buffer;
+ }
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c index 7babe51..14d9130 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c @@ -1,270 +1,265 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q15_fast.c - * Description: Fast Q15 version of convolution - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Fast Q15 convolution function (non-sqaure shape) - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in_x input tensor dimention x - * @param[in] dim_im_in_y input tensor dimention y - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel_x filter kernel size x - * @param[in] dim_kernel_y filter kernel size y - * @param[in] padding_x padding size x - * @param[in] padding_y padding size y - * @param[in] stride_x convolution stride x - * @param[in] stride_y convolution stride y - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out_x output tensor dimension x - * @param[in] dim_im_out_y output tensor dimension y - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * <b>Input dimension constraints:</b> - * - * ch_im_in is multiple of 2 - * - * ch_im_out is multiple of 2 - * - */ - -arm_status arm_convolve_HWC_q15_fast_nonsquare(const q15_t *Im_in, - const uint16_t dim_im_in_x, - const uint16_t dim_im_in_y, - const uint16_t ch_im_in, - const q15_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel_x, - const uint16_t dim_kernel_y, - const uint16_t padding_x, - const uint16_t padding_y, - const uint16_t stride_x, - const uint16_t stride_y, - const q15_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q15_t *Im_out, - const uint16_t dim_im_out_x, - const uint16_t dim_im_out_y, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - q15_t *pBuffer = bufferA; - q15_t *im_buffer = bufferA; - q15_t *pOut = Im_out; - - if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - /* This part implements the im2col function */ - for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, - * ch_im_in); */ - memcpy(pBuffer, - (q15_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, - sizeof(q15_t) * ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (i_out_x & 0x1) - { - int i; - /* initialize the matrix pointers for A */ - const q15_t *pA = wt; - - /* set up the second output pointers */ - q15_t *pOut2 = pOut + ch_im_out; - - /* this loop over rows in A */ - for (i = 0; i < ch_im_out; i += 2) - { - /* setup pointers for B */ - const q15_t *pB = im_buffer; - const q15_t *pB2 = pB + ch_im_in * dim_kernel_y * dim_kernel_x; - - /* aling the second pointer for A */ - const q15_t *pA2 = pA + ch_im_in * dim_kernel_y * dim_kernel_x; - - /* init the sum with bias */ - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 1; - /* accumulate over the vector */ - while (colCnt) - { - q31_t inA1 = arm_nn_read_q15x2_ia(&pA); - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inA2 = arm_nn_read_q15x2_ia(&pA2); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB2); - - sum = __SMLAD(inA1, inB1, sum); - sum2 = __SMLAD(inA1, inB2, sum2); - sum3 = __SMLAD(inA2, inB1, sum3); - sum4 = __SMLAD(inA2, inB2, sum4); - - colCnt--; - } /* while over colCnt */ - colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x1; - while (colCnt) - { - q15_t inA1 = *pA++; - q15_t inB1 = *pB++; - q15_t inA2 = *pA2++; - q15_t inB2 = *pB2++; - - sum += inA1 * inB1; - sum2 += inA1 * inB2; - sum3 += inA2 * inB1; - sum4 += inA2 * inB2; - colCnt--; - } /* while over colCnt */ - *pOut++ = (q15_t)__SSAT(sum >> out_shift, 16); - *pOut++ = (q15_t)__SSAT(sum3 >> out_shift, 16); - *pOut2++ = (q15_t)__SSAT(sum2 >> out_shift, 16); - *pOut2++ = (q15_t)__SSAT(sum4 >> out_shift, 16); - - /* skip the row computed with A2 */ - pA += ch_im_in * dim_kernel_y * dim_kernel_x; - } /* for over ch_im_out */ - - pOut += ch_im_out; - /* counter reset */ - pBuffer = im_buffer; - } - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out_y; j++) - { - for (k = 0; k < dim_im_out_x; k++) - { - conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel_y; m++) - { - for (n = 0; n < dim_kernel_x; n++) - { - in_row = stride_y * j + m - padding_y; - in_col = stride_x * k + n - padding_x; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel_x * dim_kernel_y + (m * dim_kernel_x + n) * ch_im_in + - l]; - } - } - } - } - Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q15_t)__SSAT((conv_out >> out_shift), 16); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_fast.c
+ * Description: Fast Q15 version of convolution
+ *
+ * $Date: 24. May 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q15 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 2
+ *
+ * ch_im_out is multipe of 2
+ *
+ */
+
+arm_status
+arm_convolve_HWC_q15_fast_nonsquare(const q15_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ q15_t *pBuffer = bufferA;
+ q15_t *im_buffer = bufferA;
+ q15_t *pOut = Im_out;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (i_out_x & 0x1)
+ {
+ int i;
+ /* initialize the matrix pointers for A */
+ const q15_t *pA = wt;
+
+ /* set up the second output pointers */
+ q15_t *pOut2 = pOut + ch_im_out;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ q15_t *pB = im_buffer;
+ const q15_t *pB2 = pB + ch_im_in * dim_kernel_y * dim_kernel_x;
+
+ /* aling the second pointer for A */
+ const q15_t *pA2 = pA + ch_im_in * dim_kernel_y * dim_kernel_x;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 1;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA2)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum2 = __SMLAD(inA1, inB2, sum2);
+ sum3 = __SMLAD(inA2, inB1, sum3);
+ sum4 = __SMLAD(inA2, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x1;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
+ *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
+
+ /* skip the row computed with A2 */
+ pA += ch_im_in * dim_kernel_y * dim_kernel_x;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+ /* counter reset */
+ pBuffer = im_buffer;
+ }
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel_x * dim_kernel_y + (m * dim_kernel_x +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c index 618f492..e53c6f9 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c @@ -1,280 +1,279 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q7_RGB.c - * Description: Q7 version of convolution for RGB image - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Q7 convolution function for RGB image - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * <b>Input dimension constraints:</b> - * - * ch_im_in equals 3 - * - * This kernel is written exclusively for convolution with ch_im_in - * equals 3. This applies on the first layer of CNNs which has input - * image with RGB format. - */ - -arm_status arm_convolve_HWC_q7_RGB(const q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - /* - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - // check if number of input channels is 3 - if (ch_im_in != 3) - { - return ARM_MATH_SIZE_MISMATCH; - } - // This part implements the im2col function - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* Equivalent to arm_fill_q15(0, pBuffer, ch_im_in) with assumption: ch_im_in = 3 */ - arm_memset_q7((q7_t *)pBuffer, (q7_t)0, 3 * sizeof(q15_t)); - pBuffer += 3; - } - else - { - /* - * Equivalent to: - * arm_q7_to_q15_no_shift( (q7_t*)Im_in+(i_ker_y*dim_im_in+i_ker_x)*3, pBuffer, 3); - */ - - const q7_t *pPixel = Im_in + (i_ker_y * dim_im_in + i_ker_x) * 3; - q31_t buf = arm_nn_read_q7x4(pPixel); - - union arm_nnword top; - union arm_nnword bottom; - - top.word = __SXTB16(buf); - bottom.word = __SXTB16(__ROR(buf, 8)); - -#ifndef ARM_MATH_BIG_ENDIAN - /* - * little-endian, | omit | 3rd | 2nd | 1st | - * MSB LSB - * top | 3rd | 1st |; bottom | omit | 2nd | - * - * version 1, need to swap 2nd and 3rd weight - * *__SIMD32(pBuffer) = top.word; - * *(pBuffer+2) = bottom.half_words[0]; - * - * version 2, no weight shuffling required - */ - *pBuffer++ = top.half_words[0]; - int32_t packed_word = __PKHBT(bottom.word, top.word, 0); - arm_memcpy_q7((q7_t *)pBuffer, (q7_t *)&packed_word, 4); -#else - /* - * big-endian, | 1st | 2nd | 3rd | omit | - * MSB LSB - * top | 2nd | omit |; bottom | 1st | 3rd | - * - * version 1, need to swap 2nd and 3rd weight - * *__SIMD32(pBuffer) = bottom.word; - * *(pBuffer+2) = top.half_words[1]; - * - * version 2, no weight shuffling required - */ - *pBuffer++ = bottom.half_words[0]; - int32_t packed_word = __PKHTB(top.word, bottom.word, 0); - arm_memcpy_q7((q7_t *)pBuffer, (q7_t *)&packed_word, 4); -#endif - pBuffer += 2; - } - } - } - - if (pBuffer == bufferA + 2 * 3 * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15( - wt, bufferA, ch_im_out, 3 * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* left-over because odd number of output pixels */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - int i; - - for (i = 0; i < ch_im_out; i++) - { - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - q15_t *pB = bufferA; - /* basically each time it process 4 entries */ - uint16_t colCnt = 3 * dim_kernel * dim_kernel >> 2; - - while (colCnt) - { - - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia((const q15_t **)&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia((const q15_t **)&pB); - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = 3 * dim_kernel * dim_kernel & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - } - } -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - // check if number of input channels is 3 - if (ch_im_in != 3) - { - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out; j++) - { - for (k = 0; k < dim_im_out; k++) - { - conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel; m++) - { - for (n = 0; n < dim_kernel; n++) - { - /* if-for implementation */ - in_row = stride * j + m - padding; - in_col = stride * k + n - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + n) * ch_im_in + l]; - } - } - } - } - Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_RGB.c
+ * Description: Q7 version of convolution for RGB image
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Q7 convolution function for RGB image
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in equals 3
+ *
+ * This kernel is written exclusively for convolution with ch_im_in
+ * equals 3. This applies on the first layer of CNNs which has input
+ * image with RGB format.
+ */
+
+arm_status
+arm_convolve_HWC_q7_RGB(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out, const uint16_t dim_im_out, q15_t * bufferA, q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ // check if number of input channels is 3
+ if (ch_im_in != 3)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+ // This part implements the im2col function
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Equivalent to arm_fill_q15(0, pBuffer, ch_im_in) with assumption: ch_im_in = 3 */
+ *__SIMD32(pBuffer) = 0x0;
+ *(pBuffer + 2) = 0;
+ pBuffer += 3;
+ } else
+ {
+ /*
+ * Equivalent to:
+ * arm_q7_to_q15_no_shift( (q7_t*)Im_in+(i_ker_y*dim_im_in+i_ker_x)*3, pBuffer, 3);
+ */
+
+ const q7_t *pPixel = Im_in + (i_ker_y * dim_im_in + i_ker_x) * 3;
+ q31_t buf = *__SIMD32(pPixel);
+
+ union arm_nnword top;
+ union arm_nnword bottom;
+
+ top.word = __SXTB16(buf);
+ bottom.word = __SXTB16(__ROR(buf, 8));
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ /*
+ * little-endian, | omit | 3rd | 2nd | 1st |
+ * MSB LSB
+ * top | 3rd | 1st |; bottom | omit | 2nd |
+ *
+ * version 1, need to swap 2nd and 3rd weight
+ * *__SIMD32(pBuffer) = top.word;
+ * *(pBuffer+2) = bottom.half_words[0];
+ *
+ * version 2, no weight shuffling required
+ */
+ *pBuffer++ = top.half_words[0];
+ *__SIMD32(pBuffer) = __PKHBT(bottom.word, top.word, 0);
+#else
+ /*
+ * big-endian, | 1st | 2nd | 3rd | omit |
+ * MSB LSB
+ * top | 2nd | omit |; bottom | 1st | 3rd |
+ *
+ * version 1, need to swap 2nd and 3rd weight
+ * *__SIMD32(pBuffer) = bottom.word;
+ * *(pBuffer+2) = top.half_words[1];
+ *
+ * version 2, no weight shuffling required
+ */
+ *pBuffer++ = bottom.half_words[0];
+ *__SIMD32(pBuffer) = __PKHTB(top.word, bottom.word, 0);
+#endif
+ pBuffer += 2;
+ }
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * 3 * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ 3 * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = 3 * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = 3 * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ // check if number of input channels is 3
+ if (ch_im_in != 3)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ /* if-for implementation */
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return (ARM_MATH_SUCCESS);
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c index e274413..7c9ec65 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c @@ -1,227 +1,230 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q7_basic.c - * Description: Q7 version of convolution - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Basic Q7 convolution function - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * This basic version is designed to work for any input tensor and weight - * dimension. - */ - -arm_status arm_convolve_HWC_q7_basic(const q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - /* - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - /* This part implements the im2col function */ - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* Filling 0 for out-of-bound paddings */ - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - /* Copying the pixel data to column */ - arm_q7_to_q15_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - /* Computation is filed for every 2 columns */ - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* left-over because odd number of output pixels */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - int i; - - for (i = 0; i < ch_im_out; i++) - { - /* Load the accumulator with bias first */ - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - - /* Point to the beging of the im2col buffer */ - const q15_t *pB = bufferA; - - /* Each time it process 4 entries */ - uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; - - while (colCnt) - { - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia(&pB); - - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - } - } -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - (void)bufferA; - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out; j++) - { - for (k = 0; k < dim_im_out; k++) - { - conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel; m++) - { - for (n = 0; n < dim_kernel; n++) - { - // if-for implementation - in_row = stride * j + m - padding; - in_col = stride * k + n - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + n) * ch_im_in + l]; - } - } - } - } - Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_basic.c
+ * Description: Q7 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q7 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * This basic version is designed to work for any input tensor and weight
+ * dimension.
+ */
+
+arm_status
+arm_convolve_HWC_q7_basic(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* Copying the pixel data to column */
+ arm_q7_to_q15_no_shift((q7_t *)
+ Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* Computation is filed for every 2 columns */
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ ch_im_in *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ /* Load the accumulator with bias first */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+
+ /* Point to the beging of the im2col buffer */
+ q15_t *pB = bufferA;
+
+ /* Each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ // if-for implementation
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c index b42a57d..24356d9 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c @@ -1,229 +1,228 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q7_basic.c - * Description: Q7 version of convolution - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Basic Q7 convolution function (non-sqaure shape) - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in_x input tensor dimention x - * @param[in] dim_im_in_y input tensor dimention y - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel_x filter kernel size x - * @param[in] dim_kernel_y filter kernel size y - * @param[in] padding_x padding size x - * @param[in] padding_y padding size y - * @param[in] stride_x convolution stride x - * @param[in] stride_y convolution stride y - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out_x output tensor dimension x - * @param[in] dim_im_out_y output tensor dimension y - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns <code>ARM_MATH_SUCCESS</code> - */ - -arm_status arm_convolve_HWC_q7_basic_nonsquare(const q7_t *Im_in, - const uint16_t dim_im_in_x, - const uint16_t dim_im_in_y, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel_x, - const uint16_t dim_kernel_y, - const uint16_t padding_x, - const uint16_t padding_y, - const uint16_t stride_x, - const uint16_t stride_y, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out_x, - const uint16_t dim_im_out_y, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - /* - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - /* This part implements the im2col function */ - for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* Filling 0 for out-of-bound paddings */ - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - /* Copying the pixel data to column */ - arm_q7_to_q15_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - /* Computation is filed for every 2 columns */ - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_y * dim_kernel_x) - { - pOut = arm_nn_mat_mult_kernel_q7_q15( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_y * dim_kernel_x, bias_shift, out_shift, bias, pOut); - - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* left-over because odd number of output pixels */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - int i; - - for (i = 0; i < ch_im_out; i++) - { - /* Load the accumulator with bias first */ - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - - /* Point to the beging of the im2col buffer */ - const q15_t *pB = bufferA; - - /* Each time it process 4 entries */ - uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 2; - - while (colCnt) - { - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia(&pB); - - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - } - } -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - (void)bufferA; - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out_y; j++) - { - for (k = 0; k < dim_im_out_x; k++) - { - conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel_y; m++) - { - for (n = 0; n < dim_kernel_x; n++) - { - // if-for implementation - in_row = stride_y * j + m - padding_y; - in_col = stride_x * k + n - padding_x; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + - l]; - } - } - } - } - Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_basic.c
+ * Description: Q7 version of convolution
+ *
+ * $Date: 13. July 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q7 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ */
+
+arm_status arm_convolve_HWC_q7_basic_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* Copying the pixel data to column */
+ arm_q7_to_q15_no_shift((q7_t *)
+ Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* Computation is filed for every 2 columns */
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_y * dim_kernel_x)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ ch_im_in *
+ dim_kernel_y * dim_kernel_x, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ /* Load the accumulator with bias first */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+
+ /* Point to the beging of the im2col buffer */
+ q15_t *pB = bufferA;
+
+ /* Each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 2;
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ // if-for implementation
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x +
+ (m * dim_kernel_x + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c index 51d98fd..e2d469f 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c @@ -1,380 +1,408 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q7_fast.c - * Description: Fast Q7 version of convolution - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Fast Q7 convolution function - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * <b>Input dimension constraints:</b> - * - * ch_im_in is multiple of 4 ( because of the SIMD32 read and swap ) - * - * ch_im_out is multiple of 2 ( bacause 2x2 mat_mult kernel ) - * - * The im2col converts the Q7 tensor input into Q15 column, which is stored in - * bufferA. There is reordering happenning during this im2col process with - * arm_q7_to_q15_reordered_no_shift. For every four elements, the second and - * third elements are swapped. - * - * The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the - * GEMM computation with the reordered columns. - * - * To speed-up the determination of the padding condition, we split the - * computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}. - * This reduces the total number of boundary condition checks and improves - * the data copying performance. - */ - -arm_status arm_convolve_HWC_q7_fast(const q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - /* - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - /* - * Here we split the entire matrix into three regions depending on the padding situation - * Top: i_out_y from 0 to padding - 1 - * Middle: i_out_y from padding to dim_im_out-padding-1 - * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1 - */ - - /* top part */ - for (i_out_y = 0; i_out_y < padding; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* middle part, here we also divide the x into left, mid and right */ - for (; i_out_y < dim_im_out - padding; i_out_y++) - { - - /* left part */ - for (i_out_x = 0; i_out_x < padding; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - - /* mid part */ - for (; i_out_x < dim_im_out - padding; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - arm_q7_to_q15_reordered_no_shift((q7_t *)Im_in + - (i_ker_y * dim_im_in + i_out_x * stride - padding) * ch_im_in, - pBuffer, - ch_im_in * dim_kernel); - pBuffer += ch_im_in * dim_kernel; - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - - /* right part */ - for (; i_out_x < dim_im_out; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - for (; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* check if there is left-over for compute */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - int i; - - for (i = 0; i < ch_im_out; i++) - { - q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift); - const q15_t *pB = bufferA; - /* each time it process 4 entries */ - uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2; - - while (colCnt) - { - - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad_reordered(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut = (q7_t)__SSAT((sum >> out_shift), 8); - pOut++; - } - } -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out; j++) - { - for (k = 0; k < dim_im_out; k++) - { - conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel; m++) - { - for (n = 0; n < dim_kernel; n++) - { - // if-for implementation - in_row = stride * j + m - padding; - in_col = stride * k + n - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel + n) * ch_im_in + l]; - } - } - } - } - Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_fast.c
+ * Description: Fast Q7 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q7 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 4 ( because of the SIMD32 read and swap )
+ *
+ * ch_im_out is multipe of 2 ( bacause 2x2 mat_mult kernel )
+ *
+ * The im2col converts the Q7 tensor input into Q15 column, which is stored in
+ * bufferA. There is reordering happenning during this im2col process with
+ * arm_q7_to_q15_reordered_no_shift. For every four elements, the second and
+ * third elements are swapped.
+ *
+ * The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the
+ * GEMM computation with the reordered columns.
+ *
+ * To speed-up the determination of the padding condition, we split the
+ * computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}.
+ * This reduces the total number of boundary condition checks and improves
+ * the data copying performance.
+ */
+
+arm_status
+arm_convolve_HWC_q7_fast(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /*
+ * Here we split the entire matrix into three regions depending on the padding situation
+ * Top: i_out_y from 0 to padding - 1
+ * Middle: i_out_y from padding to dim_im_out-padding-1
+ * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
+ */
+
+ /* top part */
+ for (i_out_y = 0; i_out_y < padding; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* middle part, here we also divide the x into left, mid and right */
+ for (; i_out_y < dim_im_out - padding; i_out_y++)
+ {
+
+ /* left part */
+ for (i_out_x = 0; i_out_x < padding; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* mid part */
+ for (; i_out_x < dim_im_out - padding; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in
+ +
+ (i_ker_y *
+ dim_im_in +
+ i_out_x *
+ stride - padding) * ch_im_in, pBuffer, ch_im_in * dim_kernel);
+ pBuffer += ch_im_in * dim_kernel;
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* right part */
+ for (; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ for (; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ // if-for implementation
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c index 25f17bb..6dc6f0b 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c @@ -1,378 +1,379 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_convolve_HWC_q7_fast_nonsquare.c - * Description: Fast Q7 version of convolution (non-sqaure shape) - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Fast Q7 convolution function (non-sqaure shape) - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in_x input tensor dimention x - * @param[in] dim_im_in_y input tensor dimention y - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel_x filter kernel size x - * @param[in] dim_kernel_y filter kernel size y - * @param[in] padding_x padding size x - * @param[in] padding_y padding size y - * @param[in] stride_x convolution stride x - * @param[in] stride_y convolution stride y - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out_x output tensor dimension x - * @param[in] dim_im_out_y output tensor dimension y - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * This function is the version with full list of optimization tricks, but with - * some constraints: - * ch_im_in is multiple of 4 - * ch_im_out is multiple of 2 - */ - -arm_status arm_convolve_HWC_q7_fast_nonsquare(const q7_t *Im_in, - const uint16_t dim_im_in_x, - const uint16_t dim_im_in_y, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel_x, - const uint16_t dim_kernel_y, - const uint16_t padding_x, - const uint16_t padding_y, - const uint16_t stride_x, - const uint16_t stride_y, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out_x, - const uint16_t dim_im_out_y, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x, i_ker_y, i_ker_x; - - /* ----------------------- - * Here we use bufferA as q15_t internally as computation are done with q15_t level - * im2col are done to output in q15_t format from q7_t input - */ - - q15_t *pBuffer = bufferA; - q7_t *pOut = Im_out; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - /* - * Here we split the entire matrix into three regions depending on the padding situation - * Top: i_out_y from 0 to padding - 1 - * Middle: i_out_y from padding to dim_im_out-padding-1 - * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1 - */ - - /* top part */ - for (i_out_y = 0; i_out_y < padding_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* middle part, here we also divide the x into left, mid and right */ - for (; i_out_y < dim_im_out_y - padding_y; i_out_y++) - { - - /* left part */ - for (i_out_x = 0; i_out_x < padding_x; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - - /* mid part */ - for (; i_out_x < dim_im_out_x - padding_x; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_out_x * stride_x - padding_x) * ch_im_in, - pBuffer, - ch_im_in * dim_kernel_x); - pBuffer += ch_im_in * dim_kernel_x; - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - - /* right part */ - for (; i_out_x < dim_im_out_x; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - for (; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - /* This part implements the im2col function */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q15(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, sizeof(q15_t) * ch_im_in); - } - else - { - arm_q7_to_q15_reordered_no_shift( - (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y) - { - pOut = arm_nn_mat_mult_kernel_q7_q15_reordered( - wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y, bias_shift, out_shift, bias, pOut); - /* counter reset */ - pBuffer = bufferA; - } - } - } - - /* check if there is left-over for compute */ - if (pBuffer != bufferA) - { - const q7_t *pA = wt; - int i; - for (i = 0; i < ch_im_out; i++) - { - q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - const q15_t *pB = bufferA; - /* basically each time it process 4 entries */ - uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2; - - while (colCnt) - { - - q31_t inA1, inA2; - q31_t inB1, inB2; - - pA = read_and_pad_reordered(pA, &inA1, &inA2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA1, inB1, sum); - inB2 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inA2, inB2, sum); - - colCnt--; - } - colCnt = (ch_im_in * dim_kernel_y * dim_kernel_x) & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - sum += inA1 * inB1; - colCnt--; - } - *pOut = (q7_t)__SSAT((sum >> out_shift), 8); - pOut++; - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i, j, k, l, m, n; - int conv_out; - int in_row, in_col; - - if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0) - { - /* check if the input dimension meets the constraints */ - return ARM_MATH_SIZE_MISMATCH; - } - - for (i = 0; i < ch_im_out; i++) - { - for (j = 0; j < dim_im_out_y; j++) - { - for (k = 0; k < dim_im_out_x; k++) - { - conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - for (m = 0; m < dim_kernel_y; m++) - { - for (n = 0; n < dim_kernel_x; n++) - { - /* if-for implementation */ - in_row = stride_y * j + m - padding_y; - in_col = stride_x * k + n - padding_x; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) - { - for (l = 0; l < ch_im_in; l++) - { - conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] * - wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + - l]; - } - } - } - } - Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_fast_nonsquare.c
+ * Description: Fast Q7 version of convolution (non-sqaure shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Fast Q7 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 4
+ * ch_im_out is multiple of 2
+ */
+
+arm_status arm_convolve_HWC_q7_fast_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /* -----------------------
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /*
+ * Here we split the entire matrix into three regions depending on the padding situation
+ * Top: i_out_y from 0 to padding - 1
+ * Middle: i_out_y from padding to dim_im_out-padding-1
+ * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
+ */
+
+ /* top part */
+ for (i_out_y = 0; i_out_y < padding_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* middle part, here we also divide the x into left, mid and right */
+ for (; i_out_y < dim_im_out_y - padding_y; i_out_y++)
+ {
+
+ /* left part */
+ for (i_out_x = 0; i_out_x < padding_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* mid part */
+ for (; i_out_x < dim_im_out_x - padding_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in +
+ (i_ker_y * dim_im_in_x + i_out_x * stride_x - padding_x) * ch_im_in,
+ pBuffer, ch_im_in * dim_kernel_x);
+ pBuffer += ch_im_in * dim_kernel_x;
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* right part */
+ for (; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ for (; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = (ch_im_in * dim_kernel_y * dim_kernel_x) & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i, j, k, l, m, n;
+ int conv_out;
+ int in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ /* if-for implementation */
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c index c9d0afc..c96dc55 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c @@ -1,336 +1,239 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_depthwise_conv_u8_basic_ver1.c - * Description: u8 depthwise convolution function - * - * $Date: 09. October 2020 - * $Revision: V.1.1.1 - * - * Target : Cortex-M CPUs - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -static void depthwise_conv_u8_mult_4(const uint8_t *input, - const int32_t input_x, - const int32_t input_y, - const int32_t input_ch, - const uint8_t *kernel, - const int32_t output_ch, - const int32_t ch_mult, - const int32_t kernel_x, - const int32_t kernel_y, - const int32_t pad_x, - const int32_t pad_y, - const int32_t stride_x, - const int32_t stride_y, - const int32_t *bias, - uint8_t *output, - const int32_t output_shift, - const int32_t output_mult, - const int32_t output_x, - const int32_t output_y, - const int32_t output_offset, - const int32_t input_offset, - const int32_t filter_offset, - const int32_t output_activation_min, - const int32_t output_activation_max) -{ - for (int32_t in_h = -pad_y, out_h = 0, out_idx = 0; out_h < output_y; in_h += stride_y, ++out_h) - { - for (int32_t in_w = -pad_x, out_w = 0, ker_h_start = MAX(0, -in_h); out_w < output_x; in_w += stride_x, ++out_w) - { - for (int32_t in_ch = 0, out_ch = 0, ker_w_start = MAX(0, -in_w); out_ch < output_ch; - ++in_ch, out_ch += ch_mult) - { - for (int mult_tile = 0; mult_tile < ch_mult; mult_tile += 4) - { - int32_t out_buff[4]; - - out_buff[0] = 0; - out_buff[1] = 0; - out_buff[2] = 0; - out_buff[3] = 0; - - for (int32_t ker_h = ker_h_start; ker_h < MIN(kernel_y, input_y - in_h); ++ker_h) - { - int32_t ker_idx = ker_h * (output_ch * kernel_x) + ker_w_start * output_ch + out_ch; - int32_t in_idx = (in_h + ker_h) * (input_ch * input_x) + in_w * input_ch + in_ch; - - for (int32_t ker_w = ker_w_start; ker_w < MIN(kernel_x, input_x - in_w); - ++ker_w, ker_idx += output_ch) - { - int32_t in_val = input[in_idx + ker_w * input_ch] + input_offset; - out_buff[0] += in_val * (kernel[ker_idx + 0 + mult_tile] + filter_offset); - out_buff[1] += in_val * (kernel[ker_idx + 1 + mult_tile] + filter_offset); - out_buff[2] += in_val * (kernel[ker_idx + 2 + mult_tile] + filter_offset); - out_buff[3] += in_val * (kernel[ker_idx + 3 + mult_tile] + filter_offset); - } - } - - if (bias != NULL) - { - out_buff[0] += bias[out_ch + 0 + mult_tile]; - out_buff[1] += bias[out_ch + 1 + mult_tile]; - out_buff[2] += bias[out_ch + 2 + mult_tile]; - out_buff[3] += bias[out_ch + 3 + mult_tile]; - } - out_buff[0] = arm_nn_requantize(out_buff[0], output_mult, output_shift); - out_buff[1] = arm_nn_requantize(out_buff[1], output_mult, output_shift); - out_buff[2] = arm_nn_requantize(out_buff[2], output_mult, output_shift); - out_buff[3] = arm_nn_requantize(out_buff[3], output_mult, output_shift); - - out_buff[0] += output_offset; - out_buff[1] += output_offset; - out_buff[2] += output_offset; - out_buff[3] += output_offset; - - out_buff[0] = MIN(MAX(out_buff[0], output_activation_min), output_activation_max); - out_buff[1] = MIN(MAX(out_buff[1], output_activation_min), output_activation_max); - out_buff[2] = MIN(MAX(out_buff[2], output_activation_min), output_activation_max); - out_buff[3] = MIN(MAX(out_buff[3], output_activation_min), output_activation_max); - - output[out_idx++] = (uint8_t)out_buff[0]; - output[out_idx++] = (uint8_t)out_buff[1]; - output[out_idx++] = (uint8_t)out_buff[2]; - output[out_idx++] = (uint8_t)out_buff[3]; - } - } - } - } -} - -static void depthwise_conv_u8_generic(const uint8_t *input, - const int32_t input_x, - const int32_t input_y, - const int32_t input_ch, - const uint8_t *kernel, - const int32_t output_ch, - const int32_t ch_mult, - const int32_t kernel_x, - const int32_t kernel_y, - const int32_t pad_x, - const int32_t pad_y, - const int32_t stride_x, - const int32_t stride_y, - const int32_t *bias, - uint8_t *output, - const int32_t output_shift, - const int32_t output_mult, - const int32_t output_x, - const int32_t output_y, - const int32_t output_offset, - const int32_t input_offset, - const int32_t filter_offset, - const int32_t output_activation_min, - const int32_t output_activation_max) -{ - (void)output_ch; - int i_out = 0; - for (int i_out_y = 0; i_out_y < output_y; i_out_y++) - { - const int16_t base_idx_y = (i_out_y * stride_y) - pad_y; - for (int i_out_x = 0; i_out_x < output_x; i_out_x++) - { - const int16_t base_idx_x = (i_out_x * stride_x) - pad_x; - for (int i_input_ch = 0; i_input_ch < input_ch; i_input_ch++) - { - for (int i_ch_mult = 0; i_ch_mult < ch_mult; i_ch_mult++) - { - const int idx_out_ch = i_ch_mult + i_input_ch * ch_mult; - int32_t acc_0; - /* Condition for kernel start dimension: (base_idx_<x,y> + ker_<x,y>_start) >= 0 */ - const int ker_y_start = MAX(0, -base_idx_y); - const int ker_x_start = MAX(0, -base_idx_x); - /* Condition for kernel end dimension: (base_idx_<x,y> + ker_<x,y>_end) < input_<x,y> */ - const int ker_y_end = MIN(kernel_y, input_y - base_idx_y); - const int ker_x_end = MIN(kernel_x, input_x - base_idx_x); - acc_0 = 0; - - for (int i_ker_y = ker_y_start; i_ker_y < ker_y_end; i_ker_y++) - { - const int32_t idx_y = base_idx_y + i_ker_y; - for (int i_ker_x = ker_x_start; i_ker_x < ker_x_end; i_ker_x++) - { - const int32_t idx_x = base_idx_x + i_ker_x; - int32_t idx_0 = (idx_y * input_x + idx_x) * input_ch + i_input_ch; - int32_t ker_idx_0 = (i_ker_y * kernel_x + i_ker_x) * (input_ch * ch_mult) + idx_out_ch; - - acc_0 += (input[idx_0] + input_offset) * (kernel[ker_idx_0] + filter_offset); - } - } - if (bias != NULL) - { - acc_0 += bias[idx_out_ch]; - } - - /* Requantize and clamp output to provided range */ - acc_0 = arm_nn_requantize(acc_0, output_mult, output_shift); - acc_0 += output_offset; - acc_0 = MAX(acc_0, output_activation_min); - acc_0 = MIN(acc_0, output_activation_max); - - output[i_out++] = acc_0; - } - } - } - } -} - -/** - * @brief uint8 depthwise convolution function with asymmetric quantization - * - * @param[in] input Pointer to input tensor - * @param[in] input_x Width of input tensor - * @param[in] input_y Height of input tensor - * @param[in] input_ch Channels in input tensor - * @param[in] kernel Pointer to kernel weights - * @param[in] kernel_x Width of kernel - * @param[in] kernel_y Height of kernel - * @param[in] ch_mult Number of channel multiplier - * @param[in] pad_x Padding sizes x - * @param[in] pad_y Padding sizes y - * @param[in] stride_x Convolution stride along the width - * @param[in] stride_y Convolution stride along the height - * @param[in] dilation_x Dilation along width. Not used and intended for future enhancement. - * @param[in] dilation_y Dilation along height. Not used and intended for future enhancement. - * @param[in] bias Pointer to optional bias values. If no bias is - * available, NULL is expected - * @param[in] input_offset Input tensor zero offset - * @param[in] filter_offset Kernel tensor zero offset - * @param[in] output_offset Output tensor zero offset - * @param[in,out] output Pointer to output tensor - * @param[in] output_x Width of output tensor - * @param[in] output_y Height of output tensor - * @param[in] output_activation_min Minimum value to clamp the output to. Range : {0, 255} - * @param[in] output_activation_max Minimum value to clamp the output to. Range : {0, 255} - * @param[in] output_shift Amount of right-shift for output - * @param[in] output_mult Output multiplier for requantization - * @return The function returns one of the following - * <code>ARM_MATH_SIZE_MISMATCH</code> - Not supported dimension of tensors - * <code>ARM_MATH_SUCCESS</code> - Successful operation - * <code>ARM_MATH_ARGUMENT_ERROR</code> - Implementation not available - * - * - */ - -arm_status arm_depthwise_conv_u8_basic_ver1(const uint8_t *input, - const uint16_t input_x, - const uint16_t input_y, - const uint16_t input_ch, - const uint8_t *kernel, - const uint16_t kernel_x, - const uint16_t kernel_y, - const int16_t ch_mult, - const int16_t pad_x, - const int16_t pad_y, - const int16_t stride_x, - const int16_t stride_y, - const int16_t dilation_x, - const int16_t dilation_y, - const int32_t *bias, - const int32_t input_offset, - const int32_t filter_offset, - const int32_t output_offset, - uint8_t *output, - const uint16_t output_x, - const uint16_t output_y, - const int32_t output_activation_min, - const int32_t output_activation_max, - const int32_t output_shift, - const int32_t output_mult) -{ - (void)dilation_x; - (void)dilation_y; - - if (ch_mult % 4 == 0) - { - depthwise_conv_u8_mult_4(input, - input_x, - input_y, - input_ch, - kernel, - ch_mult * input_ch, - ch_mult, - kernel_x, - kernel_y, - pad_x, - pad_y, - stride_x, - stride_y, - bias, - output, - output_shift, - output_mult, - output_x, - output_y, - output_offset, - input_offset, - filter_offset, - output_activation_min, - output_activation_max); - } - else - { - depthwise_conv_u8_generic(input, - input_x, - input_y, - input_ch, - kernel, - ch_mult * input_ch, - ch_mult, - kernel_x, - kernel_y, - pad_x, - pad_y, - stride_x, - stride_y, - bias, - output, - output_shift, - output_mult, - output_x, - output_y, - output_offset, - input_offset, - filter_offset, - output_activation_min, - output_activation_max); - } - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2019 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_conv_u8_basic_ver1.c
+ * Description: u8 depthwise convolution function
+ *
+ * $Date: June, 2019
+ * $Revision: V.0.8.0
+ *
+ * Target : Cortex-M cores with DSP extension
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#define DILATION_X (1)
+#define DILATION_Y (1)
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief uint8 depthwise convolution function with asymmetric quantization for even number of channel multiplier
+ * and input channels. Unless specified otherwise, arguments are mandatory. Both square and non-square inputs
+ * are accepted.
+ *
+ * @param[in] input Pointer to input tensor
+ * @param[in] input_x Width of input tensor
+ * @param[in] input_y Height of input tensor
+ * @param[in] input_ch Channels in input tensor
+ * @param[in] kernel Pointer to kernel weights
+ * @param[in] kernel_x Width of kernel
+ * @param[in] kernel_y Height of kernel
+ * @param[in] ch_mult Number of channel multiplier
+ * @param[in] pad_x Padding sizes x
+ * @param[in] pad_y Padding sizes y
+ * @param[in] stride_x Convolution stride along the width
+ * @param[in] stride_y Convolution stride along the height
+ * @param[in] dilation_x Dilation along width. Not used and intended for future enhancement.
+ * @param[in] dilation_y Dilation along height. Not used and intended for future enhancement.
+ * @param[in] bias Pointer to optional bias values. If no bias is
+ * availble, NULL is expected
+ * @param[in] input_offset Input tensor zero offset
+ * @param[in] filter_offset Kernel tensor zero offset
+ * @param[in] output_offset Output tensor zero offset
+ * @param[in,out] output Pointer to output tensor
+ * @param[in] output_x Width of output tensor
+ * @param[in] output_y Height of output tensor
+ * @param[in] output_activation_min Minimum value to clamp the output to. Range : {0, 255}
+ * @param[in] output_activation_max Minimum value to clamp the output to. Range : {0, 255}
+ * @param[in] out_shift Amount of right-shift for output
+ * @param[in] out_mult Output multiplier for requantization
+ * @return The function returns one of the following
+ * <code>ARM_MATH_SIZE_MISMATCH</code> - Not supported dimension of tensors
+ * <code>ARM_MATH_SUCCESS</code> - Successful operation
+ * <code>ARM_MATH_ARGUMENT_ERROR</code> - Implementation not available
+ *
+ * <b> Input constraints</b>
+ * ch_mult is multiple of 2
+ * kernel_x is multiple of 2
+ *
+ */
+
+arm_status arm_depthwise_conv_u8_basic_ver1(const uint8_t *input,
+ const uint16_t input_x,
+ const uint16_t input_y,
+ const uint16_t input_ch,
+ const uint8_t *kernel,
+ const uint16_t kernel_x,
+ const uint16_t kernel_y,
+ const int16_t ch_mult,
+ const int16_t pad_x,
+ const int16_t pad_y,
+ const int16_t stride_x,
+ const int16_t stride_y,
+ const int16_t dilation_x,
+ const int16_t dilation_y,
+ const int32_t *bias,
+ const int32_t input_offset,
+ const int32_t filter_offset,
+ const int32_t output_offset,
+ uint8_t *output,
+ const uint16_t output_x,
+ const uint16_t output_y,
+ const int32_t output_activation_min,
+ const int32_t output_activation_max,
+ const int32_t out_shift,
+ const int32_t out_mult)
+{
+ arm_status status = ARM_MATH_SUCCESS;
+ #if defined (ARM_MATH_DSP)
+ int i_out = 0;
+ (void)dilation_x;
+ (void)dilation_y;
+
+ const int32_t input_offset_pkd = (input_offset & 0xFFFF) | (input_offset & 0xFFFF) << 16;
+ const int32_t kernel_offset_pkd = (filter_offset & 0xFFFF) | (filter_offset & 0xFFFF) << 16;
+
+ if (0 != ch_mult % 2 || 0 != kernel_x % 2)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (int i_out_y = 0; i_out_y < output_y; i_out_y++)
+ {
+ const int16_t base_idx_y = (i_out_y * stride_y) - pad_y;
+ for (int i_out_x = 0; i_out_x < output_x; i_out_x++)
+ {
+ const int16_t base_idx_x = (i_out_x * stride_x) - pad_x;
+ for (int i_input_ch = 0; i_input_ch < input_ch; i_input_ch++)
+ {
+ for (int i_ch_mult = 0; i_ch_mult < ch_mult; i_ch_mult += 2)
+ {
+ const int idx_out_ch = i_ch_mult + i_input_ch * ch_mult;
+
+ int32_t acc_0 = 0;
+ int32_t acc_1 = 0;
+ if (NULL != bias)
+ {
+ acc_0 = bias[idx_out_ch];
+ acc_1 = bias[idx_out_ch + 1];
+ }
+
+ for (int i_ker_y = 0; i_ker_y < kernel_y; i_ker_y++)
+ {
+ const int32_t idx_y = base_idx_y + DILATION_Y * i_ker_y;
+ const int32_t y_in_range = (idx_y >= 0) && (idx_y < input_y);
+
+ for (int i_ker_x = 0; i_ker_x < kernel_x; i_ker_x += 2)
+ {
+ if (1 == y_in_range)
+ {
+ const int32_t idx_x = base_idx_x + DILATION_X * i_ker_x;
+ const int32_t idx_x1 = base_idx_x + DILATION_X * (i_ker_x + 1);
+ /* Range check for first input */
+ if (idx_x >= 0 && idx_x < input_x)
+ {
+ const int32_t idx_0 = (idx_y * input_x + idx_x) * input_ch + i_input_ch;
+
+ const int32_t ker_idx_0 =
+ (i_ker_y * kernel_x + i_ker_x) * (input_ch * ch_mult) + idx_out_ch;
+ const int32_t ker_idx_1 = ker_idx_0 + input_ch * ch_mult;
+
+ int32_t input_pkd = input[idx_0] | (input[idx_0 + input_ch] << 16);
+ int32_t kernel_pkd = kernel[ker_idx_0] | (kernel[ker_idx_1] << 16);
+
+ input_pkd = __SADD16(input_pkd, input_offset_pkd);
+ kernel_pkd = __SADD16(kernel_pkd, kernel_offset_pkd);
+ /* Range check for second input */
+ if (idx_x1 >= input_x)
+ {
+ input_pkd &= 0xFFFF;
+ }
+ acc_0 = __SMLAD(input_pkd, kernel_pkd, acc_0);
+
+ kernel_pkd = kernel[ker_idx_0 + 1] | (kernel[ker_idx_1 + 1] << 16);
+ kernel_pkd = __SADD16(kernel_pkd, kernel_offset_pkd);
+ acc_1 = __SMLAD(input_pkd, kernel_pkd, acc_1);
+ }
+ }
+ }
+ }
+
+ /* Requantize and clamp output to provided range */
+ acc_0 = arm_nn_divide_by_power_of_two(arm_nn_sat_doubling_high_mult(
+ acc_0 * (1 << LEFT_SHIFT(out_shift)), out_mult),
+ RIGHT_SHIFT(out_shift));
+
+ acc_0 += output_offset;
+
+ if (output_activation_min > acc_0)
+ {
+ acc_0 = output_activation_min;
+ }
+
+ if (acc_0 > output_activation_max)
+ {
+ acc_0 = output_activation_max;
+ }
+ output[i_out++] = acc_0;
+
+ /* Requantize and clamp output to provided range */
+ acc_1 = arm_nn_divide_by_power_of_two(arm_nn_sat_doubling_high_mult(
+ acc_1 * (1 << LEFT_SHIFT(out_shift)), out_mult),
+ RIGHT_SHIFT(out_shift));
+ acc_1 += output_offset;
+
+ if (output_activation_min > acc_1)
+ {
+ acc_1 = output_activation_min;
+ }
+
+ if (acc_1 > output_activation_max)
+ {
+ acc_1 = output_activation_max;
+ }
+ output[i_out++] = acc_1;
+ }
+ }
+ }
+ }
+#else
+ /* No available implementation. */
+ status = ARM_MATH_ARGUMENT_ERROR;
+#endif
+ return status;
+}
+
+/**
+ * @} end of NNConv group
+ */
+
+
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c index 729147f..705fa6a 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c @@ -1,422 +1,418 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_depthwise_separable_conv_HWC_q7.c - * Description: Q7 depthwise separable convolution function - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Q7 depthwise separable convolution function - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimension - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel - * - * bufferB size: 0 - * - * <b>Input dimension constraints:</b> - * - * ch_im_in equals ch_im_out - * - * Implementation: - * There are 3 nested loop here: - * Inner loop: calculate each output value with MAC instruction over an accumulator - * Mid loop: loop over different output channel - * Outer loop: loop over different output (x, y) - */ - -arm_status arm_depthwise_separable_conv_HWC_q7(const q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out, - q15_t *bufferA, - q7_t *bufferB) -{ - (void)bufferB; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_out_y, i_out_x; - int16_t i_ker_y, i_ker_x; - q7_t *colBuffer = (q7_t *)bufferA; - q7_t *pBuffer = colBuffer; - const q7_t *pBias = bias; - q7_t *pOut = Im_out; - uint16_t rowCnt; - uint16_t row_shift; - - /* do some checking here, basically ch_im_in == ch_im_out */ - if (ch_im_in != ch_im_out) - { - return ARM_MATH_SIZE_MISMATCH; - } - - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - /* we first do im2col here */ - for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++) - { - for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in) - { - /* arm_fill_q7(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, ch_im_in); - } - else - { - /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); - */ - memcpy(pBuffer, (q7_t *)Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - /* we will do the computation here for each channel */ - rowCnt = ch_im_out >> 2; - row_shift = 0; - pBias = bias; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = (dim_kernel * dim_kernel) >> 1; - q7_t *pB = colBuffer + row_shift; - const q7_t *pA = wt + row_shift; - row_shift += 4; - -#ifdef USE_INTRINSIC - -#ifndef ARM_MATH_BIG_ENDIAN - - while (colCnt) - { - q31_t inA1, inA2, inB1, inB2, opA, opB; - - inB1 = arm_nn_read_q7x4(pB); - pB += ch_im_in; - opB = arm_nn_read_q7x4(pB); - pB += ch_im_in; - inB2 = __PKHTB(opB, inB1, 16); - inB1 = __PKHBT(inB1, opB, 16); - inA1 = arm_nn_read_q7x4(pA); - pA += ch_im_in; - opB = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inA2 = __PKHTB(opB, inA1, 16); - inA1 = __PKHBT(inA1, opB, 16); - opA = __SXTB16(inA1); - opB = __SXTB16(inB1); - sum = __SMLAD(opA, opB, sum); - opA = __SXTB16(__ROR(inA1, 8)); - opB = __SXTB16(__ROR(inB1, 8)); - sum2 = __SMLAD(opA, opB, sum2); - opA = __SXTB16(inA2); - opB = __SXTB16(inB2); - sum3 = __SMLAD(opA, opB, sum3); - opA = __SXTB16(__ROR(inA2, 8)); - opB = __SXTB16(__ROR(inB2, 8)); - sum4 = __SMLAD(opA, opB, sum4); - colCnt--; - } -#else - - while (colCnt) - { - q31_t inA1, inA2, inB1, inB2, opA, opB; - - inB1 = arm_nn_read_q7x4(pB); - pB += ch_im_in; - opB = arm_nn_read_q7x4(pB); - pB += ch_im_in; - inB2 = __PKHBT(opB, inB1, 16); - inB1 = __PKHTB(inB1, opB, 16); - inA1 = arm_nn_read_q7x4(pA); - pA += ch_im_in; - opB = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inA2 = __PKHBT(opB, inA1, 16); - inA1 = __PKHTB(inA1, opB, 16); - opA = __SXTB16(inA1); - opB = __SXTB16(inB1); - sum2 = __SMLAD(opA, opB, sum2); - opA = __SXTB16(__ROR(inA1, 8)); - opB = __SXTB16(__ROR(inB1, 8)); - sum = __SMLAD(opA, opB, sum); - opA = __SXTB16(inA2); - opB = __SXTB16(inB2); - sum4 = __SMLAD(opA, opB, sum4); - opA = __SXTB16(__ROR(inA2, 8)); - opB = __SXTB16(__ROR(inB2, 8)); - sum3 = __SMLAD(opA, opB, sum3); - colCnt--; - } - -#endif /* ARM_MATH_BIG_ENDIAN */ - -#else - -#ifndef ARM_MATH_BIG_ENDIAN - /* - * r0 r1 r2 r3 r4 r5 - * inA1, inA2, inB1, inB2, opA, opB - */ - - asm volatile("COL_LOOP_%=:\n" - "ldr.w r2, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "ldr.w r5, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "pkhtb r3, r5, r2, ASR #16\n" - "pkhbt r2, r2, r5, LSL #16\n" - "ldr.w r0, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "ldr.w r5, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "pkhtb r1, r5, r0, ASR #16\n" - "pkhbt r0, r0, r5, LSL #16\n" - "sxtb16 r4, r0\n" - "sxtb16 r5, r2\n" - "smlad %[sum], r4, r5, %[sum]\n" - "mov.w r4, r0, ror #8\n" - "mov.w r5, r2, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum2], r4, r5, %[sum2]\n" - "sxtb16 r4, r1\n" - "sxtb16 r5, r3\n" - "smlad %[sum3], r4, r5, %[sum3]\n" - "mov.w r4, r1, ror #8\n" - "mov.w r5, r3, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum4], r4, r5, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt), [ ch_im_in ] "r"(ch_im_in) - : "r0", "r1", "r2", "r3", "r4", "r5"); -#else - /* - * r0 r1 r2 r3 r4 r5 - * inA1, inA2, inB1, inB2, opA, opB - */ - asm volatile("COL_LOOP_%=:\n" - "ldr.w r2, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "ldr.w r5, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "pkhbt r3, r5, r2, LSL #16\n" - "pkhtb r2, r2, r5, ASR #16\n" - "ldr.w r0, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "ldr.w r5, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "pkhbt r1, r5, r0, LSL #16\n" - "pkhtb r0, r0, r5, ASR #16\n" - "sxtb16 r4, r0\n" - "sxtb16 r5, r2\n" - "smlad %[sum2], r4, r5, %[sum2]\n" - "mov.w r4, r0, ror #8\n" - "mov.w r5, r2, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum], r4, r5, %[sum]\n" - "sxtb16 r4, r1\n" - "sxtb16 r5, r3\n" - "smlad %[sum4], r4, r5, %[sum4]\n" - "mov.w r4, r1, ror #8\n" - "mov.w r5, r3, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum3], r4, r5, %[sum3]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt), [ ch_im_in ] "r"(ch_im_in) - : "r0", "r1", "r2", "r3", "r4", "r5"); - -#endif /* ARM_MATH_BIG_ENDIAN */ - -#endif /* USE_INTRINSIC */ - - colCnt = (dim_kernel * dim_kernel) & 0x1; - while (colCnt) - { - union arm_nnword inA, inB; - inA.word = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inB.word = arm_nn_read_q7x4(pB); - pB += ch_im_in; - sum += inA.bytes[0] * inB.bytes[0]; - sum2 += inA.bytes[1] * inB.bytes[1]; - sum3 += inA.bytes[2] * inB.bytes[2]; - sum4 += inA.bytes[3] * inB.bytes[3]; - colCnt--; - } - - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum3 >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum4 >> out_shift), 8); - - rowCnt--; - } - - rowCnt = ch_im_out & 0x3; - while (rowCnt) - { - q7_t *pB = colBuffer + row_shift; - const q7_t *pA = wt + row_shift; - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = (dim_kernel * dim_kernel); - - row_shift += 1; - - while (colCnt) - { - q7_t A1 = *pA; - q7_t B1 = *pB; - pA += ch_im_in; - pB += ch_im_in; - sum += A1 * B1; - - colCnt--; - } - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - rowCnt--; - } - - /* clear counter and pointers */ - pBuffer = colBuffer; - } - } - -#else - (void)bufferA; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i_out_y, i_out_x, i_ch_out, i_ker_x, i_ker_y; - int conv_out; - - /* do some checking here, basically ch_im_in == ch_im_out */ - if (ch_im_in != ch_im_out) - { - return ARM_MATH_SIZE_MISMATCH; - } - - for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++) - { - for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) - { - // for each output - conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); - for (i_ker_y = 0; i_ker_y < dim_kernel; i_ker_y++) - { - for (i_ker_x = 0; i_ker_x < dim_kernel; i_ker_x++) - { - int in_row = stride * i_out_y + i_ker_y - padding; - int in_col = stride * i_out_x + i_ker_x - padding; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in) - { - conv_out += Im_in[(in_row * dim_im_in + in_col) * ch_im_in + i_ch_out] * - wt[(i_ker_y * dim_kernel + i_ker_x) * ch_im_out + i_ch_out]; - } - } - } - Im_out[(i_out_y * dim_im_out + i_out_x) * ch_im_out + i_ch_out] = - (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_separable_conv_HWC_q7.c
+ * Description: Q7 depthwise separable convolution function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Q7 depthwise separable convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in equals ch_im_out
+ *
+ * Implementation:
+ * There are 3 nested loop here:
+ * Inner loop: calculate each output value with MAC instruction over an accumulator
+ * Mid loop: loop over different output channel
+ * Outer loop: loop over different output (x, y)
+ */
+
+arm_status arm_depthwise_separable_conv_HWC_q7(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ker_y, i_ker_x;
+ q7_t *colBuffer = (q7_t *) bufferA;
+ q7_t *pBuffer = colBuffer;
+ const q7_t *pBias = bias;
+ q7_t *pOut = Im_out;
+ uint16_t rowCnt;
+ uint16_t row_shift;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* we first do im2col here */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q7(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, ch_im_in);
+ } else
+ {
+ /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* we will do the computation here for each channel */
+ rowCnt = ch_im_out >> 2;
+ row_shift = 0;
+ pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = (dim_kernel * dim_kernel) >> 1;
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ row_shift += 4;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHTB(opB, inB1, 16);
+ inB1 = __PKHBT(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHTB(opB, inA1, 16);
+ inA1 = __PKHBT(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum3 = __SMLAD(opA, opB, sum3);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum4 = __SMLAD(opA, opB, sum4);
+ colCnt--;
+ }
+#else
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHBT(opB, inB1, 16);
+ inB1 = __PKHTB(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHBT(opB, inA1, 16);
+ inA1 = __PKHTB(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum4 = __SMLAD(opA, opB, sum4);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum3 = __SMLAD(opA, opB, sum3);
+ colCnt--;
+ }
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ /*
+ * r0 r1 r2 r3 r4 r5
+ * inA1, inA2, inB1, inB2, opA, opB
+ */
+
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhtb r3, r5, r2, ASR #16\n"
+ "pkhbt r2, r2, r5, LSL #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhtb r1, r5, r0, ASR #16\n"
+ "pkhbt r0, r0, r5, LSL #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum]
+ "+r"(sum),[sum2] "+r"(sum2),
+ [sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),
+ [pA] "+r"(pA):[colCnt]
+ "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#else
+ /*
+ * r0 r1 r2 r3 r4 r5
+ * inA1, inA2, inB1, inB2, opA, opB
+ */
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhbt r3, r5, r2, LSL #16\n"
+ "pkhtb r2, r2, r5, ASR #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhbt r1, r5, r0, LSL #16\n"
+ "pkhtb r0, r0, r5, ASR #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum]
+ "+r"(sum),[sum2] "+r"(sum2),
+ [sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),
+ [pA] "+r"(pA):[colCnt]
+ "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = (dim_kernel * dim_kernel) & 0x1;
+ while (colCnt)
+ {
+ union arm_nnword inA, inB;
+ inA.word = *__SIMD32(pA);
+ pA += ch_im_in;
+ inB.word = *__SIMD32(pB);
+ pB += ch_im_in;
+ sum += inA.bytes[0] * inB.bytes[0];
+ sum2 += inA.bytes[1] * inB.bytes[1];
+ sum3 += inA.bytes[2] * inB.bytes[2];
+ sum4 += inA.bytes[3] * inB.bytes[3];
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+ rowCnt = ch_im_out & 0x3;
+ while (rowCnt)
+ {
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = (dim_kernel * dim_kernel);
+
+ row_shift += 1;
+
+ while (colCnt)
+ {
+ q7_t A1 = *pA;
+ q7_t B1 = *pB;
+ pA += ch_im_in;
+ pB += ch_im_in;
+ sum += A1 * B1;
+
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ rowCnt--;
+ }
+
+ /* clear counter and pointers */
+ pBuffer = colBuffer;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i_out_y, i_out_x, i_ch_out, i_ker_x, i_ker_y;
+ int conv_out;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ // for each output
+ conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ for (i_ker_y = 0; i_ker_y < dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = 0; i_ker_x < dim_kernel; i_ker_x++)
+ {
+ int in_row = stride * i_out_y + i_ker_y - padding;
+ int in_col = stride * i_out_x + i_ker_x - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ conv_out +=
+ Im_in[(in_row *
+ dim_im_in +
+ in_col) *
+ ch_im_in +
+ i_ch_out] * wt[(i_ker_y * dim_kernel + i_ker_x) * ch_im_out + i_ch_out];
+ }
+ }
+ }
+ Im_out[(i_out_y * dim_im_out +
+ i_out_x) * ch_im_out + i_ch_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c index 829acf9..5989304 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c @@ -1,427 +1,411 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_depthwise_separable_conv_HWC_q7_nonsquare.c - * Description: Q7 depthwise separable convolution function (non-square shape) - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup NNConv - * @{ - */ - -/** - * @brief Q7 depthwise separable convolution function (non-square shape) - * @param[in] Im_in pointer to input tensor - * @param[in] dim_im_in_x input tensor dimension x - * @param[in] dim_im_in_y input tensor dimension y - * @param[in] ch_im_in number of input tensor channels - * @param[in] wt pointer to kernel weights - * @param[in] ch_im_out number of filters, i.e., output tensor channels - * @param[in] dim_kernel_x filter kernel size x - * @param[in] dim_kernel_y filter kernel size y - * @param[in] padding_x padding sizes x - * @param[in] padding_y padding sizes y - * @param[in] stride_x convolution stride x - * @param[in] stride_y convolution stride y - * @param[in] bias pointer to bias - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in,out] Im_out pointer to output tensor - * @param[in] dim_im_out_x output tensor dimension x - * @param[in] dim_im_out_y output tensor dimension y - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] bufferB pointer to buffer space for output - * @return The function returns either - * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking. - * - * This function is the version with full list of optimization tricks, but with - * some constraints: - * ch_im_in is equal to ch_im_out - * - */ - -arm_status arm_depthwise_separable_conv_HWC_q7_nonsquare(const q7_t *Im_in, - const uint16_t dim_im_in_x, - const uint16_t dim_im_in_y, - const uint16_t ch_im_in, - const q7_t *wt, - const uint16_t ch_im_out, - const uint16_t dim_kernel_x, - const uint16_t dim_kernel_y, - const uint16_t padding_x, - const uint16_t padding_y, - const uint16_t stride_x, - const uint16_t stride_y, - const q7_t *bias, - const uint16_t bias_shift, - const uint16_t out_shift, - q7_t *Im_out, - const uint16_t dim_im_out_x, - const uint16_t dim_im_out_y, - q15_t *bufferA, - q7_t *bufferB) -{ - - (void)bufferB; - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - /* - * Implementation: - * There are 3 nested loop here: - * Inner loop: calculate each output value with MAC instruction over an accumulator - * Mid loop: loop over different output channel - * Outer loop: loop over different output (x, y) - * - */ - - int16_t i_out_y, i_out_x; - int16_t i_ker_y, i_ker_x; - q7_t *colBuffer = (q7_t *)bufferA; - q7_t *pBuffer = colBuffer; - const q7_t *pBias = bias; - q7_t *pOut = Im_out; - uint16_t rowCnt; - uint16_t row_shift; - - /* do some checking here, basically ch_im_in == ch_im_out */ - if (ch_im_in != ch_im_out) - { - return ARM_MATH_SIZE_MISMATCH; - } - - for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - /* we first do im2col here */ - for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; - i_ker_y++) - { - for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; - i_ker_x++) - { - if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x) - { - /* arm_fill_q7(0, pBuffer, ch_im_in); */ - memset(pBuffer, 0, ch_im_in); - } - else - { - /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, - * ch_im_in); */ - memcpy(pBuffer, (q7_t *)Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, ch_im_in); - } - pBuffer += ch_im_in; - } - } - - /* we will do the computation here for each channel */ - rowCnt = ch_im_out >> 2; - row_shift = 0; - pBias = bias; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = (dim_kernel_x * dim_kernel_y) >> 1; - q7_t *pB = colBuffer + row_shift; - const q7_t *pA = wt + row_shift; - row_shift += 4; - -#ifdef USE_INTRINSIC - -#ifndef ARM_MATH_BIG_ENDIAN - - while (colCnt) - { - q31_t inA1, inA2, inB1, inB2, opA, opB; - - inB1 = arm_nn_read_q7x4(pB); - pB += ch_im_in; - opB = arm_nn_read_q7x4(pB); - pB += ch_im_in; - inB2 = __PKHTB(opB, inB1, 16); - inB1 = __PKHBT(inB1, opB, 16); - inA1 = arm_nn_read_q7x4(pA); - pA += ch_im_in; - opB = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inA2 = __PKHTB(opB, inA1, 16); - inA1 = __PKHBT(inA1, opB, 16); - opA = __SXTB16(inA1); - opB = __SXTB16(inB1); - sum = __SMLAD(opA, opB, sum); - opA = __SXTB16(__ROR(inA1, 8)); - opB = __SXTB16(__ROR(inB1, 8)); - sum2 = __SMLAD(opA, opB, sum2); - opA = __SXTB16(inA2); - opB = __SXTB16(inB2); - sum3 = __SMLAD(opA, opB, sum3); - opA = __SXTB16(__ROR(inA2, 8)); - opB = __SXTB16(__ROR(inB2, 8)); - sum4 = __SMLAD(opA, opB, sum4); - colCnt--; - } -#else - - while (colCnt) - { - q31_t inA1, inA2, inB1, inB2, opA, opB; - - inB1 = arm_nn_read_q7x4(pB); - pB += ch_im_in; - opB = arm_nn_read_q7x4(pB); - pB += ch_im_in; - inB2 = __PKHBT(opB, inB1, 16); - inB1 = __PKHTB(inB1, opB, 16); - inA1 = arm_nn_read_q7x4(pA); - pA += ch_im_in; - opB = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inA2 = __PKHBT(opB, inA1, 16); - inA1 = __PKHTB(inA1, opB, 16); - opA = __SXTB16(inA1); - opB = __SXTB16(inB1); - sum2 = __SMLAD(opA, opB, sum2); - opA = __SXTB16(__ROR(inA1, 8)); - opB = __SXTB16(__ROR(inB1, 8)); - sum = __SMLAD(opA, opB, sum); - opA = __SXTB16(inA2); - opB = __SXTB16(inB2); - sum4 = __SMLAD(opA, opB, sum4); - opA = __SXTB16(__ROR(inA2, 8)); - opB = __SXTB16(__ROR(inB2, 8)); - sum3 = __SMLAD(opA, opB, sum3); - colCnt--; - } - -#endif /* ARM_MATH_BIG_ENDIAN */ - -#else - -#ifndef ARM_MATH_BIG_ENDIAN - // r0 r1 r2 r3 r4 r5 - // inA1, inA2, inB1, inB2, opA, opB - asm volatile("COL_LOOP:\n" - "ldr.w r2, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "ldr.w r5, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "pkhtb r3, r5, r2, ASR #16\n" - "pkhbt r2, r2, r5, LSL #16\n" - "ldr.w r0, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "ldr.w r5, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "pkhtb r1, r5, r0, ASR #16\n" - "pkhbt r0, r0, r5, LSL #16\n" - "sxtb16 r4, r0\n" - "sxtb16 r5, r2\n" - "smlad %[sum], r4, r5, %[sum]\n" - "mov.w r4, r0, ror #8\n" - "mov.w r5, r2, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum2], r4, r5, %[sum2]\n" - "sxtb16 r4, r1\n" - "sxtb16 r5, r3\n" - "smlad %[sum3], r4, r5, %[sum3]\n" - "mov.w r4, r1, ror #8\n" - "mov.w r5, r3, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum4], r4, r5, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt), [ ch_im_in ] "r"(ch_im_in) - : "r0", "r1", "r2", "r3", "r4", "r5"); -#else - // r0 r1 r2 r3 r4 r5 - // inA1, inA2, inB1, inB2, opA, opB - asm volatile("COL_LOOP:\n" - "ldr.w r2, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "ldr.w r5, [%[pB], #0]\n" - "add.w %[pB], %[pB], %[ch_im_in]\n" - "pkhbt r3, r5, r2, LSL #16\n" - "pkhtb r2, r2, r5, ASR #16\n" - "ldr.w r0, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "ldr.w r5, [%[pA], #0]\n" - "add.w %[pA], %[pA], %[ch_im_in]\n" - "pkhbt r1, r5, r0, LSL #16\n" - "pkhtb r0, r0, r5, ASR #16\n" - "sxtb16 r4, r0\n" - "sxtb16 r5, r2\n" - "smlad %[sum2], r4, r5, %[sum2]\n" - "mov.w r4, r0, ror #8\n" - "mov.w r5, r2, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum], r4, r5, %[sum]\n" - "sxtb16 r4, r1\n" - "sxtb16 r5, r3\n" - "smlad %[sum4], r4, r5, %[sum4]\n" - "mov.w r4, r1, ror #8\n" - "mov.w r5, r3, ror #8\n" - "sxtb16 r4, r4\n" - "sxtb16 r5, r5\n" - "smlad %[sum3], r4, r5, %[sum3]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt), [ ch_im_in ] "r"(ch_im_in) - : "r0", "r1", "r2", "r3", "r4", "r5"); -#endif /*ARM_MATH_BIG_ENDIAN */ - -#endif /* USE_INTRINSIC */ - - colCnt = (dim_kernel_x * dim_kernel_y) & 0x1; - while (colCnt) - { - union arm_nnword inA, inB; - inA.word = arm_nn_read_q7x4(pA); - pA += ch_im_in; - inB.word = arm_nn_read_q7x4(pB); - pB += ch_im_in; - sum += inA.bytes[0] * inB.bytes[0]; - sum2 += inA.bytes[1] * inB.bytes[1]; - sum3 += inA.bytes[2] * inB.bytes[2]; - sum4 += inA.bytes[3] * inB.bytes[3]; - colCnt--; - } - - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum3 >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum4 >> out_shift), 8); - - rowCnt--; - } - - rowCnt = ch_im_out & 0x3; - while (rowCnt) - { - q7_t *pB = colBuffer + row_shift; - const q7_t *pA = wt + row_shift; - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = (dim_kernel_x * dim_kernel_y); - - row_shift += 1; - - while (colCnt) - { - q7_t A1 = *pA; - q7_t B1 = *pB; - pA += ch_im_in; - pB += ch_im_in; - sum += A1 * B1; - - colCnt--; - } - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - rowCnt--; - } - - // clear counter and pointers - pBuffer = colBuffer; - } - } - -#else - (void)bufferA; - - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int i_out_y, i_out_x, i_ch_out; - int i_ker_y, i_ker_x; - - /* do some checking here, basically ch_im_in == ch_im_out */ - if (ch_im_in != ch_im_out) - { - return ARM_MATH_SIZE_MISMATCH; - } - - for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++) - { - for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++) - { - for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++) - { - // for each output - int conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift); - for (i_ker_y = 0; i_ker_y < dim_kernel_y; i_ker_y++) - { - for (i_ker_x = 0; i_ker_x < dim_kernel_x; i_ker_x++) - { - int in_row = stride_y * i_out_y + i_ker_y - padding_y; - int in_col = stride_x * i_out_x + i_ker_x - padding_x; - if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x) - { - conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + i_ch_out] * - wt[(i_ker_y * dim_kernel_x + i_ker_x) * ch_im_out + i_ch_out]; - } - } - } - Im_out[(i_out_y * dim_im_out_x + i_out_x) * ch_im_out + i_ch_out] = - (q7_t)__SSAT((conv_out >> out_shift), 8); - } - } - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return ARM_MATH_SUCCESS; -} - -/** - * @} end of NNConv group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_separable_conv_HWC_q7_nonsquare.c
+ * Description: Q7 depthwise separable convolution function (non-square shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Q7 depthwise separable convolution function (non-square shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding sizes x
+ * @param[in] padding_y padding sizes y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 2
+ * ch_im_out is multiple of 2
+ */
+
+arm_status arm_depthwise_separable_conv_HWC_q7_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+/*
+ * Implementation:
+ * There are 3 nested loop here:
+ * Inner loop: calculate each output value with MAC instruction over an accumulator
+ * Mid loop: loop over different output channel
+ * Outer loop: loop over different output (x, y)
+ *
+ */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ker_y, i_ker_x;
+ q7_t *colBuffer = (q7_t *) bufferA;
+ q7_t *pBuffer = colBuffer;
+ const q7_t *pBias = bias;
+ q7_t *pOut = Im_out;
+ uint16_t rowCnt;
+ uint16_t row_shift;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* we first do im2col here */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q7(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, ch_im_in);
+ } else
+ {
+ /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* we will do the computation here for each channel */
+ rowCnt = ch_im_out >> 2;
+ row_shift = 0;
+ pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = (dim_kernel_x * dim_kernel_y) >> 1;
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ row_shift += 4;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHTB(opB, inB1, 16);
+ inB1 = __PKHBT(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHTB(opB, inA1, 16);
+ inA1 = __PKHBT(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum3 = __SMLAD(opA, opB, sum3);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum4 = __SMLAD(opA, opB, sum4);
+ colCnt--;
+ }
+#else
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHBT(opB, inB1, 16);
+ inB1 = __PKHTB(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHBT(opB, inA1, 16);
+ inA1 = __PKHTB(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum4 = __SMLAD(opA, opB, sum4);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum3 = __SMLAD(opA, opB, sum3);
+ colCnt--;
+ }
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ // r0 r1 r2 r3 r4 r5
+ // inA1, inA2, inB1, inB2, opA, opB
+ asm volatile ("COL_LOOP:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhtb r3, r5, r2, ASR #16\n"
+ "pkhbt r2, r2, r5, LSL #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhtb r1, r5, r0, ASR #16\n"
+ "pkhbt r0, r0, r5, LSL #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
+ [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#else
+ // r0 r1 r2 r3 r4 r5
+ // inA1, inA2, inB1, inB2, opA, opB
+ asm volatile ("COL_LOOP:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhbt r3, r5, r2, LSL #16\n"
+ "pkhtb r2, r2, r5, ASR #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhbt r1, r5, r0, LSL #16\n"
+ "pkhtb r0, r0, r5, ASR #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
+ [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#endif /*ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = (dim_kernel_x * dim_kernel_y) & 0x1;
+ while (colCnt)
+ {
+ union arm_nnword inA, inB;
+ inA.word = *__SIMD32(pA);
+ pA += ch_im_in;
+ inB.word = *__SIMD32(pB);
+ pB += ch_im_in;
+ sum += inA.bytes[0] * inB.bytes[0];
+ sum2 += inA.bytes[1] * inB.bytes[1];
+ sum3 += inA.bytes[2] * inB.bytes[2];
+ sum4 += inA.bytes[3] * inB.bytes[3];
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+ rowCnt = ch_im_out & 0x3;
+ while (rowCnt)
+ {
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = (dim_kernel_x * dim_kernel_y);
+
+ row_shift += 1;
+
+ while (colCnt)
+ {
+ q7_t A1 = *pA;
+ q7_t B1 = *pB;
+ pA += ch_im_in;
+ pB += ch_im_in;
+ sum += A1 * B1;
+
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ rowCnt--;
+ }
+
+ // clear counter and pointers
+ pBuffer = colBuffer;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i_out_y, i_out_x, i_ch_out;
+ int i_ker_y, i_ker_x;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ // for each output
+ int conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ for (i_ker_y = 0; i_ker_y < dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = 0; i_ker_x < dim_kernel_x; i_ker_x++)
+ {
+ int in_row = stride_y * i_out_y + i_ker_y - padding_y;
+ int in_col = stride_x * i_out_x + i_ker_x - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + i_ch_out] *
+ wt[(i_ker_y * dim_kernel_x + i_ker_x) * ch_im_out + i_ch_out];
+ }
+ }
+ }
+ Im_out[(i_out_y * dim_im_out_x + i_out_x) * ch_im_out + i_ch_out] =
+ (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c index 05c95b6..24ab412 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c @@ -1,186 +1,187 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_mat_mult_kernel_q7_q15.c - * Description: Matrix-multiplication function for convolution - * - * $Date: January 26, 2021 - * $Revision: V.1.0.2 - * - * Target Processor: Cortex-M cores - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @brief Matrix-multiplication function for convolution. - * - * @details Refer to header file for details. - * - */ - -q7_t *arm_nn_mat_mult_kernel_q7_q15(const q7_t *pA, - const q15_t *pInBuffer, - const uint16_t ch_im_out, - const uint16_t numCol_A, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q7_t *pOut) -{ -#if defined(ARM_MATH_DSP) - /* set up the second output pointers */ - q7_t *pOut2 = pOut + ch_im_out; - const q7_t *pBias = bias; - - uint16_t rowCnt = ch_im_out >> 1; - /* this loop over rows in A */ - while (rowCnt) - { - /* setup pointers for B */ - const q15_t *pB = pInBuffer; - const q15_t *pB2 = pB + numCol_A; - - /* align the second pointer for A */ - const q7_t *pA2 = pA + numCol_A; - - /* init the sum with bias */ - q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = numCol_A >> 2; - /* accumulate over the vector */ - while (colCnt) - { - q31_t inA11, inA12, inA21, inA22; - - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB2); - - pA = read_and_pad(pA, &inA11, &inA12); - pA2 = read_and_pad(pA2, &inA21, &inA22); - - sum = __SMLAD(inA11, inB1, sum); - sum2 = __SMLAD(inA11, inB2, sum2); - sum3 = __SMLAD(inA21, inB1, sum3); - sum4 = __SMLAD(inA21, inB2, sum4); - - inB1 = arm_nn_read_q15x2_ia(&pB); - inB2 = arm_nn_read_q15x2_ia(&pB2); - - sum = __SMLAD(inA12, inB1, sum); - sum2 = __SMLAD(inA12, inB2, sum2); - sum3 = __SMLAD(inA22, inB1, sum3); - sum4 = __SMLAD(inA22, inB2, sum4); - - colCnt--; - } /* while over colCnt */ - colCnt = numCol_A & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - q7_t inA2 = *pA2++; - q15_t inB2 = *pB2++; - - sum += inA1 * inB1; - sum2 += inA1 * inB2; - sum3 += inA2 * inB1; - sum4 += inA2 * inB2; - colCnt--; - } /* while over colCnt */ - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum3 >> out_shift), 8); - *pOut2++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - *pOut2++ = (q7_t)__SSAT((sum4 >> out_shift), 8); - - /* skip the row computed with A2 */ - pA += numCol_A; - rowCnt--; - } /* for over ch_im_out */ - - /* compute left-over row if any */ - if (ch_im_out & 0x1) - { - /* setup pointers for B */ - const q15_t *pB = pInBuffer; - const q15_t *pB2 = pB + numCol_A; - - /* load the bias */ - q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = numCol_A >> 2; - while (colCnt) - { - q31_t inA11, inA12; - - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB2); - - pA = read_and_pad(pA, &inA11, &inA12); - - sum = __SMLAD(inA11, inB1, sum); - sum2 = __SMLAD(inA11, inB2, sum2); - - inB1 = arm_nn_read_q15x2_ia(&pB); - inB2 = arm_nn_read_q15x2_ia(&pB2); - - sum = __SMLAD(inA12, inB1, sum); - sum2 = __SMLAD(inA12, inB2, sum2); - - colCnt--; - } - colCnt = numCol_A & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - q15_t inB2 = *pB2++; - - sum += inA1 * inB1; - sum2 += inA1 * inB2; - colCnt--; - } - - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pOut2++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - } - - pOut += ch_im_out; - - /* return the new output pointer with offset */ - return pOut; -#else - (void)pA; - (void)pInBuffer; - (void)ch_im_out; - (void)numCol_A; - (void)bias_shift; - (void)out_shift; - (void)bias; - (void)pOut; - /* To be completed */ - return NULL; -#endif /* ARM_MATH_DSP */ -} +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mat_mult_kernel_q7_q15.c
+ * Description: Matrix-multiplication function for convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+ /**
+ * @brief Matrix-multiplication function for convolution
+ * @param[in] pA pointer to operand A
+ * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
+ * @param[in] ch_im_out numRow of A
+ * @param[in] numCol_A numCol of A
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias the bias
+ * @param[in,out] pOut pointer to output
+ * @return The function returns the incremented output pointer
+ *
+ * @details
+ *
+ * This function does the matrix multiplication with weight matrix
+ * and 2 columns from im2col.
+ */
+
+q7_t *arm_nn_mat_mult_kernel_q7_q15(const q7_t * pA,
+ const q15_t * pInBuffer,
+ const uint16_t ch_im_out,
+ const uint16_t numCol_A,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q7_t * pOut)
+{
+#if defined (ARM_MATH_DSP)
+ /* set up the second output pointers */
+ q7_t *pOut2 = pOut + ch_im_out;
+ const q7_t *pBias = bias;
+
+ uint16_t rowCnt = ch_im_out >> 1;
+ /* this loop over rows in A */
+ while (rowCnt)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* align the second pointer for A */
+ const q7_t *pA2 = pA + numCol_A;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA11, inA12, inA21, inA22;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
+ pA2 = (q7_t *) read_and_pad((void *)pA2, &inA21, &inA22);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+ sum3 = __SMLAD(inA21, inB1, sum3);
+ sum4 = __SMLAD(inA21, inB2, sum4);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+ sum3 = __SMLAD(inA22, inB1, sum3);
+ sum4 = __SMLAD(inA22, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q7_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ /* skip the row computed with A2 */
+ pA += numCol_A;
+ rowCnt--;
+ } /* for over ch_im_out */
+
+ /* compute left-over row if any */
+ if (ch_im_out & 0x1)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* load the bias */
+ q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ while (colCnt)
+ {
+ q31_t inA11, inA12;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+
+ colCnt--;
+ }
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ }
+
+ pOut += ch_im_out;
+
+ /* return the new output pointer with offset */
+ return pOut;
+#else
+ /* To be completed */
+ return NULL;
+#endif /* ARM_MATH_DSP */
+
+}
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c index 0870ac3..36af21a 100644 --- a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c @@ -1,137 +1,138 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_mat_mult_kernel_q7_q15_reordered.c - * Description: Matrix-multiplication function for convolution with reordered columns - * - * $Date: January 26, 2021 - * $Revision: V.1.0.2 - * - * Target Processor: Cortex-M cores - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @brief Matrix-multiplication function for convolution with re-ordered input. - * - * @details Refer to header file for details. - * - */ - -q7_t *arm_nn_mat_mult_kernel_q7_q15_reordered(const q7_t *pA, - const q15_t *pInBuffer, - const uint16_t ch_im_out, - const uint16_t numCol_A, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q7_t *pOut) -{ - -#if defined(ARM_MATH_DSP) - /* set up the second output pointers */ - q7_t *pOut2 = pOut + ch_im_out; - int i; - - /* this loop over rows in A */ - for (i = 0; i < ch_im_out; i += 2) - { - /* setup pointers for B */ - const q15_t *pB = pInBuffer; - const q15_t *pB2 = pB + numCol_A; - - /* align the second pointer for A */ - const q7_t *pA2 = pA + numCol_A; - - /* init the sum with bias */ - q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = numCol_A >> 2; - /* accumulate over the vector */ - while (colCnt) - { - q31_t inA11, inA12, inA21, inA22; - - q31_t inB1 = arm_nn_read_q15x2_ia(&pB); - q31_t inB2 = arm_nn_read_q15x2_ia(&pB2); - - pA = read_and_pad_reordered(pA, &inA11, &inA12); - pA2 = read_and_pad_reordered(pA2, &inA21, &inA22); - - sum = __SMLAD(inA11, inB1, sum); - sum2 = __SMLAD(inA11, inB2, sum2); - sum3 = __SMLAD(inA21, inB1, sum3); - sum4 = __SMLAD(inA21, inB2, sum4); - - inB1 = arm_nn_read_q15x2_ia(&pB); - inB2 = arm_nn_read_q15x2_ia(&pB2); - - sum = __SMLAD(inA12, inB1, sum); - sum2 = __SMLAD(inA12, inB2, sum2); - sum3 = __SMLAD(inA22, inB1, sum3); - sum4 = __SMLAD(inA22, inB2, sum4); - - colCnt--; - } /* while over colCnt */ - colCnt = numCol_A & 0x3; - while (colCnt) - { - q7_t inA1 = *pA++; - q15_t inB1 = *pB++; - q7_t inA2 = *pA2++; - q15_t inB2 = *pB2++; - - sum += inA1 * inB1; - sum2 += inA1 * inB2; - sum3 += inA2 * inB1; - sum4 += inA2 * inB2; - colCnt--; - } /* while over colCnt */ - *pOut++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pOut++ = (q7_t)__SSAT((sum3 >> out_shift), 8); - *pOut2++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - *pOut2++ = (q7_t)__SSAT((sum4 >> out_shift), 8); - - /* skip the row computed with A2 */ - pA += numCol_A; - } /* for over ch_im_out */ - - pOut += ch_im_out; - - /* return the new output pointer with offset */ - return pOut; -#else - (void)pA; - (void)pInBuffer; - (void)ch_im_out; - (void)numCol_A; - (void)bias_shift; - (void)out_shift; - (void)bias; - (void)pOut; - /* To be completed */ - return NULL; -#endif /* ARM_MATH_DSP */ -} +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mat_mult_kernel_q7_q15_reordered.c
+ * Description: Matrix-multiplication function for convolution with reordered columns
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnfunctions.h"
+#include "arm_math.h"
+
+ /**
+ * @brief Matrix-multiplication function for convolution with reordered columns
+ * @param[in] pA pointer to operand A
+ * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
+ * @param[in] ch_im_out numRow of A
+ * @param[in] numCol_A numCol of A
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias the bias
+ * @param[in,out] pOut pointer to output
+ * @return The function returns the incremented output pointer
+ *
+ * @details
+ *
+ * This function assumes that data in pInBuffer are reordered
+ */
+
+q7_t *arm_nn_mat_mult_kernel_q7_q15_reordered(const q7_t * pA,
+ const q15_t * pInBuffer,
+ const uint16_t ch_im_out,
+ const uint16_t numCol_A,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q7_t * pOut)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* set up the second output pointers */
+ q7_t *pOut2 = pOut + ch_im_out;
+ int i;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* align the second pointer for A */
+ const q7_t *pA2 = pA + numCol_A;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA11, inA12, inA21, inA22;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA11, &inA12);
+ pA2 = (q7_t *) read_and_pad_reordered((void *)pA2, &inA21, &inA22);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+ sum3 = __SMLAD(inA21, inB1, sum3);
+ sum4 = __SMLAD(inA21, inB2, sum4);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+ sum3 = __SMLAD(inA22, inB1, sum3);
+ sum4 = __SMLAD(inA22, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q7_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ /* skip the row computed with A2 */
+ pA += numCol_A;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+
+ /* return the new output pointer with offset */
+ return pOut;
+#else
+ /* To be completed */
+ return NULL;
+#endif /* ARM_MATH_DSP */
+}
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c index 9eb02eb..bb9a091 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15.c @@ -1,197 +1,199 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_mat_q7_vec_q15.c - * Description: Mixed Q15-Q7 fully-connected layer function - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Mixed Q15-Q7 fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: 0 - * - * Q7_Q15 version of the fully connected layer - * - * Weights are in q7_t and Activations are in q15_t - * - */ - -arm_status arm_fully_connected_mat_q7_vec_q15(const q15_t *pV, - const q7_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q15_t *pOut, - q15_t *vec_buffer) -{ - (void)vec_buffer; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q7_t *pB = pM; - const q7_t *pB2; - q15_t *pO = pOut; - const q7_t *pBias = bias; - const q15_t *pA = pV; - - uint16_t rowCnt = num_of_rows >> 1; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - pB2 = pB + dim_vec; - - while (colCnt) - { - q31_t inV, inM11, inM12, inM21, inM22; - pB = read_and_pad(pB, &inM11, &inM12); - pB2 = read_and_pad(pB2, &inM21, &inM22); - - inV = arm_nn_read_q15x2_ia(&pA); - - sum = __SMLAD(inV, inM11, sum); - sum2 = __SMLAD(inV, inM21, sum2); - - inV = arm_nn_read_q15x2_ia(&pA); - - sum = __SMLAD(inV, inM12, sum); - sum2 = __SMLAD(inV, inM22, sum2); - - colCnt--; - } - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - q7_t inM2 = *pB2++; - - sum += inV * inM; - sum2 += inV * inM2; - colCnt--; - } /* while over colCnt */ - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum2 >> out_shift), 16)); - - /*adjust the pointers and counters */ - pB += dim_vec; - rowCnt--; - } - - /* left-over part of the rows */ - rowCnt = num_of_rows & 0x1; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - - while (colCnt) - { - q31_t inV1, inV2, inM11, inM12; - - pB = read_and_pad(pB, &inM11, &inM12); - - inV1 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV1, inM11, sum); - - inV2 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV2, inM12, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - sum += inV * inM; - colCnt--; - } - - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - - rowCnt--; - } - -#else - int i, j; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - for (i = 0; i < num_of_rows; i++) - { - int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - for (j = 0; j < dim_vec; j++) - { - ip_out += pV[j] * pM[i * dim_vec + j]; - } - pOut[i] = (q15_t)__SSAT((ip_out >> out_shift), 16); - } - -#endif /* ARM_MATH_DSP */ - - /* Return to ARM_MATH_SUCCESS */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_mat_q7_vec_q15.c
+ * Description: Mixed Q15-Q7 fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Mixed Q15-Q7 fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: 0
+ *
+ * Q7_Q15 version of the fully connected layer
+ *
+ * Weights are in q7_t and Activations are in q15_t
+ *
+ */
+
+arm_status
+arm_fully_connected_mat_q7_vec_q15(const q15_t * pV,
+ const q7_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q15_t * pOut,
+ q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q7_t *pB = pM;
+ const q7_t *pB2;
+ q15_t *pO = pOut;
+ const q7_t *pBias = bias;
+ const q15_t *pA = pV;
+
+ uint16_t rowCnt = num_of_rows >> 1;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+ pB2 = pB + dim_vec;
+
+ while (colCnt)
+ {
+ q31_t inV, inM11, inM12, inM21, inM22;
+ pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12);
+ pB2 = (q7_t *) read_and_pad((void *)pB2, &inM21, &inM22);
+
+ inV = *__SIMD32(pA)++;
+
+ sum = __SMLAD(inV, inM11, sum);
+ sum2 = __SMLAD(inV, inM21, sum2);
+
+ inV = *__SIMD32(pA)++;
+
+ sum = __SMLAD(inV, inM12, sum);
+ sum2 = __SMLAD(inV, inM22, sum2);
+
+ colCnt--;
+ }
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ q7_t inM2 = *pB2++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16));
+
+ /*adjust the pointers and counters */
+ pB += dim_vec;
+ rowCnt--;
+ }
+
+ /* left-over part of the rows */
+ rowCnt = num_of_rows & 0x1;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+
+ while (colCnt)
+ {
+ q31_t inV1, inV2, inM11, inM12;
+
+ pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12);
+
+ inV1 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV1, inM11, sum);
+
+ inV2 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV2, inM12, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ sum += inV * inM;
+ colCnt--;
+ }
+
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+
+ rowCnt--;
+ }
+
+#else
+ int i, j;
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ for (i = 0; i < num_of_rows; i++)
+ {
+ int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (j = 0; j < dim_vec; j++)
+ {
+ ip_out += pV[j] * pM[i * dim_vec + j];
+ }
+ pOut[i] = (q15_t) __SSAT((ip_out >> out_shift), 16);
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to ARM_MATH_SUCCESS */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c index a2da772..b0c308b 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_mat_q7_vec_q15_opt.c @@ -1,417 +1,403 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_mat_q7_vec_q15_opt.c - * Description: Mixed Q15-Q7 opt fully-connected layer function - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Mixed Q15-Q7 opt fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: 0 - * - * Q7_Q15 version of the fully connected layer - * - * Weights are in q7_t and Activations are in q15_t - * - * Limitation: x4 version requires weight reordering to work - * - * Here we use only one pointer to read 4 rows in the weight - * matrix. So if the original q7_t matrix looks like this: - * - * | a11 | a12 | a13 | a14 | a15 | a16 | a17 | - * - * | a21 | a22 | a23 | a24 | a25 | a26 | a27 | - * - * | a31 | a32 | a33 | a34 | a35 | a36 | a37 | - * - * | a41 | a42 | a43 | a44 | a45 | a46 | a47 | - * - * | a51 | a52 | a53 | a54 | a55 | a56 | a57 | - * - * | a61 | a62 | a63 | a64 | a65 | a66 | a67 | - * - * We operates on multiple-of-4 rows, so the first four rows becomes - * - * | a11 | a21 | a12 | a22 | a31 | a41 | a32 | a42 | - * - * | a13 | a23 | a14 | a24 | a33 | a43 | a34 | a44 | - * - * | a15 | a25 | a16 | a26 | a35 | a45 | a36 | a46 | - * - * The column left over will be in-order. - * which is: - * | a17 | a27 | a37 | a47 | - * - * For the left-over rows, we do 1x1 computation, so the data remains - * as its original order. - * - * So the stored weight matrix looks like this: - * - * | a11 | a21 | a12 | a22 | a31 | a41 | - * - * | a32 | a42 | a13 | a23 | a14 | a24 | - * - * | a33 | a43 | a34 | a44 | a15 | a25 | - * - * | a16 | a26 | a35 | a45 | a36 | a46 | - * - * | a17 | a27 | a37 | a47 | a51 | a52 | - * - * | a53 | a54 | a55 | a56 | a57 | a61 | - * - * | a62 | a63 | a64 | a65 | a66 | a67 | - * - */ - -arm_status arm_fully_connected_mat_q7_vec_q15_opt(const q15_t *pV, - const q7_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q15_t *pOut, - q15_t *vec_buffer) -{ - - (void)vec_buffer; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q7_t *pB = pM; - q15_t *pO = pOut; - const q7_t *pBias = bias; - const q15_t *pA = pV; - - uint16_t rowCnt = num_of_rows >> 2; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 1; - - pA = pV; - -#ifdef USE_INTRINSIC - -#ifndef ARM_MATH_BIG_ENDIAN - - while (colCnt) - { - q31_t inM11, inM12, inM13, inM14; - q31_t inV; - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM11, inV, sum); - sum2 = __SMLAD(inM12, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM13, inV, sum3); - sum4 = __SMLAD(inM14, inV, sum4); - colCnt--; - } - -#else - - while (colCnt) - { - q31_t inM11, inM12, inM13, inM14; - q31_t inV; - - inV = *__SIMD32(pA)++; - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM12, inV, sum); - sum2 = __SMLAD(inM11, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM14, inV, sum3); - sum4 = __SMLAD(inM13, inV, sum4); - colCnt--; - } - -#endif /* ARM_MATH_BIG_ENDIAN */ - -#else - - /* - * register needed: - * loop counter: colCnt - * accumulators: sum, sum2, sum3, sum4 - * pointers: pB, pA - * weight data: inM11, inM12, inM13, inM14 - * activation data: inV - */ - -#ifndef ARM_MATH_BIG_ENDIAN - asm volatile("COL_LOOP_%=:\n" - "ldr.w r4, [%[pA]], #4\n" - "ldr.w r1, [%[pB]], #8\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r1, %[sum]\n" - "smlad %[sum2], r4, r0, %[sum2]\n" - "ldr.w r3, [%[pB], #-4]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r3, %[sum3]\n" - "smlad %[sum4], r4, r2, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt) - : "r0", "r1", "r2", "r3", "r4"); -#else - asm volatile("COL_LOOP_%=:\n" - "ldr.w r4, [%[pA]], #4\n" - "ldr.w r1, [%[pB]], #8\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r0, %[sum]\n" - "smlad %[sum2], r4, r1, %[sum2]\n" - "ldr.w r3, [%[pB], #-4]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r2, %[sum3]\n" - "smlad %[sum4], r4, r3, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt) - : "r0", "r1", "r2", "r3", "r4"); -#endif /* ARM_MATH_BIG_ENDIAN */ - -#endif /* USE_INTRINSIC */ - - colCnt = dim_vec & 0x1; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - q7_t inM2 = *pB++; - q7_t inM3 = *pB++; - q7_t inM4 = *pB++; - - sum += inV * inM; - sum2 += inV * inM2; - sum3 += inV * inM3; - sum4 += inV * inM4; - colCnt--; - } /* while over colCnt */ - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum2 >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum3 >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum4 >> out_shift), 16)); - - /* adjust the pointers and counters */ - rowCnt--; - } - - /* left-over part of the rows */ - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - - while (colCnt) - { - q31_t inV1, inV2, inM11, inM12; - - pB = read_and_pad(pB, &inM11, &inM12); - - inV1 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV1, inM11, sum); - - inV2 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV2, inM12, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - sum += inV * inM; - colCnt--; - } - - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - - rowCnt--; - } - -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - uint16_t rowCnt = num_of_rows >> 2; - const q7_t *pB = pM; - const q15_t *pA; - q15_t *pO = pOut; - const q7_t *pBias = bias; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = dim_vec >> 1; - - pA = pV; - - while (colCnt) - { - q15_t inA1 = *pA++; - q15_t inA2 = *pA++; - - q7_t inB1 = *pB++; - q7_t inB3 = *pB++; - q7_t inB2 = *pB++; - q7_t inB4 = *pB++; - - sum += inA1 * inB1 + inA2 * inB2; - sum2 += inA1 * inB3 + inA2 * inB4; - - inB1 = *pB++; - inB3 = *pB++; - inB2 = *pB++; - inB4 = *pB++; - - sum3 += inA1 * inB1 + inA2 * inB2; - sum4 += inA1 * inB3 + inA2 * inB4; - - colCnt--; - } - - colCnt = dim_vec & 0x1; - while (colCnt) - { - q15_t inA = *pA++; - q7_t inB = *pB++; - sum += inA * inB; - inB = *pB++; - sum2 += inA * inB; - inB = *pB++; - sum3 += inA * inB; - inB = *pB++; - sum4 += inA * inB; - - colCnt--; - } - *pO++ = (q15_t)__SSAT((sum >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum2 >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum3 >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum4 >> out_shift), 16); - - rowCnt--; - } - - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - int j; - - pA = pV; - for (j = 0; j < dim_vec; j++) - { - q15_t inA = *pA++; - q7_t inB = *pB++; - ip_out += inA * inB; - } - *pO++ = (q15_t)__SSAT((ip_out >> out_shift), 16); - - rowCnt--; - } - -#endif /* ARM_MATH_DSP */ - - /* Return to ARM_MATH_SUCCESS */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_mat_q7_vec_q15_opt.c
+ * Description: Mixed Q15-Q7 opt fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Mixed Q15-Q7 opt fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: 0
+ *
+ * Q7_Q15 version of the fully connected layer
+ *
+ * Weights are in q7_t and Activations are in q15_t
+ *
+ * Limitation: x4 version requires weight reordering to work
+ *
+ * Here we use only one pointer to read 4 rows in the weight
+ * matrix. So if the original q7_t matrix looks like this:
+ *
+ * | a11 | a12 | a13 | a14 | a15 | a16 | a17 |
+ *
+ * | a21 | a22 | a23 | a24 | a25 | a26 | a27 |
+ *
+ * | a31 | a32 | a33 | a34 | a35 | a36 | a37 |
+ *
+ * | a41 | a42 | a43 | a44 | a45 | a46 | a47 |
+ *
+ * | a51 | a52 | a53 | a54 | a55 | a56 | a57 |
+ *
+ * | a61 | a62 | a63 | a64 | a65 | a66 | a67 |
+ *
+ * We operates on multiple-of-4 rows, so the first four rows becomes
+ *
+ * | a11 | a21 | a12 | a22 | a31 | a41 | a32 | a42 |
+ *
+ * | a13 | a23 | a14 | a24 | a33 | a43 | a34 | a44 |
+ *
+ * | a15 | a25 | a16 | a26 | a35 | a45 | a36 | a46 |
+ *
+ * The column left over will be in-order.
+ * which is:
+ * | a17 | a27 | a37 | a47 |
+ *
+ * For the left-over rows, we do 1x1 computation, so the data remains
+ * as its original order.
+ *
+ * So the stored weight matrix looks like this:
+ *
+ * | a11 | a21 | a12 | a22 | a31 | a41 |
+ *
+ * | a32 | a42 | a13 | a23 | a14 | a24 |
+ *
+ * | a33 | a43 | a34 | a44 | a15 | a25 |
+ *
+ * | a16 | a26 | a35 | a45 | a36 | a46 |
+ *
+ * | a17 | a27 | a37 | a47 | a51 | a52 |
+ *
+ * | a53 | a54 | a55 | a56 | a57 | a61 |
+ *
+ * | a62 | a63 | a64 | a65 | a66 | a67 |
+ *
+ */
+
+arm_status
+arm_fully_connected_mat_q7_vec_q15_opt(const q15_t * pV,
+ const q7_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift, const q7_t * bias, q15_t * pOut, q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q7_t *pB = pM;
+ q15_t *pO = pOut;
+ const q7_t *pBias = bias;
+ const q15_t *pA = pV;
+
+ uint16_t rowCnt = num_of_rows >> 2;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 1;
+
+ pA = pV;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ while (colCnt)
+ {
+ q31_t inM11, inM12, inM13, inM14;
+ q31_t inV;
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM11, inV, sum);
+ sum2 = __SMLAD(inM12, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM13, inV, sum3);
+ sum4 = __SMLAD(inM14, inV, sum4);
+ colCnt--;
+ }
+
+#else
+
+ while (colCnt)
+ {
+ q31_t inM11, inM12, inM13, inM14;
+ q31_t inV;
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM12, inV, sum);
+ sum2 = __SMLAD(inM11, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM14, inV, sum3);
+ sum4 = __SMLAD(inM13, inV, sum4);
+ colCnt--;
+ }
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+ /*
+ * register needed:
+ * loop counter: colCnt
+ * accumulators: sum, sum2, sum3, sum4
+ * pointers: pB, pA
+ * weight data: inM11, inM12, inM13, inM14
+ * activation data: inV
+ */
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r4, [%[pA]], #4\n"
+ "ldr.w r1, [%[pB]], #8\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r1, %[sum]\n"
+ "smlad %[sum2], r4, r0, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-4]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r3, %[sum3]\n"
+ "smlad %[sum4], r4, r2, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum] "+r"(sum),
+ [sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4");
+#else
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r4, [%[pA]], #4\n"
+ "ldr.w r1, [%[pB]], #8\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r0, %[sum]\n"
+ "smlad %[sum2], r4, r1, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-4]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r2, %[sum3]\n"
+ "smlad %[sum4], r4, r3, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum] "+r"(sum),
+ [sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4");
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = dim_vec & 0x1;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ q7_t inM2 = *pB++;
+ q7_t inM3 = *pB++;
+ q7_t inM4 = *pB++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ sum3 += inV * inM3;
+ sum4 += inV * inM4;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum3 >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum4 >> out_shift), 16));
+
+ /* adjust the pointers and counters */
+ rowCnt--;
+ }
+
+ /* left-over part of the rows */
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+
+ while (colCnt)
+ {
+ q31_t inV1, inV2, inM11, inM12;
+
+ pB = (q7_t *) read_and_pad((void *)pB, &inM11, &inM12);
+
+ inV1 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV1, inM11, sum);
+
+ inV2 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV2, inM12, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ sum += inV * inM;
+ colCnt--;
+ }
+
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+
+ rowCnt--;
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t rowCnt = num_of_rows >> 2;
+ const q7_t *pB = pM;
+ const q15_t *pA;
+ q15_t *pO = pOut;
+ const q7_t *pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = dim_vec >> 1;
+
+ pA = pV;
+
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inA2 = *pA++;
+
+ q7_t inB1 = *pB++;
+ q7_t inB3 = *pB++;
+ q7_t inB2 = *pB++;
+ q7_t inB4 = *pB++;
+
+ sum += inA1 * inB1 + inA2 * inB2;
+ sum2 += inA1 * inB3 + inA2 * inB4;
+
+ inB1 = *pB++;
+ inB3 = *pB++;
+ inB2 = *pB++;
+ inB4 = *pB++;
+
+ sum3 += inA1 * inB1 + inA2 * inB2;
+ sum4 += inA1 * inB3 + inA2 * inB4;
+
+ colCnt--;
+ }
+
+ colCnt = dim_vec & 0x1;
+ while (colCnt)
+ {
+ q15_t inA = *pA++;
+ q7_t inB = *pB++;
+ sum += inA * inB;
+ inB = *pB++;
+ sum2 += inA * inB;
+ inB = *pB++;
+ sum3 += inA * inB;
+ inB = *pB++;
+ sum4 += inA * inB;
+
+ colCnt--;
+ }
+ *pO++ = (q15_t) __SSAT((sum >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum2 >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum3 >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum4 >> out_shift), 16);
+
+ rowCnt--;
+ }
+
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ int j;
+
+ pA = pV;
+ for (j = 0; j < dim_vec; j++)
+ {
+ q15_t inA = *pA++;
+ q7_t inB = *pB++;
+ ip_out += inA * inB;
+ }
+ *pO++ = (q15_t) __SSAT((ip_out >> out_shift), 16);
+
+ rowCnt--;
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to ARM_MATH_SUCCESS */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c index d8b6887..a4c6bba 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15.c @@ -1,195 +1,193 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_q15.c - * Description: Q15 basic fully-connected layer function - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Q15 opt fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: 0 - * - */ - -arm_status arm_fully_connected_q15(const q15_t *pV, - const q15_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q15_t *bias, - q15_t *pOut, - q15_t *vec_buffer) -{ - (void)vec_buffer; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q15_t *pB = pM; - const q15_t *pB2 = pB + dim_vec; - q15_t *pO = pOut; - const q15_t *pA; - const q15_t *pBias = bias; - uint16_t rowCnt = num_of_rows >> 1; - - /* this loop loops over different output */ - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - pB2 = pB + dim_vec; - - while (colCnt) - { - q31_t inV1, inM1, inM2; - inV1 = arm_nn_read_q15x2_ia(&pA); - inM1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inV1, inM1, sum); - inM2 = arm_nn_read_q15x2_ia(&pB2); - sum2 = __SMLAD(inV1, inM2, sum2); - - inV1 = arm_nn_read_q15x2_ia(&pA); - inM1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inV1, inM1, sum); - inM2 = arm_nn_read_q15x2_ia(&pB2); - sum2 = __SMLAD(inV1, inM2, sum2); - - colCnt--; - } - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q15_t inM = *pB++; - q15_t inM2 = *pB2++; - - sum += inV * inM; - sum2 += inV * inM2; - colCnt--; - } /* while over colCnt */ - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum2 >> out_shift), 16)); - - /* adjust the pointers and counters */ - pB = pB + dim_vec; - rowCnt--; - } - - rowCnt = num_of_rows & 0x1; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - - while (colCnt) - { - q31_t inV1, inM1; - inV1 = arm_nn_read_q15x2_ia(&pA); - inM1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inV1, inM1, sum); - - inV1 = arm_nn_read_q15x2_ia(&pA); - inM1 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inV1, inM1, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q15_t inM = *pB++; - - sum += inV * inM; - - colCnt--; - } - - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - - rowCnt--; - } - -#else - int i, j; - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - for (i = 0; i < num_of_rows; i++) - { - int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - for (j = 0; j < dim_vec; j++) - { - ip_out += pV[j] * pM[i * dim_vec + j]; - } - pOut[i] = (q15_t)__SSAT((ip_out >> out_shift), 16); - } - -#endif /* ARM_MATH_DSP */ - - /* Return to application */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_q15.c
+ * Description: Q15 basic fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Q15 opt fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: 0
+ *
+ */
+
+arm_status
+arm_fully_connected_q15(const q15_t * pV,
+ const q15_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q15_t * bias,
+ q15_t * pOut,
+ q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q15_t *pB = pM;
+ const q15_t *pB2 = pB + dim_vec;
+ q15_t *pO = pOut;
+ const q15_t *pA;
+ const q15_t *pBias = bias;
+ uint16_t rowCnt = num_of_rows >> 1;
+
+ /* this loop loops over different output */
+ while (rowCnt) {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+ pB2 = pB + dim_vec;
+
+ while (colCnt)
+ {
+ q31_t inV1, inM1, inM2;
+ inV1 = *__SIMD32(pA)++;
+ inM1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inV1, inM1, sum);
+ inM2 = *__SIMD32(pB2)++;
+ sum2 = __SMLAD(inV1, inM2, sum2);
+
+ inV1 = *__SIMD32(pA)++;
+ inM1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inV1, inM1, sum);
+ inM2 = *__SIMD32(pB2)++;
+ sum2 = __SMLAD(inV1, inM2, sum2);
+
+ colCnt--;
+ }
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q15_t inM = *pB++;
+ q15_t inM2 = *pB2++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum2>> out_shift), 16));
+
+ /* adjust the pointers and counters */
+ pB = pB + dim_vec;
+ rowCnt --;
+ }
+
+ rowCnt = num_of_rows & 0x1;
+
+ while (rowCnt) {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+
+ while (colCnt) {
+ q31_t inV1, inM1;
+ inV1 = *__SIMD32(pA)++;
+ inM1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inV1, inM1, sum);
+
+ inV1 = *__SIMD32(pA)++;
+ inM1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inV1, inM1, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while(colCnt) {
+ q15_t inV = *pA++;
+ q15_t inM = *pB++;
+
+ sum += inV * inM;
+
+ colCnt--;
+ }
+
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+
+ rowCnt --;
+ }
+
+#else
+ int i, j;
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ for (i = 0; i < num_of_rows; i++)
+ {
+ int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (j = 0; j < dim_vec; j++)
+ {
+ ip_out += pV[j] * pM[i * dim_vec + j];
+ }
+ pOut[i] = (q15_t) __SSAT((ip_out >> out_shift), 16);
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c index f6c9b16..8f3bbea 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q15_opt.c @@ -1,336 +1,332 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_q15_opt.c - * Description: Q15 opt fully-connected layer function - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Q15 opt fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: 0 - * - * Here we use only one pointer to read 4 rows in the weight - * matrix. So if the original matrix looks like this: - * - * | a11 | a12 | a13 | - * - * | a21 | a22 | a23 | - * - * | a31 | a32 | a33 | - * - * | a41 | a42 | a43 | - * - * | a51 | a52 | a53 | - * - * | a61 | a62 | a63 | - * - * We operates on multiple-of-4 rows, so the first four rows becomes - * - * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 | - * - * | a13 | a23 | a33 | a43 | - * - * Remaining rows are kept the same original order. - * - * So the stored weight matrix looks like this: - * - * - * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 | - * - * | a13 | a23 | a33 | a43 | a51 | a52 | a53 | a61 | - * - * | a62 | a63 | - */ - -arm_status arm_fully_connected_q15_opt(const q15_t *pV, - const q15_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q15_t *bias, - q15_t *pOut, - q15_t *vec_buffer) -{ - (void)vec_buffer; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q15_t *pB = pM; - q15_t *pO = pOut; - const q15_t *pBias = bias; - const q15_t *pA = pV; - - uint16_t rowCnt = num_of_rows >> 2; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 1; - - pA = pV; - -#ifdef USE_INTRINSIC - - while (colCnt) - { - q31_t inM11, inM12, inM13, inM14; - q31_t inV; - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q15x2_ia(&pB); - sum = __SMLAD(inV, inM11, sum); - inM12 = arm_nn_read_q15x2_ia(&pB); - sum2 = __SMLAD(inV, inM12, sum2); - inM13 = arm_nn_read_q15x2_ia(&pB); - sum3 = __SMLAD(inV, inM13, sum3); - inM14 = arm_nn_read_q15x2_ia(&pB); - sum4 = __SMLAD(inV, inM14, sum4); - colCnt--; - } - -#else - - /* - * register needed: - * loop counter: colCnt - * accumulators: sum, sum2, sum3, sum4 - * pointers: pB, pA - * weight data: inM11, inM12, inM13, inM14 - * activation data: inV - */ - - asm volatile("COL_LOOP_%=:\n" - "ldr.w r4, [%[pA]], #4\n" - "ldr.w r0, [%[pB]], #16\n" - "smlad %[sum], r4, r0, %[sum]\n" - "ldr.w r1, [%[pB] , #-12]\n" - "smlad %[sum2], r4, r1, %[sum2]\n" - "ldr.w r2, [%[pB] , #-8]\n" - "smlad %[sum3], r4, r2, %[sum3]\n" - "ldr.w r3, [%[pB] , #-4]\n" - "smlad %[sum4], r4, r3, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt) - : "r0", "r1", "r2", "r3", "r4"); - -#endif /* USE_INTRINSIC */ - - colCnt = dim_vec & 0x1; - while (colCnt) - { - - q15_t inV = *pA++; - q15_t inM = *pB++; - q15_t inM2 = *pB++; - q15_t inM3 = *pB++; - q15_t inM4 = *pB++; - - sum += inV * inM; - sum2 += inV * inM2; - sum3 += inV * inM3; - sum4 += inV * inM4; - colCnt--; - } /* while over colCnt */ - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum2 >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum3 >> out_shift), 16)); - *pO++ = (q15_t)(__SSAT((sum4 >> out_shift), 16)); - - /* adjust the pointers and counters */ - rowCnt--; - } - - /* left-over part of the rows */ - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - - while (colCnt) - { - q31_t inV1, inV2, inM1, inM2; - - inM1 = arm_nn_read_q15x2_ia(&pB); - inV1 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV1, inM1, sum); - - inM2 = arm_nn_read_q15x2_ia(&pB); - inV2 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV2, inM2, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q15_t inM = *pB++; - sum += inV * inM; - colCnt--; - } - - *pO++ = (q15_t)(__SSAT((sum >> out_shift), 16)); - - rowCnt--; - } - -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - uint16_t rowCnt = num_of_rows >> 2; - const q15_t *pB = pM; - const q15_t *pA; - q15_t *pO = pOut; - const q15_t *pBias = bias; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 1; - - pA = pV; - while (colCnt) - { - q15_t inA1 = *pA++; - q15_t inA2 = *pA++; - - q15_t inB1 = *pB++; - q15_t inB2 = *pB++; - sum += inA1 * inB1 + inA2 * inB2; - - inB1 = *pB++; - inB2 = *pB++; - sum2 += inA1 * inB1 + inA2 * inB2; - - inB1 = *pB++; - inB2 = *pB++; - sum3 += inA1 * inB1 + inA2 * inB2; - - inB1 = *pB++; - inB2 = *pB++; - sum4 += inA1 * inB1 + inA2 * inB2; - - colCnt--; - } - colCnt = dim_vec & 0x1; - while (colCnt) - { - q15_t inA = *pA++; - q15_t inB = *pB++; - sum += inA * inB; - inB = *pB++; - sum2 += inA * inB; - inB = *pB++; - sum3 += inA * inB; - inB = *pB++; - sum4 += inA * inB; - colCnt--; - } - *pO++ = (q15_t)__SSAT((sum >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum2 >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum3 >> out_shift), 16); - *pO++ = (q15_t)__SSAT((sum4 >> out_shift), 16); - - rowCnt--; - } - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - int j; - - pA = pV; - for (j = 0; j < dim_vec; j++) - { - q15_t inA = *pA++; - q15_t inB = *pB++; - ip_out += inA * inB; - } - *pO++ = (q15_t)__SSAT((ip_out >> out_shift), 16); - - rowCnt--; - } - -#endif /* ARM_MATH_DSP */ - - /* Return to ARM_MATH_SUCCESS */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_q15_opt.c
+ * Description: Q15 opt fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Q15 opt fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: 0
+ *
+ * Here we use only one pointer to read 4 rows in the weight
+ * matrix. So if the original matrix looks like this:
+ *
+ * | a11 | a12 | a13 |
+ *
+ * | a21 | a22 | a23 |
+ *
+ * | a31 | a32 | a33 |
+ *
+ * | a41 | a42 | a43 |
+ *
+ * | a51 | a52 | a53 |
+ *
+ * | a61 | a62 | a63 |
+ *
+ * We operates on multiple-of-4 rows, so the first four rows becomes
+ *
+ * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 |
+ *
+ * | a13 | a23 | a33 | a43 |
+ *
+ * Remaining rows are kept the same original order.
+ *
+ * So the stored weight matrix looks like this:
+ *
+ *
+ * | a11 | a12 | a21 | a22 | a31 | a32 | a41 | a42 |
+ *
+ * | a13 | a23 | a33 | a43 | a51 | a52 | a53 | a61 |
+ *
+ * | a62 | a63 |
+ */
+
+arm_status
+arm_fully_connected_q15_opt(const q15_t * pV,
+ const q15_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q15_t * bias,
+ q15_t * pOut,
+ q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q15_t *pB = pM;
+ q15_t *pO = pOut;
+ const q15_t *pBias = bias;
+ const q15_t *pA = pV;
+
+ uint16_t rowCnt = num_of_rows >> 2;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 1;
+
+ pA = pV;
+
+#ifdef USE_INTRINSIC
+
+ while (colCnt)
+ {
+ q31_t inM11, inM12, inM13, inM14;
+ q31_t inV;
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ sum = __SMLAD(inV, inM11, sum);
+ inM12 = *__SIMD32(pB)++;
+ sum2 = __SMLAD(inV, inM12, sum2);
+ inM13 = *__SIMD32(pB)++;
+ sum3 = __SMLAD(inV, inM13, sum3);
+ inM14 = *__SIMD32(pB)++;
+ sum4 = __SMLAD(inV, inM14, sum4);
+ colCnt--;
+ }
+
+#else
+
+ /*
+ * register needed:
+ * loop counter: colCnt
+ * accumulators: sum, sum2, sum3, sum4
+ * pointers: pB, pA
+ * weight data: inM11, inM12, inM13, inM14
+ * activation data: inV
+ */
+
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r4, [%[pA]], #4\n"
+ "ldr.w r0, [%[pB]], #16\n"
+ "smlad %[sum], r4, r0, %[sum]\n"
+ "ldr.w r1, [%[pB] , #-12]\n"
+ "smlad %[sum2], r4, r1, %[sum2]\n"
+ "ldr.w r2, [%[pB] , #-8]\n"
+ "smlad %[sum3], r4, r2, %[sum3]\n"
+ "ldr.w r3, [%[pB] , #-4]\n"
+ "smlad %[sum4], r4, r3, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum] "+r"(sum),
+ [sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4");
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = dim_vec & 0x1;
+ while (colCnt)
+ {
+
+ q15_t inV = *pA++;
+ q15_t inM = *pB++;
+ q15_t inM2 = *pB++;
+ q15_t inM3 = *pB++;
+ q15_t inM4 = *pB++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ sum3 += inV * inM3;
+ sum4 += inV * inM4;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum2 >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum3 >> out_shift), 16));
+ *pO++ = (q15_t) (__SSAT((sum4 >> out_shift), 16));
+
+ /* adjust the pointers and counters */
+ rowCnt--;
+ }
+
+ /* left-over part of the rows */
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+
+ while (colCnt)
+ {
+ q31_t inV1, inV2, inM1, inM2;
+
+ inM1 = *__SIMD32(pB)++;
+ inV1 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV1, inM1, sum);
+
+ inM2 = *__SIMD32(pB)++;
+ inV2 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV2, inM2, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q15_t inM = *pB++;
+ sum += inV * inM;
+ colCnt--;
+ }
+
+ *pO++ = (q15_t) (__SSAT((sum >> out_shift), 16));
+
+ rowCnt--;
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t rowCnt = num_of_rows >> 2;
+ const q15_t *pB = pM;
+ const q15_t *pA;
+ q15_t *pO = pOut;
+ const q15_t *pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 1;
+
+ pA = pV;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inA2 = *pA++;
+
+ q15_t inB1 = *pB++;
+ q15_t inB2 = *pB++;
+ sum += inA1 * inB1 + inA2 * inB2;
+
+ inB1 = *pB++;
+ inB2 = *pB++;
+ sum2 += inA1 * inB1 + inA2 * inB2;
+
+ inB1 = *pB++;
+ inB2 = *pB++;
+ sum3 += inA1 * inB1 + inA2 * inB2;
+
+ inB1 = *pB++;
+ inB2 = *pB++;
+ sum4 += inA1 * inB1 + inA2 * inB2;
+
+ colCnt--;
+ }
+ colCnt = dim_vec & 0x1;
+ while (colCnt)
+ {
+ q15_t inA = *pA++;
+ q15_t inB = *pB++;
+ sum += inA * inB;
+ inB = *pB++;
+ sum2 += inA * inB;
+ inB = *pB++;
+ sum3 += inA * inB;
+ inB = *pB++;
+ sum4 += inA * inB;
+ colCnt--;
+ }
+ *pO++ = (q15_t) __SSAT((sum >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum2 >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum3 >> out_shift), 16);
+ *pO++ = (q15_t) __SSAT((sum4 >> out_shift), 16);
+
+ rowCnt--;
+ }
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ int j;
+
+ pA = pV;
+ for (j = 0; j < dim_vec; j++)
+ {
+ q15_t inA = *pA++;
+ q15_t inB = *pB++;
+ ip_out += inA * inB;
+ }
+ *pO++ = (q15_t) __SSAT((ip_out >> out_shift), 16);
+
+ rowCnt--;
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to ARM_MATH_SUCCESS */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c index d500efe..75e924f 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7.c @@ -1,200 +1,198 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_q7.c - * Description: Q7 basic fully-connected layer function - * - * $Date: July 20, 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Q7 basic fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: dim_vec - * - * This basic function is designed to work with regular weight - * matrix without interleaving. - * - */ - -arm_status arm_fully_connected_q7(const q7_t *pV, - const q7_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q7_t *pOut, - q15_t *vec_buffer) -{ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q7_t *pB = pM; - const q7_t *pB2; - q7_t *pO = pOut; - const q7_t *pBias = bias; - const q15_t *pA; - uint16_t rowCnt = num_of_rows >> 1; - - /* expand the vector into the buffer */ - arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec); - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = dim_vec >> 2; - - pA = vec_buffer; - pB2 = pB + dim_vec; - - while (colCnt) - { - q31_t inV, inM11, inM12, inM21, inM22; - pB = read_and_pad_reordered(pB, &inM11, &inM12); - pB2 = read_and_pad_reordered(pB2, &inM21, &inM22); - - inV = arm_nn_read_q15x2_ia(&pA); - - sum = __SMLAD(inV, inM11, sum); - sum2 = __SMLAD(inV, inM21, sum2); - - inV = arm_nn_read_q15x2_ia(&pA); - - sum = __SMLAD(inV, inM12, sum); - sum2 = __SMLAD(inV, inM22, sum2); - - colCnt--; - } - colCnt = dim_vec & 0x3; - while (colCnt) - { - q7_t inV = *pA++; - q15_t inM = *pB++; - q15_t inM2 = *pB2++; - - sum += inV * inM; - sum2 += inV * inM2; - colCnt--; - } /* while over colCnt */ - *pO++ = (q7_t)(__SSAT((sum >> out_shift), 8)); - *pO++ = (q7_t)(__SSAT((sum2 >> out_shift), 8)); - - /* adjust the pointers and counters */ - pB += dim_vec; - rowCnt--; - } - - /* left-over part of the rows */ - rowCnt = num_of_rows & 0x1; - - while (rowCnt) - { - uint16_t colCnt = dim_vec >> 2; - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - pA = vec_buffer; - - while (colCnt) - { - q31_t inV1, inV2, inM11, inM12; - - pB = read_and_pad_reordered(pB, &inM11, &inM12); - - inV1 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV1, inM11, sum); - - inV2 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV2, inM12, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q7_t inV = *pA++; - q15_t inM = *pB++; - sum += inV * inM; - colCnt--; - } - - *pO++ = (q7_t)(__SSAT((sum >> out_shift), 8)); - - rowCnt--; - } - -#else - (void)vec_buffer; - int i, j; - - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - for (i = 0; i < num_of_rows; i++) - { - int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift); - for (j = 0; j < dim_vec; j++) - { - ip_out += pV[j] * pM[i * dim_vec + j]; - } - pOut[i] = (q7_t)__SSAT((ip_out >> out_shift), 8); - } - -#endif /* ARM_MATH_DSP */ - - /* Return to ARM_MATH_SUCCESS */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_q7.c
+ * Description: Q7 basic fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Q7 basic fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: dim_vec
+ *
+ * This basic function is designed to work with regular weight
+ * matrix without interleaving.
+ *
+ */
+
+arm_status
+arm_fully_connected_q7(const q7_t * pV,
+ const q7_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift, const q7_t * bias, q7_t * pOut, q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q7_t *pB = pM;
+ const q7_t *pB2;
+ q7_t *pO = pOut;
+ const q7_t *pBias = bias;
+ q15_t *pA;
+ uint16_t rowCnt = num_of_rows >> 1;
+
+ /* expand the vector into the buffer */
+ arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec);
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = vec_buffer;
+ pB2 = pB + dim_vec;
+
+ while (colCnt)
+ {
+ q31_t inV, inM11, inM12, inM21, inM22;
+ pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12);
+ pB2 = (q7_t *) read_and_pad_reordered((void *)pB2, &inM21, &inM22);
+
+ inV = *__SIMD32(pA)++;
+
+ sum = __SMLAD(inV, inM11, sum);
+ sum2 = __SMLAD(inV, inM21, sum2);
+
+ inV = *__SIMD32(pA)++;
+
+ sum = __SMLAD(inV, inM12, sum);
+ sum2 = __SMLAD(inV, inM22, sum2);
+
+ colCnt--;
+ }
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q7_t inV = *pA++;
+ q15_t inM = *pB++;
+ q15_t inM2 = *pB2++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8));
+ *pO++ = (q7_t) (__SSAT((sum2 >> out_shift), 8));
+
+ /* adjust the pointers and counters */
+ pB += dim_vec;
+ rowCnt--;
+ }
+
+ /* left-over part of the rows */
+ rowCnt = num_of_rows & 0x1;
+
+ while (rowCnt)
+ {
+ uint16_t colCnt = dim_vec >> 2;
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ pA = vec_buffer;
+
+ while (colCnt)
+ {
+ q31_t inV1, inV2, inM11, inM12;
+
+ pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12);
+
+ inV1 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV1, inM11, sum);
+
+ inV2 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV2, inM12, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q7_t inV = *pA++;
+ q15_t inM = *pB++;
+ sum += inV * inM;
+ colCnt--;
+ }
+
+ *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8));
+
+ rowCnt--;
+ }
+
+#else
+ int i, j;
+
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ for (i = 0; i < num_of_rows; i++)
+ {
+ int ip_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (j = 0; j < dim_vec; j++)
+ {
+ ip_out += pV[j] * pM[i * dim_vec + j];
+ }
+ pOut[i] = (q7_t) __SSAT((ip_out >> out_shift), 8);
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to ARM_MATH_SUCCESS */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c index 2f3d653..d197adc 100644 --- a/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c +++ b/Drivers/CMSIS/NN/Source/FullyConnectedFunctions/arm_fully_connected_q7_opt.c @@ -1,495 +1,484 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_fully_connected_q7_opt.c - * Description: Q7 basic fully-connected layer function - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup FC - * @{ - */ - -/** - * @brief Q7 opt fully-connected layer function - * @param[in] pV pointer to input vector - * @param[in] pM pointer to matrix weights - * @param[in] dim_vec length of the vector - * @param[in] num_of_rows number of rows in weight matrix - * @param[in] bias_shift amount of left-shift for bias - * @param[in] out_shift amount of right-shift for output - * @param[in] bias pointer to bias - * @param[in,out] pOut pointer to output vector - * @param[in,out] vec_buffer pointer to buffer space for input - * @return The function returns <code>ARM_MATH_SUCCESS</code> - * - * @details - * - * <b>Buffer size:</b> - * - * vec_buffer size: dim_vec - * - * This opt function is designed to work with interleaved weight - * matrix. The vector input is assumed in q7_t format, we call - * arm_q7_to_q15_no_shift_shuffle function to expand into - * q15_t format with certain weight re-ordering, refer to the function - * comments for more details. - * Here we use only one pointer to read 4 rows in the weight - * matrix. So if the original q7_t matrix looks like this: - * - * | a11 | a12 | a13 | a14 | a15 | a16 | a17 | - * - * | a21 | a22 | a23 | a24 | a25 | a26 | a27 | - * - * | a31 | a32 | a33 | a34 | a35 | a36 | a37 | - * - * | a41 | a42 | a43 | a44 | a45 | a46 | a47 | - * - * | a51 | a52 | a53 | a54 | a55 | a56 | a57 | - * - * | a61 | a62 | a63 | a64 | a65 | a66 | a67 | - * - * - * We operates on multiple-of-4 rows, so the first four rows becomes - * - * | a11 | a21 | a13 | a23 | a31 | a41 | a33 | a43 | - * - * | a12 | a22 | a14 | a24 | a32 | a42 | a34 | a44 | - * - * | a15 | a25 | a35 | a45 | a16 | a26 | a36 | a46 | - * - * So within the kernel, we first read the re-ordered vector in as: - * - * | b1 | b3 | and | b2 | b4 | - * - * the four q31_t weights will look like - * - * | a11 | a13 |, | a21 | a23 |, | a31 | a33 |, | a41 | a43 | - * - * | a12 | a14 |, | a22 | a24 |, | a32 | a34 |, | a42 | a44 | - * - * The column left over will be in-order. - * which is: - * - * | a17 | a27 | a37 | a47 | - * - * For the left-over rows, we do 1x1 computation, so the data remains - * as its original order. - * - * So the stored weight matrix looks like this: - * - * | a11 | a21 | a13 | a23 | a31 | a41 | - * - * | a33 | a43 | a12 | a22 | a14 | a24 | - * - * | a32 | a42 | a34 | a44 | a15 | a25 | - * - * | a35 | a45 | a16 | a26 | a36 | a46 | - * - * | a17 | a27 | a37 | a47 | a51 | a52 | - * - * | a53 | a54 | a55 | a56 | a57 | a61 | - * - * | a62 | a63 | a64 | a65 | a66 | a67 | - * - * - */ - -arm_status arm_fully_connected_q7_opt(const q7_t *pV, - const q7_t *pM, - const uint16_t dim_vec, - const uint16_t num_of_rows, - const uint16_t bias_shift, - const uint16_t out_shift, - const q7_t *bias, - q7_t *pOut, - q15_t *vec_buffer) -{ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - const q7_t *pB = pM; - q7_t *pO = pOut; - const q7_t *pBias = bias; - const q15_t *pA; - uint16_t rowCnt = num_of_rows >> 2; - - arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec); - - while (rowCnt) - { - - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = vec_buffer; - -#ifdef USE_INTRINSIC - -#ifndef ARM_MATH_BIG_ENDIAN - while (colCnt) - { - q31_t inM11, inM12, inM13, inM14; - q31_t inV; - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM11, inV, sum); - sum2 = __SMLAD(inM12, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM13, inV, sum3); - sum4 = __SMLAD(inM14, inV, sum4); - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM11, inV, sum); - sum2 = __SMLAD(inM12, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM13, inV, sum3); - sum4 = __SMLAD(inM14, inV, sum4); - colCnt--; - } -#else - while (colCnt) - { - q31_t inM11, inM12, inM13, inM14; - q31_t inV; - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM12, inV, sum); - sum2 = __SMLAD(inM11, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM14, inV, sum3); - sum4 = __SMLAD(inM13, inV, sum4); - - inV = arm_nn_read_q15x2_ia(&pA); - inM11 = arm_nn_read_q7x4_ia(&pB); - inM12 = __SXTB16(__ROR(inM11, 8)); - inM11 = __SXTB16(inM11); - sum = __SMLAD(inM12, inV, sum); - sum2 = __SMLAD(inM11, inV, sum2); - inM13 = arm_nn_read_q7x4_ia(&pB); - inM14 = __SXTB16(__ROR(inM13, 8)); - inM13 = __SXTB16(inM13); - sum3 = __SMLAD(inM14, inV, sum3); - sum4 = __SMLAD(inM13, inV, sum4); - colCnt--; - } -#endif /* ARM_MATH_BIG_ENDIAN */ - -#else - - /* - * register needed: - * loop counter: colCnt - * accumulators: sum, sum2, sum3, sum4 - * pointers: pB, pA - * weight data: inM11, inM12, inM13, inM14 - * activation data: inV - */ - -#ifndef ARM_MATH_BIG_ENDIAN - asm volatile("COL_LOOP_%=:\n" - "ldr.w r4, [%[pA]], #8\n" - "ldr.w r1, [%[pB]], #16\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r1, %[sum]\n" - "smlad %[sum2], r4, r0, %[sum2]\n" - "ldr.w r3, [%[pB], #-12]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r3, %[sum3]\n" - "smlad %[sum4], r4, r2, %[sum4]\n" - "ldr.w r4, [%[pA], #-4]\n" - "ldr.w r1, [%[pB], #-8]\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r1, %[sum]\n" - "smlad %[sum2], r4, r0, %[sum2]\n" - "ldr.w r3, [%[pB], #-4]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r3, %[sum3]\n" - "smlad %[sum4], r4, r2, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt) - : "r0", "r1", "r2", "r3", "r4"); -#else - asm volatile("COL_LOOP_%=:\n" - "ldr.w r4, [%[pA]], #8\n" - "ldr.w r1, [%[pB]], #16\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r0, %[sum]\n" - "smlad %[sum2], r4, r1, %[sum2]\n" - "ldr.w r3, [%[pB], #-12]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r2, %[sum3]\n" - "smlad %[sum4], r4, r3, %[sum4]\n" - "ldr.w r4, [%[pA], #-4]\n" - "ldr.w r1, [%[pB], #-8]\n" - "mov.w r0, r1, ror #8\n" - "sxtb16 r0, r0\n" - "sxtb16 r1, r1\n" - "smlad %[sum], r4, r0, %[sum]\n" - "smlad %[sum2], r4, r1, %[sum2]\n" - "ldr.w r3, [%[pB], #-4]\n" - "mov.w r2, r3, ror #8\n" - "sxtb16 r2, r2\n" - "sxtb16 r3, r3\n" - "smlad %[sum3], r4, r2, %[sum3]\n" - "smlad %[sum4], r4, r3, %[sum4]\n" - "subs %[colCnt], #1\n" - "bne COL_LOOP_%=\n" - : [ sum ] "+r"(sum), - [ sum2 ] "+r"(sum2), - [ sum3 ] "+r"(sum3), - [ sum4 ] "+r"(sum4), - [ pB ] "+r"(pB), - [ pA ] "+r"(pA) - : [ colCnt ] "r"(colCnt) - : "r0", "r1", "r2", "r3", "r4"); -#endif /* ARM_MATH_BIG_ENDIAN */ - -#endif /* USE_INTRINSIC */ - - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - q7_t inM2 = *pB++; - q7_t inM3 = *pB++; - q7_t inM4 = *pB++; - - sum += inV * inM; - sum2 += inV * inM2; - sum3 += inV * inM3; - sum4 += inV * inM4; - colCnt--; - } /* while over colCnt */ - *pO++ = (q7_t)(__SSAT((sum >> out_shift), 8)); - *pO++ = (q7_t)(__SSAT((sum2 >> out_shift), 8)); - *pO++ = (q7_t)(__SSAT((sum3 >> out_shift), 8)); - *pO++ = (q7_t)(__SSAT((sum4 >> out_shift), 8)); - - /* adjust the pointers and counters */ - rowCnt--; - } - - /* left-over part of the rows */ - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - uint16_t colCnt = dim_vec >> 2; - - pA = vec_buffer; - - while (colCnt) - { - q31_t inV1, inV2, inM11, inM12; - - pB = read_and_pad_reordered(pB, &inM11, &inM12); - - inV1 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV1, inM11, sum); - - inV2 = arm_nn_read_q15x2_ia(&pA); - sum = __SMLAD(inV2, inM12, sum); - - colCnt--; - } - - /* left-over of the vector */ - colCnt = dim_vec & 0x3; - while (colCnt) - { - q15_t inV = *pA++; - q7_t inM = *pB++; - sum += inV * inM; - colCnt--; - } - - *pO++ = (q7_t)(__SSAT((sum >> out_shift), 8)); - - rowCnt--; - } - -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - (void)vec_buffer; - uint16_t rowCnt = num_of_rows >> 2; - const q7_t *pB = pM; - const q7_t *pA; - q7_t *pO = pOut; - const q7_t *pBias = bias; - - while (rowCnt) - { - q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - uint16_t colCnt = dim_vec >> 2; - - pA = pV; - - while (colCnt) - { - q7_t inA1 = *pA++; - q7_t inA3 = *pA++; - q7_t inA2 = *pA++; - q7_t inA4 = *pA++; - - q7_t inB1 = *pB++; - q7_t inB3 = *pB++; - q7_t inB2 = *pB++; - q7_t inB4 = *pB++; - - sum += inA1 * inB1 + inA2 * inB2; - sum2 += inA1 * inB3 + inA2 * inB4; - - inB1 = *pB++; - inB3 = *pB++; - inB2 = *pB++; - inB4 = *pB++; - - sum3 += inA1 * inB1 + inA2 * inB2; - sum4 += inA1 * inB3 + inA2 * inB4; - - inB1 = *pB++; - inB3 = *pB++; - inB2 = *pB++; - inB4 = *pB++; - - sum += inA3 * inB1 + inA4 * inB2; - sum2 += inA3 * inB3 + inA4 * inB4; - - inB1 = *pB++; - inB3 = *pB++; - inB2 = *pB++; - inB4 = *pB++; - - sum3 += inA3 * inB1 + inA4 * inB2; - sum4 += inA3 * inB3 + inA4 * inB4; - - colCnt--; - } - colCnt = dim_vec & 0x3; - while (colCnt) - { - q7_t inA = *pA++; - q7_t inB = *pB++; - sum += inA * inB; - inB = *pB++; - sum2 += inA * inB; - inB = *pB++; - sum3 += inA * inB; - inB = *pB++; - sum4 += inA * inB; - - colCnt--; - } - *pO++ = (q7_t)__SSAT((sum >> out_shift), 8); - *pO++ = (q7_t)__SSAT((sum2 >> out_shift), 8); - *pO++ = (q7_t)__SSAT((sum3 >> out_shift), 8); - *pO++ = (q7_t)__SSAT((sum4 >> out_shift), 8); - - rowCnt--; - } - - rowCnt = num_of_rows & 0x3; - - while (rowCnt) - { - int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift); - - int j; - - pA = pV; - for (j = 0; j < dim_vec; j++) - { - q7_t inA = *pA++; - q7_t inB = *pB++; - ip_out += inA * inB; - } - *pO++ = (q7_t)__SSAT((ip_out >> out_shift), 8); - - rowCnt--; - } - -#endif /* ARM_MATH_DSP */ - - /* Return to ARM_MATH_SUCCESS */ - return (ARM_MATH_SUCCESS); -} - -/** - * @} end of FC group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_fully_connected_q7_opt.c
+ * Description: Q7 basic fully-connected layer function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup FC
+ * @{
+ */
+
+ /**
+ * @brief Q7 opt fully-connected layer function
+ * @param[in] pV pointer to input vector
+ * @param[in] pM pointer to matrix weights
+ * @param[in] dim_vec length of the vector
+ * @param[in] num_of_rows number of rows in weight matrix
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias pointer to bias
+ * @param[in,out] pOut pointer to output vector
+ * @param[in,out] vec_buffer pointer to buffer space for input
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * vec_buffer size: dim_vec
+ *
+ * This opt function is designed to work with interleaved weight
+ * matrix. The vector input is assumed in q7_t format, we call
+ * arm_q7_to_q15_no_shift_shuffle function to expand into
+ * q15_t format with certain weight re-ordering, refer to the function
+ * comments for more details.
+ * Here we use only one pointer to read 4 rows in the weight
+ * matrix. So if the original q7_t matrix looks like this:
+ *
+ * | a11 | a12 | a13 | a14 | a15 | a16 | a17 |
+ *
+ * | a21 | a22 | a23 | a24 | a25 | a26 | a27 |
+ *
+ * | a31 | a32 | a33 | a34 | a35 | a36 | a37 |
+ *
+ * | a41 | a42 | a43 | a44 | a45 | a46 | a47 |
+ *
+ * | a51 | a52 | a53 | a54 | a55 | a56 | a57 |
+ *
+ * | a61 | a62 | a63 | a64 | a65 | a66 | a67 |
+ *
+ *
+ * We operates on multiple-of-4 rows, so the first four rows becomes
+ *
+ * | a11 | a21 | a13 | a23 | a31 | a41 | a33 | a43 |
+ *
+ * | a12 | a22 | a14 | a24 | a32 | a42 | a34 | a44 |
+ *
+ * | a15 | a25 | a35 | a45 | a16 | a26 | a36 | a46 |
+ *
+ * So within the kernel, we first read the re-ordered vector in as:
+ *
+ * | b1 | b3 | and | b2 | b4 |
+ *
+ * the four q31_t weights will look like
+ *
+ * | a11 | a13 |, | a21 | a23 |, | a31 | a33 |, | a41 | a43 |
+ *
+ * | a12 | a14 |, | a22 | a24 |, | a32 | a34 |, | a42 | a44 |
+ *
+ * The column left over will be in-order.
+ * which is:
+ *
+ * | a17 | a27 | a37 | a47 |
+ *
+ * For the left-over rows, we do 1x1 computation, so the data remains
+ * as its original order.
+ *
+ * So the stored weight matrix looks like this:
+ *
+ * | a11 | a21 | a13 | a23 | a31 | a41 |
+ *
+ * | a33 | a43 | a12 | a22 | a14 | a24 |
+ *
+ * | a32 | a42 | a34 | a44 | a15 | a25 |
+ *
+ * | a35 | a45 | a16 | a26 | a36 | a46 |
+ *
+ * | a17 | a27 | a37 | a47 | a51 | a52 |
+ *
+ * | a53 | a54 | a55 | a56 | a57 | a61 |
+ *
+ * | a62 | a63 | a64 | a65 | a66 | a67 |
+ *
+ *
+ */
+
+arm_status
+arm_fully_connected_q7_opt(const q7_t * pV,
+ const q7_t * pM,
+ const uint16_t dim_vec,
+ const uint16_t num_of_rows,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q7_t * pOut,
+ q15_t * vec_buffer)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ const q7_t *pB = pM;
+ q7_t *pO = pOut;
+ const q7_t *pBias = bias;
+ q15_t *pA;
+ uint16_t rowCnt = num_of_rows >> 2;
+
+ arm_q7_to_q15_reordered_no_shift(pV, vec_buffer, dim_vec);
+
+ while (rowCnt)
+ {
+
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = vec_buffer;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ while (colCnt)
+ {
+ q31_t inM11, inM12, inM13, inM14;
+ q31_t inV;
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM11, inV, sum);
+ sum2 = __SMLAD(inM12, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM13, inV, sum3);
+ sum4 = __SMLAD(inM14, inV, sum4);
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM11, inV, sum);
+ sum2 = __SMLAD(inM12, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM13, inV, sum3);
+ sum4 = __SMLAD(inM14, inV, sum4);
+ colCnt--;
+ }
+#else
+ while (colCnt)
+ {
+ q31_t inM11, inM12, inM13, inM14;
+ q31_t inV;
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM12, inV, sum);
+ sum2 = __SMLAD(inM11, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM14, inV, sum3);
+ sum4 = __SMLAD(inM13, inV, sum4);
+
+ inV = *__SIMD32(pA)++;
+ inM11 = *__SIMD32(pB)++;
+ inM12 = __SXTB16(__ROR(inM11, 8));
+ inM11 = __SXTB16(inM11);
+ sum = __SMLAD(inM12, inV, sum);
+ sum2 = __SMLAD(inM11, inV, sum2);
+ inM13 = *__SIMD32(pB)++;
+ inM14 = __SXTB16(__ROR(inM13, 8));
+ inM13 = __SXTB16(inM13);
+ sum3 = __SMLAD(inM14, inV, sum3);
+ sum4 = __SMLAD(inM13, inV, sum4);
+ colCnt--;
+ }
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+ /*
+ * register needed:
+ * loop counter: colCnt
+ * accumulators: sum, sum2, sum3, sum4
+ * pointers: pB, pA
+ * weight data: inM11, inM12, inM13, inM14
+ * activation data: inV
+ */
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r4, [%[pA]], #8\n"
+ "ldr.w r1, [%[pB]], #16\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r1, %[sum]\n"
+ "smlad %[sum2], r4, r0, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-12]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r3, %[sum3]\n"
+ "smlad %[sum4], r4, r2, %[sum4]\n"
+ "ldr.w r4, [%[pA], #-4]\n"
+ "ldr.w r1, [%[pB], #-8]\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r1, %[sum]\n"
+ "smlad %[sum2], r4, r0, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-4]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r3, %[sum3]\n"
+ "smlad %[sum4], r4, r2, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum] "+r"(sum),
+ [sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4");
+#else
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r4, [%[pA]], #8\n"
+ "ldr.w r1, [%[pB]], #16\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r0, %[sum]\n"
+ "smlad %[sum2], r4, r1, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-12]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r2, %[sum3]\n"
+ "smlad %[sum4], r4, r3, %[sum4]\n"
+ "ldr.w r4, [%[pA], #-4]\n"
+ "ldr.w r1, [%[pB], #-8]\n"
+ "mov.w r0, r1, ror #8\n"
+ "sxtb16 r0, r0\n"
+ "sxtb16 r1, r1\n"
+ "smlad %[sum], r4, r0, %[sum]\n"
+ "smlad %[sum2], r4, r1, %[sum2]\n"
+ "ldr.w r3, [%[pB], #-4]\n"
+ "mov.w r2, r3, ror #8\n"
+ "sxtb16 r2, r2\n"
+ "sxtb16 r3, r3\n"
+ "smlad %[sum3], r4, r2, %[sum3]\n"
+ "smlad %[sum4], r4, r3, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum] "+r"(sum),
+ [sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt):"r0", "r1", "r2", "r3", "r4");
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ q7_t inM2 = *pB++;
+ q7_t inM3 = *pB++;
+ q7_t inM4 = *pB++;
+
+ sum += inV * inM;
+ sum2 += inV * inM2;
+ sum3 += inV * inM3;
+ sum4 += inV * inM4;
+ colCnt--;
+ } /* while over colCnt */
+ *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8));
+ *pO++ = (q7_t) (__SSAT((sum2 >> out_shift), 8));
+ *pO++ = (q7_t) (__SSAT((sum3 >> out_shift), 8));
+ *pO++ = (q7_t) (__SSAT((sum4 >> out_shift), 8));
+
+ /* adjust the pointers and counters */
+ rowCnt--;
+ }
+
+ /* left-over part of the rows */
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = vec_buffer;
+
+ while (colCnt)
+ {
+ q31_t inV1, inV2, inM11, inM12;
+
+ pB = (q7_t *) read_and_pad_reordered((void *)pB, &inM11, &inM12);
+
+ inV1 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV1, inM11, sum);
+
+ inV2 = *__SIMD32(pA)++;
+ sum = __SMLAD(inV2, inM12, sum);
+
+ colCnt--;
+ }
+
+ /* left-over of the vector */
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q15_t inV = *pA++;
+ q7_t inM = *pB++;
+ sum += inV * inM;
+ colCnt--;
+ }
+
+ *pO++ = (q7_t) (__SSAT((sum >> out_shift), 8));
+
+ rowCnt--;
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t rowCnt = num_of_rows >> 2;
+ const q7_t *pB = pM;
+ const q7_t *pA;
+ q7_t *pO = pOut;
+ const q7_t *pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = dim_vec >> 2;
+
+ pA = pV;
+
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q7_t inA3 = *pA++;
+ q7_t inA2 = *pA++;
+ q7_t inA4 = *pA++;
+
+ q7_t inB1 = *pB++;
+ q7_t inB3 = *pB++;
+ q7_t inB2 = *pB++;
+ q7_t inB4 = *pB++;
+
+ sum += inA1 * inB1 + inA2 * inB2;
+ sum2 += inA1 * inB3 + inA2 * inB4;
+
+ inB1 = *pB++;
+ inB3 = *pB++;
+ inB2 = *pB++;
+ inB4 = *pB++;
+
+ sum3 += inA1 * inB1 + inA2 * inB2;
+ sum4 += inA1 * inB3 + inA2 * inB4;
+
+ inB1 = *pB++;
+ inB3 = *pB++;
+ inB2 = *pB++;
+ inB4 = *pB++;
+
+ sum += inA3 * inB1 + inA4 * inB2;
+ sum2 += inA3 * inB3 + inA4 * inB4;
+
+ inB1 = *pB++;
+ inB3 = *pB++;
+ inB2 = *pB++;
+ inB4 = *pB++;
+
+ sum3 += inA3 * inB1 + inA4 * inB2;
+ sum4 += inA3 * inB3 + inA4 * inB4;
+
+ colCnt--;
+ }
+ colCnt = dim_vec & 0x3;
+ while (colCnt)
+ {
+ q7_t inA = *pA++;
+ q7_t inB = *pB++;
+ sum += inA * inB;
+ inB = *pB++;
+ sum2 += inA * inB;
+ inB = *pB++;
+ sum3 += inA * inB;
+ inB = *pB++;
+ sum4 += inA * inB;
+
+ colCnt--;
+ }
+ *pO++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pO++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pO++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pO++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+ rowCnt = num_of_rows & 0x3;
+
+ while (rowCnt)
+ {
+ int ip_out = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ int j;
+
+ pA = pV;
+ for (j = 0; j < dim_vec; j++)
+ {
+ q7_t inA = *pA++;
+ q7_t inB = *pB++;
+ ip_out += inA * inB;
+ }
+ *pO++ = (q7_t) __SSAT((ip_out >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to ARM_MATH_SUCCESS */
+ return (ARM_MATH_SUCCESS);
+
+}
+
+/**
+ * @} end of FC group
+ */
diff --git a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c index d6a45ef..de7668b 100644 --- a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c +++ b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q15.c @@ -1,73 +1,147 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_mult_q15.c - * Description: Q15 vector multiplication with variable output shifts - * - * $Date: 20. July 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupSupport - */ - -/** - * @addtogroup NNBasicMath - * @{ - */ - -/** - * @brief Q7 vector multiplication with variable output shifts - * @param[in] *pSrcA pointer to the first input vector - * @param[in] *pSrcB pointer to the second input vector - * @param[out] *pDst pointer to the output vector - * @param[in] out_shift amount of right-shift for output - * @param[in] blockSize number of samples in each vector - * - * <b>Scaling and Overflow Behavior:</b> - * \par - * The function uses saturating arithmetic. - * Results outside of the allowable Q15 range [0x8000 0x7FFF] will be saturated. - */ - -void arm_nn_mult_q15(q15_t *pSrcA, q15_t *pSrcB, q15_t *pDst, const uint16_t out_shift, uint32_t blockSize) -{ - uint32_t blkCnt = blockSize; /* loop counters */ - - while (blkCnt > 0U) - { - /* C = A * B */ - /* Multiply the inputs and store the result in the destination buffer */ - *pDst++ = (q15_t)__SSAT(((q31_t)((q31_t)(*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 16); - - /* Decrement the blockSize loop counter */ - blkCnt--; - } -} - -/** - * @} end of NNBasicMath group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mult_q15.c
+ * Description: Q15 vector multiplication with variable output shifts
+ *
+ * $Date: 13. July 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupSupport
+ */
+
+/**
+ * @addtogroup NNBasicMath
+ * @{
+ */
+
+
+/**
+ * @brief Q7 vector multiplication with variable output shifts
+ * @param[in] *pSrcA pointer to the first input vector
+ * @param[in] *pSrcB pointer to the second input vector
+ * @param[out] *pDst pointer to the output vector
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] blockSize number of samples in each vector
+ * @return none.
+ *
+ * <b>Scaling and Overflow Behavior:</b>
+ * \par
+ * The function uses saturating arithmetic.
+ * Results outside of the allowable Q15 range [0x8000 0x7FFF] will be saturated.
+ */
+
+void arm_nn_mult_q15(
+ q15_t * pSrcA,
+ q15_t * pSrcB,
+ q15_t * pDst,
+ const uint16_t out_shift,
+ uint32_t blockSize)
+{
+ uint32_t blkCnt; /* loop counters */
+
+#if defined (ARM_MATH_DSP)
+
+/* Run the below code for Cortex-M4 and Cortex-M3 */
+ q31_t inA1, inA2, inB1, inB2; /* temporary input variables */
+ q15_t out1, out2, out3, out4; /* temporary output variables */
+ q31_t mul1, mul2, mul3, mul4; /* temporary variables */
+
+ /* loop Unrolling */
+ blkCnt = blockSize >> 2U;
+
+ /* First part of the processing with loop unrolling. Compute 4 outputs at a time.
+ ** a second loop below computes the remaining 1 to 3 samples. */
+ while (blkCnt > 0U)
+ {
+ /* read two samples at a time from sourceA */
+ inA1 = *__SIMD32(pSrcA)++;
+ /* read two samples at a time from sourceB */
+ inB1 = *__SIMD32(pSrcB)++;
+ /* read two samples at a time from sourceA */
+ inA2 = *__SIMD32(pSrcA)++;
+ /* read two samples at a time from sourceB */
+ inB2 = *__SIMD32(pSrcB)++;
+
+ /* multiply mul = sourceA * sourceB */
+ mul1 = (q31_t) ((q15_t) (inA1 >> 16) * (q15_t) (inB1 >> 16));
+ mul2 = (q31_t) ((q15_t) inA1 * (q15_t) inB1);
+ mul3 = (q31_t) ((q15_t) (inA2 >> 16) * (q15_t) (inB2 >> 16));
+ mul4 = (q31_t) ((q15_t) inA2 * (q15_t) inB2);
+
+ /* saturate result to 16 bit */
+ out1 = (q15_t) __SSAT((mul1 + NN_ROUND(out_shift)) >> out_shift, 16);
+ out2 = (q15_t) __SSAT((mul2 + NN_ROUND(out_shift)) >> out_shift, 16);
+ out3 = (q15_t) __SSAT((mul3 + NN_ROUND(out_shift)) >> out_shift, 16);
+ out4 = (q15_t) __SSAT((mul4 + NN_ROUND(out_shift)) >> out_shift, 16);
+
+ /* store the result */
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ *__SIMD32(pDst)++ = __PKHBT(out2, out1, 16);
+ *__SIMD32(pDst)++ = __PKHBT(out4, out3, 16);
+
+#else
+
+ *__SIMD32(pDst)++ = __PKHBT(out2, out1, 16);
+ *__SIMD32(pDst)++ = __PKHBT(out4, out3, 16);
+
+#endif /* #ifndef ARM_MATH_BIG_ENDIAN */
+
+ /* Decrement the blockSize loop counter */
+ blkCnt--;
+ }
+
+ /* If the blockSize is not a multiple of 4, compute any remaining output samples here.
+ ** No loop unrolling is used. */
+ blkCnt = blockSize % 0x4U;
+
+#else
+
+ /* Run the below code for Cortex-M0 */
+
+ /* Initialize blkCnt with number of samples */
+ blkCnt = blockSize;
+
+#endif /* #if defined (ARM_MATH_DSP) */
+
+
+ while (blkCnt > 0U)
+ {
+ /* C = A * B */
+ /* Multiply the inputs and store the result in the destination buffer */
+ *pDst++ = (q15_t) __SSAT((((q31_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 16);
+
+ /* Decrement the blockSize loop counter */
+ blkCnt--;
+ }
+}
+
+/**
+ * @} end of NNBasicMath group
+ */
+
diff --git a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c index fdced4c..1b4e02c 100644 --- a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c +++ b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nn_mult_q7.c @@ -1,73 +1,119 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nn_mult_q7.c - * Description: Q7 vector multiplication with variable output shifts - * - * $Date: 20. July 2021 - * $Revision: V.1.1.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupSupport - */ - -/** - * @addtogroup NNBasicMath - * @{ - */ - -/** - * @brief Q7 vector multiplication with variable output shifts - * @param[in] *pSrcA pointer to the first input vector - * @param[in] *pSrcB pointer to the second input vector - * @param[out] *pDst pointer to the output vector - * @param[in] out_shift amount of right-shift for output - * @param[in] blockSize number of samples in each vector - * - * <b>Scaling and Overflow Behavior:</b> - * \par - * The function uses saturating arithmetic. - * Results outside of the allowable Q7 range [0x80 0x7F] will be saturated. - */ - -void arm_nn_mult_q7(q7_t *pSrcA, q7_t *pSrcB, q7_t *pDst, const uint16_t out_shift, uint32_t blockSize) -{ - uint32_t blkCnt = blockSize; /* loop counters */ - - while (blkCnt > 0U) - { - /* C = A * B */ - /* Multiply the inputs and store the result in the destination buffer */ - *pDst++ = (q7_t)__SSAT(((q15_t)((q15_t)(*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8); - - /* Decrement the blockSize loop counter */ - blkCnt--; - } -} - -/** - * @} end of NNBasicMath group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mult_q7.c
+ * Description: Q7 vector multiplication with variable output shifts
+ *
+ * $Date: 13. July 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupSupport
+ */
+
+/**
+ * @addtogroup NNBasicMath
+ * @{
+ */
+
+/**
+ * @brief Q7 vector multiplication with variable output shifts
+ * @param[in] *pSrcA pointer to the first input vector
+ * @param[in] *pSrcB pointer to the second input vector
+ * @param[out] *pDst pointer to the output vector
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] blockSize number of samples in each vector
+ * @return none.
+ *
+ * <b>Scaling and Overflow Behavior:</b>
+ * \par
+ * The function uses saturating arithmetic.
+ * Results outside of the allowable Q7 range [0x80 0x7F] will be saturated.
+ */
+
+void arm_nn_mult_q7(
+ q7_t * pSrcA,
+ q7_t * pSrcB,
+ q7_t * pDst,
+ const uint16_t out_shift,
+ uint32_t blockSize)
+{
+ uint32_t blkCnt; /* loop counters */
+
+#if defined (ARM_MATH_DSP)
+
+/* Run the below code for Cortex-M4 and Cortex-M3 */
+ q7_t out1, out2, out3, out4; /* Temporary variables to store the product */
+
+ /* loop Unrolling */
+ blkCnt = blockSize >> 2U;
+
+ /* First part of the processing with loop unrolling. Compute 4 outputs at a time.
+ ** a second loop below computes the remaining 1 to 3 samples. */
+ while (blkCnt > 0U)
+ {
+ /* C = A * B */
+ /* Multiply the inputs and store the results in temporary variables */
+ out1 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8);
+ out2 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8);
+ out3 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8);
+ out4 = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8);
+
+ /* Store the results of 4 inputs in the destination buffer in single cycle by packing */
+ *__SIMD32(pDst)++ = __PACKq7(out1, out2, out3, out4);
+
+ /* Decrement the blockSize loop counter */
+ blkCnt--;
+ }
+
+ /* If the blockSize is not a multiple of 4, compute any remaining output samples here.
+ ** No loop unrolling is used. */
+ blkCnt = blockSize % 0x4U;
+
+#else
+
+ /* Run the below code for Cortex-M0 */
+
+ /* Initialize blkCnt with number of samples */
+ blkCnt = blockSize;
+
+#endif /* #if defined (ARM_MATH_DSP) */
+
+
+ while (blkCnt > 0U)
+ {
+ /* C = A * B */
+ /* Multiply the inputs and store the result in the destination buffer */
+ *pDst++ = (q7_t) __SSAT((((q15_t) (*pSrcA++) * (*pSrcB++) + NN_ROUND(out_shift)) >> out_shift), 8);
+
+ /* Decrement the blockSize loop counter */
+ blkCnt--;
+ }
+}
+
+/**
+ * @} end of NNBasicMath group
+ */
diff --git a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nntables.c b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nntables.c index 5a8cea2..cabd9b1 100644 --- a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nntables.c +++ b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_nntables.c @@ -1,203 +1,297 @@ -/* - * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_nntables.c - * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift - * - * $Date: 17. January 2018 - * $Revision: V.1.0.0 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnsupportfunctions.h" - -/** - * @brief tables for various activation functions - * - * This file include the declaration of common tables. - * Most of them are used for activation functions - * - * Assumption: - * Unified table: input is 3.x format, i.e, range of [-8, 8) - * sigmoid(8) = 0.9996646498695336 - * tanh(8) = 0.9999997749296758 - * The accuracy here should be good enough - * - * 2-stage HL table: - * - * The entire input range is divided into two parts: - * - * Low range table: 0x000x xxxx or 0x111x xxxx - * table entry will be the binary number excluding the first - * two digits, i.e., 0x0x xxxx or 0x1x xxxx - * - * - * - * High range table 0x0010 0000 -- 0x0111 1111 - * 0x1000 0000 -- 0x1101 1111 - * - * For positive numbers, table entry will be - * 0x0010 0000 -- 0x0111 1111 minus 0x0010 0000 - * i.e., 0x0000 0000 - 0x0101 11111 - * - * same thing for the negative numbers, table entry will be - * 0x1000 0000 -- 0x1101 1111 minux 0x0010 0000 - * i.e., 0x0110 0000 - 0x1011 1111 - */ - -const q7_t sigmoidTable_q7[256] = { - 0x40, 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e, 0x50, 0x52, 0x53, 0x55, 0x57, 0x59, 0x5a, 0x5c, 0x5e, 0x5f, 0x61, - 0x62, 0x63, 0x65, 0x66, 0x67, 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70, 0x71, 0x72, 0x72, 0x73, 0x74, 0x74, - 0x75, 0x76, 0x76, 0x77, 0x77, 0x78, 0x78, 0x79, 0x79, 0x7a, 0x7a, 0x7a, 0x7b, 0x7b, 0x7b, 0x7c, 0x7c, 0x7c, 0x7c, - 0x7c, 0x7d, 0x7d, 0x7d, 0x7d, 0x7d, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, - 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x01, 0x01, 0x01, - 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, - 0x02, 0x02, 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x04, 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06, 0x06, - 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09, 0x0a, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, 0x0e, 0x0e, 0x0f, 0x10, 0x11, 0x12, - 0x13, 0x14, 0x15, 0x16, 0x17, 0x19, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f, 0x21, 0x22, 0x24, 0x26, 0x27, 0x29, 0x2b, 0x2d, - 0x2e, 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e, -}; - -const q15_t sigmoidTable_q15[256] = { - 0x4000, 0x4200, 0x43ff, 0x45fc, 0x47f5, 0x49eb, 0x4bdc, 0x4dc8, 0x4fad, 0x518a, 0x5360, 0x552c, 0x56ef, 0x58a8, - 0x5a57, 0x5bfb, 0x5d93, 0x5f20, 0x60a1, 0x6216, 0x637f, 0x64db, 0x662b, 0x676f, 0x68a6, 0x69d2, 0x6af1, 0x6c05, - 0x6d0d, 0x6e09, 0x6efb, 0x6fe2, 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7, 0x764a, 0x76d6, - 0x775b, 0x77d8, 0x784f, 0x78c0, 0x792a, 0x798f, 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03, - 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d, 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, - 0x7e69, 0x7e81, 0x7e98, 0x7eae, 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17, 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, - 0x7f55, 0x7f5f, 0x7f69, 0x7f72, 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa, 0x7faf, 0x7fb4, - 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, 0x7fc8, 0x7fcc, 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0, - 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, 0x7fe9, 0x7fea, 0x7feb, 0x7fed, 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, - 0x7ff4, 0x7ff4, 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0015, 0x0016, - 0x0017, 0x0019, 0x001a, 0x001c, 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e, 0x0031, 0x0034, - 0x0038, 0x003b, 0x003f, 0x0043, 0x0048, 0x004c, 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d, - 0x0085, 0x008e, 0x0097, 0x00a1, 0x00ab, 0x00b6, 0x00c2, 0x00ce, 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, - 0x013e, 0x0152, 0x0168, 0x017f, 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a, 0x024d, 0x0273, 0x029a, 0x02c4, - 0x02f1, 0x0320, 0x0353, 0x0388, 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8, 0x0612, 0x0671, - 0x06d6, 0x0740, 0x07b1, 0x0828, 0x08a5, 0x092a, 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70, - 0x0f42, 0x101e, 0x1105, 0x11f7, 0x12f3, 0x13fb, 0x150f, 0x162e, 0x175a, 0x1891, 0x19d5, 0x1b25, 0x1c81, 0x1dea, - 0x1f5f, 0x20e0, 0x226d, 0x2405, 0x25a9, 0x2758, 0x2911, 0x2ad4, 0x2ca0, 0x2e76, 0x3053, 0x3238, 0x3424, 0x3615, - 0x380b, 0x3a04, 0x3c01, 0x3e00, -}; - -const q15_t sigmoidLTable_q15[128] = { - 0x4000, 0x4100, 0x4200, 0x42ff, 0x43ff, 0x44fd, 0x45fc, 0x46f9, 0x47f5, 0x48f1, 0x49eb, 0x4ae5, 0x4bdc, - 0x4cd3, 0x4dc8, 0x4ebb, 0x4fad, 0x509c, 0x518a, 0x5276, 0x5360, 0x5447, 0x552c, 0x560f, 0x56ef, 0x57cd, - 0x58a8, 0x5981, 0x5a57, 0x5b2a, 0x5bfb, 0x5cc9, 0x5d93, 0x5e5b, 0x5f20, 0x5fe2, 0x60a1, 0x615d, 0x6216, - 0x62cc, 0x637f, 0x642e, 0x64db, 0x6584, 0x662b, 0x66ce, 0x676f, 0x680c, 0x68a6, 0x693d, 0x69d2, 0x6a63, - 0x6af1, 0x6b7c, 0x6c05, 0x6c8a, 0x6d0d, 0x6d8d, 0x6e09, 0x6e84, 0x6efb, 0x6f70, 0x6fe2, 0x7051, 0x0f42, - 0x0faf, 0x101e, 0x1090, 0x1105, 0x117c, 0x11f7, 0x1273, 0x12f3, 0x1376, 0x13fb, 0x1484, 0x150f, 0x159d, - 0x162e, 0x16c3, 0x175a, 0x17f4, 0x1891, 0x1932, 0x19d5, 0x1a7c, 0x1b25, 0x1bd2, 0x1c81, 0x1d34, 0x1dea, - 0x1ea3, 0x1f5f, 0x201e, 0x20e0, 0x21a5, 0x226d, 0x2337, 0x2405, 0x24d6, 0x25a9, 0x267f, 0x2758, 0x2833, - 0x2911, 0x29f1, 0x2ad4, 0x2bb9, 0x2ca0, 0x2d8a, 0x2e76, 0x2f64, 0x3053, 0x3145, 0x3238, 0x332d, 0x3424, - 0x351b, 0x3615, 0x370f, 0x380b, 0x3907, 0x3a04, 0x3b03, 0x3c01, 0x3d01, 0x3e00, 0x3f00, -}; - -const q15_t sigmoidHTable_q15[192] = { - 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7, 0x764a, 0x76d6, 0x775b, 0x77d8, 0x784f, 0x78c0, - 0x792a, 0x798f, 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03, 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, - 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d, 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, 0x7e69, 0x7e81, 0x7e98, 0x7eae, - 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17, 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, 0x7f55, 0x7f5f, 0x7f69, 0x7f72, - 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa, 0x7faf, 0x7fb4, 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, - 0x7fc8, 0x7fcc, 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0, 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, - 0x7fe9, 0x7fea, 0x7feb, 0x7fed, 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, 0x7ff4, 0x7ff4, 0x000b, 0x000c, - 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011, 0x0012, 0x0013, 0x0015, 0x0016, 0x0017, 0x0019, 0x001a, 0x001c, - 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e, 0x0031, 0x0034, 0x0038, 0x003b, 0x003f, 0x0043, - 0x0048, 0x004c, 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d, 0x0085, 0x008e, 0x0097, 0x00a1, - 0x00ab, 0x00b6, 0x00c2, 0x00ce, 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, 0x013e, 0x0152, 0x0168, 0x017f, - 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a, 0x024d, 0x0273, 0x029a, 0x02c4, 0x02f1, 0x0320, 0x0353, 0x0388, - 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8, 0x0612, 0x0671, 0x06d6, 0x0740, 0x07b1, 0x0828, - 0x08a5, 0x092a, 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70, -}; - -const q7_t tanhTable_q7[256] = { - 0x00, 0x08, 0x10, 0x18, 0x1f, 0x27, 0x2e, 0x35, 0x3b, 0x41, 0x47, 0x4c, 0x51, 0x56, 0x5a, 0x5e, 0x61, 0x65, 0x68, - 0x6a, 0x6d, 0x6f, 0x71, 0x72, 0x74, 0x75, 0x76, 0x78, 0x78, 0x79, 0x7a, 0x7b, 0x7b, 0x7c, 0x7c, 0x7d, 0x7d, 0x7e, - 0x7e, 0x7e, 0x7e, 0x7e, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, - 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, - 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x81, 0x81, - 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x82, 0x82, 0x82, 0x82, 0x82, 0x83, 0x83, 0x84, 0x84, 0x85, 0x85, 0x86, 0x87, - 0x88, 0x88, 0x8a, 0x8b, 0x8c, 0x8e, 0x8f, 0x91, 0x93, 0x96, 0x98, 0x9b, 0x9f, 0xa2, 0xa6, 0xaa, 0xaf, 0xb4, 0xb9, - 0xbf, 0xc5, 0xcb, 0xd2, 0xd9, 0xe1, 0xe8, 0xf0, 0xf8, -}; - -const q15_t tanhTable_q15[256] = { - 0x0000, 0x07fd, 0x0feb, 0x17b9, 0x1f59, 0x26bf, 0x2ddf, 0x34ae, 0x3b27, 0x4142, 0x46fd, 0x4c56, 0x514d, 0x55e2, - 0x5a1a, 0x5df6, 0x617c, 0x64b0, 0x6797, 0x6a37, 0x6c95, 0x6eb5, 0x709e, 0x7254, 0x73dc, 0x753a, 0x7672, 0x7788, - 0x787f, 0x795b, 0x7a1e, 0x7acb, 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f, 0x7e49, 0x7e7d, - 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, 0x7f30, 0x7f48, 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc, - 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7, 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, - 0x7ff6, 0x7ff7, 0x7ff8, 0x7ff9, 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd, 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, - 0x7ffe, 0x7ffe, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, - 0x8001, 0x8001, 0x8001, 0x8002, 0x8002, 0x8002, 0x8002, 0x8003, 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, - 0x8006, 0x8007, 0x8008, 0x8009, 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013, 0x8016, 0x8019, 0x801c, 0x8020, - 0x8024, 0x8029, 0x802f, 0x8035, 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f, 0x80a2, 0x80b8, - 0x80d0, 0x80ec, 0x810b, 0x812e, 0x8156, 0x8183, 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412, - 0x849b, 0x8535, 0x85e2, 0x86a5, 0x8781, 0x8878, 0x898e, 0x8ac6, 0x8c24, 0x8dac, 0x8f62, 0x914b, 0x936b, 0x95c9, - 0x9869, 0x9b50, 0x9e84, 0xa20a, 0xa5e6, 0xaa1e, 0xaeb3, 0xb3aa, 0xb903, 0xbebe, 0xc4d9, 0xcb52, 0xd221, 0xd941, - 0xe0a7, 0xe847, 0xf015, 0xf803, -}; - -const q15_t tanhLTable_q15[128] = { - 0x0000, 0x0400, 0x07fd, 0x0bf7, 0x0feb, 0x13d7, 0x17b9, 0x1b90, 0x1f59, 0x2314, 0x26bf, 0x2a58, 0x2ddf, - 0x3151, 0x34ae, 0x37f6, 0x3b27, 0x3e40, 0x4142, 0x442c, 0x46fd, 0x49b6, 0x4c56, 0x4edd, 0x514d, 0x53a3, - 0x55e2, 0x580a, 0x5a1a, 0x5c13, 0x5df6, 0x5fc4, 0x617c, 0x6320, 0x64b0, 0x662d, 0x6797, 0x68f0, 0x6a37, - 0x6b6e, 0x6c95, 0x6dac, 0x6eb5, 0x6fb0, 0x709e, 0x717f, 0x7254, 0x731e, 0x73dc, 0x7490, 0x753a, 0x75da, - 0x7672, 0x7701, 0x7788, 0x7807, 0x787f, 0x78f0, 0x795b, 0x79bf, 0x7a1e, 0x7a77, 0x7acb, 0x7b1b, 0x849b, - 0x84e5, 0x8535, 0x8589, 0x85e2, 0x8641, 0x86a5, 0x8710, 0x8781, 0x87f9, 0x8878, 0x88ff, 0x898e, 0x8a26, - 0x8ac6, 0x8b70, 0x8c24, 0x8ce2, 0x8dac, 0x8e81, 0x8f62, 0x9050, 0x914b, 0x9254, 0x936b, 0x9492, 0x95c9, - 0x9710, 0x9869, 0x99d3, 0x9b50, 0x9ce0, 0x9e84, 0xa03c, 0xa20a, 0xa3ed, 0xa5e6, 0xa7f6, 0xaa1e, 0xac5d, - 0xaeb3, 0xb123, 0xb3aa, 0xb64a, 0xb903, 0xbbd4, 0xbebe, 0xc1c0, 0xc4d9, 0xc80a, 0xcb52, 0xceaf, 0xd221, - 0xd5a8, 0xd941, 0xdcec, 0xe0a7, 0xe470, 0xe847, 0xec29, 0xf015, 0xf409, 0xf803, 0xfc00, -}; - -const q15_t tanhHTable_q15[192] = { - 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f, 0x7e49, 0x7e7d, 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, - 0x7f30, 0x7f48, 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc, 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, - 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7, 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, 0x7ff6, 0x7ff7, 0x7ff8, 0x7ff9, - 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd, 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, 0x7ffe, 0x7ffe, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, - 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, - 0x8000, 0x8000, 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8002, - 0x8002, 0x8002, 0x8002, 0x8003, 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, 0x8006, 0x8007, 0x8008, 0x8009, - 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013, 0x8016, 0x8019, 0x801c, 0x8020, 0x8024, 0x8029, 0x802f, 0x8035, - 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f, 0x80a2, 0x80b8, 0x80d0, 0x80ec, 0x810b, 0x812e, - 0x8156, 0x8183, 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412, -}; +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nntables.c
+ * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnsupportfunctions.h"
+
+/**
+ * @brief tables for various activation functions
+ *
+ * This file include the declaration of common tables.
+ * Most of them are used for activation functions
+ *
+ * Assumption:
+ * Unified table: input is 3.x format, i.e, range of [-8, 8)
+ * sigmoid(8) = 0.9996646498695336
+ * tanh(8) = 0.9999997749296758
+ * The accuracy here should be good enough
+ *
+ * 2-stage HL table:
+ *
+ * The entire input range is divided into two parts:
+ *
+ * Low range table: 0x000x xxxx or 0x111x xxxx
+ * table entry will be the binary number excluding the first
+ * two digits, i.e., 0x0x xxxx or 0x1x xxxx
+ *
+ *
+ *
+ * High range table 0x0010 0000 -- 0x0111 1111
+ * 0x1000 0000 -- 0x1101 1111
+ *
+ * For positive numbers, table entry will be
+ * 0x0010 0000 -- 0x0111 1111 minus 0x0010 0000
+ * i.e., 0x0000 0000 - 0x0101 11111
+ *
+ * same thing for the negative numbers, table entry will be
+ * 0x1000 0000 -- 0x1101 1111 minux 0x0010 0000
+ * i.e., 0x0110 0000 - 0x1011 1111
+ */
+
+const q7_t sigmoidTable_q7[256] = {
+ 0x40, 0x42, 0x44, 0x46, 0x48, 0x4a, 0x4c, 0x4e,
+ 0x50, 0x52, 0x53, 0x55, 0x57, 0x59, 0x5a, 0x5c,
+ 0x5e, 0x5f, 0x61, 0x62, 0x63, 0x65, 0x66, 0x67,
+ 0x69, 0x6a, 0x6b, 0x6c, 0x6d, 0x6e, 0x6f, 0x70,
+ 0x71, 0x72, 0x72, 0x73, 0x74, 0x74, 0x75, 0x76,
+ 0x76, 0x77, 0x77, 0x78, 0x78, 0x79, 0x79, 0x7a,
+ 0x7a, 0x7a, 0x7b, 0x7b, 0x7b, 0x7c, 0x7c, 0x7c,
+ 0x7c, 0x7c, 0x7d, 0x7d, 0x7d, 0x7d, 0x7d, 0x7e,
+ 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7e, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00, 0x00,
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01, 0x01,
+ 0x01, 0x01, 0x02, 0x02, 0x02, 0x02, 0x02, 0x02,
+ 0x02, 0x02, 0x03, 0x03, 0x03, 0x03, 0x03, 0x04,
+ 0x04, 0x04, 0x04, 0x04, 0x05, 0x05, 0x05, 0x06,
+ 0x06, 0x06, 0x07, 0x07, 0x08, 0x08, 0x09, 0x09,
+ 0x0a, 0x0a, 0x0b, 0x0c, 0x0c, 0x0d, 0x0e, 0x0e,
+ 0x0f, 0x10, 0x11, 0x12, 0x13, 0x14, 0x15, 0x16,
+ 0x17, 0x19, 0x1a, 0x1b, 0x1d, 0x1e, 0x1f, 0x21,
+ 0x22, 0x24, 0x26, 0x27, 0x29, 0x2b, 0x2d, 0x2e,
+ 0x30, 0x32, 0x34, 0x36, 0x38, 0x3a, 0x3c, 0x3e,
+};
+
+const q15_t sigmoidTable_q15[256] = {
+ 0x4000, 0x4200, 0x43ff, 0x45fc, 0x47f5, 0x49eb, 0x4bdc, 0x4dc8,
+ 0x4fad, 0x518a, 0x5360, 0x552c, 0x56ef, 0x58a8, 0x5a57, 0x5bfb,
+ 0x5d93, 0x5f20, 0x60a1, 0x6216, 0x637f, 0x64db, 0x662b, 0x676f,
+ 0x68a6, 0x69d2, 0x6af1, 0x6c05, 0x6d0d, 0x6e09, 0x6efb, 0x6fe2,
+ 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7,
+ 0x764a, 0x76d6, 0x775b, 0x77d8, 0x784f, 0x78c0, 0x792a, 0x798f,
+ 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03,
+ 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d,
+ 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, 0x7e69, 0x7e81,
+ 0x7e98, 0x7eae, 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17,
+ 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, 0x7f55, 0x7f5f, 0x7f69, 0x7f72,
+ 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa,
+ 0x7faf, 0x7fb4, 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, 0x7fc8, 0x7fcc,
+ 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0,
+ 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, 0x7fe9, 0x7fea, 0x7feb, 0x7fed,
+ 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, 0x7ff4, 0x7ff4,
+ 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011,
+ 0x0012, 0x0013, 0x0015, 0x0016, 0x0017, 0x0019, 0x001a, 0x001c,
+ 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e,
+ 0x0031, 0x0034, 0x0038, 0x003b, 0x003f, 0x0043, 0x0048, 0x004c,
+ 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d,
+ 0x0085, 0x008e, 0x0097, 0x00a1, 0x00ab, 0x00b6, 0x00c2, 0x00ce,
+ 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, 0x013e, 0x0152,
+ 0x0168, 0x017f, 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a,
+ 0x024d, 0x0273, 0x029a, 0x02c4, 0x02f1, 0x0320, 0x0353, 0x0388,
+ 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8,
+ 0x0612, 0x0671, 0x06d6, 0x0740, 0x07b1, 0x0828, 0x08a5, 0x092a,
+ 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70,
+ 0x0f42, 0x101e, 0x1105, 0x11f7, 0x12f3, 0x13fb, 0x150f, 0x162e,
+ 0x175a, 0x1891, 0x19d5, 0x1b25, 0x1c81, 0x1dea, 0x1f5f, 0x20e0,
+ 0x226d, 0x2405, 0x25a9, 0x2758, 0x2911, 0x2ad4, 0x2ca0, 0x2e76,
+ 0x3053, 0x3238, 0x3424, 0x3615, 0x380b, 0x3a04, 0x3c01, 0x3e00,
+};
+
+const q15_t sigmoidLTable_q15[128] = {
+ 0x4000, 0x4100, 0x4200, 0x42ff, 0x43ff, 0x44fd, 0x45fc, 0x46f9,
+ 0x47f5, 0x48f1, 0x49eb, 0x4ae5, 0x4bdc, 0x4cd3, 0x4dc8, 0x4ebb,
+ 0x4fad, 0x509c, 0x518a, 0x5276, 0x5360, 0x5447, 0x552c, 0x560f,
+ 0x56ef, 0x57cd, 0x58a8, 0x5981, 0x5a57, 0x5b2a, 0x5bfb, 0x5cc9,
+ 0x5d93, 0x5e5b, 0x5f20, 0x5fe2, 0x60a1, 0x615d, 0x6216, 0x62cc,
+ 0x637f, 0x642e, 0x64db, 0x6584, 0x662b, 0x66ce, 0x676f, 0x680c,
+ 0x68a6, 0x693d, 0x69d2, 0x6a63, 0x6af1, 0x6b7c, 0x6c05, 0x6c8a,
+ 0x6d0d, 0x6d8d, 0x6e09, 0x6e84, 0x6efb, 0x6f70, 0x6fe2, 0x7051,
+ 0x0f42, 0x0faf, 0x101e, 0x1090, 0x1105, 0x117c, 0x11f7, 0x1273,
+ 0x12f3, 0x1376, 0x13fb, 0x1484, 0x150f, 0x159d, 0x162e, 0x16c3,
+ 0x175a, 0x17f4, 0x1891, 0x1932, 0x19d5, 0x1a7c, 0x1b25, 0x1bd2,
+ 0x1c81, 0x1d34, 0x1dea, 0x1ea3, 0x1f5f, 0x201e, 0x20e0, 0x21a5,
+ 0x226d, 0x2337, 0x2405, 0x24d6, 0x25a9, 0x267f, 0x2758, 0x2833,
+ 0x2911, 0x29f1, 0x2ad4, 0x2bb9, 0x2ca0, 0x2d8a, 0x2e76, 0x2f64,
+ 0x3053, 0x3145, 0x3238, 0x332d, 0x3424, 0x351b, 0x3615, 0x370f,
+ 0x380b, 0x3907, 0x3a04, 0x3b03, 0x3c01, 0x3d01, 0x3e00, 0x3f00,
+};
+
+const q15_t sigmoidHTable_q15[192] = {
+ 0x70be, 0x7190, 0x7258, 0x7316, 0x73cc, 0x7478, 0x751b, 0x75b7,
+ 0x764a, 0x76d6, 0x775b, 0x77d8, 0x784f, 0x78c0, 0x792a, 0x798f,
+ 0x79ee, 0x7a48, 0x7a9d, 0x7aed, 0x7b39, 0x7b80, 0x7bc4, 0x7c03,
+ 0x7c3f, 0x7c78, 0x7cad, 0x7ce0, 0x7d0f, 0x7d3c, 0x7d66, 0x7d8d,
+ 0x7db3, 0x7dd6, 0x7df7, 0x7e16, 0x7e33, 0x7e4f, 0x7e69, 0x7e81,
+ 0x7e98, 0x7eae, 0x7ec2, 0x7ed5, 0x7ee7, 0x7ef8, 0x7f08, 0x7f17,
+ 0x7f25, 0x7f32, 0x7f3e, 0x7f4a, 0x7f55, 0x7f5f, 0x7f69, 0x7f72,
+ 0x7f7b, 0x7f83, 0x7f8a, 0x7f91, 0x7f98, 0x7f9e, 0x7fa4, 0x7faa,
+ 0x7faf, 0x7fb4, 0x7fb8, 0x7fbd, 0x7fc1, 0x7fc5, 0x7fc8, 0x7fcc,
+ 0x7fcf, 0x7fd2, 0x7fd5, 0x7fd7, 0x7fda, 0x7fdc, 0x7fde, 0x7fe0,
+ 0x7fe2, 0x7fe4, 0x7fe6, 0x7fe7, 0x7fe9, 0x7fea, 0x7feb, 0x7fed,
+ 0x7fee, 0x7fef, 0x7ff0, 0x7ff1, 0x7ff2, 0x7ff3, 0x7ff4, 0x7ff4,
+ 0x000b, 0x000c, 0x000c, 0x000d, 0x000e, 0x000f, 0x0010, 0x0011,
+ 0x0012, 0x0013, 0x0015, 0x0016, 0x0017, 0x0019, 0x001a, 0x001c,
+ 0x001e, 0x0020, 0x0022, 0x0024, 0x0026, 0x0029, 0x002b, 0x002e,
+ 0x0031, 0x0034, 0x0038, 0x003b, 0x003f, 0x0043, 0x0048, 0x004c,
+ 0x0051, 0x0056, 0x005c, 0x0062, 0x0068, 0x006f, 0x0076, 0x007d,
+ 0x0085, 0x008e, 0x0097, 0x00a1, 0x00ab, 0x00b6, 0x00c2, 0x00ce,
+ 0x00db, 0x00e9, 0x00f8, 0x0108, 0x0119, 0x012b, 0x013e, 0x0152,
+ 0x0168, 0x017f, 0x0197, 0x01b1, 0x01cd, 0x01ea, 0x0209, 0x022a,
+ 0x024d, 0x0273, 0x029a, 0x02c4, 0x02f1, 0x0320, 0x0353, 0x0388,
+ 0x03c1, 0x03fd, 0x043c, 0x0480, 0x04c7, 0x0513, 0x0563, 0x05b8,
+ 0x0612, 0x0671, 0x06d6, 0x0740, 0x07b1, 0x0828, 0x08a5, 0x092a,
+ 0x09b6, 0x0a49, 0x0ae5, 0x0b88, 0x0c34, 0x0cea, 0x0da8, 0x0e70,
+};
+
+const q7_t tanhTable_q7[256] = {
+ 0x00, 0x08, 0x10, 0x18, 0x1f, 0x27, 0x2e, 0x35,
+ 0x3b, 0x41, 0x47, 0x4c, 0x51, 0x56, 0x5a, 0x5e,
+ 0x61, 0x65, 0x68, 0x6a, 0x6d, 0x6f, 0x71, 0x72,
+ 0x74, 0x75, 0x76, 0x78, 0x78, 0x79, 0x7a, 0x7b,
+ 0x7b, 0x7c, 0x7c, 0x7d, 0x7d, 0x7e, 0x7e, 0x7e,
+ 0x7e, 0x7e, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f, 0x7f,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80,
+ 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x80, 0x81,
+ 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x81, 0x82,
+ 0x82, 0x82, 0x82, 0x82, 0x83, 0x83, 0x84, 0x84,
+ 0x85, 0x85, 0x86, 0x87, 0x88, 0x88, 0x8a, 0x8b,
+ 0x8c, 0x8e, 0x8f, 0x91, 0x93, 0x96, 0x98, 0x9b,
+ 0x9f, 0xa2, 0xa6, 0xaa, 0xaf, 0xb4, 0xb9, 0xbf,
+ 0xc5, 0xcb, 0xd2, 0xd9, 0xe1, 0xe8, 0xf0, 0xf8,
+};
+
+const q15_t tanhTable_q15[256] = {
+ 0x0000, 0x07fd, 0x0feb, 0x17b9, 0x1f59, 0x26bf, 0x2ddf, 0x34ae,
+ 0x3b27, 0x4142, 0x46fd, 0x4c56, 0x514d, 0x55e2, 0x5a1a, 0x5df6,
+ 0x617c, 0x64b0, 0x6797, 0x6a37, 0x6c95, 0x6eb5, 0x709e, 0x7254,
+ 0x73dc, 0x753a, 0x7672, 0x7788, 0x787f, 0x795b, 0x7a1e, 0x7acb,
+ 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f,
+ 0x7e49, 0x7e7d, 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, 0x7f30, 0x7f48,
+ 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc,
+ 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7,
+ 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, 0x7ff6, 0x7ff7,
+ 0x7ff8, 0x7ff9, 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd,
+ 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, 0x7ffe, 0x7ffe, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001,
+ 0x8001, 0x8001, 0x8001, 0x8002, 0x8002, 0x8002, 0x8002, 0x8003,
+ 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, 0x8006, 0x8007,
+ 0x8008, 0x8009, 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013,
+ 0x8016, 0x8019, 0x801c, 0x8020, 0x8024, 0x8029, 0x802f, 0x8035,
+ 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f,
+ 0x80a2, 0x80b8, 0x80d0, 0x80ec, 0x810b, 0x812e, 0x8156, 0x8183,
+ 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412,
+ 0x849b, 0x8535, 0x85e2, 0x86a5, 0x8781, 0x8878, 0x898e, 0x8ac6,
+ 0x8c24, 0x8dac, 0x8f62, 0x914b, 0x936b, 0x95c9, 0x9869, 0x9b50,
+ 0x9e84, 0xa20a, 0xa5e6, 0xaa1e, 0xaeb3, 0xb3aa, 0xb903, 0xbebe,
+ 0xc4d9, 0xcb52, 0xd221, 0xd941, 0xe0a7, 0xe847, 0xf015, 0xf803,
+};
+
+const q15_t tanhLTable_q15[128] = {
+ 0x0000, 0x0400, 0x07fd, 0x0bf7, 0x0feb, 0x13d7, 0x17b9, 0x1b90,
+ 0x1f59, 0x2314, 0x26bf, 0x2a58, 0x2ddf, 0x3151, 0x34ae, 0x37f6,
+ 0x3b27, 0x3e40, 0x4142, 0x442c, 0x46fd, 0x49b6, 0x4c56, 0x4edd,
+ 0x514d, 0x53a3, 0x55e2, 0x580a, 0x5a1a, 0x5c13, 0x5df6, 0x5fc4,
+ 0x617c, 0x6320, 0x64b0, 0x662d, 0x6797, 0x68f0, 0x6a37, 0x6b6e,
+ 0x6c95, 0x6dac, 0x6eb5, 0x6fb0, 0x709e, 0x717f, 0x7254, 0x731e,
+ 0x73dc, 0x7490, 0x753a, 0x75da, 0x7672, 0x7701, 0x7788, 0x7807,
+ 0x787f, 0x78f0, 0x795b, 0x79bf, 0x7a1e, 0x7a77, 0x7acb, 0x7b1b,
+ 0x849b, 0x84e5, 0x8535, 0x8589, 0x85e2, 0x8641, 0x86a5, 0x8710,
+ 0x8781, 0x87f9, 0x8878, 0x88ff, 0x898e, 0x8a26, 0x8ac6, 0x8b70,
+ 0x8c24, 0x8ce2, 0x8dac, 0x8e81, 0x8f62, 0x9050, 0x914b, 0x9254,
+ 0x936b, 0x9492, 0x95c9, 0x9710, 0x9869, 0x99d3, 0x9b50, 0x9ce0,
+ 0x9e84, 0xa03c, 0xa20a, 0xa3ed, 0xa5e6, 0xa7f6, 0xaa1e, 0xac5d,
+ 0xaeb3, 0xb123, 0xb3aa, 0xb64a, 0xb903, 0xbbd4, 0xbebe, 0xc1c0,
+ 0xc4d9, 0xc80a, 0xcb52, 0xceaf, 0xd221, 0xd5a8, 0xd941, 0xdcec,
+ 0xe0a7, 0xe470, 0xe847, 0xec29, 0xf015, 0xf409, 0xf803, 0xfc00,
+};
+
+const q15_t tanhHTable_q15[192] = {
+ 0x7b65, 0x7bee, 0x7c66, 0x7cd1, 0x7d30, 0x7d84, 0x7dce, 0x7e0f,
+ 0x7e49, 0x7e7d, 0x7eaa, 0x7ed2, 0x7ef5, 0x7f14, 0x7f30, 0x7f48,
+ 0x7f5e, 0x7f71, 0x7f82, 0x7f91, 0x7f9e, 0x7fa9, 0x7fb3, 0x7fbc,
+ 0x7fc4, 0x7fcb, 0x7fd1, 0x7fd7, 0x7fdc, 0x7fe0, 0x7fe4, 0x7fe7,
+ 0x7fea, 0x7fed, 0x7fef, 0x7ff1, 0x7ff3, 0x7ff4, 0x7ff6, 0x7ff7,
+ 0x7ff8, 0x7ff9, 0x7ffa, 0x7ffa, 0x7ffb, 0x7ffc, 0x7ffc, 0x7ffd,
+ 0x7ffd, 0x7ffd, 0x7ffe, 0x7ffe, 0x7ffe, 0x7ffe, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff, 0x7fff,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000, 0x8000,
+ 0x8000, 0x8000, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001, 0x8001,
+ 0x8001, 0x8001, 0x8001, 0x8002, 0x8002, 0x8002, 0x8002, 0x8003,
+ 0x8003, 0x8003, 0x8004, 0x8004, 0x8005, 0x8006, 0x8006, 0x8007,
+ 0x8008, 0x8009, 0x800a, 0x800c, 0x800d, 0x800f, 0x8011, 0x8013,
+ 0x8016, 0x8019, 0x801c, 0x8020, 0x8024, 0x8029, 0x802f, 0x8035,
+ 0x803c, 0x8044, 0x804d, 0x8057, 0x8062, 0x806f, 0x807e, 0x808f,
+ 0x80a2, 0x80b8, 0x80d0, 0x80ec, 0x810b, 0x812e, 0x8156, 0x8183,
+ 0x81b7, 0x81f1, 0x8232, 0x827c, 0x82d0, 0x832f, 0x839a, 0x8412,
+};
diff --git a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c index 6f2f575..e043b38 100644 --- a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c +++ b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_no_shift.c @@ -1,121 +1,134 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_q7_to_q15_no_shift.c - * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift - * - * $Date: May 29, 2020 - * $Revision: V.1.0.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupSupport - */ - -/** - * @addtogroup nndata_convert - * @{ - */ - -/** - * @brief Converts the elements of the Q7 vector to Q15 vector without left-shift - * @param[in] *pSrc points to the Q7 input vector - * @param[out] *pDst points to the Q15 output vector - * @param[in] blockSize length of the input vector - * - * \par Description: - * - * The equation used for the conversion process is: - * - * <pre> - * pDst[n] = (q15_t) pSrc[n]; 0 <= n < blockSize. - * </pre> - * - */ - -void arm_q7_to_q15_no_shift(const q7_t *pSrc, q15_t *pDst, uint32_t blockSize) -{ - const q7_t *pIn = pSrc; - uint32_t blkCnt; - -#if defined(ARM_MATH_DSP) - q31_t in; - q31_t in1, in2; - q31_t out1, out2; - - /*loop Unrolling */ - blkCnt = blockSize >> 2u; - - /* First part of the processing with loop unrolling. Compute 4 outputs at a time. */ - while (blkCnt > 0u) - { - in = arm_nn_read_q7x4_ia(&pIn); - - /* rotatate in by 8 and extend two q7_t values to q15_t values */ - in1 = __SXTB16(__ROR((uint32_t)in, 8)); - - /* extend remaining two q7_t values to q15_t values */ - in2 = __SXTB16(in); - -#ifndef ARM_MATH_BIG_ENDIAN - out2 = (int32_t)__PKHTB(in1, in2, 16); - out1 = (int32_t)__PKHBT(in2, in1, 16); -#else - out1 = (int32_t)__PKHTB(in1, in2, 16); - out2 = (int32_t)__PKHBT(in2, in1, 16); -#endif - arm_nn_write_q15x2_ia(&pDst, out1); - arm_nn_write_q15x2_ia(&pDst, out2); - - /* Decrement the loop counter */ - blkCnt--; - } - - /* If the blockSize is not a multiple of 4, compute any remaining output samples here. - ** No loop unrolling is used. */ - blkCnt = blockSize % 0x4u; - -#else - - /* Run the below code for Cortex-M0 */ - - /* Loop over blockSize number of values */ - blkCnt = blockSize; - -#endif /* #ifndef ARM_MATH_CM0_FAMILY */ - - while (blkCnt > 0u) - { - /* convert from q7 to q15 and then store the results in the destination buffer */ - *pDst++ = (q15_t)*pIn++; - - /* Decrement the loop counter */ - blkCnt--; - } -} - -/** - * @} end of nndata_convert group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_q7_to_q15_no_shift.c
+ * Description: Converts the elements of the Q7 vector to Q15 vector without left-shift
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnsupportfunctions.h"
+
+/**
+ * @ingroup groupSupport
+ */
+
+/**
+ * @addtogroup nndata_convert
+ * @{
+ */
+
+/**
+ * @brief Converts the elements of the Q7 vector to Q15 vector without left-shift
+ * @param[in] *pSrc points to the Q7 input vector
+ * @param[out] *pDst points to the Q15 output vector
+ * @param[in] blockSize length of the input vector
+ * @return none.
+ *
+ * \par Description:
+ *
+ * The equation used for the conversion process is:
+ *
+ * <pre>
+ * pDst[n] = (q15_t) pSrc[n]; 0 <= n < blockSize.
+ * </pre>
+ *
+ */
+
+void arm_q7_to_q15_no_shift(const q7_t * pSrc, q15_t * pDst, uint32_t blockSize)
+{
+ const q7_t *pIn = pSrc; /* Src pointer */
+ uint32_t blkCnt; /* loop counter */
+
+#ifndef ARM_MATH_CM0_FAMILY
+ q31_t in;
+ q31_t in1, in2;
+ q31_t out1, out2;
+
+ /* Run the below code for Cortex-M4 and Cortex-M3 */
+
+ /*loop Unrolling */
+ blkCnt = blockSize >> 2u;
+
+ /* First part of the processing with loop unrolling. Compute 4 outputs at a time.
+ ** a second loop below computes the remaining 1 to 3 samples. */
+ while (blkCnt > 0u)
+ {
+ /* C = (q15_t) A << 8 */
+ /* convert from q7 to q15 and then store the results in the destination buffer */
+ in = *__SIMD32(pIn)++;
+
+ /* rotatate in by 8 and extend two q7_t values to q15_t values */
+ in1 = __SXTB16(__ROR(in, 8));
+
+ /* extend remainig two q7_t values to q15_t values */
+ in2 = __SXTB16(in);
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ out2 = __PKHTB(in1, in2, 16);
+ out1 = __PKHBT(in2, in1, 16);
+
+#else
+
+ out1 = __PKHTB(in1, in2, 16);
+ out2 = __PKHBT(in2, in1, 16);
+
+#endif
+
+ *__SIMD32(pDst)++ = out1;
+ *__SIMD32(pDst)++ = out2;
+
+ /* Decrement the loop counter */
+ blkCnt--;
+ }
+
+ /* If the blockSize is not a multiple of 4, compute any remaining output samples here.
+ ** No loop unrolling is used. */
+ blkCnt = blockSize % 0x4u;
+
+#else
+
+ /* Run the below code for Cortex-M0 */
+
+ /* Loop over blockSize number of values */
+ blkCnt = blockSize;
+
+#endif /* #ifndef ARM_MATH_CM0_FAMILY */
+
+ while (blkCnt > 0u)
+ {
+ /* C = (q15_t) A << 8 */
+ /* convert from q7 to q15 and then store the results in the destination buffer */
+ *pDst++ = (q15_t) * pIn++;
+
+ /* Decrement the loop counter */
+ blkCnt--;
+ }
+
+}
+
+/**
+ * @} end of nndata_convert group
+ */
diff --git a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c index 8abbc3a..52f5f8e 100644 --- a/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c +++ b/Drivers/CMSIS/NN/Source/NNSupportFunctions/arm_q7_to_q15_reordered_no_shift.c @@ -1,143 +1,145 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_q7_to_q15_reordered_no_shift.c - * Description: Converts the elements of the Q7 vector to reordered Q15 vector without left-shift - * - * $Date: July 20, 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnsupportfunctions.h" - -/** - * @ingroup groupSupport - */ - -/** - * @addtogroup nndata_convert - * @{ - */ - -/** - * @brief Converts the elements of the Q7 vector to reordered Q15 vector without left-shift - * @param[in] *pSrc points to the Q7 input vector - * @param[out] *pDst points to the Q15 output vector - * @param[in] blockSize length of the input vector - * - * @details - * - * This function does the q7 to q15 expansion with re-ordering - * - * <pre> - * | A1 | A2 | A3 | A4 | - * - * 0 7 8 15 16 23 24 31 - * </pre> - * - * is converted into: - * - * <pre> - * | A1 | A3 | and | A2 | A4 | - * - * 0 15 16 31 0 15 16 31 - * </pre> - * - * - * This looks strange but is natural considering how sign-extension is done at - * assembly level. - * - * The expansion of other other oprand will follow the same rule so that the end - * results are the same. - * - * The tail (i.e., last (N % 4) elements) will still be in original order. - * - */ - -void arm_q7_to_q15_reordered_no_shift(const q7_t *pSrc, q15_t *pDst, uint32_t blockSize) -{ - const q7_t *pIn = pSrc; /* Src pointer */ - uint32_t blkCnt; /* loop counter */ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - q31_t in; - q31_t in1, in2; - - /* Run the below code for Cortex-M4 and Cortex-M3 */ - - /*loop Unrolling */ - blkCnt = blockSize >> 2u; - - /* First part of the processing with loop unrolling. Compute 4 outputs at a time. - ** a second loop below computes the remaining 1 to 3 samples. */ - while (blkCnt > 0u) - { - /* C = (q15_t) A << 8 */ - /* convert from q7 to q15 and then store the results in the destination buffer */ - in = arm_nn_read_q7x4_ia(&pIn); - - /* rotatate in by 8 and extend two q7_t values to q15_t values */ - in1 = __SXTB16(__ROR((uint32_t)in, 8)); - - /* extend remainig two q7_t values to q15_t values */ - in2 = __SXTB16(in); - -#ifndef ARM_MATH_BIG_ENDIAN - arm_nn_write_q7x4_ia((q7_t **)&pDst, in2); - arm_nn_write_q7x4_ia((q7_t **)&pDst, in1); -#else - arm_nn_write_q7x4_ia((q7_t **)&pDst, in1); - arm_nn_write_q7x4_ia((q7_t **)&pDst, in2); -#endif - - /* Decrement the loop counter */ - blkCnt--; - } - - /* If the blockSize is not a multiple of 4, compute any remaining output samples here. - ** No loop unrolling is used. */ - blkCnt = blockSize % 0x4u; - -#else - - /* Run the below code for Cortex-M0 */ - - /* Loop over blockSize number of values */ - blkCnt = blockSize; - -#endif /* #ifndef ARM_MATH_CM0_FAMILY */ - - while (blkCnt > 0u) - { - /* C = (q15_t) A << 8 */ - /* convert from q7 to q15 and then store the results in the destination buffer */ - *pDst++ = (q15_t)*pIn++; - - /* Decrement the loop counter */ - blkCnt--; - } -} - -/** - * @} end of q7_to_x group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_q7_to_q15_reordered_no_shift.c
+ * Description: Converts the elements of the Q7 vector to reordered Q15 vector without left-shift
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnsupportfunctions.h"
+
+/**
+ * @ingroup groupSupport
+ */
+
+/**
+ * @addtogroup nndata_convert
+ * @{
+ */
+
+/**
+ * @brief Converts the elements of the Q7 vector to reordered Q15 vector without left-shift
+ * @param[in] *pSrc points to the Q7 input vector
+ * @param[out] *pDst points to the Q15 output vector
+ * @param[in] blockSize length of the input vector
+ * @return none.
+ *
+ * @details
+ *
+ * This function does the q7 to q15 expansion with re-ordering
+ *
+ * <pre>
+ * | A1 | A2 | A3 | A4 |
+ *
+ * 0 7 8 15 16 23 24 31
+ * </pre>
+ *
+ * is converted into:
+ *
+ * <pre>
+ * | A1 | A3 | and | A2 | A4 |
+ *
+ * 0 15 16 31 0 15 16 31
+ * </pre>
+ *
+ *
+ * This looks strange but is natural considering how sign-extension is done at
+ * assembly level.
+ *
+ * The expansion of other other oprand will follow the same rule so that the end
+ * results are the same.
+ *
+ * The tail (i.e., last (N % 4) elements) will still be in original order.
+ *
+ */
+
+void arm_q7_to_q15_reordered_no_shift(const q7_t * pSrc, q15_t * pDst, uint32_t blockSize)
+{
+ const q7_t *pIn = pSrc; /* Src pointer */
+ uint32_t blkCnt; /* loop counter */
+
+#ifndef ARM_MATH_CM0_FAMILY
+ q31_t in;
+ q31_t in1, in2;
+
+ /* Run the below code for Cortex-M4 and Cortex-M3 */
+
+ /*loop Unrolling */
+ blkCnt = blockSize >> 2u;
+
+ /* First part of the processing with loop unrolling. Compute 4 outputs at a time.
+ ** a second loop below computes the remaining 1 to 3 samples. */
+ while (blkCnt > 0u)
+ {
+ /* C = (q15_t) A << 8 */
+ /* convert from q7 to q15 and then store the results in the destination buffer */
+ in = *__SIMD32(pIn)++;
+
+ /* rotatate in by 8 and extend two q7_t values to q15_t values */
+ in1 = __SXTB16(__ROR(in, 8));
+
+ /* extend remainig two q7_t values to q15_t values */
+ in2 = __SXTB16(in);
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ *__SIMD32(pDst)++ = in2;
+ *__SIMD32(pDst)++ = in1;
+#else
+ *__SIMD32(pDst)++ = in1;
+ *__SIMD32(pDst)++ = in2;
+#endif
+
+ /* Decrement the loop counter */
+ blkCnt--;
+ }
+
+ /* If the blockSize is not a multiple of 4, compute any remaining output samples here.
+ ** No loop unrolling is used. */
+ blkCnt = blockSize % 0x4u;
+
+#else
+
+ /* Run the below code for Cortex-M0 */
+
+ /* Loop over blockSize number of values */
+ blkCnt = blockSize;
+
+#endif /* #ifndef ARM_MATH_CM0_FAMILY */
+
+ while (blkCnt > 0u)
+ {
+ /* C = (q15_t) A << 8 */
+ /* convert from q7 to q15 and then store the results in the destination buffer */
+ *pDst++ = (q15_t) * pIn++;
+
+ /* Decrement the loop counter */
+ blkCnt--;
+ }
+
+}
+
+/**
+ * @} end of q7_to_x group
+ */
diff --git a/Drivers/CMSIS/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c b/Drivers/CMSIS/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c index 5a3b1af..71fb771 100644 --- a/Drivers/CMSIS/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c +++ b/Drivers/CMSIS/NN/Source/PoolingFunctions/arm_pool_q7_HWC.c @@ -1,464 +1,460 @@ -/* - * Copyright (C) 2010-2021 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_pool_q7_HWC.c - * Description: Pooling function implementations - * - * $Date: 20. July 2021 - * $Revision: V.1.1.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" -#include "arm_nnsupportfunctions.h" - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - -/** - * @brief A few utility functions used by pooling functions - * - * - */ - -static void buffer_scale_back_q15_to_q7(q15_t *buffer, q7_t *target, uint16_t length, uint16_t scale) -{ - int i; - - for (i = 0; i < length; i++) - { - target[i] = (q7_t)(buffer[i] / scale); - } -} - -static void compare_and_replace_if_larger_q7(q7_t *base, // base data - const q7_t *target, // compare target - const uint16_t length // data length -) -{ - q7_t *pIn = base; - const q7_t *pCom = target; - union arm_nnword in; - union arm_nnword com; - uint16_t cnt = length >> 2; - - while (cnt > 0u) - { - in.word = arm_nn_read_q7x4((const q7_t *)pIn); - com.word = arm_nn_read_q7x4_ia((const q7_t **)&pCom); - - // if version - if (com.bytes[0] > in.bytes[0]) - in.bytes[0] = com.bytes[0]; - if (com.bytes[1] > in.bytes[1]) - in.bytes[1] = com.bytes[1]; - if (com.bytes[2] > in.bytes[2]) - in.bytes[2] = com.bytes[2]; - if (com.bytes[3] > in.bytes[3]) - in.bytes[3] = com.bytes[3]; - - arm_nn_write_q7x4_ia(&pIn, in.word); - - cnt--; - } - - cnt = length & 0x3; - while (cnt > 0u) - { - if (*pCom > *pIn) - { - *pIn = *pCom; - } - pIn++; - pCom++; - cnt--; - } -} - -static void accumulate_q7_to_q15(q15_t *base, q7_t *target, const uint16_t length) -{ - q15_t *pCnt = base; - q7_t *pV = target; - q31_t v1, v2, vo1, vo2; - uint16_t cnt = length >> 2; - q31_t in; - - while (cnt > 0u) - { - q31_t value = arm_nn_read_q7x4_ia((const q7_t **)&pV); - v1 = __SXTB16(__ROR(value, 8)); - v2 = __SXTB16(value); -#ifndef ARM_MATH_BIG_ENDIAN - - vo2 = __PKHTB(v1, v2, 16); - vo1 = __PKHBT(v2, v1, 16); - -#else - - vo1 = __PKHTB(v1, v2, 16); - vo2 = __PKHBT(v2, v1, 16); - -#endif - - in = arm_nn_read_q15x2(pCnt); - arm_nn_write_q15x2_ia(&pCnt, __QADD16(vo1, in)); - - in = arm_nn_read_q15x2(pCnt); - arm_nn_write_q15x2_ia(&pCnt, __QADD16(vo2, in)); - - cnt--; - } - cnt = length & 0x3; - while (cnt > 0u) - { - *pCnt++ += *pV++; - cnt--; - } -} - -#endif // ARM_MATH_DSP - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Pooling - * @{ - */ - -/** - * @brief Q7 max pooling function - * @param[in, out] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA Not used - * @param[in,out] Im_out pointer to output tensor - * - * @details - * - * The pooling function is implemented as split x-pooling then - * y-pooling. - * - * This pooling function is input-destructive. Input data is undefined - * after calling this function. - * - */ - -void arm_maxpool_q7_HWC(q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const uint16_t dim_im_out, - q7_t *bufferA, - q7_t *Im_out) -{ - (void)bufferA; -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - int16_t i_x, i_y; - - /* first does the pooling along x axis */ - for (i_y = 0; i_y < dim_im_in; i_y++) - { - - for (i_x = 0; i_x < dim_im_out; i_x++) - { - /* for each output pixel */ - q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in; - q7_t *win_start; - q7_t *win_stop; - if (i_x * stride - padding < 0) - { - win_start = target; - } - else - { - win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in; - } - - if (i_x * stride - padding + dim_kernel >= dim_im_in) - { - win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in; - } - else - { - win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in; - } - - /* first step is to copy over initial data */ - /* arm_copy_q7(win_start, target, ch_im_in); */ - memmove(target, win_start, ch_im_in); - - /* start the max operation from the second part */ - win_start += ch_im_in; - for (; win_start < win_stop; win_start += ch_im_in) - { - compare_and_replace_if_larger_q7(target, win_start, ch_im_in); - } - } - } - - /* then does the pooling along y axis */ - for (i_y = 0; i_y < dim_im_out; i_y++) - { - - /* for each output row */ - q7_t *target = Im_out + i_y * dim_im_out * ch_im_in; - q7_t *row_start; - q7_t *row_end; - /* setting the starting row */ - if (i_y * stride - padding < 0) - { - row_start = Im_in; - } - else - { - row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in; - } - /* setting the stopping row */ - if (i_y * stride - padding + dim_kernel >= dim_im_in) - { - row_end = Im_in + dim_im_in * dim_im_in * ch_im_in; - } - else - { - row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in; - } - - /* copy over the first row */ - /* arm_copy_q7(row_start, target, dim_im_out * ch_im_in); */ - memmove(target, row_start, dim_im_out * ch_im_in); - - /* move over to next row */ - row_start += ch_im_in * dim_im_in; - - for (; row_start < row_end; row_start += dim_im_in * ch_im_in) - { - compare_and_replace_if_larger_q7(target, row_start, dim_im_out * ch_im_in); - } - } - -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - int16_t i_ch_in, i_x, i_y; - int16_t k_x, k_y; - - for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++) - { - for (i_y = 0; i_y < dim_im_out; i_y++) - { - for (i_x = 0; i_x < dim_im_out; i_x++) - { - int max = -129; - for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++) - { - for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++) - { - if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in) - { - if (Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)] > max) - { - max = Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)]; - } - } - } - } - Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = max; - } - } - } - -#endif /* ARM_MATH_DSP */ -} - -/** - * @brief Q7 average pooling function - * @param[in,out] Im_in pointer to input tensor - * @param[in] dim_im_in input tensor dimention - * @param[in] ch_im_in number of input tensor channels - * @param[in] dim_kernel filter kernel size - * @param[in] padding padding sizes - * @param[in] stride convolution stride - * @param[in] dim_im_out output tensor dimension - * @param[in,out] bufferA pointer to buffer space for input - * @param[in,out] Im_out pointer to output tensor - * - * @details - * - * <b>Buffer size:</b> - * - * bufferA size: 2*dim_im_out*ch_im_in - * - * The pooling function is implemented as split x-pooling then - * y-pooling. - * - * This pooling function is input-destructive. Input data is undefined - * after calling this function. - * - */ - -void arm_avepool_q7_HWC(q7_t *Im_in, - const uint16_t dim_im_in, - const uint16_t ch_im_in, - const uint16_t dim_kernel, - const uint16_t padding, - const uint16_t stride, - const uint16_t dim_im_out, - q7_t *bufferA, - q7_t *Im_out) -{ - -#if defined(ARM_MATH_DSP) && !defined(ARM_MATH_MVEI) - /* Run the following code for Cortex-M4 and Cortex-M7 */ - - q15_t *buffer = (q15_t *)bufferA; - int16_t i_x, i_y; - int16_t count = 0; - - /* first does the pooling along x axis */ - for (i_y = 0; i_y < dim_im_in; i_y++) - { - - for (i_x = 0; i_x < dim_im_out; i_x++) - { - /* for each output pixel */ - q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in; - q7_t *win_start; - q7_t *win_stop; - if (i_x * stride - padding < 0) - { - win_start = target; - } - else - { - win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in; - } - - if (i_x * stride - padding + dim_kernel >= dim_im_in) - { - win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in; - } - else - { - win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in; - } - - /* first step is to copy over initial data */ - arm_q7_to_q15_no_shift(win_start, buffer, ch_im_in); - count = 1; - - /* start the max operation from the second part */ - win_start += ch_im_in; - for (; win_start < win_stop; win_start += ch_im_in) - { - accumulate_q7_to_q15(buffer, win_start, ch_im_in); - count++; - } - buffer_scale_back_q15_to_q7(buffer, target, ch_im_in, count); - } - } - - /* then does the pooling along y axis */ - for (i_y = 0; i_y < dim_im_out; i_y++) - { - /* for each output row */ - q7_t *target = Im_out + i_y * dim_im_out * ch_im_in; - q7_t *row_start; - q7_t *row_end; - /* setting the starting row */ - if (i_y * stride - padding < 0) - { - row_start = Im_in; - } - else - { - row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in; - } - /* setting the stopping row */ - if (i_y * stride - padding + dim_kernel >= dim_im_in) - { - row_end = Im_in + dim_im_in * dim_im_in * ch_im_in; - } - else - { - row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in; - } - - /* copy over the first row */ - arm_q7_to_q15_no_shift(row_start, buffer, dim_im_out * ch_im_in); - count = 1; - - /* move over to next row */ - row_start += ch_im_in * dim_im_in; - - for (; row_start < row_end; row_start += dim_im_in * ch_im_in) - { - accumulate_q7_to_q15(buffer, row_start, dim_im_out * ch_im_in); - count++; - } - buffer_scale_back_q15_to_q7(buffer, target, dim_im_out * ch_im_in, count); - } - -#else - /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */ - - (void)bufferA; - int16_t i_ch_in, i_x, i_y; - int16_t k_x, k_y; - - for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++) - { - for (i_y = 0; i_y < dim_im_out; i_y++) - { - for (i_x = 0; i_x < dim_im_out; i_x++) - { - int sum = 0; - int count = 0; - for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++) - { - for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++) - { - if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in) - { - sum += Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)]; - count++; - } - } - } - Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = sum / count; - } - } - } - -#endif /* ARM_MATH_DSP */ -} - -/** - * @} end of Pooling group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_pool_q7_HWC.c
+ * Description: Pooling function implementations
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+#if defined (ARM_MATH_DSP)
+
+/**
+ * @brief A few utility functions used by pooling functions
+ *
+ *
+ */
+
+static void buffer_scale_back_q15_to_q7(q15_t * buffer, q7_t * target, uint16_t length, uint16_t scale)
+{
+ int i;
+
+ for (i = 0; i < length; i++)
+ {
+ target[i] = (q7_t) (buffer[i] / scale);
+ }
+}
+
+static void compare_and_replace_if_larger_q7(q7_t * base, // base data
+ const q7_t * target, // compare target
+ const uint16_t length // data length
+ )
+{
+ q7_t *pIn = base;
+ const q7_t *pCom = target;
+ union arm_nnword in;
+ union arm_nnword com;
+ uint16_t cnt = length >> 2;
+
+ while (cnt > 0u)
+ {
+ in.word = *__SIMD32(pIn);
+ com.word = *__SIMD32(pCom)++;
+
+ // if version
+ if (com.bytes[0] > in.bytes[0])
+ in.bytes[0] = com.bytes[0];
+ if (com.bytes[1] > in.bytes[1])
+ in.bytes[1] = com.bytes[1];
+ if (com.bytes[2] > in.bytes[2])
+ in.bytes[2] = com.bytes[2];
+ if (com.bytes[3] > in.bytes[3])
+ in.bytes[3] = com.bytes[3];
+
+ *__SIMD32(pIn)++ = in.word;
+
+ cnt--;
+ }
+
+ cnt = length & 0x3;
+ while (cnt > 0u)
+ {
+ if (*pCom > *pIn)
+ {
+ *pIn = *pCom;
+ }
+ pIn++;
+ pCom++;
+ cnt--;
+ }
+}
+
+static void accumulate_q7_to_q15(q15_t * base, q7_t * target, const uint16_t length)
+{
+ q15_t *pCnt = base;
+ q7_t *pV = target;
+ q31_t v1, v2, vo1, vo2;
+ uint16_t cnt = length >> 2;
+ q31_t in;
+
+ while (cnt > 0u)
+ {
+ q31_t value = *__SIMD32(pV)++;
+ v1 = __SXTB16(__ROR(value, 8));
+ v2 = __SXTB16(value);
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ vo2 = __PKHTB(v1, v2, 16);
+ vo1 = __PKHBT(v2, v1, 16);
+
+#else
+
+ vo1 = __PKHTB(v1, v2, 16);
+ vo2 = __PKHBT(v2, v1, 16);
+
+#endif
+
+ in = *__SIMD32(pCnt);
+ *__SIMD32(pCnt)++ = __QADD16(vo1, in);
+
+ in = *__SIMD32(pCnt);
+ *__SIMD32(pCnt)++ = __QADD16(vo2, in);
+
+ cnt--;
+ }
+ cnt = length & 0x3;
+ while (cnt > 0u)
+ {
+ *pCnt++ += *pV++;
+ cnt--;
+ }
+}
+
+#endif // ARM_MATH_DSP
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Pooling
+ * @{
+ */
+
+ /**
+ * @brief Q7 max pooling function
+ * @param[in, out] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] Im_out pointer to output tensor
+ * @return none.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 0
+ *
+ * The pooling function is implemented as split x-pooling then
+ * y-pooling.
+ *
+ * This pooling function is input-destructive. Input data is undefined
+ * after calling this function.
+ *
+ */
+
+void
+arm_maxpool_q7_HWC(q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride, const uint16_t dim_im_out, q7_t * bufferA, q7_t * Im_out)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_x, i_y;
+
+ /* first does the pooling along x axis */
+ for (i_y = 0; i_y < dim_im_in; i_y++)
+ {
+
+ for (i_x = 0; i_x < dim_im_out; i_x++)
+ {
+ /* for each output pixel */
+ q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in;
+ q7_t *win_start;
+ q7_t *win_stop;
+ if (i_x * stride - padding < 0)
+ {
+ win_start = target;
+ } else
+ {
+ win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in;
+ }
+
+ if (i_x * stride - padding + dim_kernel >= dim_im_in)
+ {
+ win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in;
+ } else
+ {
+ win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in;
+ }
+
+ /* first step is to copy over initial data */
+ /* arm_copy_q7(win_start, target, ch_im_in); */
+ memmove(target, win_start, ch_im_in);
+
+ /* start the max operation from the second part */
+ win_start += ch_im_in;
+ for (; win_start < win_stop; win_start += ch_im_in)
+ {
+ compare_and_replace_if_larger_q7(target, win_start, ch_im_in);
+ }
+ }
+ }
+
+ /* then does the pooling along y axis */
+ for (i_y = 0; i_y < dim_im_out; i_y++)
+ {
+
+ /* for each output row */
+ q7_t *target = Im_out + i_y * dim_im_out * ch_im_in;
+ q7_t *row_start;
+ q7_t *row_end;
+ /* setting the starting row */
+ if (i_y * stride - padding < 0)
+ {
+ row_start = Im_in;
+ } else
+ {
+ row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in;
+ }
+ /* setting the stopping row */
+ if (i_y * stride - padding + dim_kernel >= dim_im_in)
+ {
+ row_end = Im_in + dim_im_in * dim_im_in * ch_im_in;
+ } else
+ {
+ row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in;
+ }
+
+ /* copy over the first row */
+ /* arm_copy_q7(row_start, target, dim_im_out * ch_im_in); */
+ memmove(target, row_start, dim_im_out * ch_im_in);
+
+ /* move over to next row */
+ row_start += ch_im_in * dim_im_in;
+
+ for (; row_start < row_end; row_start += dim_im_in * ch_im_in)
+ {
+ compare_and_replace_if_larger_q7(target, row_start, dim_im_out * ch_im_in);
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ int16_t i_ch_in, i_x, i_y;
+ int16_t k_x, k_y;
+
+ for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++)
+ {
+ for (i_y = 0; i_y < dim_im_out; i_y++)
+ {
+ for (i_x = 0; i_x < dim_im_out; i_x++)
+ {
+ int max = -129;
+ for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++)
+ {
+ for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++)
+ {
+ if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in)
+ {
+ if (Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)] > max)
+ {
+ max = Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)];
+ }
+ }
+ }
+ }
+ Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = max;
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+}
+
+ /**
+ * @brief Q7 average pooling function
+ * @param[in,out] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] Im_out pointer to output tensor
+ * @return none.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*dim_im_out*ch_im_in
+ *
+ * The pooling function is implemented as split x-pooling then
+ * y-pooling.
+ *
+ * This pooling function is input-destructive. Input data is undefined
+ * after calling this function.
+ *
+ */
+
+void
+arm_avepool_q7_HWC(q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride, const uint16_t dim_im_out, q7_t * bufferA, q7_t * Im_out)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ q15_t *buffer = (q15_t *) bufferA;
+ int16_t i_x, i_y;
+ int16_t count = 0;
+
+ /* first does the pooling along x axis */
+ for (i_y = 0; i_y < dim_im_in; i_y++)
+ {
+
+ for (i_x = 0; i_x < dim_im_out; i_x++)
+ {
+ /* for each output pixel */
+ q7_t *target = Im_in + (i_y * dim_im_in + i_x) * ch_im_in;
+ q7_t *win_start;
+ q7_t *win_stop;
+ if (i_x * stride - padding < 0)
+ {
+ win_start = target;
+ } else
+ {
+ win_start = Im_in + (i_y * dim_im_in + i_x * stride - padding) * ch_im_in;
+ }
+
+ if (i_x * stride - padding + dim_kernel >= dim_im_in)
+ {
+ win_stop = Im_in + (i_y * dim_im_in + dim_im_in) * ch_im_in;
+ } else
+ {
+ win_stop = Im_in + (i_y * dim_im_in + i_x * stride - padding + dim_kernel) * ch_im_in;
+ }
+
+ /* first step is to copy over initial data */
+ arm_q7_to_q15_no_shift(win_start, buffer, ch_im_in);
+ count = 1;
+
+ /* start the max operation from the second part */
+ win_start += ch_im_in;
+ for (; win_start < win_stop; win_start += ch_im_in)
+ {
+ accumulate_q7_to_q15(buffer, win_start, ch_im_in);
+ count++;
+ }
+ buffer_scale_back_q15_to_q7(buffer, target, ch_im_in, count);
+ }
+ }
+
+ /* then does the pooling along y axis */
+ for (i_y = 0; i_y < dim_im_out; i_y++)
+ {
+ /* for each output row */
+ q7_t *target = Im_out + i_y * dim_im_out * ch_im_in;
+ q7_t *row_start;
+ q7_t *row_end;
+ /* setting the starting row */
+ if (i_y * stride - padding < 0)
+ {
+ row_start = Im_in;
+ } else
+ {
+ row_start = Im_in + (i_y * stride - padding) * dim_im_in * ch_im_in;
+ }
+ /* setting the stopping row */
+ if (i_y * stride - padding + dim_kernel >= dim_im_in)
+ {
+ row_end = Im_in + dim_im_in * dim_im_in * ch_im_in;
+ } else
+ {
+ row_end = Im_in + (i_y * stride - padding + dim_kernel) * dim_im_in * ch_im_in;
+ }
+
+ /* copy over the first row */
+ arm_q7_to_q15_no_shift(row_start, buffer, dim_im_out * ch_im_in);
+ count = 1;
+
+ /* move over to next row */
+ row_start += ch_im_in * dim_im_in;
+
+ for (; row_start < row_end; row_start += dim_im_in * ch_im_in)
+ {
+ accumulate_q7_to_q15(buffer, row_start, dim_im_out * ch_im_in);
+ count++;
+ }
+ buffer_scale_back_q15_to_q7(buffer, target, dim_im_out * ch_im_in, count);
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ int16_t i_ch_in, i_x, i_y;
+ int16_t k_x, k_y;
+
+ for (i_ch_in = 0; i_ch_in < ch_im_in; i_ch_in++)
+ {
+ for (i_y = 0; i_y < dim_im_out; i_y++)
+ {
+ for (i_x = 0; i_x < dim_im_out; i_x++)
+ {
+ int sum = 0;
+ int count = 0;
+ for (k_y = i_y * stride - padding; k_y < i_y * stride - padding + dim_kernel; k_y++)
+ {
+ for (k_x = i_x * stride - padding; k_x < i_x * stride - padding + dim_kernel; k_x++)
+ {
+ if (k_y >= 0 && k_x >= 0 && k_y < dim_im_in && k_x < dim_im_in)
+ {
+ sum += Im_in[i_ch_in + ch_im_in * (k_x + k_y * dim_im_in)];
+ count++;
+ }
+ }
+ }
+ Im_out[i_ch_in + ch_im_in * (i_x + i_y * dim_im_out)] = sum / count;
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+}
+
+/**
+ * @} end of Pooling group
+ */
diff --git a/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q15.c b/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q15.c index 18f3e83..22fa62b 100644 --- a/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q15.c +++ b/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q15.c @@ -1,118 +1,120 @@ -/* - * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_softmax_q15.c - * Description: Q15 softmax function - * - * $Date: 09. October 2020 - * $Revision: V.1.0.1 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Softmax - * @{ - */ - -/** - * @brief Q15 softmax function - * @param[in] vec_in pointer to input vector - * @param[in] dim_vec input vector dimention - * @param[out] p_out pointer to output vector - * - * @details - * - * Here, instead of typical e based softmax, we use - * 2-based softmax, i.e.,: - * - * y_i = 2^(x_i) / sum(2^x_j) - * - * The relative output will be different here. - * But mathematically, the gradient will be the same - * with a log(2) scaling factor. - * - */ - -void arm_softmax_q15(const q15_t *vec_in, const uint16_t dim_vec, q15_t *p_out) -{ - q31_t sum; - int16_t i; - uint8_t shift; - q31_t base; - base = -1 * 0x100000; - for (i = 0; i < dim_vec; i++) - { - if (vec_in[i] > base) - { - base = vec_in[i]; - } - } - - /* we ignore really small values - * anyway, they will be 0 after shrinking - * to q15_t - */ - base = base - 16; - - sum = 0; - - for (i = 0; i < dim_vec; i++) - { - if (vec_in[i] > base) - { - shift = (uint8_t)__USAT(vec_in[i] - base, 5); - sum += 0x1 << shift; - } - } - - /* This is effectively (0x1 << 32) / sum */ - int64_t div_base = 0x100000000LL; - int output_base = (int32_t)(div_base / sum); - - /* Final confidence will be output_base >> ( 17 - (vec_in[i] - base) ) - * so 32768 (0x1<<15) -> 100% confidence when sum = 0x1 << 16, output_base = 0x1 << 16 - * and vec_in[i]-base = 16 - */ - for (i = 0; i < dim_vec; i++) - { - if (vec_in[i] > base) - { - /* Here minimum value of 17+base-vec[i] will be 1 */ - shift = (uint8_t)__USAT(17 + base - vec_in[i], 5); - p_out[i] = (q15_t)__SSAT((output_base >> shift), 16); - } - else - { - p_out[i] = 0; - } - } -} - -/** - * @} end of Softmax group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_softmax_q15.c
+ * Description: Q15 softmax function
+ *
+ * $Date: 20. February 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Softmax
+ * @{
+ */
+
+ /**
+ * @brief Q15 softmax function
+ * @param[in] vec_in pointer to input vector
+ * @param[in] dim_vec input vector dimention
+ * @param[out] p_out pointer to output vector
+ * @return none.
+ *
+ * @details
+ *
+ * Here, instead of typical e based softmax, we use
+ * 2-based softmax, i.e.,:
+ *
+ * y_i = 2^(x_i) / sum(2^x_j)
+ *
+ * The relative output will be different here.
+ * But mathematically, the gradient will be the same
+ * with a log(2) scaling factor.
+ *
+ */
+
+void arm_softmax_q15(const q15_t * vec_in, const uint16_t dim_vec, q15_t * p_out)
+{
+ q31_t sum;
+ int16_t i;
+ uint8_t shift;
+ q31_t base;
+ base = -1 * 0x100000;
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ base = vec_in[i];
+ }
+ }
+
+ /* we ignore really small values
+ * anyway, they will be 0 after shrinking
+ * to q15_t
+ */
+ base = base - 16;
+
+ sum = 0;
+
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ shift = (uint8_t)__USAT(vec_in[i] - base, 5);
+ sum += 0x1 << shift;
+ }
+ }
+
+ /* This is effectively (0x1 << 32) / sum */
+ int64_t div_base = 0x100000000LL;
+ int output_base = (int32_t)(div_base / sum);
+
+ /* Final confidence will be output_base >> ( 17 - (vec_in[i] - base) )
+ * so 32768 (0x1<<15) -> 100% confidence when sum = 0x1 << 16, output_base = 0x1 << 16
+ * and vec_in[i]-base = 16
+ */
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ /* Here minimum value of 17+base-vec[i] will be 1 */
+ shift = (uint8_t)__USAT(17+base-vec_in[i], 5);
+ p_out[i] = (q15_t) __SSAT((output_base >> shift), 16);
+ } else
+ {
+ p_out[i] = 0;
+ }
+ }
+
+}
+
+/**
+ * @} end of Softmax group
+ */
diff --git a/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q7.c b/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q7.c index 58eb990..06a69e1 100644 --- a/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q7.c +++ b/Drivers/CMSIS/NN/Source/SoftmaxFunctions/arm_softmax_q7.c @@ -1,107 +1,121 @@ -/* - * Copyright (C) 2010-2020 Arm Limited or its affiliates. All rights reserved. - * - * SPDX-License-Identifier: Apache-2.0 - * - * Licensed under the Apache License, Version 2.0 (the License); you may - * not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an AS IS BASIS, WITHOUT - * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -/* ---------------------------------------------------------------------- - * Project: CMSIS NN Library - * Title: arm_softmax_q7.c - * Description: Q7 softmax function - * - * $Date: 09. October 2020 - * $Revision: V.1.0.2 - * - * Target Processor: Cortex-M cores - * - * -------------------------------------------------------------------- */ - -#include "arm_nnfunctions.h" - -/** - * @ingroup groupNN - */ - -/** - * @addtogroup Softmax - * @{ - */ - -/** - * @brief Q7 softmax function - * @param[in] vec_in pointer to input vector - * @param[in] dim_vec input vector dimention - * @param[out] p_out pointer to output vector - * - * @details - * - * Here, instead of typical natural logarithm e based softmax, we use - * 2-based softmax here, i.e.,: - * - * y_i = 2^(x_i) / sum(2^x_j) - * - * The relative output will be different here. - * But mathematically, the gradient will be the same - * with a log(2) scaling factor. - * - */ - -void arm_softmax_q7(const q7_t *vec_in, const uint16_t dim_vec, q7_t *p_out) -{ - q31_t sum; - int16_t i; - uint8_t shift; - q15_t base; - base = -128; - - /* We first search for the maximum */ - for (i = 0; i < dim_vec; i++) - { - if (vec_in[i] > base) - { - base = vec_in[i]; - } - } - - /* - * So the base is set to max-8, meaning - * that we ignore really small values. - * anyway, they will be 0 after shrinking to q7_t. - */ - base = base - (1 << 3); - - sum = 0; - - for (i = 0; i < dim_vec; i++) - { - shift = (uint8_t)__USAT(vec_in[i] - base, 3); - sum += 0x1 << shift; - } - - /* This is effectively (0x1 << 20) / sum */ - int output_base = (1 << 20) / sum; - - for (i = 0; i < dim_vec; i++) - { - - /* Here minimum value of 13+base-vec_in[i] will be 5 */ - shift = (uint8_t)__USAT(13 + base - vec_in[i], 5); - p_out[i] = (q7_t)__SSAT((output_base >> shift), 8); - } -} - -/** - * @} end of Softmax group - */ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_softmax_q7.c
+ * Description: Q7 softmax function
+ *
+ * $Date: 20. February 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup Softmax
+ * @{
+ */
+
+ /**
+ * @brief Q7 softmax function
+ * @param[in] vec_in pointer to input vector
+ * @param[in] dim_vec input vector dimention
+ * @param[out] p_out pointer to output vector
+ * @return none.
+ *
+ * @details
+ *
+ * Here, instead of typical natural logarithm e based softmax, we use
+ * 2-based softmax here, i.e.,:
+ *
+ * y_i = 2^(x_i) / sum(2^x_j)
+ *
+ * The relative output will be different here.
+ * But mathematically, the gradient will be the same
+ * with a log(2) scaling factor.
+ *
+ */
+
+void arm_softmax_q7(const q7_t * vec_in, const uint16_t dim_vec, q7_t * p_out)
+{
+ q31_t sum;
+ int16_t i;
+ uint8_t shift;
+ q15_t base;
+ base = -257;
+
+ /* We first search for the maximum */
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ base = vec_in[i];
+ }
+ }
+
+ /*
+ * So the base is set to max-8, meaning
+ * that we ignore really small values.
+ * anyway, they will be 0 after shrinking to q7_t.
+ */
+ base = base - 8;
+
+ sum = 0;
+
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ shift = (uint8_t)__USAT(vec_in[i] - base, 5);
+ sum += 0x1 << shift;
+ }
+ }
+
+ /* This is effectively (0x1 << 20) / sum */
+ int output_base = 0x100000 / sum;
+
+ /*
+ * Final confidence will be output_base >> ( 13 - (vec_in[i] - base) )
+ * so 128 (0x1<<7) -> 100% confidence when sum = 0x1 << 8, output_base = 0x1 << 12
+ * and vec_in[i]-base = 8
+ */
+ for (i = 0; i < dim_vec; i++)
+ {
+ if (vec_in[i] > base)
+ {
+ /* Here minimum value of 13+base-vec_in[i] will be 5 */
+ shift = (uint8_t)__USAT(13+base-vec_in[i], 5);
+ p_out[i] = (q7_t) __SSAT((output_base >> shift), 8);
+ } else {
+ p_out[i] = 0;
+ }
+ }
+}
+
+/**
+ * @} end of Softmax group
+ */
|