diff options
Diffstat (limited to 'Drivers/CMSIS/NN/Source/ConvolutionFunctions')
14 files changed, 3879 insertions, 0 deletions
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c new file mode 100644 index 0000000..4c69e7c --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_1x1_HWC_q7_fast_nonsquare.c @@ -0,0 +1,235 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_1x1_HWC_q7_fast_nonsquare.c
+ * Description: Fast Q7 version of 1x1 convolution (non-square shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Fast Q7 version of 1x1 convolution (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is optimized for convolution with 1x1 kernel size (i.e., dim_kernel_x=1
+ * and dim_kernel_y=1). It can be used for the second half of MobileNets [1] after depthwise
+ * separable convolution.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 4
+ * ch_im_out is multiple of 2
+ *
+ * [1] MobileNets: Efficient Convolutional Neural Networks for Mobile Vision Applications
+ * https://arxiv.org/abs/1704.04861
+ */
+
+arm_status arm_convolve_1x1_HWC_q7_fast_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ch_out;
+
+ /* -----------------------
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
+ || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_out_y * dim_im_in_x + i_out_x) * ch_im_in, pBuffer,
+ ch_im_in);
+ pBuffer += ch_im_in;
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ q31_t sum = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ int i, j, k, l, m, n;
+ int conv_out;
+ int in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0 || dim_kernel_x != 1 || dim_kernel_y != 1
+ || padding_x != 0 || padding_y != 0 || stride_x != 1 || stride_y != 1)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ // if-for implementation
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_y + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c new file mode 100644 index 0000000..ee08d74 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_basic.c @@ -0,0 +1,207 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_basic.c
+ * Description: Q15 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q15 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * This basic version is designed to work for any input tensor and weight
+ * dimension.
+ */
+
+arm_status
+arm_convolve_HWC_q15_basic(const q15_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ uint16_t im2col_out_pixel_index = 0;
+ q15_t *pBuffer = bufferA;
+ q15_t *pOut = Im_out;
+ q15_t *im_buffer = bufferA;
+ const q15_t *pA;
+ int i;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ pA = wt;
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = im_buffer;
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA)++;
+ q31_t inB2 = *__SIMD32(pB)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q15_t) __SSAT((sum >> out_shift), 16);
+ pOut++;
+ }
+
+ /* counter reset */
+ pBuffer = im_buffer;
+ im2col_out_pixel_index++;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c new file mode 100644 index 0000000..a02aaa0 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast.c @@ -0,0 +1,255 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_fast.c
+ * Description: Fast Q15 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q15 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 2
+ *
+ * ch_im_out is multipe of 2
+ *
+ */
+
+arm_status
+arm_convolve_HWC_q15_fast(const q15_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ q15_t *pBuffer = bufferA;
+ q15_t *im_buffer = bufferA;
+ q15_t *pOut = Im_out;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (i_out_x & 0x1)
+ {
+ int i;
+ /* initialize the matrix pointers for A */
+ const q15_t *pA = wt;
+
+ /* set up the second output pointers */
+ q15_t *pOut2 = pOut + ch_im_out;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ q15_t *pB = im_buffer;
+ const q15_t *pB2 = pB + ch_im_in * dim_kernel * dim_kernel;
+
+ /* aling the second pointer for A */
+ const q15_t *pA2 = pA + ch_im_in * dim_kernel * dim_kernel;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 1;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA2)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum2 = __SMLAD(inA1, inB2, sum2);
+ sum3 = __SMLAD(inA2, inB1, sum3);
+ sum4 = __SMLAD(inA2, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x1;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
+ *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
+
+ /* skip the row computed with A2 */
+ pA += ch_im_in * dim_kernel * dim_kernel;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+ /* counter reset */
+ pBuffer = im_buffer;
+ }
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c new file mode 100644 index 0000000..14d9130 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q15_fast_nonsquare.c @@ -0,0 +1,265 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q15_fast.c
+ * Description: Fast Q15 version of convolution
+ *
+ * $Date: 24. May 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q15 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 2
+ *
+ * ch_im_out is multipe of 2
+ *
+ */
+
+arm_status
+arm_convolve_HWC_q15_fast_nonsquare(const q15_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q15_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q15_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q15_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ q15_t *pBuffer = bufferA;
+ q15_t *im_buffer = bufferA;
+ q15_t *pOut = Im_out;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* arm_copy_q15((q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q15_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, sizeof(q15_t)*ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (i_out_x & 0x1)
+ {
+ int i;
+ /* initialize the matrix pointers for A */
+ const q15_t *pA = wt;
+
+ /* set up the second output pointers */
+ q15_t *pOut2 = pOut + ch_im_out;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ q15_t *pB = im_buffer;
+ const q15_t *pB2 = pB + ch_im_in * dim_kernel_y * dim_kernel_x;
+
+ /* aling the second pointer for A */
+ const q15_t *pA2 = pA + ch_im_in * dim_kernel_y * dim_kernel_x;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)bias[i + 1] << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 1;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA1 = *__SIMD32(pA)++;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inA2 = *__SIMD32(pA2)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA1, inB1, sum);
+ sum2 = __SMLAD(inA1, inB2, sum2);
+ sum3 = __SMLAD(inA2, inB1, sum3);
+ sum4 = __SMLAD(inA2, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x1;
+ while (colCnt)
+ {
+ q15_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q15_t) __SSAT(sum >> out_shift, 16);
+ *pOut++ = (q15_t) __SSAT(sum3 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum2 >> out_shift, 16);
+ *pOut2++ = (q15_t) __SSAT(sum4 >> out_shift, 16);
+
+ /* skip the row computed with A2 */
+ pA += ch_im_in * dim_kernel_y * dim_kernel_x;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+ /* counter reset */
+ pBuffer = im_buffer;
+ }
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 2 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel_x * dim_kernel_y + (m * dim_kernel_x +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q15_t) __SSAT((conv_out >> out_shift), 16);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c new file mode 100644 index 0000000..e53c6f9 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_RGB.c @@ -0,0 +1,279 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_RGB.c
+ * Description: Q7 version of convolution for RGB image
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Q7 convolution function for RGB image
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in equals 3
+ *
+ * This kernel is written exclusively for convolution with ch_im_in
+ * equals 3. This applies on the first layer of CNNs which has input
+ * image with RGB format.
+ */
+
+arm_status
+arm_convolve_HWC_q7_RGB(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out, const uint16_t dim_im_out, q15_t * bufferA, q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ // check if number of input channels is 3
+ if (ch_im_in != 3)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+ // This part implements the im2col function
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Equivalent to arm_fill_q15(0, pBuffer, ch_im_in) with assumption: ch_im_in = 3 */
+ *__SIMD32(pBuffer) = 0x0;
+ *(pBuffer + 2) = 0;
+ pBuffer += 3;
+ } else
+ {
+ /*
+ * Equivalent to:
+ * arm_q7_to_q15_no_shift( (q7_t*)Im_in+(i_ker_y*dim_im_in+i_ker_x)*3, pBuffer, 3);
+ */
+
+ const q7_t *pPixel = Im_in + (i_ker_y * dim_im_in + i_ker_x) * 3;
+ q31_t buf = *__SIMD32(pPixel);
+
+ union arm_nnword top;
+ union arm_nnword bottom;
+
+ top.word = __SXTB16(buf);
+ bottom.word = __SXTB16(__ROR(buf, 8));
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ /*
+ * little-endian, | omit | 3rd | 2nd | 1st |
+ * MSB LSB
+ * top | 3rd | 1st |; bottom | omit | 2nd |
+ *
+ * version 1, need to swap 2nd and 3rd weight
+ * *__SIMD32(pBuffer) = top.word;
+ * *(pBuffer+2) = bottom.half_words[0];
+ *
+ * version 2, no weight shuffling required
+ */
+ *pBuffer++ = top.half_words[0];
+ *__SIMD32(pBuffer) = __PKHBT(bottom.word, top.word, 0);
+#else
+ /*
+ * big-endian, | 1st | 2nd | 3rd | omit |
+ * MSB LSB
+ * top | 2nd | omit |; bottom | 1st | 3rd |
+ *
+ * version 1, need to swap 2nd and 3rd weight
+ * *__SIMD32(pBuffer) = bottom.word;
+ * *(pBuffer+2) = top.half_words[1];
+ *
+ * version 2, no weight shuffling required
+ */
+ *pBuffer++ = bottom.half_words[0];
+ *__SIMD32(pBuffer) = __PKHTB(top.word, bottom.word, 0);
+#endif
+ pBuffer += 2;
+ }
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * 3 * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ 3 * dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = 3 * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = 3 * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ // check if number of input channels is 3
+ if (ch_im_in != 3)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ /* if-for implementation */
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return (ARM_MATH_SUCCESS);
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c new file mode 100644 index 0000000..7c9ec65 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic.c @@ -0,0 +1,230 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_basic.c
+ * Description: Q7 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q7 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * This basic version is designed to work for any input tensor and weight
+ * dimension.
+ */
+
+arm_status
+arm_convolve_HWC_q7_basic(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* Copying the pixel data to column */
+ arm_q7_to_q15_no_shift((q7_t *)
+ Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* Computation is filed for every 2 columns */
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ ch_im_in *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ /* Load the accumulator with bias first */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+
+ /* Point to the beging of the im2col buffer */
+ q15_t *pB = bufferA;
+
+ /* Each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ // if-for implementation
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c new file mode 100644 index 0000000..24356d9 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_basic_nonsquare.c @@ -0,0 +1,228 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_basic.c
+ * Description: Q7 version of convolution
+ *
+ * $Date: 13. July 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Basic Q7 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns <code>ARM_MATH_SUCCESS</code>
+ */
+
+arm_status arm_convolve_HWC_q7_basic_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ /* This part implements the im2col function */
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* Filling 0 for out-of-bound paddings */
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ /* Copying the pixel data to column */
+ arm_q7_to_q15_no_shift((q7_t *)
+ Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* Computation is filed for every 2 columns */
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_y * dim_kernel_x)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15(wt, bufferA,
+ ch_im_out,
+ ch_im_in *
+ dim_kernel_y * dim_kernel_x, bias_shift, out_shift, bias, pOut);
+
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* left-over because odd number of output pixels */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ /* Load the accumulator with bias first */
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+
+ /* Point to the beging of the im2col buffer */
+ q15_t *pB = bufferA;
+
+ /* Each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_y * dim_kernel_x >> 2;
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel_y * dim_kernel_x & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ }
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ // if-for implementation
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x +
+ (m * dim_kernel_x + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c new file mode 100644 index 0000000..e2d469f --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast.c @@ -0,0 +1,408 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_fast.c
+ * Description: Fast Q7 version of convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+ /**
+ * @brief Fast Q7 convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in is multiple of 4 ( because of the SIMD32 read and swap )
+ *
+ * ch_im_out is multipe of 2 ( bacause 2x2 mat_mult kernel )
+ *
+ * The im2col converts the Q7 tensor input into Q15 column, which is stored in
+ * bufferA. There is reordering happenning during this im2col process with
+ * arm_q7_to_q15_reordered_no_shift. For every four elements, the second and
+ * third elements are swapped.
+ *
+ * The computation kernel arm_nn_mat_mult_kernel_q7_q15_reordered does the
+ * GEMM computation with the reordered columns.
+ *
+ * To speed-up the determination of the padding condition, we split the
+ * computation into 3x3 parts, i.e., {top, mid, bottom} X {left, mid, right}.
+ * This reduces the total number of boundary condition checks and improves
+ * the data copying performance.
+ */
+
+arm_status
+arm_convolve_HWC_q7_fast(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /*
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /*
+ * Here we split the entire matrix into three regions depending on the padding situation
+ * Top: i_out_y from 0 to padding - 1
+ * Middle: i_out_y from padding to dim_im_out-padding-1
+ * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
+ */
+
+ /* top part */
+ for (i_out_y = 0; i_out_y < padding; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* middle part, here we also divide the x into left, mid and right */
+ for (; i_out_y < dim_im_out - padding; i_out_y++)
+ {
+
+ /* left part */
+ for (i_out_x = 0; i_out_x < padding; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* mid part */
+ for (; i_out_x < dim_im_out - padding; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in
+ +
+ (i_ker_y *
+ dim_im_in +
+ i_out_x *
+ stride - padding) * ch_im_in, pBuffer, ch_im_in * dim_kernel);
+ pBuffer += ch_im_in * dim_kernel;
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* right part */
+ for (; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ for (; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift
+ ((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel * dim_kernel)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt,
+ bufferA,
+ ch_im_out,
+ ch_im_in
+ *
+ dim_kernel * dim_kernel, bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)bias[i] << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel * dim_kernel >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = ch_im_in * dim_kernel * dim_kernel & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+
+ uint16_t i, j, k, l, m, n;
+ int conv_out;
+ signed char in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out; j++)
+ {
+ for (k = 0; k < dim_im_out; k++)
+ {
+ conv_out = (bias[i] << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel; m++)
+ {
+ for (n = 0; n < dim_kernel; n++)
+ {
+ // if-for implementation
+ in_row = stride * j + m - padding;
+ in_col = stride * k + n - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out +=
+ Im_in[(in_row * dim_im_in + in_col) * ch_im_in +
+ l] * wt[i * ch_im_in * dim_kernel * dim_kernel + (m * dim_kernel +
+ n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c new file mode 100644 index 0000000..6dc6f0b --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_convolve_HWC_q7_fast_nonsquare.c @@ -0,0 +1,379 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_convolve_HWC_q7_fast_nonsquare.c
+ * Description: Fast Q7 version of convolution (non-sqaure shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Fast Q7 convolution function (non-sqaure shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding size x
+ * @param[in] padding_y padding size y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 4
+ * ch_im_out is multiple of 2
+ */
+
+arm_status arm_convolve_HWC_q7_fast_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x, i_ker_y, i_ker_x;
+
+ /* -----------------------
+ * Here we use bufferA as q15_t internally as computation are done with q15_t level
+ * im2col are done to output in q15_t format from q7_t input
+ */
+
+ q15_t *pBuffer = bufferA;
+ q7_t *pOut = Im_out;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ /*
+ * Here we split the entire matrix into three regions depending on the padding situation
+ * Top: i_out_y from 0 to padding - 1
+ * Middle: i_out_y from padding to dim_im_out-padding-1
+ * Bottom: i_out_y from dim_im_out-padding to dim_im_out-1
+ */
+
+ /* top part */
+ for (i_out_y = 0; i_out_y < padding_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* middle part, here we also divide the x into left, mid and right */
+ for (; i_out_y < dim_im_out_y - padding_y; i_out_y++)
+ {
+
+ /* left part */
+ for (i_out_x = 0; i_out_x < padding_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* mid part */
+ for (; i_out_x < dim_im_out_x - padding_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in +
+ (i_ker_y * dim_im_in_x + i_out_x * stride_x - padding_x) * ch_im_in,
+ pBuffer, ch_im_in * dim_kernel_x);
+ pBuffer += ch_im_in * dim_kernel_x;
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+
+ /* right part */
+ for (; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ for (; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* This part implements the im2col function */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q15(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, sizeof(q15_t)*ch_im_in);
+ } else
+ {
+ arm_q7_to_q15_reordered_no_shift((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in,
+ pBuffer, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ if (pBuffer == bufferA + 2 * ch_im_in * dim_kernel_x * dim_kernel_y)
+ {
+ pOut =
+ arm_nn_mat_mult_kernel_q7_q15_reordered(wt, bufferA, ch_im_out, ch_im_in * dim_kernel_x * dim_kernel_y,
+ bias_shift, out_shift, bias, pOut);
+ /* counter reset */
+ pBuffer = bufferA;
+ }
+ }
+ }
+
+ /* check if there is left-over for compute */
+ if (pBuffer != bufferA)
+ {
+ const q7_t *pA = wt;
+ int i;
+ for (i = 0; i < ch_im_out; i++)
+ {
+ q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q15_t *pB = bufferA;
+ /* basically each time it process 4 entries */
+ uint16_t colCnt = ch_im_in * dim_kernel_x * dim_kernel_y >> 2;
+
+ while (colCnt)
+ {
+
+ q31_t inA1, inA2;
+ q31_t inB1, inB2;
+
+ pA = (const q7_t *)read_and_pad_reordered((void *)pA, &inA1, &inA2);
+
+ inB1 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA1, inB1, sum);
+ inB2 = *__SIMD32(pB)++;
+ sum = __SMLAD(inA2, inB2, sum);
+
+ colCnt--;
+ }
+ colCnt = (ch_im_in * dim_kernel_y * dim_kernel_x) & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ sum += inA1 * inB1;
+ colCnt--;
+ }
+ *pOut = (q7_t) __SSAT((sum >> out_shift), 8);
+ pOut++;
+
+ }
+
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i, j, k, l, m, n;
+ int conv_out;
+ int in_row, in_col;
+
+ if (ch_im_in % 4 != 0 || ch_im_out % 2 != 0)
+ {
+ /* check if the input dimension meets the constraints */
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i = 0; i < ch_im_out; i++)
+ {
+ for (j = 0; j < dim_im_out_y; j++)
+ {
+ for (k = 0; k < dim_im_out_x; k++)
+ {
+ conv_out = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ for (m = 0; m < dim_kernel_y; m++)
+ {
+ for (n = 0; n < dim_kernel_x; n++)
+ {
+ /* if-for implementation */
+ in_row = stride_y * j + m - padding_y;
+ in_col = stride_x * k + n - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ for (l = 0; l < ch_im_in; l++)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + l] *
+ wt[i * ch_im_in * dim_kernel_y * dim_kernel_x + (m * dim_kernel_x + n) * ch_im_in + l];
+ }
+ }
+ }
+ }
+ Im_out[i + (j * dim_im_out_x + k) * ch_im_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c new file mode 100644 index 0000000..c96dc55 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_conv_u8_basic_ver1.c @@ -0,0 +1,239 @@ +/*
+ * Copyright (C) 2010-2019 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_conv_u8_basic_ver1.c
+ * Description: u8 depthwise convolution function
+ *
+ * $Date: June, 2019
+ * $Revision: V.0.8.0
+ *
+ * Target : Cortex-M cores with DSP extension
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+#include <stdint.h>
+#include <stdio.h>
+
+#define DILATION_X (1)
+#define DILATION_Y (1)
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief uint8 depthwise convolution function with asymmetric quantization for even number of channel multiplier
+ * and input channels. Unless specified otherwise, arguments are mandatory. Both square and non-square inputs
+ * are accepted.
+ *
+ * @param[in] input Pointer to input tensor
+ * @param[in] input_x Width of input tensor
+ * @param[in] input_y Height of input tensor
+ * @param[in] input_ch Channels in input tensor
+ * @param[in] kernel Pointer to kernel weights
+ * @param[in] kernel_x Width of kernel
+ * @param[in] kernel_y Height of kernel
+ * @param[in] ch_mult Number of channel multiplier
+ * @param[in] pad_x Padding sizes x
+ * @param[in] pad_y Padding sizes y
+ * @param[in] stride_x Convolution stride along the width
+ * @param[in] stride_y Convolution stride along the height
+ * @param[in] dilation_x Dilation along width. Not used and intended for future enhancement.
+ * @param[in] dilation_y Dilation along height. Not used and intended for future enhancement.
+ * @param[in] bias Pointer to optional bias values. If no bias is
+ * availble, NULL is expected
+ * @param[in] input_offset Input tensor zero offset
+ * @param[in] filter_offset Kernel tensor zero offset
+ * @param[in] output_offset Output tensor zero offset
+ * @param[in,out] output Pointer to output tensor
+ * @param[in] output_x Width of output tensor
+ * @param[in] output_y Height of output tensor
+ * @param[in] output_activation_min Minimum value to clamp the output to. Range : {0, 255}
+ * @param[in] output_activation_max Minimum value to clamp the output to. Range : {0, 255}
+ * @param[in] out_shift Amount of right-shift for output
+ * @param[in] out_mult Output multiplier for requantization
+ * @return The function returns one of the following
+ * <code>ARM_MATH_SIZE_MISMATCH</code> - Not supported dimension of tensors
+ * <code>ARM_MATH_SUCCESS</code> - Successful operation
+ * <code>ARM_MATH_ARGUMENT_ERROR</code> - Implementation not available
+ *
+ * <b> Input constraints</b>
+ * ch_mult is multiple of 2
+ * kernel_x is multiple of 2
+ *
+ */
+
+arm_status arm_depthwise_conv_u8_basic_ver1(const uint8_t *input,
+ const uint16_t input_x,
+ const uint16_t input_y,
+ const uint16_t input_ch,
+ const uint8_t *kernel,
+ const uint16_t kernel_x,
+ const uint16_t kernel_y,
+ const int16_t ch_mult,
+ const int16_t pad_x,
+ const int16_t pad_y,
+ const int16_t stride_x,
+ const int16_t stride_y,
+ const int16_t dilation_x,
+ const int16_t dilation_y,
+ const int32_t *bias,
+ const int32_t input_offset,
+ const int32_t filter_offset,
+ const int32_t output_offset,
+ uint8_t *output,
+ const uint16_t output_x,
+ const uint16_t output_y,
+ const int32_t output_activation_min,
+ const int32_t output_activation_max,
+ const int32_t out_shift,
+ const int32_t out_mult)
+{
+ arm_status status = ARM_MATH_SUCCESS;
+ #if defined (ARM_MATH_DSP)
+ int i_out = 0;
+ (void)dilation_x;
+ (void)dilation_y;
+
+ const int32_t input_offset_pkd = (input_offset & 0xFFFF) | (input_offset & 0xFFFF) << 16;
+ const int32_t kernel_offset_pkd = (filter_offset & 0xFFFF) | (filter_offset & 0xFFFF) << 16;
+
+ if (0 != ch_mult % 2 || 0 != kernel_x % 2)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (int i_out_y = 0; i_out_y < output_y; i_out_y++)
+ {
+ const int16_t base_idx_y = (i_out_y * stride_y) - pad_y;
+ for (int i_out_x = 0; i_out_x < output_x; i_out_x++)
+ {
+ const int16_t base_idx_x = (i_out_x * stride_x) - pad_x;
+ for (int i_input_ch = 0; i_input_ch < input_ch; i_input_ch++)
+ {
+ for (int i_ch_mult = 0; i_ch_mult < ch_mult; i_ch_mult += 2)
+ {
+ const int idx_out_ch = i_ch_mult + i_input_ch * ch_mult;
+
+ int32_t acc_0 = 0;
+ int32_t acc_1 = 0;
+ if (NULL != bias)
+ {
+ acc_0 = bias[idx_out_ch];
+ acc_1 = bias[idx_out_ch + 1];
+ }
+
+ for (int i_ker_y = 0; i_ker_y < kernel_y; i_ker_y++)
+ {
+ const int32_t idx_y = base_idx_y + DILATION_Y * i_ker_y;
+ const int32_t y_in_range = (idx_y >= 0) && (idx_y < input_y);
+
+ for (int i_ker_x = 0; i_ker_x < kernel_x; i_ker_x += 2)
+ {
+ if (1 == y_in_range)
+ {
+ const int32_t idx_x = base_idx_x + DILATION_X * i_ker_x;
+ const int32_t idx_x1 = base_idx_x + DILATION_X * (i_ker_x + 1);
+ /* Range check for first input */
+ if (idx_x >= 0 && idx_x < input_x)
+ {
+ const int32_t idx_0 = (idx_y * input_x + idx_x) * input_ch + i_input_ch;
+
+ const int32_t ker_idx_0 =
+ (i_ker_y * kernel_x + i_ker_x) * (input_ch * ch_mult) + idx_out_ch;
+ const int32_t ker_idx_1 = ker_idx_0 + input_ch * ch_mult;
+
+ int32_t input_pkd = input[idx_0] | (input[idx_0 + input_ch] << 16);
+ int32_t kernel_pkd = kernel[ker_idx_0] | (kernel[ker_idx_1] << 16);
+
+ input_pkd = __SADD16(input_pkd, input_offset_pkd);
+ kernel_pkd = __SADD16(kernel_pkd, kernel_offset_pkd);
+ /* Range check for second input */
+ if (idx_x1 >= input_x)
+ {
+ input_pkd &= 0xFFFF;
+ }
+ acc_0 = __SMLAD(input_pkd, kernel_pkd, acc_0);
+
+ kernel_pkd = kernel[ker_idx_0 + 1] | (kernel[ker_idx_1 + 1] << 16);
+ kernel_pkd = __SADD16(kernel_pkd, kernel_offset_pkd);
+ acc_1 = __SMLAD(input_pkd, kernel_pkd, acc_1);
+ }
+ }
+ }
+ }
+
+ /* Requantize and clamp output to provided range */
+ acc_0 = arm_nn_divide_by_power_of_two(arm_nn_sat_doubling_high_mult(
+ acc_0 * (1 << LEFT_SHIFT(out_shift)), out_mult),
+ RIGHT_SHIFT(out_shift));
+
+ acc_0 += output_offset;
+
+ if (output_activation_min > acc_0)
+ {
+ acc_0 = output_activation_min;
+ }
+
+ if (acc_0 > output_activation_max)
+ {
+ acc_0 = output_activation_max;
+ }
+ output[i_out++] = acc_0;
+
+ /* Requantize and clamp output to provided range */
+ acc_1 = arm_nn_divide_by_power_of_two(arm_nn_sat_doubling_high_mult(
+ acc_1 * (1 << LEFT_SHIFT(out_shift)), out_mult),
+ RIGHT_SHIFT(out_shift));
+ acc_1 += output_offset;
+
+ if (output_activation_min > acc_1)
+ {
+ acc_1 = output_activation_min;
+ }
+
+ if (acc_1 > output_activation_max)
+ {
+ acc_1 = output_activation_max;
+ }
+ output[i_out++] = acc_1;
+ }
+ }
+ }
+ }
+#else
+ /* No available implementation. */
+ status = ARM_MATH_ARGUMENT_ERROR;
+#endif
+ return status;
+}
+
+/**
+ * @} end of NNConv group
+ */
+
+
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c new file mode 100644 index 0000000..705fa6a --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7.c @@ -0,0 +1,418 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_separable_conv_HWC_q7.c
+ * Description: Q7 depthwise separable convolution function
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Q7 depthwise separable convolution function
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in input tensor dimention
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel filter kernel size
+ * @param[in] padding padding sizes
+ * @param[in] stride convolution stride
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out output tensor dimension
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * @details
+ *
+ * <b>Buffer size:</b>
+ *
+ * bufferA size: 2*ch_im_in*dim_kernel*dim_kernel
+ *
+ * bufferB size: 0
+ *
+ * <b>Input dimension constraints:</b>
+ *
+ * ch_im_in equals ch_im_out
+ *
+ * Implementation:
+ * There are 3 nested loop here:
+ * Inner loop: calculate each output value with MAC instruction over an accumulator
+ * Mid loop: loop over different output channel
+ * Outer loop: loop over different output (x, y)
+ */
+
+arm_status arm_depthwise_separable_conv_HWC_q7(const q7_t * Im_in,
+ const uint16_t dim_im_in,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel,
+ const uint16_t padding,
+ const uint16_t stride,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ker_y, i_ker_x;
+ q7_t *colBuffer = (q7_t *) bufferA;
+ q7_t *pBuffer = colBuffer;
+ const q7_t *pBias = bias;
+ q7_t *pOut = Im_out;
+ uint16_t rowCnt;
+ uint16_t row_shift;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ /* we first do im2col here */
+ for (i_ker_y = i_out_y * stride - padding; i_ker_y < i_out_y * stride - padding + dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride - padding; i_ker_x < i_out_x * stride - padding + dim_kernel; i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in || i_ker_x < 0 || i_ker_x >= dim_im_in)
+ {
+ /* arm_fill_q7(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, ch_im_in);
+ } else
+ {
+ /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in + i_ker_x) * ch_im_in, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* we will do the computation here for each channel */
+ rowCnt = ch_im_out >> 2;
+ row_shift = 0;
+ pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = (dim_kernel * dim_kernel) >> 1;
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ row_shift += 4;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHTB(opB, inB1, 16);
+ inB1 = __PKHBT(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHTB(opB, inA1, 16);
+ inA1 = __PKHBT(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum3 = __SMLAD(opA, opB, sum3);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum4 = __SMLAD(opA, opB, sum4);
+ colCnt--;
+ }
+#else
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHBT(opB, inB1, 16);
+ inB1 = __PKHTB(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHBT(opB, inA1, 16);
+ inA1 = __PKHTB(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum4 = __SMLAD(opA, opB, sum4);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum3 = __SMLAD(opA, opB, sum3);
+ colCnt--;
+ }
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ /*
+ * r0 r1 r2 r3 r4 r5
+ * inA1, inA2, inB1, inB2, opA, opB
+ */
+
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhtb r3, r5, r2, ASR #16\n"
+ "pkhbt r2, r2, r5, LSL #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhtb r1, r5, r0, ASR #16\n"
+ "pkhbt r0, r0, r5, LSL #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum]
+ "+r"(sum),[sum2] "+r"(sum2),
+ [sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),
+ [pA] "+r"(pA):[colCnt]
+ "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#else
+ /*
+ * r0 r1 r2 r3 r4 r5
+ * inA1, inA2, inB1, inB2, opA, opB
+ */
+ asm volatile ("COL_LOOP_%=:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhbt r3, r5, r2, LSL #16\n"
+ "pkhtb r2, r2, r5, ASR #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhbt r1, r5, r0, LSL #16\n"
+ "pkhtb r0, r0, r5, ASR #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP_%=\n":[sum]
+ "+r"(sum),[sum2] "+r"(sum2),
+ [sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),
+ [pA] "+r"(pA):[colCnt]
+ "r"(colCnt),[ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = (dim_kernel * dim_kernel) & 0x1;
+ while (colCnt)
+ {
+ union arm_nnword inA, inB;
+ inA.word = *__SIMD32(pA);
+ pA += ch_im_in;
+ inB.word = *__SIMD32(pB);
+ pB += ch_im_in;
+ sum += inA.bytes[0] * inB.bytes[0];
+ sum2 += inA.bytes[1] * inB.bytes[1];
+ sum3 += inA.bytes[2] * inB.bytes[2];
+ sum4 += inA.bytes[3] * inB.bytes[3];
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+ rowCnt = ch_im_out & 0x3;
+ while (rowCnt)
+ {
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = (dim_kernel * dim_kernel);
+
+ row_shift += 1;
+
+ while (colCnt)
+ {
+ q7_t A1 = *pA;
+ q7_t B1 = *pB;
+ pA += ch_im_in;
+ pB += ch_im_in;
+ sum += A1 * B1;
+
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ rowCnt--;
+ }
+
+ /* clear counter and pointers */
+ pBuffer = colBuffer;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i_out_y, i_out_x, i_ch_out, i_ker_x, i_ker_y;
+ int conv_out;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out; i_out_x++)
+ {
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ // for each output
+ conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ for (i_ker_y = 0; i_ker_y < dim_kernel; i_ker_y++)
+ {
+ for (i_ker_x = 0; i_ker_x < dim_kernel; i_ker_x++)
+ {
+ int in_row = stride * i_out_y + i_ker_y - padding;
+ int in_col = stride * i_out_x + i_ker_x - padding;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in && in_col < dim_im_in)
+ {
+ conv_out +=
+ Im_in[(in_row *
+ dim_im_in +
+ in_col) *
+ ch_im_in +
+ i_ch_out] * wt[(i_ker_y * dim_kernel + i_ker_x) * ch_im_out + i_ch_out];
+ }
+ }
+ }
+ Im_out[(i_out_y * dim_im_out +
+ i_out_x) * ch_im_out + i_ch_out] = (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c new file mode 100644 index 0000000..5989304 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_depthwise_separable_conv_HWC_q7_nonsquare.c @@ -0,0 +1,411 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_depthwise_separable_conv_HWC_q7_nonsquare.c
+ * Description: Q7 depthwise separable convolution function (non-square shape)
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ *
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+/**
+ * @ingroup groupNN
+ */
+
+/**
+ * @addtogroup NNConv
+ * @{
+ */
+
+/**
+ * @brief Q7 depthwise separable convolution function (non-square shape)
+ * @param[in] Im_in pointer to input tensor
+ * @param[in] dim_im_in_x input tensor dimention x
+ * @param[in] dim_im_in_y input tensor dimention y
+ * @param[in] ch_im_in number of input tensor channels
+ * @param[in] wt pointer to kernel weights
+ * @param[in] ch_im_out number of filters, i.e., output tensor channels
+ * @param[in] dim_kernel_x filter kernel size x
+ * @param[in] dim_kernel_y filter kernel size y
+ * @param[in] padding_x padding sizes x
+ * @param[in] padding_y padding sizes y
+ * @param[in] stride_x convolution stride x
+ * @param[in] stride_y convolution stride y
+ * @param[in] bias pointer to bias
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in,out] Im_out pointer to output tensor
+ * @param[in] dim_im_out_x output tensor dimension x
+ * @param[in] dim_im_out_y output tensor dimension y
+ * @param[in,out] bufferA pointer to buffer space for input
+ * @param[in,out] bufferB pointer to buffer space for output
+ * @return The function returns either
+ * <code>ARM_MATH_SIZE_MISMATCH</code> or <code>ARM_MATH_SUCCESS</code> based on the outcome of size checking.
+ *
+ * This function is the version with full list of optimization tricks, but with
+ * some contraints:
+ * ch_im_in is multiple of 2
+ * ch_im_out is multiple of 2
+ */
+
+arm_status arm_depthwise_separable_conv_HWC_q7_nonsquare(const q7_t * Im_in,
+ const uint16_t dim_im_in_x,
+ const uint16_t dim_im_in_y,
+ const uint16_t ch_im_in,
+ const q7_t * wt,
+ const uint16_t ch_im_out,
+ const uint16_t dim_kernel_x,
+ const uint16_t dim_kernel_y,
+ const uint16_t padding_x,
+ const uint16_t padding_y,
+ const uint16_t stride_x,
+ const uint16_t stride_y,
+ const q7_t * bias,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ q7_t * Im_out,
+ const uint16_t dim_im_out_x,
+ const uint16_t dim_im_out_y,
+ q15_t * bufferA,
+ q7_t * bufferB)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* Run the following code for Cortex-M4 and Cortex-M7 */
+
+/*
+ * Implementation:
+ * There are 3 nested loop here:
+ * Inner loop: calculate each output value with MAC instruction over an accumulator
+ * Mid loop: loop over different output channel
+ * Outer loop: loop over different output (x, y)
+ *
+ */
+
+ int16_t i_out_y, i_out_x;
+ int16_t i_ker_y, i_ker_x;
+ q7_t *colBuffer = (q7_t *) bufferA;
+ q7_t *pBuffer = colBuffer;
+ const q7_t *pBias = bias;
+ q7_t *pOut = Im_out;
+ uint16_t rowCnt;
+ uint16_t row_shift;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ /* we first do im2col here */
+ for (i_ker_y = i_out_y * stride_y - padding_y; i_ker_y < i_out_y * stride_y - padding_y + dim_kernel_y;
+ i_ker_y++)
+ {
+ for (i_ker_x = i_out_x * stride_x - padding_x; i_ker_x < i_out_x * stride_x - padding_x + dim_kernel_x;
+ i_ker_x++)
+ {
+ if (i_ker_y < 0 || i_ker_y >= dim_im_in_y || i_ker_x < 0 || i_ker_x >= dim_im_in_x)
+ {
+ /* arm_fill_q7(0, pBuffer, ch_im_in); */
+ memset(pBuffer, 0, ch_im_in);
+ } else
+ {
+ /* arm_copy_q7((q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, pBuffer, ch_im_in); */
+ memcpy(pBuffer, (q7_t *) Im_in + (i_ker_y * dim_im_in_x + i_ker_x) * ch_im_in, ch_im_in);
+ }
+ pBuffer += ch_im_in;
+ }
+ }
+
+ /* we will do the computation here for each channel */
+ rowCnt = ch_im_out >> 2;
+ row_shift = 0;
+ pBias = bias;
+
+ while (rowCnt)
+ {
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = (dim_kernel_x * dim_kernel_y) >> 1;
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ row_shift += 4;
+
+#ifdef USE_INTRINSIC
+
+#ifndef ARM_MATH_BIG_ENDIAN
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHTB(opB, inB1, 16);
+ inB1 = __PKHBT(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHTB(opB, inA1, 16);
+ inA1 = __PKHBT(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum3 = __SMLAD(opA, opB, sum3);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum4 = __SMLAD(opA, opB, sum4);
+ colCnt--;
+ }
+#else
+
+ while (colCnt)
+ {
+ q31_t inA1, inA2, inB1, inB2, opA, opB;
+
+ inB1 = *__SIMD32(pB);
+ pB += ch_im_in;
+ opB = *__SIMD32(pB);
+ pB += ch_im_in;
+ inB2 = __PKHBT(opB, inB1, 16);
+ inB1 = __PKHTB(inB1, opB, 16);
+ inA1 = *__SIMD32(pA);
+ pA += ch_im_in;
+ opB = *__SIMD32(pA);
+ pA += ch_im_in;
+ inA2 = __PKHBT(opB, inA1, 16);
+ inA1 = __PKHTB(inA1, opB, 16);
+ opA = __SXTB16(inA1);
+ opB = __SXTB16(inB1);
+ sum2 = __SMLAD(opA, opB, sum2);
+ opA = __SXTB16(__ROR(inA1, 8));
+ opB = __SXTB16(__ROR(inB1, 8));
+ sum = __SMLAD(opA, opB, sum);
+ opA = __SXTB16(inA2);
+ opB = __SXTB16(inB2);
+ sum4 = __SMLAD(opA, opB, sum4);
+ opA = __SXTB16(__ROR(inA2, 8));
+ opB = __SXTB16(__ROR(inB2, 8));
+ sum3 = __SMLAD(opA, opB, sum3);
+ colCnt--;
+ }
+
+#endif /* ARM_MATH_BIG_ENDIAN */
+
+#else
+
+#ifndef ARM_MATH_BIG_ENDIAN
+ // r0 r1 r2 r3 r4 r5
+ // inA1, inA2, inB1, inB2, opA, opB
+ asm volatile ("COL_LOOP:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhtb r3, r5, r2, ASR #16\n"
+ "pkhbt r2, r2, r5, LSL #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhtb r1, r5, r0, ASR #16\n"
+ "pkhbt r0, r0, r5, LSL #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
+ [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#else
+ // r0 r1 r2 r3 r4 r5
+ // inA1, inA2, inB1, inB2, opA, opB
+ asm volatile ("COL_LOOP:\n"
+ "ldr.w r2, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "ldr.w r5, [%[pB], #0]\n"
+ "add.w %[pB], %[pB], %[ch_im_in]\n"
+ "pkhbt r3, r5, r2, LSL #16\n"
+ "pkhtb r2, r2, r5, ASR #16\n"
+ "ldr.w r0, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "ldr.w r5, [%[pA], #0]\n"
+ "add.w %[pA], %[pA], %[ch_im_in]\n"
+ "pkhbt r1, r5, r0, LSL #16\n"
+ "pkhtb r0, r0, r5, ASR #16\n"
+ "sxtb16 r4, r0\n"
+ "sxtb16 r5, r2\n"
+ "smlad %[sum2], r4, r5, %[sum2]\n"
+ "mov.w r4, r0, ror #8\n"
+ "mov.w r5, r2, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum], r4, r5, %[sum]\n"
+ "sxtb16 r4, r1\n"
+ "sxtb16 r5, r3\n"
+ "smlad %[sum4], r4, r5, %[sum4]\n"
+ "mov.w r4, r1, ror #8\n"
+ "mov.w r5, r3, ror #8\n"
+ "sxtb16 r4, r4\n"
+ "sxtb16 r5, r5\n"
+ "smlad %[sum3], r4, r5, %[sum3]\n"
+ "subs %[colCnt], #1\n"
+ "bne COL_LOOP\n":[sum] "+r"(sum),[sum2] "+r"(sum2),[sum3] "+r"(sum3),
+ [sum4] "+r"(sum4),[pB] "+r"(pB),[pA] "+r"(pA):[colCnt] "r"(colCnt),
+ [ch_im_in] "r"(ch_im_in):"r0", "r1", "r2", "r3", "r4", "r5");
+#endif /*ARM_MATH_BIG_ENDIAN */
+
+#endif /* USE_INTRINSIC */
+
+ colCnt = (dim_kernel_x * dim_kernel_y) & 0x1;
+ while (colCnt)
+ {
+ union arm_nnword inA, inB;
+ inA.word = *__SIMD32(pA);
+ pA += ch_im_in;
+ inB.word = *__SIMD32(pB);
+ pB += ch_im_in;
+ sum += inA.bytes[0] * inB.bytes[0];
+ sum2 += inA.bytes[1] * inB.bytes[1];
+ sum3 += inA.bytes[2] * inB.bytes[2];
+ sum4 += inA.bytes[3] * inB.bytes[3];
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ rowCnt--;
+ }
+
+ rowCnt = ch_im_out & 0x3;
+ while (rowCnt)
+ {
+ q7_t *pB = colBuffer + row_shift;
+ const q7_t *pA = wt + row_shift;
+ q31_t sum = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ uint16_t colCnt = (dim_kernel_x * dim_kernel_y);
+
+ row_shift += 1;
+
+ while (colCnt)
+ {
+ q7_t A1 = *pA;
+ q7_t B1 = *pB;
+ pA += ch_im_in;
+ pB += ch_im_in;
+ sum += A1 * B1;
+
+ colCnt--;
+ }
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ rowCnt--;
+ }
+
+ // clear counter and pointers
+ pBuffer = colBuffer;
+ }
+ }
+
+#else
+ /* Run the following code as reference implementation for Cortex-M0 and Cortex-M3 */
+ int i_out_y, i_out_x, i_ch_out;
+ int i_ker_y, i_ker_x;
+
+ /* do some checking here, basically ch_im_in == ch_im_out */
+ if (ch_im_in != ch_im_out)
+ {
+ return ARM_MATH_SIZE_MISMATCH;
+ }
+
+ for (i_out_y = 0; i_out_y < dim_im_out_y; i_out_y++)
+ {
+ for (i_out_x = 0; i_out_x < dim_im_out_x; i_out_x++)
+ {
+ for (i_ch_out = 0; i_ch_out < ch_im_out; i_ch_out++)
+ {
+ // for each output
+ int conv_out = ((q31_t)(bias[i_ch_out]) << bias_shift) + NN_ROUND(out_shift);
+ for (i_ker_y = 0; i_ker_y < dim_kernel_y; i_ker_y++)
+ {
+ for (i_ker_x = 0; i_ker_x < dim_kernel_x; i_ker_x++)
+ {
+ int in_row = stride_y * i_out_y + i_ker_y - padding_y;
+ int in_col = stride_x * i_out_x + i_ker_x - padding_x;
+ if (in_row >= 0 && in_col >= 0 && in_row < dim_im_in_y && in_col < dim_im_in_x)
+ {
+ conv_out += Im_in[(in_row * dim_im_in_x + in_col) * ch_im_in + i_ch_out] *
+ wt[(i_ker_y * dim_kernel_x + i_ker_x) * ch_im_out + i_ch_out];
+ }
+ }
+ }
+ Im_out[(i_out_y * dim_im_out_x + i_out_x) * ch_im_out + i_ch_out] =
+ (q7_t) __SSAT((conv_out >> out_shift), 8);
+ }
+ }
+ }
+
+#endif /* ARM_MATH_DSP */
+
+
+ /* Return to application */
+ return ARM_MATH_SUCCESS;
+
+}
+
+/**
+ * @} end of NNConv group
+ */
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c new file mode 100644 index 0000000..24ab412 --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15.c @@ -0,0 +1,187 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mat_mult_kernel_q7_q15.c
+ * Description: Matrix-multiplication function for convolution
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ * -------------------------------------------------------------------- */
+
+#include "arm_math.h"
+#include "arm_nnfunctions.h"
+
+ /**
+ * @brief Matrix-multiplication function for convolution
+ * @param[in] pA pointer to operand A
+ * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
+ * @param[in] ch_im_out numRow of A
+ * @param[in] numCol_A numCol of A
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias the bias
+ * @param[in,out] pOut pointer to output
+ * @return The function returns the incremented output pointer
+ *
+ * @details
+ *
+ * This function does the matrix multiplication with weight matrix
+ * and 2 columns from im2col.
+ */
+
+q7_t *arm_nn_mat_mult_kernel_q7_q15(const q7_t * pA,
+ const q15_t * pInBuffer,
+ const uint16_t ch_im_out,
+ const uint16_t numCol_A,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q7_t * pOut)
+{
+#if defined (ARM_MATH_DSP)
+ /* set up the second output pointers */
+ q7_t *pOut2 = pOut + ch_im_out;
+ const q7_t *pBias = bias;
+
+ uint16_t rowCnt = ch_im_out >> 1;
+ /* this loop over rows in A */
+ while (rowCnt)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* align the second pointer for A */
+ const q7_t *pA2 = pA + numCol_A;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA11, inA12, inA21, inA22;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
+ pA2 = (q7_t *) read_and_pad((void *)pA2, &inA21, &inA22);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+ sum3 = __SMLAD(inA21, inB1, sum3);
+ sum4 = __SMLAD(inA21, inB2, sum4);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+ sum3 = __SMLAD(inA22, inB1, sum3);
+ sum4 = __SMLAD(inA22, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q7_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ /* skip the row computed with A2 */
+ pA += numCol_A;
+ rowCnt--;
+ } /* for over ch_im_out */
+
+ /* compute left-over row if any */
+ if (ch_im_out & 0x1)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* load the bias */
+ q31_t sum = ((q31_t)(*pBias) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(*pBias++) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ while (colCnt)
+ {
+ q31_t inA11, inA12;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad((void *)pA, &inA11, &inA12);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+
+ colCnt--;
+ }
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ colCnt--;
+ }
+
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ }
+
+ pOut += ch_im_out;
+
+ /* return the new output pointer with offset */
+ return pOut;
+#else
+ /* To be completed */
+ return NULL;
+#endif /* ARM_MATH_DSP */
+
+}
diff --git a/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c new file mode 100644 index 0000000..36af21a --- /dev/null +++ b/Drivers/CMSIS/NN/Source/ConvolutionFunctions/arm_nn_mat_mult_kernel_q7_q15_reordered.c @@ -0,0 +1,138 @@ +/*
+ * Copyright (C) 2010-2018 Arm Limited or its affiliates. All rights reserved.
+ *
+ * SPDX-License-Identifier: Apache-2.0
+ *
+ * Licensed under the Apache License, Version 2.0 (the License); you may
+ * not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ * www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an AS IS BASIS, WITHOUT
+ * WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/* ----------------------------------------------------------------------
+ * Project: CMSIS NN Library
+ * Title: arm_nn_mat_mult_kernel_q7_q15_reordered.c
+ * Description: Matrix-multiplication function for convolution with reordered columns
+ *
+ * $Date: 17. January 2018
+ * $Revision: V.1.0.0
+ *
+ * Target Processor: Cortex-M cores
+ * -------------------------------------------------------------------- */
+
+#include "arm_nnfunctions.h"
+#include "arm_math.h"
+
+ /**
+ * @brief Matrix-multiplication function for convolution with reordered columns
+ * @param[in] pA pointer to operand A
+ * @param[in] pInBuffer pointer to operand B, always conssists of 2 vectors
+ * @param[in] ch_im_out numRow of A
+ * @param[in] numCol_A numCol of A
+ * @param[in] bias_shift amount of left-shift for bias
+ * @param[in] out_shift amount of right-shift for output
+ * @param[in] bias the bias
+ * @param[in,out] pOut pointer to output
+ * @return The function returns the incremented output pointer
+ *
+ * @details
+ *
+ * This function assumes that data in pInBuffer are reordered
+ */
+
+q7_t *arm_nn_mat_mult_kernel_q7_q15_reordered(const q7_t * pA,
+ const q15_t * pInBuffer,
+ const uint16_t ch_im_out,
+ const uint16_t numCol_A,
+ const uint16_t bias_shift,
+ const uint16_t out_shift,
+ const q7_t * bias,
+ q7_t * pOut)
+{
+
+#if defined (ARM_MATH_DSP)
+ /* set up the second output pointers */
+ q7_t *pOut2 = pOut + ch_im_out;
+ int i;
+
+ /* this loop over rows in A */
+ for (i = 0; i < ch_im_out; i += 2)
+ {
+ /* setup pointers for B */
+ const q15_t *pB = pInBuffer;
+ const q15_t *pB2 = pB + numCol_A;
+
+ /* align the second pointer for A */
+ const q7_t *pA2 = pA + numCol_A;
+
+ /* init the sum with bias */
+ q31_t sum = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum2 = ((q31_t)(bias[i]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum3 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
+ q31_t sum4 = ((q31_t)(bias[i + 1]) << bias_shift) + NN_ROUND(out_shift);
+
+ uint16_t colCnt = numCol_A >> 2;
+ /* accumulate over the vector */
+ while (colCnt)
+ {
+ q31_t inA11, inA12, inA21, inA22;
+ q31_t inB1 = *__SIMD32(pB)++;
+ q31_t inB2 = *__SIMD32(pB2)++;
+
+ pA = (q7_t *) read_and_pad_reordered((void *)pA, &inA11, &inA12);
+ pA2 = (q7_t *) read_and_pad_reordered((void *)pA2, &inA21, &inA22);
+
+ sum = __SMLAD(inA11, inB1, sum);
+ sum2 = __SMLAD(inA11, inB2, sum2);
+ sum3 = __SMLAD(inA21, inB1, sum3);
+ sum4 = __SMLAD(inA21, inB2, sum4);
+
+ inB1 = *__SIMD32(pB)++;
+ inB2 = *__SIMD32(pB2)++;
+
+ sum = __SMLAD(inA12, inB1, sum);
+ sum2 = __SMLAD(inA12, inB2, sum2);
+ sum3 = __SMLAD(inA22, inB1, sum3);
+ sum4 = __SMLAD(inA22, inB2, sum4);
+
+ colCnt--;
+ } /* while over colCnt */
+ colCnt = numCol_A & 0x3;
+ while (colCnt)
+ {
+ q7_t inA1 = *pA++;
+ q15_t inB1 = *pB++;
+ q7_t inA2 = *pA2++;
+ q15_t inB2 = *pB2++;
+
+ sum += inA1 * inB1;
+ sum2 += inA1 * inB2;
+ sum3 += inA2 * inB1;
+ sum4 += inA2 * inB2;
+ colCnt--;
+ } /* while over colCnt */
+ *pOut++ = (q7_t) __SSAT((sum >> out_shift), 8);
+ *pOut++ = (q7_t) __SSAT((sum3 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum2 >> out_shift), 8);
+ *pOut2++ = (q7_t) __SSAT((sum4 >> out_shift), 8);
+
+ /* skip the row computed with A2 */
+ pA += numCol_A;
+ } /* for over ch_im_out */
+
+ pOut += ch_im_out;
+
+ /* return the new output pointer with offset */
+ return pOut;
+#else
+ /* To be completed */
+ return NULL;
+#endif /* ARM_MATH_DSP */
+}
|