From 0fde0b870b3077157f5d73f77e1da1ecf86541e8 Mon Sep 17 00:00:00 2001 From: Ashutosh Parkhi Date: Wed, 8 Jun 2022 17:42:05 +0100 Subject: [PATCH 1/3] Made CMSIS-NN tests pylint compliant Change-Id: I6bc536a80a24a1603e9f75f8ee9a26d0d88f10df --- tests/lint/pylint.sh | 2 + tests/python/contrib/test_cmsisnn/__init__.py | 17 ++ .../contrib/test_cmsisnn/test_binary_ops.py | 22 +- .../contrib/test_cmsisnn/test_conv2d.py | 21 +- .../test_cmsisnn/test_extract_constants.py | 217 ++++++++++-------- .../test_cmsisnn/test_fully_connected.py | 28 ++- .../test_cmsisnn/test_generate_constants.py | 19 +- .../test_cmsisnn/test_invalid_graphs.py | 14 +- .../contrib/test_cmsisnn/test_networks.py | 18 +- .../contrib/test_cmsisnn/test_pooling.py | 17 +- .../test_scalar_to_tensor_constant.py | 201 ++++++++-------- .../contrib/test_cmsisnn/test_softmax.py | 11 +- tests/python/contrib/test_cmsisnn/utils.py | 7 +- 13 files changed, 320 insertions(+), 274 deletions(-) create mode 100644 tests/python/contrib/test_cmsisnn/__init__.py diff --git a/tests/lint/pylint.sh b/tests/lint/pylint.sh index 6c958a923139..11d4b9f4b968 100755 --- a/tests/lint/pylint.sh +++ b/tests/lint/pylint.sh @@ -20,3 +20,5 @@ set -euxo pipefail python3 -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc python3 -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc +python3 -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc --exit-zero + diff --git a/tests/python/contrib/test_cmsisnn/__init__.py b/tests/python/contrib/test_cmsisnn/__init__.py new file mode 100644 index 000000000000..f9a622464a47 --- /dev/null +++ b/tests/python/contrib/test_cmsisnn/__init__.py @@ -0,0 +1,17 @@ +# Licensed to the Apache Software Foundation (ASF) under one +# or more contributor license agreements. See the NOTICE file +# distributed with this work for additional information +# regarding copyright ownership. The ASF licenses this file +# to you under the Apache License, Version 2.0 (the +# "License"); you may not use this file except in compliance +# with the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, +# software distributed under the License is distributed on an +# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +# KIND, either express or implied. See the License for the +# specific language governing permissions and limitations +# under the License. +"""Infrastructure and tests for CMSIS-NN""" diff --git a/tests/python/contrib/test_cmsisnn/test_binary_ops.py b/tests/python/contrib/test_cmsisnn/test_binary_ops.py index 49c76870157e..fec18c197e04 100644 --- a/tests/python/contrib/test_cmsisnn/test_binary_ops.py +++ b/tests/python/contrib/test_cmsisnn/test_binary_ops.py @@ -18,17 +18,19 @@ """CMSIS-NN integration tests: binary ops""" import itertools -import sys import numpy as np -from enum import Enum import pytest import tvm from tvm import relay from tvm.relay.op.contrib import cmsisnn +from tvm.testing.aot import generate_ref_data, AOTTestModel, compile_and_run +from tvm.micro.testing.aot_test_utils import ( + AOT_USMP_CORSTONE300_RUNNER, +) -from utils import ( +from .utils import ( skip_if_no_reference_system, make_module, make_qnn_relu, @@ -36,11 +38,6 @@ assert_partitioned_function, assert_no_external_function, ) -from tvm.testing.aot import generate_ref_data, AOTTestModel, compile_and_run -from tvm.micro.testing.aot_test_utils import ( - AOT_CORSTONE300_RUNNER, - AOT_USMP_CORSTONE300_RUNNER, -) def generate_tensor_constant(): @@ -104,6 +101,7 @@ def make_model( def test_op_int8( op, relu_type, input_0_scale, input_0_zero_point, input_1_scale, input_1_zero_point ): + """Tests QNN Conv2D operator for CMSIS-NN""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -147,8 +145,10 @@ def test_op_int8( ) -# At least one of the inputs is a constant, both can't be variables, both can't be scalars def parameterize_for_constant_inputs(test): + """Generates parameters in such a way so that at least one of the inputs is a constant, + both can't be variables, both can't be scalars. + """ op = [relay.qnn.op.mul, relay.qnn.op.add] input_0 = [generate_variable("input_0"), generate_tensor_constant(), generate_scalar_constant()] input_1 = [generate_variable("input_1"), generate_tensor_constant(), generate_scalar_constant()] @@ -178,6 +178,7 @@ def parameterize_for_constant_inputs(test): @tvm.testing.requires_cmsisnn @parameterize_for_constant_inputs def test_constant_input_int8(op, input_0, input_1): + """Tests binary ops where one of the operands is a constant""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -231,9 +232,9 @@ def test_constant_input_int8(op, input_0, input_1): def test_both_scalar_inputs_int8( op, ): + """Tests binary ops where both operands are scalars""" input_scale = 0.256 input_zero_point = 33 - dtype = "int8" model = make_model( op, generate_scalar_constant(), @@ -257,6 +258,7 @@ def test_invalid_parameters( op, input_dtype, ): + """Tests binary ops for non int8 dtypes""" input_scale = 0.256 input_zero_point = 33 model = make_model( diff --git a/tests/python/contrib/test_cmsisnn/test_conv2d.py b/tests/python/contrib/test_cmsisnn/test_conv2d.py index 90261e540a7d..58baea88f054 100644 --- a/tests/python/contrib/test_cmsisnn/test_conv2d.py +++ b/tests/python/contrib/test_cmsisnn/test_conv2d.py @@ -26,8 +26,7 @@ from tvm.testing.aot import generate_ref_data, AOTTestModel, compile_models, compile_and_run from tvm.micro.testing.aot_test_utils import AOT_USMP_CORSTONE300_RUNNER -from utils import ( - skip_if_no_reference_system, +from .utils import ( make_module, get_range_for_dtype_str, get_same_padding, @@ -76,7 +75,7 @@ def make_model( shape = (shape[0], shape[1] + p[0] + p[2], shape[2] + p[1] + p[3], shape[3]) rng = np.random.default_rng(12321) - w = tvm.nd.array( + weight = tvm.nd.array( rng.integers( np.iinfo(kernel_dtype).min, high=np.iinfo(kernel_dtype).max, @@ -84,7 +83,7 @@ def make_model( dtype=kernel_dtype, ) ) - weight_const = relay.const(w, kernel_dtype) + weight_const = relay.const(weight, kernel_dtype) conv = relay.qnn.op.conv2d( invar, weight_const, @@ -102,8 +101,8 @@ def make_model( padding=p, out_dtype="int32", ) - b = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) - bias_const = relay.const(b, "int32") + bias = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) + bias_const = relay.const(bias, "int32") last_op = relay.nn.bias_add(conv, bias_const, axis=3) if enable_bias else conv requant_input_sc = [sc * input_scale for sc in kernel_scale] last_op = relay.qnn.op.requantize( @@ -115,7 +114,7 @@ def make_model( out_dtype=dtype, ) last_op = make_qnn_relu(last_op, relu_type, output_scale, output_zero_point, dtype) - params = {"w": w, "b": b} + params = {"w": weight, "b": bias} return last_op, params @@ -230,6 +229,7 @@ def test_conv2d_symmetric_padding_int8( kernel_scale, out_channels, ): + """Tests QNN Conv2D where the padding is symmetric on both sides of input""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -319,6 +319,7 @@ def test_conv2d_asymmetric_padding_int8( kernel_scale, out_channels, ): + """Tests QNN Conv2D where the padding is asymmetric on different sides of input""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -390,6 +391,7 @@ def test_conv2d_asymmetric_padding_int8( ) +# pylint: disable=import-outside-toplevel @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("ifm_shape", [(1, 55, 55, 3)]) @pytest.mark.parametrize("kernel_shape", [(3, 2), (1, 3)]) @@ -397,6 +399,7 @@ def test_conv2d_asymmetric_padding_int8( @pytest.mark.parametrize("padding", ["SAME", "VALID"]) @pytest.mark.parametrize("activation", ["NONE", "RELU"]) def test_conv2d_int8_tflite(ifm_shape, kernel_shape, strides, dilation, padding, activation): + """Compares TVM output against TFLite output""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -460,6 +463,7 @@ def test_depthwise_int8( out_channels, depth_multiplier, ): + """Tests QNN Depthwise int8 op via CMSIS-NN""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -537,6 +541,7 @@ def test_depthwise_int8( def parameterize_for_invalid_model(test): + """Generates non int8 inputs""" in_dtype = ["uint8", "int8"] kernel_dtype = ["uint8", "int8"] kernel_zero_point = [-33, 10, 0] @@ -560,12 +565,12 @@ def test_invalid_parameters( kernel_dtype, kernel_zero_point, ): + """Tests Depthwise op for non int8 inputs""" ifm_shape = (1, 28, 28, 12) out_channels = 2 input_scale = 1 input_zero_point = 24 kernel_scale = [0.11, 0.0237] - in_min, in_max = get_range_for_dtype_str(in_dtype) kernel_layout = "HWIO" kernel_shape = [3, 3, ifm_shape[3], out_channels] diff --git a/tests/python/contrib/test_cmsisnn/test_extract_constants.py b/tests/python/contrib/test_cmsisnn/test_extract_constants.py index 789d400faf97..8831596d40e6 100644 --- a/tests/python/contrib/test_cmsisnn/test_extract_constants.py +++ b/tests/python/contrib/test_cmsisnn/test_extract_constants.py @@ -16,8 +16,6 @@ # under the License. """CMSIS-NN integration tests: extract_constants pass""" -import itertools -import math import numpy as np import pytest import tvm @@ -28,6 +26,8 @@ class CheckFunctionsForConstants(tvm.relay.ExprVisitor): + """Provides methods to test number of constants present in a function""" + def __init__(self): super().__init__() self.num_constants_ = 0 @@ -38,7 +38,7 @@ def visit_call(self, call): if isinstance(arg, relay.Constant) and arg.data.numpy().ndim > 0: self.num_constants_ += 1 - def check_num_constants(self, func): + def check_num_constants(self): assert self.num_constants_ == 0, "Functions should not have constant arguments in Calls" @@ -56,118 +56,132 @@ def set_composite_func_attr(func, name): @tvm.testing.requires_cmsisnn def test_external_function(): - y0_data = np.random.uniform(0, 1, (8, 8)).astype("float32") - x0 = relay.var("x0", shape=(8, 8)) - y0_const = relay.const(y0_data, "float32") - z0 = x0 + y0_const - ef = relay.Function([x0], z0, relay.TensorType((8, 8), "float32")) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - - x = relay.var("x", shape=(8, 8)) - c = relay.Call(ev, [x]) - mf = relay.Function([x], c, relay.TensorType((8, 8), "float32")) - mv = relay.GlobalVar("main") + """Tests the pass ExternConstants when the function is a global function""" + input1_data = np.random.uniform(0, 1, (8, 8)).astype("float32") + input0 = relay.var("input0", shape=(8, 8)) + input1_const = relay.const(input1_data, "float32") + binary_op = input0 + input1_const + extern_func = relay.Function([input0], binary_op, relay.TensorType((8, 8), "float32")) + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + + arg = relay.var("arg", shape=(8, 8)) + call_extern_func = relay.Call(global_var, [arg]) + main_func = relay.Function([arg], call_extern_func, relay.TensorType((8, 8), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = ExtractConstantsFromPartitionedFunction()(mod) - CheckFunctionsForConstants().check_num_constants(mod[ev]) + constant_verifier = CheckFunctionsForConstants() + constant_verifier.visit_function(mod[global_var]) + constant_verifier.check_num_constants() relay.transform.InferType()(mod) @tvm.testing.requires_cmsisnn def test_nested_function(): - y1_data = np.random.uniform(0, 1, (8, 8)).astype("float32") - x1 = relay.var("x1", shape=(8, 8)) - y1_const = relay.const(y1_data, "float32") - z1 = x1 + y1_const - w1 = z1 * relay.const(5.0, "float32") - lf = relay.Function([x1], w1, relay.TensorType((8, 8), "float32")) - lf = set_composite_func_attr(lf, "cmsis-nn") - - x0 = relay.var("x0", shape=(8, 8)) - c0 = relay.Call(lf, [x0]) - ef = relay.Function([x0], c0, relay.TensorType((8, 8), "float32")) - - x = relay.var("x", shape=(8, 8)) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, [x]) - mf = relay.Function([x], c, relay.TensorType((8, 8), "float32")) - mv = relay.GlobalVar("main") + """Tests the pass ExternConstants when a composite function + is present within global function + """ + input1_data = np.random.uniform(0, 1, (8, 8)).astype("float32") + input0 = relay.var("input0", shape=(8, 8)) + input1_const = relay.const(input1_data, "float32") + binary_op0 = input0 + input1_const + binary_op1 = binary_op0 * relay.const(5.0, "float32") + local_func = relay.Function([input0], binary_op1, relay.TensorType((8, 8), "float32")) + local_func = set_composite_func_attr(local_func, "cmsis-nn") + + arg = relay.var("arg", shape=(8, 8)) + call_local_func = relay.Call(local_func, [arg]) + extern_func = relay.Function([arg], call_local_func, relay.TensorType((8, 8), "float32")) + + global_arg = relay.var("garg", shape=(8, 8)) + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, [global_arg]) + main_func = relay.Function([global_arg], call_extern_func, relay.TensorType((8, 8), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = ExtractConstantsFromPartitionedFunction()(mod) - CheckFunctionsForConstants().check_num_constants(mod[ev]) + constant_verifier = CheckFunctionsForConstants() + constant_verifier.visit_function(mod[global_var]) + constant_verifier.check_num_constants() relay.transform.InferType()(mod) @tvm.testing.requires_cmsisnn def test_multiple_functions(): - y20_data = np.random.uniform(0, 1, (8, 8)).astype("float32") - x20 = relay.var("x20", shape=(8, 8)) - y20_const = relay.const(y20_data, "float32") - z20 = x20 + y20_const - f20 = relay.Function([x20], z20, relay.TensorType((8, 8), "float32")) - f20 = set_composite_func_attr(f20, "cmsis-nn") - - y21_data = np.random.uniform(0, 1, (8, 8)).astype("float32") - x21 = relay.var("x21", shape=(8, 8)) - y21_const = relay.const(y21_data, "float32") - z21 = x21 + y21_const - f21 = relay.Function([x21], z21, relay.TensorType((8, 8), "float32")) - f21 = set_composite_func_attr(f21, "cmsis-nn") - - x10 = relay.var("x10", shape=(8, 8)) - c10 = relay.Call(f20, [x10]) - c11 = relay.Call(f21, [c10]) - ef = relay.Function([x10], c11, relay.TensorType((8, 8), "float32")) - x0 = relay.var("x0", shape=(8, 8)) - ev = relay.GlobalVar("cmsis-nn") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, [x0]) - mf = relay.Function([x0], c, relay.TensorType((8, 8), "float32")) - mv = relay.GlobalVar("main") + """Tests the pass ExternConstants when global function + contains multiple composite functions inside it + """ + f0_input1_data = np.random.uniform(0, 1, (8, 8)).astype("float32") + f0_input0 = relay.var("f0_in0", shape=(8, 8)) + f0_input1_const = relay.const(f0_input1_data, "float32") + f0_binary_op = f0_input0 + f0_input1_const + f0_func = relay.Function([f0_input0], f0_binary_op, relay.TensorType((8, 8), "float32")) + f0_func = set_composite_func_attr(f0_func, "cmsis-nn") + + f1_input1_data = np.random.uniform(0, 1, (8, 8)).astype("float32") + f1_input0 = relay.var("f1_in0", shape=(8, 8)) + f1_input1_const = relay.const(f1_input1_data, "float32") + f1_binary_op = f1_input0 + f1_input1_const + f1_func = relay.Function([f1_input0], f1_binary_op, relay.TensorType((8, 8), "float32")) + f1_func = set_composite_func_attr(f1_func, "cmsis-nn") + + arg0 = relay.var("arg0", shape=(8, 8)) + call_local_func0 = relay.Call(f0_func, [arg0]) + call_local_func1 = relay.Call(f1_func, [call_local_func0]) + extern_func = relay.Function([arg0], call_local_func1, relay.TensorType((8, 8), "float32")) + input0 = relay.var("input0", shape=(8, 8)) + global_var = relay.GlobalVar("cmsis-nn") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, [input0]) + main_func = relay.Function([input0], call_extern_func, relay.TensorType((8, 8), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = ExtractConstantsFromPartitionedFunction()(mod) - CheckFunctionsForConstants().check_num_constants(mod[ev]) + constant_verifier = CheckFunctionsForConstants() + constant_verifier.visit_function(mod[global_var]) + constant_verifier.check_num_constants() relay.transform.InferType()(mod) @tvm.testing.requires_cmsisnn def test_main_function(): - x0 = relay.var("x0", shape=(8, 8)) - y0 = relay.var("y0", shape=(8, 8)) - z0 = x0 + y0 - ef = relay.Function([x0, y0], z0, relay.TensorType((8, 8), "float32")) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - - x = relay.var("x", shape=(8, 8)) - y_data = np.random.uniform(0, 1, (8, 8)).astype("float32") - y_const = relay.const(y_data, "float32") - z = x + y_const - c = relay.Call(ev, [x, z]) - mf = relay.Function([x], c, relay.TensorType((8, 8), "float32")) - mv = relay.GlobalVar("main") + """Tests the pass ExternConstants on main function""" + input0 = relay.var("input0", shape=(8, 8)) + input1 = relay.var("input1", shape=(8, 8)) + binary_op = input0 + input1 + extern_func = relay.Function([input0, input1], binary_op, relay.TensorType((8, 8), "float32")) + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + + arg = relay.var("arg", shape=(8, 8)) + input_data = np.random.uniform(0, 1, (8, 8)).astype("float32") + input_const = relay.const(input_data, "float32") + binary_op = arg + input_const + call_extern_func = relay.Call(global_var, [arg, binary_op]) + main_func = relay.Function([arg], call_extern_func, relay.TensorType((8, 8), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = ExtractConstantsFromPartitionedFunction()(mod) check_for_constants = CheckFunctionsForConstants() - check_for_constants.visit_call(mod[mv].body) + check_for_constants.visit_call(mod[main_var].body) assert ( check_for_constants.num_constants_ == 1 ), "main() should have same number of arguments as before" @@ -176,6 +190,7 @@ def test_main_function(): @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("external_compiler", ["cmsis-nn", "other_compiler"]) def test_multiple_functions_non_cmsisnn_compiler(external_compiler): + """Tests the pass ExternConstants on non CMSIS-NN targets""" y20_data = np.random.uniform(0, 1, (8, 8)).astype("float32") x20 = relay.var("x20", shape=(8, 8)) y20_const = relay.const(y20_data, "float32") @@ -183,8 +198,8 @@ def test_multiple_functions_non_cmsisnn_compiler(external_compiler): f20 = relay.Function([x20], z20, relay.TensorType((8, 8), "float32")) f20 = set_composite_func_attr(f20, "cmsis-nn.qnn_op_1") x10 = relay.var("x10", shape=(8, 8)) - c10 = relay.Call(f20, [x10]) - ef0 = relay.Function([x10], c10, relay.TensorType((8, 8), "float32")) + call_local_func0 = relay.Call(f20, [x10]) + extern_func0 = relay.Function([x10], call_local_func0, relay.TensorType((8, 8), "float32")) y21_data = np.random.uniform(0, 1, (8, 8)).astype("float32") x21 = relay.var("x21", shape=(8, 8)) @@ -193,27 +208,27 @@ def test_multiple_functions_non_cmsisnn_compiler(external_compiler): f21 = relay.Function([x21], z21, relay.TensorType((8, 8), "float32")) f21 = set_composite_func_attr(f21, "cmsis-nn.qnn_op_2") x11 = relay.var("x11", shape=(8, 8)) - c11 = relay.Call(f21, [x11]) - ef1 = relay.Function([x11], c11, relay.TensorType((8, 8), "float32")) - - x0 = relay.var("x0", shape=(8, 8)) - ev0 = relay.GlobalVar("external_function_0") - ef0 = set_external_func_attr(ef0, external_compiler, ev0.name_hint) - c0 = relay.Call(ev0, [x0]) - ev1 = relay.GlobalVar("external_function_1") - ef1 = set_external_func_attr(ef1, external_compiler, ev1.name_hint) - c1 = relay.Call(ev1, [c0]) - mf = relay.Function([x0], c1, relay.TensorType((8, 8), "float32")) - mv = relay.GlobalVar("main") + call_local_func1 = relay.Call(f21, [x11]) + extern_func1 = relay.Function([x11], call_local_func1, relay.TensorType((8, 8), "float32")) + + input0 = relay.var("input0", shape=(8, 8)) + global_var0 = relay.GlobalVar("external_function_0") + extern_func0 = set_external_func_attr(extern_func0, external_compiler, global_var0.name_hint) + call_extern_func0 = relay.Call(global_var0, [input0]) + global_var1 = relay.GlobalVar("external_function_1") + extern_func1 = set_external_func_attr(extern_func1, external_compiler, global_var1.name_hint) + call_extern_func1 = relay.Call(global_var1, [call_extern_func0]) + main_func = relay.Function([input0], call_extern_func1, relay.TensorType((8, 8), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev0] = ef0 - mod[ev1] = ef1 - mod[mv] = mf + mod[global_var0] = extern_func0 + mod[global_var1] = extern_func1 + mod[main_var] = main_func mod = ExtractConstantsFromPartitionedFunction()(mod) check_for_constants = CheckFunctionsForConstants() - check_for_constants.visit_call(mod[mv].body) + check_for_constants.visit_call(mod[main_var].body) num_extracted_constants = 0 if external_compiler == "cmsis-nn": diff --git a/tests/python/contrib/test_cmsisnn/test_fully_connected.py b/tests/python/contrib/test_cmsisnn/test_fully_connected.py index c5d97f807b04..3a2061096dc1 100644 --- a/tests/python/contrib/test_cmsisnn/test_fully_connected.py +++ b/tests/python/contrib/test_cmsisnn/test_fully_connected.py @@ -27,11 +27,9 @@ from tvm.micro.testing.aot_test_utils import ( AOT_USMP_CORSTONE300_RUNNER, ) -from utils import ( - skip_if_no_reference_system, +from .utils import ( make_module, get_range_for_dtype_str, - get_same_padding, get_conv2d_qnn_params, make_qnn_relu, assert_partitioned_function, @@ -55,9 +53,9 @@ def make_model( relu_type="NONE", ): """Return a model and any parameters it may have""" - a = relay.var("input", shape=in_shape, dtype=dtype) + input_ = relay.var("input", shape=in_shape, dtype=dtype) rng = np.random.default_rng(12321) - w = tvm.nd.array( + weight = tvm.nd.array( rng.integers( np.iinfo(kernel_dtype).min, high=np.iinfo(kernel_dtype).max, @@ -65,9 +63,9 @@ def make_model( dtype=kernel_dtype, ) ) - weight_const = relay.const(w, kernel_dtype) - fc = relay.qnn.op.dense( - a, + weight_const = relay.const(weight, kernel_dtype) + dense = relay.qnn.op.dense( + input_, weight_const, input_zero_point=relay.const(input_zero_point, "int32"), kernel_zero_point=relay.const(kernel_zero_point, "int32"), @@ -77,9 +75,9 @@ def make_model( out_dtype="int32", ) - b = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) - bias_const = relay.const(b, "int32") - last_op = relay.nn.bias_add(fc, bias_const) if enable_bias else fc + bias = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) + bias_const = relay.const(bias, "int32") + last_op = relay.nn.bias_add(dense, bias_const) if enable_bias else dense requant_input_sc = input_scale * kernel_scale last_op = relay.qnn.op.requantize( last_op, @@ -90,7 +88,7 @@ def make_model( out_dtype=dtype, ) last_op = make_qnn_relu(last_op, relu_type, output_scale, output_zero_point, dtype) - params = {"w": w, "b": b} + params = {"w": weight, "b": bias} return last_op, params @@ -98,7 +96,6 @@ def make_model( @pytest.mark.parametrize("in_shape", [(2, 28), (1, 64)]) @pytest.mark.parametrize("out_channels", [12, 128]) @pytest.mark.parametrize("enable_bias", [False, True]) -@pytest.mark.parametrize("relu_type", ["RELU"]) @pytest.mark.parametrize( "input_zero_point, input_scale, kernel_scale", [(10, 0.0128, 0.11), (-64, 0.0256, 1.37)], @@ -110,8 +107,8 @@ def test_op_int8( input_scale, kernel_scale, out_channels, - relu_type, ): + """Test QNN fully connected layer""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -170,6 +167,7 @@ def test_op_int8( def parameterize_for_invalid_model(test): + """Generates parameters for non int8 inputs to fully connected layer""" in_dtype = ["uint8", "int8"] kernel_dtype = ["uint8", "int8"] kernel_zero_point = [-33, 10, 0] @@ -193,12 +191,12 @@ def test_invalid_parameters( kernel_dtype, kernel_zero_point, ): + """Tests fully connected layer with non int8 inputs""" in_shape = (2, 28) out_channels = 2 input_scale = 1 input_zero_point = 24 kernel_scale = [0.11, 0.0237] - in_min, in_max = get_range_for_dtype_str(in_dtype) kernel_shape = [out_channels, in_shape[1]] conv2d_kernel_shape = [1, 1, kernel_shape[0], kernel_shape[1]] diff --git a/tests/python/contrib/test_cmsisnn/test_generate_constants.py b/tests/python/contrib/test_cmsisnn/test_generate_constants.py index cded0f03566d..e6faa1a243f5 100644 --- a/tests/python/contrib/test_cmsisnn/test_generate_constants.py +++ b/tests/python/contrib/test_cmsisnn/test_generate_constants.py @@ -16,7 +16,6 @@ # under the License. """CMSIS-NN integration tests: generate_constants pass""" -import itertools import math import numpy as np import pytest @@ -25,9 +24,8 @@ from tvm import relay from tvm.relay.op.contrib import cmsisnn -from utils import ( +from .utils import ( make_module, - get_range_for_dtype_str, get_same_padding, get_conv2d_qnn_params, make_qnn_relu, @@ -43,6 +41,8 @@ def quantize_scale(scale): class CheckGeneratedConstants(tvm.relay.ExprVisitor): + """Provides methods to compare against expected quantization parameters""" + def __init__(self, enable_bias, multiplier, shift): super().__init__() self.num_constant_args_ = 0 @@ -53,7 +53,6 @@ def __init__(self, enable_bias, multiplier, shift): def visit_call(self, call): super().visit_call(call) if isinstance(call.op, tvm.ir.expr.GlobalVar): - # extern_fn_call(input, weight, multiplier, weight_scale, bias_optional, input_scale, shift) multiplier = call.args[2] shift = call.args[6] if self.enable_bias_ else call.args[5] assert isinstance( @@ -107,7 +106,7 @@ def make_model( weight_shape = (kernel_h, kernel_w, shape[3] // groups, out_channels) rng = np.random.default_rng(12321) - w = tvm.nd.array( + weight = tvm.nd.array( rng.integers( np.iinfo(kernel_dtype).min, high=np.iinfo(kernel_dtype).max, @@ -115,7 +114,7 @@ def make_model( dtype=kernel_dtype, ) ) - weight_const = relay.const(w, kernel_dtype) + weight_const = relay.const(weight, kernel_dtype) conv = relay.qnn.op.conv2d( a, weight_const, @@ -133,8 +132,8 @@ def make_model( padding=p, out_dtype="int32", ) - b = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) - bias_const = relay.const(b, "int32") + bias = tvm.nd.array(rng.integers(0, high=10, size=(out_channels,), dtype="int32")) + bias_const = relay.const(bias, "int32") last_op = relay.nn.bias_add(conv, bias_const, axis=3) if enable_bias else conv requant_input_sc = [sc * input_scale for sc in kernel_scale] last_op = relay.qnn.op.requantize( @@ -146,7 +145,7 @@ def make_model( out_dtype=dtype, ) last_op = make_qnn_relu(last_op, relu_type, output_scale, output_zero_point, dtype) - params = {"w": w, "b": b} + params = {"w": weight, "b": bias} return last_op, params @@ -163,6 +162,7 @@ def test_op_int8( kernel_scale, out_channels, ): + """Tests for CMSIS-NN constants when the dtype is int8""" ifm_shape = (1, 28, 28, 3) padding = "VALID" strides = (1, 1) @@ -175,7 +175,6 @@ def test_op_int8( kernel_w = kernel_size[1] dtype = "int8" relu_type = "RELU" - in_min, in_max = get_range_for_dtype_str(dtype) weight_shape = (kernel_h, kernel_w, ifm_shape[3] // groups, out_channels) diff --git a/tests/python/contrib/test_cmsisnn/test_invalid_graphs.py b/tests/python/contrib/test_cmsisnn/test_invalid_graphs.py index d0a8547d32ac..c66f9d0e0726 100644 --- a/tests/python/contrib/test_cmsisnn/test_invalid_graphs.py +++ b/tests/python/contrib/test_cmsisnn/test_invalid_graphs.py @@ -16,17 +16,14 @@ # under the License. """CMSIS-NN integration tests: Tests invalid graphs""" -import itertools import numpy as np -import pytest import tvm -from tvm import relay from tvm.testing.aot import AOTTestModel, compile_and_run, generate_ref_data from tvm.micro.testing.aot_test_utils import ( AOT_USMP_CORSTONE300_RUNNER, ) -from utils import ( +from .utils import ( skip_if_no_reference_system, get_range_for_dtype_str, ) @@ -35,13 +32,14 @@ @skip_if_no_reference_system @tvm.testing.requires_cmsisnn def test_empty_function(): - ORIGINAL_MODEL = """ + """Test partitioned function without composite function""" + original_model = """ #[version = "0.0.5"] def @main(%data : Tensor[(16, 29), int8]) -> Tensor[(16, 29), int8] { add(%data, %data) } """ - CMSISNN_MODEL = """ + cmsisnn_model = """ #[version = "0.0.5"] def @tvmgen_default_cmsis_nn_main_1(%i1: Tensor[(16, 29), int8], Inline=1, Compiler="cmsis-nn", global_symbol="tvmgen_default_cmsis_nn_main_1", Primitive=1) -> Tensor[(16, 29), int8] { add(%i1, %i1) @@ -51,8 +49,8 @@ def @main(%data : Tensor[(16, 29), int8]) -> Tensor[(16, 29), int8] { %1 } """ - orig_mod = tvm.parser.fromtext(ORIGINAL_MODEL) - cmsisnn_mod = tvm.parser.fromtext(CMSISNN_MODEL) + orig_mod = tvm.parser.fromtext(original_model) + cmsisnn_mod = tvm.parser.fromtext(cmsisnn_model) params = {} # validate the output diff --git a/tests/python/contrib/test_cmsisnn/test_networks.py b/tests/python/contrib/test_cmsisnn/test_networks.py index 3b1e2331f2ff..41bde238b0d8 100644 --- a/tests/python/contrib/test_cmsisnn/test_networks.py +++ b/tests/python/contrib/test_cmsisnn/test_networks.py @@ -17,8 +17,6 @@ """CMSIS-NN: testing with networks""" -import sys - import pytest import numpy as np @@ -26,20 +24,21 @@ from tvm import relay from tvm.contrib.download import download_testdata from tvm.relay.op.contrib import cmsisnn - -from utils import skip_if_no_reference_system, get_range_for_dtype_str from tvm.testing.aot import AOTTestModel, compile_and_run, generate_ref_data from tvm.micro.testing.aot_test_utils import ( AOT_CORSTONE300_RUNNER, AOT_USMP_CORSTONE300_RUNNER, ) +from .utils import skip_if_no_reference_system, get_range_for_dtype_str - +# pylint: disable=import-outside-toplevel def _convert_to_relay( tflite_model_buf, input_data, input_node, ): + """Converts TFLite model to Relay module and params""" + def convert_to_list(x): if not isinstance(x, list): x = [x] @@ -62,9 +61,9 @@ def convert_to_list(x): shape_dict = {} dtype_dict = {} - for i, e in enumerate(input_node): - shape_dict[e] = input_data[i].shape - dtype_dict[e] = input_data[i].dtype.name + for i, name in enumerate(input_node): + shape_dict[name] = input_data[i].shape + dtype_dict[name] = input_data[i].dtype.name mod, params = relay.frontend.from_tflite( tflite_model, shape_dict=shape_dict, dtype_dict=dtype_dict @@ -78,8 +77,9 @@ def convert_to_list(x): @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("test_runner", [AOT_CORSTONE300_RUNNER, AOT_USMP_CORSTONE300_RUNNER]) def test_cnn_small(test_runner): + """Download a small network and tests TVM via CMSIS-NN output against TFLite output""" # download the model - base_url = "https://github.com/ARM-software/ML-zoo/raw/48a22ee22325d15d2371a6df24eb7d67e21dcc97/models/keyword_spotting/cnn_small/tflite_int8" + base_url = "https://github.com/ARM-software/ML-zoo/raw/48a22ee22325d15d2371a6df24eb7d67e21dcc97/models/keyword_spotting/cnn_small/tflite_int8" # pylint: disable=line-too-long file_to_download = "cnn_s_quantized.tflite" file_saved = "cnn_s_quantized_15Dec2021.tflite" model_file = download_testdata("{}/{}".format(base_url, file_to_download), file_saved) diff --git a/tests/python/contrib/test_cmsisnn/test_pooling.py b/tests/python/contrib/test_cmsisnn/test_pooling.py index 1fd280b7d81a..6b719cdc9938 100644 --- a/tests/python/contrib/test_cmsisnn/test_pooling.py +++ b/tests/python/contrib/test_cmsisnn/test_pooling.py @@ -16,7 +16,6 @@ # under the License. """CMSIS-NN integration tests: Conv2D""" -import itertools import numpy as np import pytest import tvm @@ -25,12 +24,10 @@ from tvm.testing.aot import AOTTestModel, compile_and_run, generate_ref_data from tvm.micro.testing.aot_test_utils import AOT_USMP_CORSTONE300_RUNNER -from utils import ( - skip_if_no_reference_system, +from .utils import ( make_module, get_range_for_dtype_str, get_same_padding, - get_conv2d_qnn_params, make_qnn_relu, assert_partitioned_function, assert_no_external_function, @@ -49,7 +46,9 @@ def make_model( relu_type="RELU", layout="NHWC", ): - """Return a model and any parameters it may have, all parameters are defaulted to known good values""" + """Return a model and any parameters it may have, + all parameters are defaulted to known good values + """ op = relay.var("input", shape=shape, dtype=dtype) pad_ = (0, 0, 0, 0) if padding == "SAME": @@ -61,12 +60,12 @@ def make_model( pad_value=zero_point, pad_mode="constant", ) - if pool_op == relay.nn.avg_pool2d: + if pool_op.__name__ == relay.nn.avg_pool2d.__name__: op = relay.cast(op, "int32") op = pool_op( op, pool_size=pool_size, strides=strides, padding=pad_, ceil_mode=True, layout=layout ) - if pool_op == relay.nn.avg_pool2d: + if pool_op.__name__ == relay.nn.avg_pool2d.__name__: op = relay.cast(op, dtype) op = make_qnn_relu(op, relu_type, scale, zero_point, dtype) return op @@ -91,6 +90,7 @@ def test_op_int8( zero_point, scale, ): + """Tests QNN pooling op for int8 inputs""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -138,6 +138,7 @@ def test_op_int8( @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("op", [relay.nn.avg_pool2d, relay.nn.max_pool2d]) def test_invalid_datatype(op): + """Checks CMSIS-NN partitioning for non int8 dtype""" model = make_model(pool_op=op, dtype="int64") orig_mod = make_module(model) @@ -148,6 +149,7 @@ def test_invalid_datatype(op): @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("op", [relay.nn.avg_pool2d, relay.nn.max_pool2d]) def test_invalid_batch_size(op): + """Checks CMSIS-NN partitioning when batch size is not 1""" model = make_model( pool_op=op, shape=(2, 28, 28, 12), @@ -161,6 +163,7 @@ def test_invalid_batch_size(op): @tvm.testing.requires_cmsisnn @pytest.mark.parametrize("op", [relay.nn.avg_pool2d, relay.nn.max_pool2d]) def test_invalid_layout(op): + """Checks CMSIS-NN partitioning when layout is not NHWC""" model = make_model(pool_op=op, layout="NCHW") orig_mod = make_module(model) diff --git a/tests/python/contrib/test_cmsisnn/test_scalar_to_tensor_constant.py b/tests/python/contrib/test_cmsisnn/test_scalar_to_tensor_constant.py index 35bdabf3171c..557a65aeffca 100644 --- a/tests/python/contrib/test_cmsisnn/test_scalar_to_tensor_constant.py +++ b/tests/python/contrib/test_cmsisnn/test_scalar_to_tensor_constant.py @@ -16,10 +16,7 @@ # under the License. """CMSIS-NN integration tests: scalar_to_tensor_constant pass""" -import sys - import numpy as np -import pytest import tvm import tvm.testing from tvm import relay @@ -56,6 +53,8 @@ def make_binary_op( class CheckFunctionsForConstants(tvm.relay.ExprVisitor): + """Provides method to test number of scalar constants present in a function""" + def __init__(self): super().__init__() self.num_constants_ = 0 @@ -66,7 +65,7 @@ def visit_call(self, call): if isinstance(arg, relay.Constant) and arg.data.numpy().ndim > 0: self.num_constants_ += 1 - def check_num_constants(self, func): + def check_num_constants(self): assert self.num_constants_ == 0, "Functions should not have constant arguments in Calls" @@ -84,44 +83,45 @@ def set_composite_func_attr(func, name): @tvm.testing.requires_cmsisnn def test_single_scalar_position_0(): + """Tests conversion to tensor constant when first operand is a scalar""" dtype = "int8" shape = (8, 8) - x0 = generate_variable("x0", None, dtype) - x1 = generate_variable("x1", shape, dtype) - z1 = make_binary_op( + operand0 = generate_variable("operand0", None, dtype) + operand1 = generate_variable("operand1", shape, dtype) + binary_op = make_binary_op( relay.qnn.op.add, - x0, - x1, + operand0, + operand1, input_0_scale=0.0128, input_0_zero_point=32, input_1_scale=0.256, input_1_zero_point=-64, ) - lf = relay.Function([x0, x1], z1, relay.TensorType(shape, dtype)) - lf = set_composite_func_attr(lf, "cmsis-nn.qnn_add") + local_func = relay.Function([operand0, operand1], binary_op, relay.TensorType(shape, dtype)) + local_func = set_composite_func_attr(local_func, "cmsis-nn.qnn_add") - y0 = relay.expr.const(3, dtype) - y1 = relay.var("y1", shape=shape, dtype=dtype) - c0 = relay.Call(lf, [y0, y1]) - ef = relay.Function([y1], c0, relay.TensorType(shape, dtype)) + arg0 = relay.expr.const(3, dtype) + arg1 = relay.var("arg1", shape=shape, dtype=dtype) + call_local_func = relay.Call(local_func, [arg0, arg1]) + extern_func = relay.Function([arg1], call_local_func, relay.TensorType(shape, dtype)) x = relay.var("x", shape=shape, dtype=dtype) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, [x]) - mf = relay.Function([x], c, relay.TensorType(shape, dtype)) - mv = relay.GlobalVar("main") + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, [x]) + main_func = relay.Function([x], call_extern_func, relay.TensorType(shape, dtype)) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = relay.transform.InferType()(mod) mod = ScalarToTensorConstants()(mod) mod = relay.transform.InferType()(mod) check_for_constants = CheckFunctionsForConstants() - check_for_constants.visit_call(mod[ev].body) + check_for_constants.visit_call(mod[global_var].body) assert ( check_for_constants.num_constants_ == 1 ), "Scalar constant wasn't converted into tensor constant" @@ -129,44 +129,45 @@ def test_single_scalar_position_0(): @tvm.testing.requires_cmsisnn def test_single_scalar_position_1(): + """Tests conversion to tensor constant when second operand is a scalar""" dtype = "int8" shape = (8, 8) - x0 = generate_variable("x0", shape, dtype) - x1 = generate_variable("x1", None, dtype) - z1 = make_binary_op( + operand0 = generate_variable("operand0", shape, dtype) + operand1 = generate_variable("operand1", None, dtype) + binary_op = make_binary_op( relay.qnn.op.add, - x0, - x1, + operand0, + operand1, input_0_scale=0.0128, input_0_zero_point=32, input_1_scale=0.256, input_1_zero_point=-64, ) - lf = relay.Function([x0, x1], z1, relay.TensorType(shape, dtype)) - lf = set_composite_func_attr(lf, "cmsis-nn.qnn_add") + local_func = relay.Function([operand0, operand1], binary_op, relay.TensorType(shape, dtype)) + local_func = set_composite_func_attr(local_func, "cmsis-nn.qnn_add") - y0 = relay.var("y0", shape=shape, dtype=dtype) - y1 = relay.expr.const(3, dtype) - c0 = relay.Call(lf, [y0, y1]) - ef = relay.Function([y0], c0, relay.TensorType(shape, dtype)) + arg0 = relay.var("arg0", shape=shape, dtype=dtype) + arg1 = relay.expr.const(3, dtype) + call_local_func = relay.Call(local_func, [arg0, arg1]) + extern_func = relay.Function([arg0], call_local_func, relay.TensorType(shape, dtype)) x = relay.var("x", shape=shape, dtype=dtype) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, [x]) - mf = relay.Function([x], c, relay.TensorType(shape, dtype)) - mv = relay.GlobalVar("main") + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, [x]) + main_func = relay.Function([x], call_extern_func, relay.TensorType(shape, dtype)) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = relay.transform.InferType()(mod) mod = ScalarToTensorConstants()(mod) mod = relay.transform.InferType()(mod) check_for_constants = CheckFunctionsForConstants() - check_for_constants.visit_call(mod[ev].body) + check_for_constants.visit_call(mod[global_var].body) assert ( check_for_constants.num_constants_ == 1 ), "Scalar constant wasn't converted into tensor constant" @@ -174,83 +175,85 @@ def test_single_scalar_position_1(): @tvm.testing.requires_cmsisnn def test_primary_operands_all_scalars(): + """Tests conversion to tensor constants all operands are scalars""" dtype = "int8" shape = None - x0 = generate_variable("x0", None, dtype) - x1 = generate_variable("x1", None, dtype) - z1 = make_binary_op( + operand0 = generate_variable("operand0", None, dtype) + operand1 = generate_variable("operand1", None, dtype) + binary_op = make_binary_op( relay.qnn.op.add, - x0, - x1, + operand0, + operand1, input_0_scale=0.0128, input_0_zero_point=32, input_1_scale=0.256, input_1_zero_point=-64, ) - lf = relay.Function([x0, x1], z1, relay.TensorType(shape, dtype)) - lf = set_composite_func_attr(lf, "cmsis-nn.qnn_add") + local_func = relay.Function([operand0, operand1], binary_op, relay.TensorType(shape, dtype)) + local_func = set_composite_func_attr(local_func, "cmsis-nn.qnn_add") - y0 = relay.expr.const(7, dtype) - y1 = relay.expr.const(3, dtype) - c0 = relay.Call(lf, [y0, y1]) - ef = relay.Function([], c0, relay.TensorType(shape, dtype)) + arg0 = relay.expr.const(7, dtype) + arg1 = relay.expr.const(3, dtype) + call_local_func = relay.Call(local_func, [arg0, arg1]) + extern_func = relay.Function([], call_local_func, relay.TensorType(shape, dtype)) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, []) - mf = relay.Function([], c, relay.TensorType(shape, dtype)) - mv = relay.GlobalVar("main") + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, []) + main_func = relay.Function([], call_extern_func, relay.TensorType(shape, dtype)) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = relay.transform.InferType()(mod) mod = ScalarToTensorConstants()(mod) new_mod = relay.transform.InferType()(mod) - assert tvm.ir.structural_equal(mod[ev].body, new_mod[ev].body) + assert tvm.ir.structural_equal(mod[global_var].body, new_mod[global_var].body) @tvm.testing.requires_cmsisnn def test_all_primary_operands_tensor_constants(): + """Tests conversion to tensor constants all operands are tensors""" dtype = "int8" shape = (1, 3, 3, 32) - x0 = generate_variable("x0", shape, dtype) - x1 = generate_variable("x1", shape, dtype) - z1 = make_binary_op( + operand0 = generate_variable("operand0", shape, dtype) + operand1 = generate_variable("operand1", shape, dtype) + binary_op = make_binary_op( relay.qnn.op.add, - x0, - x1, + operand0, + operand1, input_0_scale=0.0128, input_0_zero_point=32, input_1_scale=0.256, input_1_zero_point=-64, ) - lf = relay.Function([x0, x1], z1, relay.TensorType(shape, dtype)) - lf = set_composite_func_attr(lf, "cmsis-nn.qnn_add") + local_func = relay.Function([operand0, operand1], binary_op, relay.TensorType(shape, dtype)) + local_func = set_composite_func_attr(local_func, "cmsis-nn.qnn_add") rng = np.random.default_rng(12345) - y0 = relay.const(rng.integers(-128, high=127, size=shape, dtype=dtype)) - y1 = relay.const(rng.integers(-128, high=127, size=shape, dtype=dtype)) - c0 = relay.Call(lf, [y0, y1]) - ef = relay.Function([], c0, relay.TensorType(shape, dtype)) + arg0 = relay.const(rng.integers(-128, high=127, size=shape, dtype=dtype)) + arg1 = relay.const(rng.integers(-128, high=127, size=shape, dtype=dtype)) + call_local_func = relay.Call(local_func, [arg0, arg1]) + extern_func = relay.Function([], call_local_func, relay.TensorType(shape, dtype)) - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "cmsis-nn", ev.name_hint) - c = relay.Call(ev, []) - mf = relay.Function([], c, relay.TensorType(shape, dtype)) - mv = relay.GlobalVar("main") + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "cmsis-nn", global_var.name_hint) + call_extern_func = relay.Call(global_var, []) + main_func = relay.Function([], call_extern_func, relay.TensorType(shape, dtype)) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = relay.transform.InferType()(mod) mod = ScalarToTensorConstants()(mod) new_mod = relay.transform.InferType()(mod) - assert tvm.ir.structural_equal(mod[ev].body, new_mod[ev].body) + assert tvm.ir.structural_equal(mod[global_var].body, new_mod[global_var].body) @tvm.testing.requires_cmsisnn @@ -258,26 +261,28 @@ def test_non_cmsisnn_ext_func(): """Non CMSISNN functions should not be altered.""" def get_mod(): - x1 = relay.var("x1", shape=None) - x2 = relay.var("x2", shape=None) - z1 = x1 + x2 - lf = relay.Function([x1, x2], z1, relay.TensorType((), "float32")) - lf = set_composite_func_attr(lf, "cmsis-nn.qnn_add") - - y0 = relay.expr.const(5, "float32") - y1 = relay.expr.const(3, "float32") - c0 = relay.Call(lf, [y0, y1]) - ef = relay.Function([], c0, relay.TensorType((), "float32")) - - ev = relay.GlobalVar("external_function") - ef = set_external_func_attr(ef, "foo", ev.name_hint) - c = relay.Call(ev, []) - mf = relay.Function([], c, relay.TensorType((), "float32")) - mv = relay.GlobalVar("main") + operand1 = relay.var("operand1", shape=None) + operand2 = relay.var("operand2", shape=None) + binary_op = operand1 + operand2 + local_func = relay.Function( + [operand1, operand2], binary_op, relay.TensorType((), "float32") + ) + local_func = set_composite_func_attr(local_func, "cmsis-nn.qnn_add") + + arg0 = relay.expr.const(5, "float32") + arg1 = relay.expr.const(3, "float32") + call_local_func = relay.Call(local_func, [arg0, arg1]) + extern_func = relay.Function([], call_local_func, relay.TensorType((), "float32")) + + global_var = relay.GlobalVar("external_function") + extern_func = set_external_func_attr(extern_func, "foo", global_var.name_hint) + call_extern_func = relay.Call(global_var, []) + main_func = relay.Function([], call_extern_func, relay.TensorType((), "float32")) + main_var = relay.GlobalVar("main") mod = tvm.IRModule() - mod[ev] = ef - mod[mv] = mf + mod[global_var] = extern_func + mod[main_var] = main_func mod = relay.transform.InferType()(mod) return mod diff --git a/tests/python/contrib/test_cmsisnn/test_softmax.py b/tests/python/contrib/test_cmsisnn/test_softmax.py index 840d0e6f4436..c6d2e4ec4537 100644 --- a/tests/python/contrib/test_cmsisnn/test_softmax.py +++ b/tests/python/contrib/test_cmsisnn/test_softmax.py @@ -16,8 +16,6 @@ # under the License. """CMSIS-NN integration tests: Softmax""" - -import sys import itertools import numpy as np @@ -26,16 +24,16 @@ import tvm.testing from tvm import relay from tvm.relay.op.contrib import cmsisnn +from tvm.testing.aot import AOTTestModel, compile_and_run, generate_ref_data +from tvm.micro.testing.aot_test_utils import AOT_USMP_CORSTONE300_RUNNER -from utils import ( +from .utils import ( skip_if_no_reference_system, make_module, get_range_for_dtype_str, assert_partitioned_function, assert_no_external_function, ) -from tvm.testing.aot import AOTTestModel, compile_and_run, generate_ref_data -from tvm.micro.testing.aot_test_utils import AOT_USMP_CORSTONE300_RUNNER def make_model( @@ -62,6 +60,7 @@ def make_model( @pytest.mark.parametrize(["zero_point", "scale"], [[33, 0.256], [-64, 0.0128]]) @tvm.testing.requires_cmsisnn def test_op_int8(zero_point, scale): + """Tests int8 QNN Softmax for CMSIS-NN""" interface_api = "c" use_unpacked_api = True test_runner = AOT_USMP_CORSTONE300_RUNNER @@ -92,6 +91,7 @@ def test_op_int8(zero_point, scale): def parameterize_for_invalid_model(test): + """Generates parameters for non int8 input and output of Softmax""" in_dtype = ["uint8", "int8"] out_dtype = ["uint8", "int8"] zero_point = [-128, 64] @@ -119,6 +119,7 @@ def parameterize_for_invalid_model(test): @parameterize_for_invalid_model @tvm.testing.requires_cmsisnn def test_invalid_parameters(in_dtype, out_dtype, zero_point, scale, out_zero_point, out_scale): + """Tests for non int8 input and output of Softmax""" model = make_model( [1, 16, 16, 3], in_dtype, out_dtype, zero_point, scale, out_zero_point, out_scale ) diff --git a/tests/python/contrib/test_cmsisnn/utils.py b/tests/python/contrib/test_cmsisnn/utils.py index 83c67cd95b1c..2da335a8ba76 100644 --- a/tests/python/contrib/test_cmsisnn/utils.py +++ b/tests/python/contrib/test_cmsisnn/utils.py @@ -17,11 +17,9 @@ """CMSIS-NN functions for testing networks""" -import platform import math +from typing import List, Union, Tuple import numpy as np -import pytest -from typing import List, Dict, Optional, Any, Union, Tuple import tvm from tvm import relay @@ -52,6 +50,7 @@ def visit_call(self, call): def assert_partitioned_function(orig_mod, cmsisnn_mod): + """If kCompiler attribute is missing, this function raises assertion""" attrs = [ cmsisnn_mod[var.name_hint].attrs for var in cmsisnn_mod.get_global_vars() @@ -205,6 +204,7 @@ def get_conv2d_qnn_params( return output_scale, output_zp +# pylint: disable=inconsistent-return-statements def make_qnn_relu(expr, fused_activation_fn, scale, zero_point, dtype): """Mimics convert_qnn_fused_activation_function from TFLite frontend""" quantize = lambda x: float(int(round(x / scale)) + zero_point) @@ -225,3 +225,4 @@ def make_qnn_relu(expr, fused_activation_fn, scale, zero_point, dtype): ) if fused_activation_fn == "RELU": return tvm.relay.op.clip(expr, a_min=max(qmin, quantize(0.0)), a_max=qmax) + assert False, "Unsupported activation function" From d34f5b13161b80041e91b99267aa496e997634c1 Mon Sep 17 00:00:00 2001 From: Ashutosh Parkhi Date: Thu, 9 Jun 2022 12:06:23 +0100 Subject: [PATCH 2/3] Removed comments that disabled pylint checks Change-Id: Iee513a4a5bef1db5b78e1d25a30ac7202f8b0e92 --- tests/lint/pylint.sh | 2 +- tests/python/contrib/test_cmsisnn/test_networks.py | 6 +++++- tests/python/contrib/test_cmsisnn/utils.py | 4 ++-- 3 files changed, 8 insertions(+), 4 deletions(-) diff --git a/tests/lint/pylint.sh b/tests/lint/pylint.sh index 11d4b9f4b968..b442c33c0ff6 100755 --- a/tests/lint/pylint.sh +++ b/tests/lint/pylint.sh @@ -20,5 +20,5 @@ set -euxo pipefail python3 -m pylint python/tvm --rcfile="$(dirname "$0")"/pylintrc python3 -m pylint vta/python/vta --rcfile="$(dirname "$0")"/pylintrc python3 -m pylint tests/python/unittest/test_tvmscript_type.py --rcfile="$(dirname "$0")"/pylintrc -python3 -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc --exit-zero +python3 -m pylint tests/python/contrib/test_cmsisnn --rcfile="$(dirname "$0")"/pylintrc diff --git a/tests/python/contrib/test_cmsisnn/test_networks.py b/tests/python/contrib/test_cmsisnn/test_networks.py index 41bde238b0d8..6f9f3743a622 100644 --- a/tests/python/contrib/test_cmsisnn/test_networks.py +++ b/tests/python/contrib/test_cmsisnn/test_networks.py @@ -79,7 +79,11 @@ def convert_to_list(x): def test_cnn_small(test_runner): """Download a small network and tests TVM via CMSIS-NN output against TFLite output""" # download the model - base_url = "https://github.com/ARM-software/ML-zoo/raw/48a22ee22325d15d2371a6df24eb7d67e21dcc97/models/keyword_spotting/cnn_small/tflite_int8" # pylint: disable=line-too-long + base_url = ( + "https://github.com/ARM-software/ML-zoo/raw/" + "48a22ee22325d15d2371a6df24eb7d67e21dcc97" + "/models/keyword_spotting/cnn_small/tflite_int8" + ) file_to_download = "cnn_s_quantized.tflite" file_saved = "cnn_s_quantized_15Dec2021.tflite" model_file = download_testdata("{}/{}".format(base_url, file_to_download), file_saved) diff --git a/tests/python/contrib/test_cmsisnn/utils.py b/tests/python/contrib/test_cmsisnn/utils.py index 2da335a8ba76..e69329ebc5a4 100644 --- a/tests/python/contrib/test_cmsisnn/utils.py +++ b/tests/python/contrib/test_cmsisnn/utils.py @@ -204,7 +204,6 @@ def get_conv2d_qnn_params( return output_scale, output_zp -# pylint: disable=inconsistent-return-statements def make_qnn_relu(expr, fused_activation_fn, scale, zero_point, dtype): """Mimics convert_qnn_fused_activation_function from TFLite frontend""" quantize = lambda x: float(int(round(x / scale)) + zero_point) @@ -225,4 +224,5 @@ def make_qnn_relu(expr, fused_activation_fn, scale, zero_point, dtype): ) if fused_activation_fn == "RELU": return tvm.relay.op.clip(expr, a_min=max(qmin, quantize(0.0)), a_max=qmax) - assert False, "Unsupported activation function" + + raise ValueError("Invalid argument provided with fused_activation_fn") From 745b984973ea17561f508a9b6f2881fec501bfbf Mon Sep 17 00:00:00 2001 From: Ashutosh Parkhi Date: Thu, 9 Jun 2022 16:08:04 +0100 Subject: [PATCH 3/3] Pylint checks for missing docstring, unused variable Change-Id: Iff029c3899b289f0c7aaf80ba6c2b648c6ba33eb --- tests/python/contrib/test_cmsisnn/test_conv2d.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/python/contrib/test_cmsisnn/test_conv2d.py b/tests/python/contrib/test_cmsisnn/test_conv2d.py index 58baea88f054..462eb8834719 100644 --- a/tests/python/contrib/test_cmsisnn/test_conv2d.py +++ b/tests/python/contrib/test_cmsisnn/test_conv2d.py @@ -133,9 +133,9 @@ def test_conv2d_number_primfunc_args( kernel_scale, out_channels, ): + """Tests number of arguments in Conv2D primfunc""" interface_api = "c" use_unpacked_api = True - test_runner = AOT_USMP_CORSTONE300_RUNNER ifm_shape = (1, 64, 100, 4) kernel_size = (3, 3) @@ -203,7 +203,7 @@ def test_conv2d_number_primfunc_args( expected_num_params = 6 if enable_bias else 5 cmsisnn_tir_mod = None for target, mod in compiled_models[0].executor_factory.lowered_ir_mods.items(): - if "cmsis-nn" == target.kind.name: + if target.kind.name == "cmsis-nn": cmsisnn_tir_mod = mod cmsisnn_func = cmsisnn_tir_mod["tvmgen_default_cmsis_nn_main_0"]