1 # Greentea host test script for Mbed TLS on-target test suite testing.
3 # Copyright (C) 2018, Arm Limited, All Rights Reserved
4 # SPDX-License-Identifier: Apache-2.0
6 # Licensed under the Apache License, Version 2.0 (the "License"); you may
7 # not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
10 # http://www.apache.org/licenses/LICENSE-2.0
12 # Unless required by applicable law or agreed to in writing, software
13 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
14 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 # See the License for the specific language governing permissions and
16 # limitations under the License.
18 # This file is part of Mbed TLS (https://tls.mbed.org)
22 Mbed TLS on-target test suite tests are implemented as Greentea
23 tests. Greentea tests are implemented in two parts: target test and
24 host test. Target test is a C application that is built for the
25 target platform and executes on the target. Host test is a Python
26 class derived from mbed_host_tests.BaseHostTest. Target communicates
27 with the host over serial for the test data and sends back the result.
29 Python tool mbedgt (Greentea) is responsible for flashing the test
30 binary on to the target and dynamically loading this host test module.
32 Greentea documentation can be found here:
33 https://github.com/ARMmbed/greentea
41 from mbed_host_tests import BaseHostTest, event_callback # pylint: disable=import-error
44 class TestDataParserError(Exception):
45 """Indicates error in test data, read from .data file."""
49 class TestDataParser(object):
51 Parses test name, dependencies, test function name and test parameters
61 def parse(self, data_file):
65 :param data_file: Data file path
67 with open(data_file, 'r') as data_f:
71 def __escaped_split(inp_str, split_char):
73 Splits inp_str on split_char except when escaped.
75 :param inp_str: String to split
76 :param split_char: Split character
77 :return: List of splits
79 split_colon_fn = lambda x: re.sub(r'\\' + split_char, split_char, x)
80 if len(split_char) > 1:
81 raise ValueError('Expected split character. Found string!')
82 out = map(split_colon_fn, re.split(r'(?<!\\)' + split_char, inp_str))
83 out = [x for x in out if x]
86 def __parse(self, data_f):
88 Parses data file using supplied file object.
90 :param data_f: Data file object
102 line = data_f.next().strip()
103 match = re.search('depends_on:(.*)', line)
105 dependencies = [int(x) for x in match.group(1).split(':')]
106 line = data_f.next().strip()
109 line = line.replace('\\n', '\n')
110 parts = self.__escaped_split(line, ':')
111 function_name = int(parts[0])
113 args_count = len(args)
114 if args_count % 2 != 0:
115 err_str_fmt = "Number of test arguments({}) should be even: {}"
116 raise TestDataParserError(err_str_fmt.format(args_count, line))
117 grouped_args = [(args[i * 2], args[(i * 2) + 1])
118 for i in range(len(args)/2)]
119 self.tests.append((name, function_name, dependencies,
122 def get_test_data(self):
129 class MbedTlsTest(BaseHostTest):
131 Host test for Mbed TLS unit tests. This script is loaded at
132 run time by Greentea for executing Mbed TLS test suites. Each
133 communication from the target is received in this object as
134 an event, which is then handled by the event handler method
135 decorated by the associated event. Ex: @event_callback('GO').
137 Target test sends requests for dispatching next test. It reads
138 tests from the intermediate data file and sends test function
139 identifier, dependency identifiers, expression identifiers and
140 the test data in binary form. Target test checks dependencies
141 , evaluate integer constant expressions and dispatches the test
142 function with received test parameters. After test function is
143 finished, target sends the result. This class handles the result
144 event and prints verdict in the form that Greentea understands.
147 # status/error codes from suites/helpers.function
148 DEPENDENCY_SUPPORTED = 0
149 KEY_VALUE_MAPPING_FOUND = DEPENDENCY_SUPPORTED
150 DISPATCH_TEST_SUCCESS = DEPENDENCY_SUPPORTED
152 KEY_VALUE_MAPPING_NOT_FOUND = -1 # Expression Id not found.
153 DEPENDENCY_NOT_SUPPORTED = -2 # Dependency not supported.
154 DISPATCH_TEST_FN_NOT_FOUND = -3 # Test function not found.
155 DISPATCH_INVALID_TEST_DATA = -4 # Invalid parameter type.
156 DISPATCH_UNSUPPORTED_SUITE = -5 # Test suite not supported/enabled.
160 Constructor initialises test index to 0.
162 super(MbedTlsTest, self).__init__()
166 self.suite_passed = True
167 self.error_str = dict()
168 self.error_str[self.DEPENDENCY_SUPPORTED] = \
169 'DEPENDENCY_SUPPORTED'
170 self.error_str[self.KEY_VALUE_MAPPING_NOT_FOUND] = \
171 'KEY_VALUE_MAPPING_NOT_FOUND'
172 self.error_str[self.DEPENDENCY_NOT_SUPPORTED] = \
173 'DEPENDENCY_NOT_SUPPORTED'
174 self.error_str[self.DISPATCH_TEST_FN_NOT_FOUND] = \
175 'DISPATCH_TEST_FN_NOT_FOUND'
176 self.error_str[self.DISPATCH_INVALID_TEST_DATA] = \
177 'DISPATCH_INVALID_TEST_DATA'
178 self.error_str[self.DISPATCH_UNSUPPORTED_SUITE] = \
179 'DISPATCH_UNSUPPORTED_SUITE'
183 Setup hook implementation. Reads test suite data file and parses out
186 binary_path = self.get_config_item('image_path')
187 script_dir = os.path.split(os.path.abspath(__file__))[0]
188 suite_name = os.path.splitext(os.path.basename(binary_path))[0]
189 data_file = ".".join((suite_name, 'datax'))
190 data_file = os.path.join(script_dir, '..', 'mbedtls',
191 suite_name, data_file)
192 if os.path.exists(data_file):
193 self.log("Running tests from %s" % data_file)
194 parser = TestDataParser()
195 parser.parse(data_file)
196 self.tests = parser.get_test_data()
197 self.print_test_info()
199 self.log("Data file not found: %s" % data_file)
200 self.notify_complete(False)
202 def print_test_info(self):
204 Prints test summary read by Greentea to detect test cases.
206 self.log('{{__testcase_count;%d}}' % len(self.tests))
207 for name, _, _, _ in self.tests:
208 self.log('{{__testcase_name;%s}}' % name)
211 def align_32bit(data_bytes):
213 4 byte aligns input byte array.
217 data_bytes += bytearray((4 - (len(data_bytes))) % 4)
220 def hex_str_bytes(hex_str):
222 Converts Hex string representation to byte array
224 :param hex_str: Hex in string format.
225 :return: Output Byte array
227 if hex_str[0] != '"' or hex_str[len(hex_str) - 1] != '"':
228 raise TestDataParserError("HEX test parameter missing '\"':"
230 hex_str = hex_str.strip('"')
231 if len(hex_str) % 2 != 0:
232 raise TestDataParserError("HEX parameter len should be mod of "
235 data_bytes = binascii.unhexlify(hex_str)
239 def int32_to_big_endian_bytes(i):
241 Coverts i to byte array in big endian format.
243 :param i: Input integer
244 :return: Output bytes array in big endian or network order
246 data_bytes = bytearray([((i >> x) & 0xff) for x in [24, 16, 8, 0]])
249 def test_vector_to_bytes(self, function_id, dependencies, parameters):
251 Converts test vector into a byte array that can be sent to the target.
253 :param function_id: Test Function Identifier
254 :param dependencies: Dependency list
255 :param parameters: Test function input parameters
256 :return: Byte array and its length
258 data_bytes = bytearray([len(dependencies)])
260 data_bytes += bytearray(dependencies)
261 data_bytes += bytearray([function_id, len(parameters)])
262 for typ, param in parameters:
263 if typ == 'int' or typ == 'exp':
265 data_bytes += 'I' if typ == 'int' else 'E'
266 self.align_32bit(data_bytes)
267 data_bytes += self.int32_to_big_endian_bytes(i)
269 param = param.strip('"')
270 i = len(param) + 1 # + 1 for null termination
272 self.align_32bit(data_bytes)
273 data_bytes += self.int32_to_big_endian_bytes(i)
274 data_bytes += bytearray(list(param))
275 data_bytes += '\0' # Null terminate
277 binary_data = self.hex_str_bytes(param)
279 self.align_32bit(data_bytes)
281 data_bytes += self.int32_to_big_endian_bytes(i)
282 data_bytes += binary_data
283 length = self.int32_to_big_endian_bytes(len(data_bytes))
284 return data_bytes, length
286 def run_next_test(self):
288 Fetch next test information and execute the test.
293 if self.test_index < len(self.tests):
294 name, function_id, dependencies, args = self.tests[self.test_index]
295 self.run_test(name, function_id, dependencies, args)
297 self.notify_complete(self.suite_passed)
299 def run_test(self, name, function_id, dependencies, args):
301 Execute the test on target by sending next test information.
303 :param name: Test name
304 :param function_id: function identifier
305 :param dependencies: Dependencies list
306 :param args: test parameters
309 self.log("Running: %s" % name)
311 param_bytes, length = self.test_vector_to_bytes(function_id,
313 self.send_kv(length, param_bytes)
316 def get_result(value):
318 Converts result from string type to integer
319 :param value: Result code in string
320 :return: Integer result code. Value is from the test status
321 constants defined under the MbedTlsTest class.
326 ValueError("Result should return error number. "
327 "Instead received %s" % value)
329 @event_callback('GO')
330 def on_go(self, _key, _value, _timestamp):
332 Sent by the target to start first test.
334 :param _key: Event key
335 :param _value: Value. ignored
336 :param _timestamp: Timestamp ignored.
342 def on_result(self, _key, value, _timestamp):
344 Handle result. Prints test start, finish required by Greentea
345 to detect test execution.
347 :param _key: Event key
348 :param value: Value. ignored
349 :param _timestamp: Timestamp ignored.
352 int_val = self.get_result(value)
353 name, _, _, _ = self.tests[self.test_index]
354 self.log('{{__testcase_start;%s}}' % name)
355 self.log('{{__testcase_finish;%s;%d;%d}}' % (name, int_val == 0,
358 self.suite_passed = False
362 def on_failure(self, _key, value, _timestamp):
364 Handles test execution failure. That means dependency not supported or
365 Test function not supported. Hence marking test as skipped.
367 :param _key: Event key
368 :param value: Value. ignored
369 :param _timestamp: Timestamp ignored.
372 int_val = self.get_result(value)
373 if int_val in self.error_str:
374 err = self.error_str[int_val]
376 err = 'Unknown error'
377 # For skip status, do not write {{__testcase_finish;...}}
378 self.log("Error: %s" % err)