third_party/pigweed/repo/pw_tokenizer/py/database_test.py

   1 #!/usr/bin/env python3
   2 # Copyright 2020 The Pigweed Authors
   3 #
   4 # Licensed under the Apache License, Version 2.0 (the "License"); you may not
   5 # use this file except in compliance with the License. You may obtain a copy of
   6 # the License at
   7 #
   8 #     https://www.apache.org/licenses/LICENSE-2.0
   9 #
  10 # Unless required by applicable law or agreed to in writing, software
  11 # distributed under the License is distributed on an "AS IS" BASIS, WITHOUT
  12 # WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
  13 # License for the specific language governing permissions and limitations under
  14 # the License.
  15 """Tests for the database module."""
  16
  17 import json
  18 import io
  19 from pathlib import Path
  20 import shutil
  21 import sys
  22 import tempfile
  23 import unittest
  24 from unittest import mock
  25
  26 from pw_tokenizer import database
  27
  28 # This is an ELF file with only the pw_tokenizer sections. It was created
  29 # from a tokenize_test binary built for the STM32F429i Discovery board. The
  30 # pw_tokenizer sections were extracted with this command:
  31 #
  32 #   arm-none-eabi-objcopy -S --only-section ".pw_tokenize*" <ELF> <OUTPUT>
  33 #
  34 TOKENIZED_ENTRIES_ELF = Path(
  35     __file__).parent / 'example_binary_with_tokenized_strings.elf'
  36 LEGACY_PLAIN_STRING_ELF = Path(
  37     __file__).parent / 'example_legacy_binary_with_tokenized_strings.elf'
  38
  39 CSV_DEFAULT_DOMAIN = '''\
  40 00000000,          ,""
  41 141c35d5,          ,"The answer: ""%s"""
  42 29aef586,          ,"1234"
  43 2b78825f,          ,"[:-)"
  44 2e668cd6,          ,"Jello, world!"
  45 31631781,          ,"%d"
  46 61fd1e26,          ,"%ld"
  47 68ab92da,          ,"%s there are %x (%.2f) of them%c"
  48 7b940e2a,          ,"Hello %s! %hd %e"
  49 7da55d52,          ,">:-[]"
  50 7f35a9a5,          ,"TestName"
  51 851beeb6,          ,"%u %d"
  52 881436a0,          ,"The answer is: %s"
  53 88808930,          ,"%u%d%02x%X%hu%hhd%d%ld%lu%lld%llu%c%c%c"
  54 92723f44,          ,"???"
  55 a09d6698,          ,"won-won-won-wonderful"
  56 aa9ffa66,          ,"void pw::tokenizer::{anonymous}::TestName()"
  57 ad002c97,          ,"%llx"
  58 b3653e13,          ,"Jello!"
  59 cc6d3131,          ,"Jello?"
  60 e13b0f94,          ,"%llu"
  61 e65aefef,          ,"Won't fit : %s%d"
  62 '''
  63
  64 CSV_TEST_DOMAIN = """\
  65 17fa86d3,          ,"hello"
  66 18c5017c,          ,"yes"
  67 59b2701c,          ,"The answer was: %s"
  68 881436a0,          ,"The answer is: %s"
  69 d18ada0f,          ,"something"
  70 """
  71
  72 CSV_ALL_DOMAINS = '''\
  73 00000000,          ,""
  74 141c35d5,          ,"The answer: ""%s"""
  75 17fa86d3,          ,"hello"
  76 18c5017c,          ,"yes"
  77 29aef586,          ,"1234"
  78 2b78825f,          ,"[:-)"
  79 2e668cd6,          ,"Jello, world!"
  80 31631781,          ,"%d"
  81 59b2701c,          ,"The answer was: %s"
  82 61fd1e26,          ,"%ld"
  83 68ab92da,          ,"%s there are %x (%.2f) of them%c"
  84 7b940e2a,          ,"Hello %s! %hd %e"
  85 7da55d52,          ,">:-[]"
  86 7f35a9a5,          ,"TestName"
  87 851beeb6,          ,"%u %d"
  88 881436a0,          ,"The answer is: %s"
  89 88808930,          ,"%u%d%02x%X%hu%hhd%d%ld%lu%lld%llu%c%c%c"
  90 92723f44,          ,"???"
  91 a09d6698,          ,"won-won-won-wonderful"
  92 aa9ffa66,          ,"void pw::tokenizer::{anonymous}::TestName()"
  93 ad002c97,          ,"%llx"
  94 b3653e13,          ,"Jello!"
  95 cc6d3131,          ,"Jello?"
  96 d18ada0f,          ,"something"
  97 e13b0f94,          ,"%llu"
  98 e65aefef,          ,"Won't fit : %s%d"
  99 '''
 100
 101 EXPECTED_REPORT = {
 102     str(TOKENIZED_ENTRIES_ELF): {
 103         '': {
 104             'present_entries': 22,
 105             'present_size_bytes': 289,
 106             'total_entries': 22,
 107             'total_size_bytes': 289,
 108             'collisions': {}
 109         },
 110         'TEST_DOMAIN': {
 111             'present_entries': 5,
 112             'present_size_bytes': 57,
 113             'total_entries': 5,
 114             'total_size_bytes': 57,
 115             'collisions': {}
 116         }
 117     }
 118 }
 119
 120
 121 def run_cli(*args) -> None:
 122     original_argv = sys.argv
 123     sys.argv = ['database.py', *(str(a) for a in args)]
 124     # pylint: disable=protected-access
 125     try:
 126         database._main(*database._parse_args())
 127     finally:
 128         # Remove the log handler added by _main to avoid duplicate logs.
 129         if database._LOG.handlers:
 130             database._LOG.handlers.pop()
 131         # pylint: enable=protected-access
 132
 133         sys.argv = original_argv
 134
 135
 136 def _mock_output() -> io.TextIOWrapper:
 137     output = io.BytesIO()
 138     output.name = '<fake stdout>'
 139     return io.TextIOWrapper(output, write_through=True)
 140
 141
 142 class DatabaseCommandLineTest(unittest.TestCase):
 143     """Tests the database.py command line interface."""
 144     def setUp(self):
 145         self._dir = Path(tempfile.mkdtemp('_pw_tokenizer_test'))
 146         self._csv = self._dir / 'db.csv'
 147         self._elf = TOKENIZED_ENTRIES_ELF
 148
 149         self._csv_test_domain = CSV_TEST_DOMAIN
 150
 151     def tearDown(self):
 152         shutil.rmtree(self._dir)
 153
 154     def test_create_csv(self):
 155         run_cli('create', '--database', self._csv, self._elf)
 156
 157         self.assertEqual(CSV_DEFAULT_DOMAIN.splitlines(),
 158                          self._csv.read_text().splitlines())
 159
 160     def test_create_csv_test_domain(self):
 161         run_cli('create', '--database', self._csv, f'{self._elf}#TEST_DOMAIN')
 162
 163         self.assertEqual(self._csv_test_domain.splitlines(),
 164                          self._csv.read_text().splitlines())
 165
 166     def test_create_csv_all_domains(self):
 167         run_cli('create', '--database', self._csv, f'{self._elf}#.*')
 168
 169         self.assertEqual(CSV_ALL_DOMAINS.splitlines(),
 170                          self._csv.read_text().splitlines())
 171
 172     def test_create_force(self):
 173         self._csv.write_text(CSV_ALL_DOMAINS)
 174
 175         with self.assertRaises(FileExistsError):
 176             run_cli('create', '--database', self._csv, self._elf)
 177
 178         run_cli('create', '--force', '--database', self._csv, self._elf)
 179
 180     def test_create_binary(self):
 181         binary = self._dir / 'db.bin'
 182         run_cli('create', '--type', 'binary', '--database', binary, self._elf)
 183
 184         # Write the binary database as CSV to verify its contents.
 185         run_cli('create', '--database', self._csv, binary)
 186
 187         self.assertEqual(CSV_DEFAULT_DOMAIN.splitlines(),
 188                          self._csv.read_text().splitlines())
 189
 190     def test_add_does_not_recalculate_tokens(self):
 191         db_with_custom_token = '01234567,          ,"hello"'
 192
 193         to_add = self._dir / 'add_this.csv'
 194         to_add.write_text(db_with_custom_token + '\n')
 195         self._csv.touch()
 196
 197         run_cli('add', '--database', self._csv, to_add)
 198         self.assertEqual(db_with_custom_token.splitlines(),
 199                          self._csv.read_text().splitlines())
 200
 201     def test_mark_removals(self):
 202         self._csv.write_text(CSV_ALL_DOMAINS)
 203
 204         run_cli('mark_removals', '--database', self._csv, '--date',
 205                 '1998-09-04', self._elf)
 206
 207         # Add the removal date to the four tokens not in the default domain
 208         new_csv = CSV_ALL_DOMAINS
 209         new_csv = new_csv.replace('17fa86d3,          ,"hello"',
 210                                   '17fa86d3,1998-09-04,"hello"')
 211         new_csv = new_csv.replace('18c5017c,          ,"yes"',
 212                                   '18c5017c,1998-09-04,"yes"')
 213         new_csv = new_csv.replace('59b2701c,          ,"The answer was: %s"',
 214                                   '59b2701c,1998-09-04,"The answer was: %s"')
 215         new_csv = new_csv.replace('d18ada0f,          ,"something"',
 216                                   'd18ada0f,1998-09-04,"something"')
 217         self.assertNotEqual(CSV_ALL_DOMAINS, new_csv)
 218
 219         self.assertEqual(new_csv.splitlines(),
 220                          self._csv.read_text().splitlines())
 221
 222     def test_purge(self):
 223         self._csv.write_text(CSV_ALL_DOMAINS)
 224
 225         # Mark everything not in TEST_DOMAIN as removed.
 226         run_cli('mark_removals', '--database', self._csv,
 227                 f'{self._elf}#TEST_DOMAIN')
 228
 229         # Delete all entries except those in TEST_DOMAIN.
 230         run_cli('purge', '--database', self._csv)
 231
 232         self.assertEqual(self._csv_test_domain.splitlines(),
 233                          self._csv.read_text().splitlines())
 234
 235     @mock.patch('sys.stdout', new_callable=_mock_output)
 236     def test_report(self, mock_stdout):
 237         run_cli('report', self._elf)
 238
 239         self.assertEqual(json.loads(mock_stdout.buffer.getvalue()),
 240                          EXPECTED_REPORT)
 241
 242     def test_replace(self):
 243         sub = 'replace/ment'
 244         run_cli('create', '--database', self._csv, self._elf, '--replace',
 245                 r'(?i)\b[jh]ello\b/' + sub)
 246         self.assertEqual(
 247             CSV_DEFAULT_DOMAIN.replace('Jello', sub).replace('Hello', sub),
 248             self._csv.read_text())
 249
 250
 251 class LegacyDatabaseCommandLineTest(DatabaseCommandLineTest):
 252     """Test an ELF with the legacy plain string storage format."""
 253     def setUp(self):
 254         super().setUp()
 255         self._elf = LEGACY_PLAIN_STRING_ELF
 256
 257         # The legacy approach for storing tokenized strings in an ELF always
 258         # adds an entry for "", even if the empty string was never tokenized.
 259         self._csv_test_domain = '00000000,          ,""\n' + CSV_TEST_DOMAIN
 260
 261     @mock.patch('sys.stdout', new_callable=_mock_output)
 262     def test_report(self, mock_stdout):
 263         run_cli('report', self._elf)
 264
 265         report = EXPECTED_REPORT[str(TOKENIZED_ENTRIES_ELF)].copy()
 266
 267         # Count the implicitly added "" entry in TEST_DOMAIN.
 268         report['TEST_DOMAIN']['present_entries'] += 1
 269         report['TEST_DOMAIN']['present_size_bytes'] += 1
 270         report['TEST_DOMAIN']['total_entries'] += 1
 271         report['TEST_DOMAIN']['total_size_bytes'] += 1
 272
 273         # Rename "" to the legacy name "default"
 274         report['default'] = report['']
 275         del report['']
 276
 277         self.assertEqual({str(LEGACY_PLAIN_STRING_ELF): report},
 278                          json.loads(mock_stdout.buffer.getvalue()))
 279
 280
 281 if __name__ == '__main__':
 282     unittest.main()