compiler/pota-quantization-value-test/compare_tensors.py

   1 #!/usr/bin/env python3
   2 import h5py as h5
   3 import numpy as np
   4 import argparse
   5 import os.path
   6 import json
   7 import sys
   8
   9 #
  10 # This script checks if the min/max values recorded in the circle model are the same with the expected values
  11 #
  12 # Basic usage:
  13 #   compare_tensors.py --input_h5 <path/to/iput/h5> --expect_dir <path/to/expect/dir> --mode <compare_mode>
  14 #   ex: compare_minmax.py --input_h5 Add_000.h5 --expect_dir expected_outputs/Add_000 --mode fake_quantization
  15
  16 parser = argparse.ArgumentParser()
  17 parser.add_argument('--input_h5', type=str, required=True)
  18 parser.add_argument('--expect_dir', type=str, required=True)
  19 parser.add_argument('--mode', type=str, required=True)
  20 args = parser.parse_args()
  21
  22 supported_modes = ["fake_quantization", "record_minmax", "quantization"]
  23
  24 model = args.input_h5
  25 expect_dir = args.expect_dir
  26 mode = args.mode
  27
  28 failed_cases = 0
  29
  30 if mode not in supported_modes:
  31     raise SystemExit("Unsupported mode. --mode should be one of " + str(supported_modes))
  32
  33
  34 def compare_fake_quantization(tensor, tensor_name, expect_dir):
  35     global failed_cases
  36     with open(expect_dir + "/" + tensor_name + ".json", "r") as expect_file:
  37         json_load = json.load(expect_file)
  38     expected_weights = np.array(json_load["weights"])
  39     input_weights = tensor["weights"][:]
  40     if np.allclose(input_weights, expected_weights, rtol=1.e-5, atol=1.e-5) == False:
  41         print("Fake-quantized weights of " + tensor_name + " (" + str(input_weights) +
  42               ") do not match with expected value (" + str(expected_weights) + ").")
  43         failed_cases += 1
  44
  45
  46 def compare_record_minmax(tensor, tensor_name, expect_dir):
  47     global failed_cases
  48     with open(expect_dir + "/" + tensor_name + ".json", "r") as expect_file:
  49         json_load = json.load(expect_file)
  50     expected_min = np.array(json_load["min"])
  51     expected_max = np.array(json_load["max"])
  52     input_min = tensor["min"][:]
  53     input_max = tensor["max"][:]
  54     if np.allclose(input_min, expected_min, rtol=1.e-5, atol=1.e-5) == False:
  55         print("Recorded min of " + tensor_name + " (" + str(input_min) +
  56               ") does not match with expected value (" + str(expected_min) + ").")
  57         failed_cases += 1
  58     if np.allclose(input_max, expected_max, rtol=1.e-5, atol=1.e-5) == False:
  59         print("Recorded max of " + tensor_name + " (" + str(input_max) +
  60               ") does not match with expected value (" + str(expected_max) + ").")
  61         failed_cases += 1
  62
  63
  64 def compare_quantization(tensor, tensor_name, expect_dir):
  65     global failed_cases
  66     with open(expect_dir + "/" + tensor_name + ".json", "r") as expect_file:
  67         json_load = json.load(expect_file)
  68     for key in json_load:
  69         if key == "weights":
  70             expected_weights = np.array(json_load["weights"])
  71             input_weights = tensor["weights"][:]
  72             if np.allclose(input_weights, expected_weights, rtol=0, atol=1) == False:
  73                 print("Quantized weights of " + tensor_name + " (" + str(input_weights) +
  74                       ") do not match with expected value (" + str(expected_weights) +
  75                       ").")
  76                 failed_cases += 1
  77
  78         if key == "scale":
  79             expected_scale = np.array(json_load["scale"])
  80             input_scale = tensor["scale"][:]
  81             if np.allclose(input_scale, expected_scale, rtol=1.e-5, atol=1.e-5) == False:
  82                 print("Quantized scale of " + tensor_name + " (" + str(input_scale) +
  83                       ") do not match with expected value (" + str(expected_scale) + ").")
  84                 failed_cases += 1
  85
  86         if key == "zero_point":
  87             expected_zero_point = np.array(json_load["zero_point"])
  88             input_zero_point = tensor["zero_point"][:]
  89             if np.allclose(
  90                     input_zero_point, expected_zero_point, rtol=0, atol=1) == False:
  91                 print("Quantized zero_point of " + tensor_name + " (" +
  92                       str(input_zero_point) + ") do not match with expected value (" +
  93                       str(expected_zero_point) + ").")
  94                 failed_cases += 1
  95
  96
  97 with h5.File(model, "r") as input:
  98     for tensor_name in input.keys():
  99         # We only check the given golden data
 100         if os.path.isfile(expect_dir + "/" + tensor_name + ".json"):
 101             print("Compare " + tensor_name)
 102             if mode == "fake_quantization":
 103                 compare_fake_quantization(input[tensor_name], tensor_name, expect_dir)
 104             elif mode == "record_minmax":
 105                 compare_record_minmax(input[tensor_name], tensor_name, expect_dir)
 106             elif mode == "quantization":
 107                 compare_quantization(input[tensor_name], tensor_name, expect_dir)
 108             else:
 109                 raise SystemExit("Unsupproted mode.")
 110
 111 sys.exit(failed_cases)