import os
import glob
import numpy as np
+import json
from pathlib import Path
-from visqlib.Util import to_filename
+from collections import defaultdict
class QErrorComputer:
def __init__(self, fp32_dir, fq_dir):
self._fp32_dir = fp32_dir
self._fq_dir = fq_dir
- self.qerror_map = dict()
+ self.qerror_map = defaultdict(float)
self._num_processed_data = 0
def collect_data_path(self, fp32_dir, fq_dir):
# Assumption: FM data are saved as follows
#
# fp32_dir/
- # tensors.txt
+ # tensors.json
# <DATA_INDEX>/
- # <TENSOR_NAME>.npy
+ # <TENSOR_ID>.npy
#
# fq_dir/
- # tensors.txt
+ # tensors.json
# <DATA_INDEX>/
- # <TENSOR_NAME>.npy
+ # <TENSOR_ID>.npy
+ #
+ # NOTE tensors.json has a dictionary {TENSOR_NAME -> TENSOR_ID}
self._num_data = len(list(filter(os.path.isdir, glob.glob(fp32_dir + '/*'))))
if self._num_data != len(list(filter(os.path.isdir, glob.glob(fq_dir + '/*')))):
raise RuntimeError("Number of data mistmatches")
self._num_processed_data += self._num_data
- self._filename_to_tensor = dict()
- with open(Path(fp32_dir) / 'tensors.txt') as f:
- tensors = set([line.rstrip() for line in f])
- for tensor in tensors:
- # Check if filename is unique
- # Fix name finding logic unless
- assert to_filename(tensor) not in self._filename_to_tensor
- self._filename_to_tensor[to_filename(tensor)] = tensor
+ self._tid_to_tname = dict() # {tensor id -> tensor name}
+ with open(Path(fp32_dir) / 'tensors.json') as f:
+ tname_to_tid = json.load(f)
+
+ for tname, tid in tname_to_tid.items():
+ self._tid_to_tname[tid] = tname
# Save paths to fp32 data and fq data for each tensor
# dict
fq_data_path = fq_dir + '/' + str(data_idx) + '/' + fp32_path.with_suffix(
'.npy').name
fq_path = Path(fq_data_path)
- filename = fp32_path.stem
- tensor_name = self._filename_to_tensor[filename]
+ tid = int(fp32_path.stem)
+ tensor_name = self._tid_to_tname[tid]
# Only save the tensors which have both fp32 data and fq data
if fq_path.is_file() and fp32_path.is_file():
# To prevent this, relaxed PEIR with epsilon(10^(-6)) is used.
rPEIR = PEAK_ERROR / (INTERVAL + 0.000001)
- if tensor_name in self.qerror_map:
- self.qerror_map[tensor_name] += rPEIR
- else:
- self.qerror_map[tensor_name] = rPEIR
+ self.qerror_map[tensor_name] += rPEIR
+ # Return
+ # qerror_map (dict: tensor_name(string) -> qerror(float))
+ # qerror_min (float)
+ # qerror_max (float)
def get_final_result(self):
qerror_map = dict()
for tensor_name, acc in self.qerror_map.items():
qerror_map[tensor_name] = acc / self._num_processed_data
- return qerror_map
+ # Fixed qerror_min (0), qerror_max (1)
+ return qerror_map, 0.0, 1.0
def run(self):
self.advance_on(self._fp32_dir, self._fq_dir)
MSE = np.square(fp32_data - fq_data).mean()
- if tensor_name in self.qerror_map:
- self.qerror_map[tensor_name] += MSE
- else:
- self.qerror_map[tensor_name] = MSE
+ self.qerror_map[tensor_name] += MSE
self.qerror_min = min(MSE, self.qerror_min)
self.qerror_max = max(MSE, self.qerror_max)
+ # Return
+ # qerror_map (dict: tensor_name(string) -> qerror(float))
+ # qerror_min (float)
+ # qerror_max (float)
def get_final_result(self):
qerror_map = dict()
for tensor_name, acc in self.qerror_map.items():
total_error = np.sum(np.abs(fp32_data - fq_data))
- if tensor_name in self.qerror_map:
- self.qerror_map[tensor_name] += total_error
- else:
- self.qerror_map[tensor_name] = total_error
+ self.qerror_map[tensor_name] += total_error
self.qerror_min = min(total_error, self.qerror_min)
self.qerror_max = max(total_error, self.qerror_max)
+ # Return
+ # qerror_map (dict: tensor_name(string) -> qerror(float))
+ # qerror_min (float)
+ # qerror_max (float)
def get_final_result(self):
qerror_map = dict()
for tensor_name, acc in self.qerror_map.items():
def run(self):
self.advance_on(self._fp32_dir, self._fq_dir)
return self.get_final_result()
+
+
+# Scaled Root Mean Square Error (SRMSE)
+# SRMSE = sqrt(MSE) / scale
+class SRMSEComputer(QErrorComputer):
+ def __init__(self, fp32_dir, fq_dir):
+ super().__init__(fp32_dir, fq_dir)
+ if fq_dir != None:
+ self.scale_file = Path(fq_dir) / 'scales.txt'
+
+ # Incrementally compute Qerror while traversing all data in fp32_dir and fq_dir
+ def advance_on(self, fp32_dir, fq_dir):
+ if fq_dir != None:
+ self.scale_file = Path(fq_dir) / 'scales.txt'
+ self._fq_dir = fq_dir
+
+ data_paths = self.collect_data_path(fp32_dir, fq_dir)
+
+ for tensor_name, data_path in data_paths.items():
+ for (fp32_data_path, fq_data_path) in data_path:
+ fp32_data = np.load(fp32_data_path)
+ fq_data = np.load(fq_data_path)
+
+ MSE = np.square(fp32_data - fq_data).mean()
+
+ self.qerror_map[tensor_name] += MSE
+
+ # Return
+ # qerror_map (dict: tensor_name(string) -> qerror(float))
+ # qerror_min (float)
+ # qerror_max (float)
+ def get_final_result(self):
+ with open(self.scale_file) as f:
+ # scale_map: {tensor_name(str) -> scale(float)}
+ scale_map = json.load(f)
+
+ qerror_max = 0.0
+ qerror_map = dict()
+ for tensor_name, acc in self.qerror_map.items():
+ MSE = acc / self._num_processed_data
+ SRMSE = np.sqrt(MSE) / scale_map[tensor_name]
+ qerror_map[tensor_name] = SRMSE
+ qerror_max = max(SRMSE, qerror_max)
+
+ return qerror_map, 0.0, qerror_max
+
+ def run(self):
+ self.advance_on(self._fp32_dir, self._fq_dir)
+ return self.get_final_result()
projects / platform / core / ml / nnfw.git / blobdiff