// TODO(sami): convert references to pointers
struct ConvertGraphParams {
ConvertGraphParams(
- tensorflow::Graph& inp_graph, const std::vector<string>& output_node_names,
- const std::set<int>& subgraph_node_id_numbers, size_t max_supported_batch_size,
- size_t max_consumed_workspace_size_bytes,
+ tensorflow::Graph& inp_graph,
+ const std::vector<string>& output_node_names,
+ const std::set<int>& subgraph_node_id_numbers,
+ size_t max_supported_batch_size, size_t max_consumed_workspace_size_bytes,
const tensorflow::grappler::GraphProperties& current_graph_properties,
std::unordered_map<string, std::pair<int, string>>* output_edges,
int engine_precision_mode)
auto dst_input = in_edge->dst_input();
VLOG(1) << " update edge " << trt_node->name() << ":" << src_output
<< " -> " << dst_node->name() << ":" << dst_input;
- TF_RETURN_IF_ERROR(params->graph.UpdateEdge(
- trt_node, src_output, dst_node, dst_input));
+ TF_RETURN_IF_ERROR(
+ params->graph.UpdateEdge(trt_node, src_output, dst_node, dst_input));
}
return tensorflow::Status::OK();
}
Raises:
RuntimeError: if the returned status message is malformed.
"""
- supported_precision_modes = {"FP32": 0,
- "FP16": 1,
- "INT8": 2}
+ supported_precision_modes = {"FP32": 0, "FP16": 1, "INT8": 2}
if precision_mode.upper() not in supported_precision_modes:
raise ValueError(("precision mode '{}' is not supported."
- "It should be one of {}"
- ).format(precision_mode,
- "{'FP32', 'FP16', 'INT8'}"))
+ "It should be one of {}").format(
+ precision_mode, "{'FP32', 'FP16', 'INT8'}"))
mode = supported_precision_modes[precision_mode.upper()]
+
def py2bytes(inp):
return inp
del output_graph_def_string # Save some memory
return output_graph_def
+
def calib_graph_to_infer_graph(calibration_graph_def):
"""Convert an existing calibration graph containing calibration data
to inference graph"""
out = calib_convert(graph_str)
status = to_string(out[0])
output_graph_def_string = out[1]
- del graph_str #save some memory
+ del graph_str #save some memory
if len(status) < 2:
raise _impl.UnknownError(None, None, status)
if status[:2] != "OK":
msg = status.split(";")
if len(msg) == 1:
raise RuntimeError("Status message is malformed {}".format(status))
- raise _impl._make_specific_exception(None, None, ";".join(msg[1:]), int(msg[0]))
+ raise _impl._make_specific_exception(None, None, ";".join(msg[1:]),
+ int(msg[0]))
output_graph_def = graph_pb2.GraphDef()
output_graph_def.ParseFromString(output_graph_def_string)
- del output_graph_def_string #save some memory
+ del output_graph_def_string #save some memory
return output_graph_def
dtype=dtypes.float32, shape=(None, 24, 24, 2), name="input")
e = cop.constant(
[[[[1., 0.5, 4., 6., 0.5, 1.], [1., 0.5, 1., 1., 0.5, 1.]]]],
- name="weights", dtype=dtypes.float32)
+ name="weights",
+ dtype=dtypes.float32)
conv = nn.conv2d(
input=a, filter=e, strides=[1, 2, 2, 1], padding="SAME", name="conv")
- b = cop.constant([4., 1.5, 2., 3., 5., 7.], name="bias",
- dtype=dtypes.float32)
+ b = cop.constant(
+ [4., 1.5, 2., 3., 5., 7.], name="bias", dtype=dtypes.float32)
t = nn.bias_add(conv, b, name="biasAdd")
relu = nn.relu(t, "relu")
idty = aops.identity(relu, "ID")
val = sess.run(out, {inp: dumm_inp})
return val
+
# Use real data that is representatitive of the inference dataset
# for calibration. For this test script it is random data
out = out.outputs[0]
with csess.Session(
config=cpb2.ConfigProto(gpu_options=gpu_options), graph=g) as sess:
- # run over real calibration data here, we are mimicking a
- # calibration set of 30 different batches. Use as much calibration data as you want
+ # run over real calibration data here, we are mimicking a calibration set of
+ # 30 different batches. Use as much calibration data as you want
for _ in range(30):
val = sess.run(out, {inp: dumm_inp})
return val
dummy_input = np.random.random_sample(inp_dims)
orig_graph = get_simple_graph_def() # use a frozen graph for inference
# Get optimized graph
- trt_graph = trt.create_inference_graph(input_graph_def=orig_graph,
- outputs=["output"],
- max_batch_size=inp_dims[0],
- max_workspace_size_bytes=1 << 25,
- precision_mode="FP32", # TRT Engine precision "FP32","FP16" or "INT8"
- minimum_segment_size=2 # minimum number of nodes in an engine
- )
+ trt_graph = trt.create_inference_graph(
+ input_graph_def=orig_graph,
+ outputs=["output"],
+ max_batch_size=inp_dims[0],
+ max_workspace_size_bytes=1 << 25,
+ precision_mode="FP32", # TRT Engine precision "FP32","FP16" or "INT8"
+ minimum_segment_size=2 # minimum number of nodes in an engine
+ )
o1 = run_graph(orig_graph, dummy_input)
o2 = run_graph(trt_graph, dummy_input)
o3 = run_graph(trt_graph, dummy_input)
assert np.array_equal(o1, o2)
assert np.array_equal(o3, o2) # sanity check
- fp16_graph = trt.create_inference_graph(input_graph_def=orig_graph,
- outputs=["output"],
- max_batch_size=inp_dims[0],
- max_workspace_size_bytes=1 << 25,
- precision_mode="FP16", # TRT Engine precision "FP32","FP16" or "INT8"
- minimum_segment_size=2 # minimum number of nodes in an engine
- )
- int8_calib_gdef = trt.create_inference_graph(input_graph_def=orig_graph,
- outputs=["output"],
- max_batch_size=inp_dims[0],
- max_workspace_size_bytes=1 << 25,
- precision_mode="INT8", # TRT Engine precision "FP32","FP16" or "INT8"
- minimum_segment_size=2 # minimum number of nodes in an engine
- )
+ fp16_graph = trt.create_inference_graph(
+ input_graph_def=orig_graph,
+ outputs=["output"],
+ max_batch_size=inp_dims[0],
+ max_workspace_size_bytes=1 << 25,
+ precision_mode="FP16", # TRT Engine precision "FP32","FP16" or "INT8"
+ minimum_segment_size=2 # minimum number of nodes in an engine
+ )
+ int8_calib_gdef = trt.create_inference_graph(
+ input_graph_def=orig_graph,
+ outputs=["output"],
+ max_batch_size=inp_dims[0],
+ max_workspace_size_bytes=1 << 25,
+ precision_mode="INT8", # TRT Engine precision "FP32","FP16" or "INT8"
+ minimum_segment_size=2 # minimum number of nodes in an engine
+ )
o4 = run_graph(fp16_graph, dummy_input)
_ = run_calibration(int8_calib_gdef, dummy_input)
int8_graph = trt.calib_graph_to_infer_graph(int8_calib_gdef)
projects / platform / upstream / tensorflow.git / commitdiff