[ML][Pipeline] Implement CustomFilter callback
ACR: TWDAPI-274
This is the second part of CustomFilter implementation. It adds
transfering the data between JS and C++.
[Verification] Code tested with the snippets below works fine
// Valid CustomFilter callback - the happy scenario
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
var outputTD = outputTI.getTensorsData();
var rawInputData = input.getTensorRawData(0);
for (var i = 0; i < rawInputData.data.size; ++i) {
rawInputData.data[i] = rawInputData.data[i] + 1;
}
outputTD.setTensorRawData(0, rawInputData.data);
return new tizen.ml.CustomFilterOutput(0, outputTD);
}
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// PAUSED
pipeline.start()
// PLAYING
// <CustomFilter and SinkListener callbacks' outputs 3 times>
////////////////////////////////////////////////////////////
// Valid CustomFilter callback - the happy scenario; ignore the data
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
return new tizen.ml.CustomFilterOutput(1, null); // ignore data
}
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// Custom filter called
// Custom filter called
// PAUSED
pipeline.start()
// PLAYING
////////////////////////////////////////////////////////////
// Valid CustomFilter callback - CustomFilter returns an error
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
return new tizen.ml.CustomFilterOutput(-1, null);
}
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// PAUSED
pipeline.start()
// PLAYING
////////////////////////////////////////////////////////////
// Invalid CustomFilter callback output - status == 0, but no
TensorsData
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
return new tizen.ml.CustomFilterOutput(0, null);
}
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// custom filter error:
// {name: "AbortError", message: "CustomFilter has thrown exception:
// {'code':0, 'name':'InvalidValuesError' ..."}
////////////////////////////////////////////////////////////
// Invalid CustomFilter callback output - non-CustomFilterOutput
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
return 123;
}
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// custom filter error:
// {name: "TypeMismatchError",
// message: "The value returned from CustomFilter is not a
// CustomFilterOutput object"}
////////////////////////////////////////////////////////////
// CustomFilter callback returns TensorsData with dimensions other
// than specified in tizen.ml.pipeline.registerCustomFilter(...) call
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
var invalidOutputTI = new tizen.ml.TensorsInfo();
invalidOutputTI.addTensorInfo('ti1', 'UINT8', [5, 2]);
var outputTD = invalidOutputTI.getTensorsData();
outputTD.setTensorRawData(0, [1, 2, 3, 4, 5, 6, 7, 8, 9, 10]);
return new tizen.ml.CustomFilterOutput(0, outputTD);
}
// register - the happy scenario
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// custom filter error: {name: "InvalidValuesError",
// message: "Output's TensorsInfo is not equal to expected"}
pipeline.start()
////////////////////////////////////////////////////////////
// CustomFilter callback returns non-TensorsData as output
var inputTI = new tizen.ml.TensorsInfo();
inputTI.addTensorInfo('ti1', 'UINT8', [4, 20, 15, 1]);
var outputTI = new tizen.ml.TensorsInfo();
outputTI.addTensorInfo('ti1', 'UINT8', [1200]);
var flattenPlusOne = function(input) {
console.log("Custom filter called");
return new tizen.ml.CustomFilterOutput(0, "this should be TensorsData");
}
// register - the happy scenario
tizen.ml.pipeline.registerCustomFilter('testfilter2', flattenPlusOne, inputTI,
outputTI, function errorCallback(error) {
console.warn('custom filter error:') ; console.warn(error);
});
var pipeline_def = "videotestsrc num-buffers=3 "
+ "! video/x-raw,width=20,height=15,format=BGRA "
+ "! tensor_converter "
+ "! tensor_filter framework=custom-easy model=testfilter2 "
+ "! appsink name=mysink";
var pipeline = tizen.ml.pipeline.createPipeline(pipeline_def,
state => {console.log(state);})
pipeline.registerSinkListener('mysink', function(sinkName, data) {
console.log('SinkListener for "' + sinkName + '" sink called');
console.log(data);
})
// READY
// Custom filter called
// custom filter error:
// {name: "AbortError", message: "CustomFilter has thrown exception: {
..."}
////////////////////////////////////////////////////////////
// Invalid CustomFilterOutput.status
new tizen.ml.CustomFilterOutput(666, null);
// InvalidValuesError,
// message: "CustomFilterOutput.status === 1 is the only legal positive value"
Change-Id: Icf2edee853eb97dde54ecd83f00164b302aa29ca
Signed-off-by: Pawel Wasowski <p.wasowski2@samsung.com>