Merge changes from 2020.4.0.1 (#1809)

* [GNA] Propagate QoS timeout to the calling app (#1188)

* [GNA] Support timeout value set in Wait (#1499)

* [GNA] stored request id for completed sync infer request in order to get status later using wait() (#1458)

* stored request id for completed async infer request in order to get it's status later

* preserved status not started for multiple sequential calls to wait()

Co-authored-by: Denis Orlov <denis.orlov@intel.com>
* [GNA] Fix callbacks (#1607)

Co-authored-by: Eugene Smirnov <eugene.smirnov@intel.com>

diff --git a/inference-engine/src/gna_plugin/gna_device.cpp b/inference-engine/src/gna_plugin/gna_device.cpp
index e6170a86c588e9b1bf22f2b0f1dad8ff57143aaf..d01e84da0229b0cf157d4a4c140abf9ec75f2e59 100644 (file)
--- a/inference-engine/src/gna_plugin/gna_device.cpp
+++ b/inference-engine/src/gna_plugin/gna_device.cpp
@@ -269,19 +269,23 @@ const std::map <const std::pair<Gna2OperationType, int32_t>, const std::string>
 };
 #endif
 
-void GNADeviceHelper::wait(uint32_t reqId) {
+bool GNADeviceHelper::wait(uint32_t reqId, int64_t millisTimeout) {
 #if GNA_LIB_VER == 2
-    const auto status = Gna2RequestWait(reqId, GNA_TIMEOUT);
+    const auto status = Gna2RequestWait(reqId, millisTimeout);
+    if (status == Gna2StatusDriverQoSTimeoutExceeded) {
+        return false;
+    }
     checkGna2Status(status);
 #else
     if (isPerformanceMeasuring) {
-        nGNAStatus = GNAWaitPerfRes(nGNAHandle, GNA_TIMEOUT, reqId, &nGNAPerfResults);
+        nGNAStatus = GNAWaitPerfRes(nGNAHandle, millisTimeout, reqId, &nGNAPerfResults);
     } else {
-        nGNAStatus = GNAWait(nGNAHandle, GNA_TIMEOUT, reqId);
+        nGNAStatus = GNAWait(nGNAHandle, millisTimeout, reqId);
     }
     checkStatus();
 #endif
     updateGnaPerfCounters();
+    return true;
 }
 
 #if GNA_LIB_VER == 1

diff --git a/inference-engine/src/gna_plugin/gna_device.hpp b/inference-engine/src/gna_plugin/gna_device.hpp
index 99ce1a1b08b3337e344345a23581c06959e1e9d9..8c803bb37bab4e08ff5364775fcbad5f88f404ef 100644 (file)
--- a/inference-engine/src/gna_plugin/gna_device.hpp
+++ b/inference-engine/src/gna_plugin/gna_device.hpp
@@ -52,7 +52,6 @@ class GNADeviceHelper {
 
 #define MAX_TIMEOUT 500000
 #endif
-    const uint32_t GNA_TIMEOUT = MAX_TIMEOUT;
     bool isPerformanceMeasuring = false;
     bool deviceOpened = false;
 public:
@@ -115,7 +114,7 @@ public:
     static void checkGna2Status(Gna2Status status);
     static void checkGna2Status(Gna2Status status, const Gna2Model& gnaModel);
 #endif
-    void wait(uint32_t id);
+    bool wait(uint32_t id, int64_t millisTimeout = MAX_TIMEOUT);
 
     struct DumpResult {
 #if GNA_LIB_VER == 2

diff --git a/inference-engine/src/gna_plugin/gna_infer_request.hpp b/inference-engine/src/gna_plugin/gna_infer_request.hpp
index 9641c17011689c893ca9321fe255a7d8b44ba612..5d34db4bea714a3dccb30c64f6f01d96eae3d189 100644 (file)
--- a/inference-engine/src/gna_plugin/gna_infer_request.hpp
+++ b/inference-engine/src/gna_plugin/gna_infer_request.hpp
@@ -48,7 +48,16 @@ class GNAInferRequest : public InferenceEngine::AsyncInferRequestInternal {
     void InferImpl() override {
         // execute input pre-processing.
         execDataPreprocessing(_inputs);
-        plg->Infer(_inputs, _outputs);
+        // result returned from sync infer wait method
+        auto result = plg->Infer(_inputs, _outputs);
+
+        // if result is false we are dealing with QoS feature
+        // if result is ok, next call to wait() will return Ok, if request not in gna_queue
+        if (!result) {
+            inferRequestIdx = -1;
+        } else {
+            inferRequestIdx = -2;
+        }
     }
 
     /**
@@ -73,11 +82,12 @@ class GNAInferRequest : public InferenceEngine::AsyncInferRequestInternal {
         if (_callback) {
             auto infer_request = _publicInterface.lock();
             IE_ASSERT(infer_request != nullptr);
-            auto res = Wait(0);
+            auto res = Wait(InferenceEngine::IInferRequest::WaitMode::RESULT_READY);
             _callback(infer_request, res);
         }
     }
 
+
     InferenceEngine::StatusCode Wait(int64_t millis_timeout) override {
         if (inferRequestIdx == -1) {
             return InferenceEngine::INFER_NOT_STARTED;
@@ -85,8 +95,20 @@ class GNAInferRequest : public InferenceEngine::AsyncInferRequestInternal {
             THROW_IE_EXCEPTION << PARAMETER_MISMATCH_str;
         }
 
-        plg->Wait(inferRequestIdx);
-        return InferenceEngine::OK;
+        bool qosOK;
+        if (millis_timeout == InferenceEngine::IInferRequest::WaitMode::RESULT_READY) {
+            qosOK = plg->Wait(inferRequestIdx);
+        } else {
+            qosOK = plg->WaitFor(inferRequestIdx, millis_timeout);
+        }
+
+        if (qosOK) {
+            return InferenceEngine::OK;
+        } else {
+            // need to preserve invalid state here to avoid next Wait() from clearing it
+            inferRequestIdx = -1;
+            return InferenceEngine::INFER_NOT_STARTED;
+        }
     }
 };
 }  // namespace GNAPluginNS

diff --git a/inference-engine/src/gna_plugin/gna_plugin.cpp b/inference-engine/src/gna_plugin/gna_plugin.cpp
index 06835b4f4f32016a13ab6c257465c8be8744e0be..3dc0392cac16361aa4f7f44b44bf71d2e79dae88 100644 (file)
--- a/inference-engine/src/gna_plugin/gna_plugin.cpp
+++ b/inference-engine/src/gna_plugin/gna_plugin.cpp
@@ -967,16 +967,23 @@ uint32_t GNAPlugin::QueueInference(const InferenceEngine::BlobMap &inputs, Infer
     return idx;
 }
 
-void GNAPlugin::Wait(uint32_t request_idx) {
+bool GNAPlugin::Wait(uint32_t request_idx) {
+    return WaitFor(request_idx, MAX_TIMEOUT);
+}
+
+bool GNAPlugin::WaitFor(uint32_t request_idx, int64_t millisTimeout) {
 #if GNA_LIB_VER == 2
     auto& nnets = gnaRequestConfigToRequestIdMap;
 #endif
-    if (nnets.size() <= request_idx) return;    // TODO: GNA2: check whether necessary
+    if (nnets.size() <= request_idx) return true;    // TODO: GNA2: check whether necessary
     // already synced TODO: might be copy required ???
-    if (std::get<1>(nnets[request_idx]) == -1) return;
+    if (std::get<1>(nnets[request_idx]) == -1) return true;
 
     if (gnadevice) {
-        gnadevice->wait(std::get<1>(nnets[request_idx]));
+        if (!gnadevice->wait(std::get<1>(nnets[request_idx]), millisTimeout)) {
+            std::get<1>(nnets[request_idx]) = -1;
+            return false;
+        }
     }
 
     std::get<1>(nnets[request_idx]) = -1;
@@ -1063,13 +1070,14 @@ void GNAPlugin::Wait(uint32_t request_idx) {
         }
         output_idx++;
     }
+    return true;
 }
 
 void GNAPlugin::Reset() {
     graphCompiler.Reset();
 }
 
-void GNAPlugin::Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob &output) {
+bool GNAPlugin::Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob &output) {
     BlobMap bmInput;
     BlobMap bmOutput;
     if (inputsDataMap.size() != 1) {
@@ -1080,11 +1088,11 @@ void GNAPlugin::Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob
     bmInput[inputsDataMap.begin()->first] = std::shared_ptr<Blob>(const_cast<Blob*>(&input), [](Blob*){});
     IE_ASSERT(!outputsDataMap.empty());
     bmOutput[outputsDataMap.begin()->first] = std::shared_ptr<Blob>(&output, [](Blob*){});
-    Infer(bmInput, bmOutput);
+    return Infer(bmInput, bmOutput);
 }
 
-void GNAPlugin::Infer(const InferenceEngine::BlobMap &input, InferenceEngine::BlobMap &result) {
-    Wait(QueueInference(input, result));
+bool GNAPlugin::Infer(const InferenceEngine::BlobMap &input, InferenceEngine::BlobMap &result) {
+    return  Wait(QueueInference(input, result));
 }
 
 Blob::Ptr GNAPlugin::GetOutputBlob(const std::string& name, InferenceEngine::Precision precision) {

diff --git a/inference-engine/src/gna_plugin/gna_plugin.hpp b/inference-engine/src/gna_plugin/gna_plugin.hpp
index 8dcb8048f66df7bee430b554b2440f77d60ff946..13ef1db960483857dc471752077ae89d475db824 100644 (file)
--- a/inference-engine/src/gna_plugin/gna_plugin.hpp
+++ b/inference-engine/src/gna_plugin/gna_plugin.hpp
@@ -98,7 +98,7 @@ class GNAPlugin : public InferenceEngine::IInferencePlugin {
 
     void LoadNetwork(InferenceEngine::ICNNNetwork &network);
 
-    void Infer(const InferenceEngine::BlobMap &input, InferenceEngine::BlobMap &result);
+    bool Infer(const InferenceEngine::BlobMap &input, InferenceEngine::BlobMap &result);
     void GetPerformanceCounts(std::map<std::string, InferenceEngine::InferenceEngineProfileInfo> &perfMap);
     void AddExtension(InferenceEngine::IExtensionPtr extension) override;
 
@@ -109,7 +109,7 @@ class GNAPlugin : public InferenceEngine::IInferencePlugin {
     InferenceEngine::ExecutableNetwork LoadNetwork(const InferenceEngine::ICNNNetwork &network,
                                   const std::map<std::string, std::string> &config_map,
                                   InferenceEngine::RemoteContext::Ptr context) override { THROW_GNA_EXCEPTION << "Not implemented"; }
-    void Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob &result);
+    bool Infer(const InferenceEngine::Blob &input, InferenceEngine::Blob &result);
     void SetCore(InferenceEngine::ICore*) noexcept override {}
     InferenceEngine::ICore* GetCore() const noexcept override {return nullptr;}
     void Reset();
@@ -117,7 +117,8 @@ class GNAPlugin : public InferenceEngine::IInferencePlugin {
                       const std::map<std::string, std::string>& config,
                       InferenceEngine::QueryNetworkResult &res) const override;
     uint32_t QueueInference(const InferenceEngine::BlobMap &input, InferenceEngine::BlobMap &result);
-    void Wait(uint32_t idx = 0);
+    bool Wait(uint32_t idx);
+    bool WaitFor(uint32_t idx, int64_t millisTimeout);
 
     InferenceEngine::Parameter GetConfig(const std::string& name,
                                          const std::map<std::string, InferenceEngine::Parameter> & options) const override;