speech-recognition: experimental dictionary switching, audio sample collecting.

Experimental dictionary switching now works with the fake backend.
Quite a horrible kludge (especially the re-entrancy handling), needs
a big cleanup. Audio samples are now collected and passed to the command
notification callbacks.

diff --git a/src/daemon/client.c b/src/daemon/client.c
index 8f9820b..c57b0c6 100644 (file)
--- a/src/daemon/client.c
+++ b/src/daemon/client.c
@@ -302,18 +302,13 @@ void client_resource_event(srs_client_t *c, srs_resset_event_t e)
 }
 
 
-void client_notify_command(srs_client_t *c, int index)
+void client_notify_command(srs_client_t *c, int index,
+                           int ntoken, const char **tokens,
+                           void *samplebuf, size_t samplelen,
+                           uint32_t *start, uint32_t *end)
 {
-    srs_command_t *cmd;
-    char          *tokens[SRS_MAX_TOKENS];
-    int            ntoken;
-
     if (c->enabled && /*c->allowed && */ 0 <= index && index < c->ncommand) {
-        cmd = c->commands + index;
-
-        for (ntoken = 0; ntoken < cmd->ntoken; ntoken++)
-            tokens[ntoken] = cmd->tokens[ntoken];
-
-        c->ops.notify_command(c, ntoken, tokens);
+        c->ops.notify_command(c, index, ntoken, (char **)tokens,
+                              samplebuf, samplelen, start, end);
     }
 }

diff --git a/src/daemon/client.h b/src/daemon/client.h
index 19eec5a..8f71f15 100644 (file)
--- a/src/daemon/client.h
+++ b/src/daemon/client.h
@@ -119,7 +119,9 @@ typedef enum {
 
 typedef struct {
     int (*notify_focus)(srs_client_t *c, srs_voice_focus_t focus);
-    int (*notify_command)(srs_client_t *c, int ntoken, char **tokens);
+    int (*notify_command)(srs_client_t *c, int idx, int ntoken,
+                          char **tokens, void *samplebuf, size_t samplelen,
+                          uint32_t *start, uint32_t *end);
 } srs_client_ops_t;
 
 
@@ -169,6 +171,8 @@ void client_reset_resources(srs_context_t *srs);
 void client_resource_event(srs_client_t *c, srs_resset_event_t event);
 
 /** Deliver a command notification event to the client. */
-void client_notify_command(srs_client_t *c, int index);
+void client_notify_command(srs_client_t *c, int idx, int ntoken,
+                           const char **tokens, void *samplebuf,
+                           size_t samplelen, uint32_t *start, uint32_t *end);
 
 #endif /* __SRS_DAEMON_CLIENT_H__ */

diff --git a/src/daemon/dbusif.c b/src/daemon/dbusif.c
index 50dc1fb..25d3565 100644 (file)
--- a/src/daemon/dbusif.c
+++ b/src/daemon/dbusif.c
@@ -45,7 +45,9 @@ static int unregister_cb(mrp_dbus_t *dbus, DBusMessage *msg, void *user_data);
 static int focus_cb(mrp_dbus_t *dbus, DBusMessage *msg, void *user_data);
 
 static int focus_notify(srs_client_t *c, srs_voice_focus_t focus);
-static int command_notify(srs_client_t *c, int ntoken, char **tokens);
+static int command_notify(srs_client_t *c, int idx, int ntoken, char **tokens,
+                          void *samplebuf, size_t samplelen, uint32_t *start,
+                          uint32_t *end);
 
 #define reply_register   simple_reply
 #define reply_unregister simple_reply
@@ -414,7 +416,9 @@ static int focus_notify(srs_client_t *c, srs_voice_focus_t focus)
 }
 
 
-static int command_notify(srs_client_t *c, int ntoken, char **tokens)
+static int command_notify(srs_client_t *c, int idx, int ntoken, char **tokens,
+                          void *samplebuf, size_t samplelen, uint32_t *start,
+                          uint32_t *end)
 {
     srs_context_t *srs   = c->srs;
     const char    *dest  = c->id;
@@ -425,8 +429,14 @@ static int command_notify(srs_client_t *c, int ntoken, char **tokens)
     char           buf[1024], *cmd, *p, *t;
     int            i, n, l;
 
+    MRP_UNUSED(idx);
+    MRP_UNUSED(samplebuf);
+    MRP_UNUSED(samplelen);
+    MRP_UNUSED(start);
+    MRP_UNUSED(end);
+
     p = cmd = buf;
-    l = sizeof(cmd) - 1;
+    l = sizeof(buf) - 1;
     t = "";
 
     for (i = 0; i < ntoken; i++) {
@@ -436,7 +446,7 @@ static int command_notify(srs_client_t *c, int ntoken, char **tokens)
             return FALSE;
 
         p += n;
-        l += n;
+        l -= n;
         t  = " ";
     }
 

diff --git a/src/daemon/recognizer.c b/src/daemon/recognizer.c
index a959af2..60aa6e8 100644 (file)
--- a/src/daemon/recognizer.c
+++ b/src/daemon/recognizer.c
@@ -221,6 +221,7 @@ static void free_srec_result(srs_srec_result_t *res)
 {
     srs_srec_match_t *m;
     mrp_list_hook_t  *p, *n;
+    int               i;
 
     switch (res->type) {
     case SRS_SREC_RESULT_MATCH:
@@ -241,32 +242,146 @@ static void free_srec_result(srs_srec_result_t *res)
         break;
     }
 
+    for (i = 0; i < res->ntoken; i++)
+        mrp_free(res->tokens[i]);
+    mrp_free(res->tokens);
+
+    mrp_free(res->start);
+    mrp_free(res->end);
+    mrp_free(res->samplebuf);
+
+    for (i = 0; i < res->ndict; i++)
+        mrp_free(res->dicts[i]);
+    mrp_free(res->dicts);
+
+
     mrp_list_delete(&res->hook);
     mrp_free(res);
 }
 
 
+static int switch_dict(srs_srec_t *srec, const char *dict)
+{
+    return srec->api.select_decoder(dict, srec->api_data) ? 0 : -1;
+}
+
+
+static int push_dict(srs_srec_t *srec, srs_srec_result_t *res)
+{
+    const char *active, *next;
+
+    active = srec->api.active_decoder(srec->api_data);
+    next   = res->result.dict.dict;
+
+    if (mrp_reallocz(res->dicts, res->ndict, res->ndict + 1) == NULL ||
+        (res->dicts[res->ndict] = mrp_strdup(active))        == NULL)
+        return -1;
+
+    if (res->dicts[res->ndict] == NULL)
+        return -1;
+
+    res->ndict++;
+
+    return switch_dict(srec, next);
+}
+
+
+static int pop_dict(srs_srec_t *srec, srs_srec_result_t *res)
+{
+    char *prev;
+
+    if (res->ndict <= 0) {
+        errno = ENOENT;
+        return -1;
+    }
+
+    prev = res->dicts[res->ndict - 1];
+
+    if (switch_dict(srec, prev) == 0) {
+        mrp_reallocz(res->dicts, res->ndict, res->ndict - 1);
+        res->ndict--;
+        mrp_free(prev);
+
+        return 0;
+    }
+    else
+        return -1;
+}
+
+
 static void process_match_result(srs_srec_t *srec, srs_srec_result_t *res)
 {
     mrp_list_hook_t  *p, *n;
     srs_srec_match_t *match;
+    int               i;
 
     mrp_list_foreach(&res->result.matches, p, n) {
         match = mrp_list_entry(p, typeof(*match), hook);
-        client_notify_command(match->client, match->index);
+
+        for (i = 0; i < res->ntoken; i++) {
+            mrp_log_info("  #%d token ('%s'): %u - %u", i,
+                         res->tokens[i], res->start[i], res->end[i]);
+        }
+
+        client_notify_command(match->client, match->index,
+                              res->ntoken, (const char **)res->tokens,
+                              res->samplebuf, res->samplelen,
+                              res->start, res->end);
+
+        while (res->ndict > 0)
+            pop_dict(srec, res);
     }
 }
 
 
-static void process_dict_result(srs_srec_t *srec, srs_srec_result_t *res)
+static int process_dict_result(srs_srec_t *srec, srs_srec_result_t *res)
 {
-    printf("*** should process dictionary operation ***\n");
-    return;
+    if (srec->result != NULL && srec->result != res) {
+        mrp_log_error("Conflicting results (%p != %p) for dictionary switch.",
+                      res, srec->result);
+        return 0;
+    }
+
+    switch (res->result.dict.op) {
+    case SRS_DICT_OP_POP:
+        if (pop_dict(srec, res) < 0) {
+            mrp_log_error("Failed to pop dictionary.");
+            return 0;
+        }
+        break;
+
+    case SRS_DICT_OP_PUSH:
+        if (push_dict(srec, res) < 0) {
+            mrp_log_error("Failed to push dictionary '%s'.",
+                          res->result.dict.dict);
+            return 0;
+        }
+        break;
+
+    case SRS_DICT_OP_SWITCH:
+        if (switch_dict(srec, res->result.dict.dict) < 0) {
+            mrp_log_error("Failed to switch to dictionary '%s'.",
+                          res->result.dict.dict);
+            return 0;
+        }
+        break;
+
+    default:
+        break;
+    }
+
+    srec->result     = res;
+    res->sampleoffs += res->result.dict.rescan;
+
+    return res->result.dict.rescan;
 }
 
 
 static void process_ambiguity(srs_srec_t *srec, srs_srec_result_t *res)
 {
+    MRP_UNUSED(srec);
+    MRP_UNUSED(res);
+
     return;
 }
 
@@ -278,7 +393,8 @@ static int srec_notify_cb(srs_srec_utterance_t *utt, void *notify_data)
     srs_srec_candidate_t *c;
     srs_srec_result_t    *res;
     srs_srec_token_t     *t;
-    int                   i, j;
+    int                   flush, i, j;
+    uint32_t              start, end;
 
     mrp_log_info("Got %zd recognition candidates in from %s backend:",
                  utt->ncand, srec->name);
@@ -287,13 +403,31 @@ static int srec_notify_cb(srs_srec_utterance_t *utt, void *notify_data)
         c = utt->cands[i];
         mrp_log_info("Candidate #%d:", i);
         for (j = 0, t = c->tokens; j < (int)c->ntoken; j++, t++) {
-            mrp_log_info("    token #%d: '%s'", j, t->token);
+            mrp_log_info("    token #%d: '%s' (%u - %u)", j, t->token,
+                         t->start, t->end);
         }
     }
 
     dis = find_disamb(srec->srs, SRS_DEFAULT_DISAMBIGUATOR);
 
     if (dis != NULL) {
+        if (srec->result == NULL) {
+            res = srec->result = mrp_allocz(sizeof(*srec->result));
+
+            if (res == NULL)
+                return SRS_SREC_FLUSH_ALL;
+
+            mrp_list_init(&res->hook);
+            mrp_list_init(&res->result.matches);
+
+            c     = utt->cands[0];
+            start = c->tokens[0].start;
+            end   = c->tokens[c->ntoken-1].end;
+
+            res->samplebuf = srec->api.sampledup(start, end, &res->samplelen,
+                                                 srec->api_data);
+        }
+
         res = srec->result;
 
         if (dis->api.disambiguate(utt, &res, dis->api_data) == 0 && res) {
@@ -302,25 +436,29 @@ static int srec_notify_cb(srs_srec_utterance_t *utt, void *notify_data)
             switch (res->type) {
             case SRS_SREC_RESULT_MATCH:
                 process_match_result(srec, res);
+                free_srec_result(res);
+                srec->result = NULL;
+                flush = SRS_SREC_FLUSH_ALL;
                 break;
 
             case SRS_SREC_RESULT_DICT:
-                process_dict_result(srec, res);
+                flush = process_dict_result(srec, res);
                 break;
 
             case SRS_SREC_RESULT_AMBIGUOUS:
                 process_ambiguity(srec, res);
+                free_srec_result(res);
+                flush = SRS_SREC_FLUSH_ALL;
                 break;
 
             default:
+                flush = SRS_SREC_FLUSH_ALL;
                 break;
             }
-
-            free_srec_result(res);
         }
     }
 
-    return SRS_SREC_FLUSH_ALL;
+    return flush;
 }
 
 

diff --git a/src/daemon/recognizer.h b/src/daemon/recognizer.h
index 090ca3d..6030899 100644 (file)
--- a/src/daemon/recognizer.h
+++ b/src/daemon/recognizer.h
@@ -65,7 +65,7 @@ typedef struct {
     /** Set language model to be used. */
     int (*select_decoder)(const char *decoder, void *user_data);
     /** Get the used language model. */
-    const char *(*get_decoder)(void *user_data);
+    const char *(*active_decoder)(void *user_data);
 } srs_srec_api_t;
 
 /*
@@ -160,7 +160,12 @@ typedef struct {
 struct srs_srec_result_s {
     srs_srec_result_type_t   type;       /* result type */
     mrp_list_hook_t          hook;       /* to list of results */
+    void                    *samplebuf;  /* utterance audio */
+    size_t                   samplelen;  /* audio buffer length */
+    uint32_t                 sampleoffs; /* extra audio offset */
     char                   **tokens;     /* matched tokens */
+    uint32_t                *start;      /* token start offset */
+    uint32_t                *end;        /* token end offsets */
     int                      ntoken;     /* number of tokens */
     char                   **dicts;      /* dictionary stack */
     int                      ndict;      /* stack depth */