test/unit/test_ngram/test_lm_score.c

   1 #include <ngram_model.h>
   2 #include <logmath.h>
   3 #include <strfuncs.h>
   4
   5 #include "test_macros.h"
   6
   7 #include <stdio.h>
   8 #include <string.h>
   9 #include <math.h>
  10
  11 void
  12 run_tests(ngram_model_t *model)
  13 {
  14         int32 n_used;
  15
  16         ngram_tg_score(model,
  17                        ngram_wid(model, "daines"),
  18                        ngram_wid(model, "huggins"),
  19                        ngram_wid(model, "huggins"), &n_used);
  20         TEST_EQUAL(n_used, 2);
  21         ngram_tg_score(model,
  22                        ngram_wid(model, "david"),
  23                        ngram_wid(model, "david"),
  24                        ngram_wid(model, "david"), &n_used);
  25         TEST_EQUAL(n_used, 1);
  26
  27         /* Apply weights. */
  28         ngram_model_apply_weights(model, 7.5, 0.5, 1.0);
  29         /* -9452 * 7.5 + log(0.5) = -77821 */
  30         TEST_EQUAL_LOG(ngram_score(model, "daines", "huggins", "david", NULL),
  31                    -77821);
  32         /* Recover original score. */
  33         TEST_EQUAL_LOG(ngram_prob(model, "daines", "huggins", "david", NULL),
  34                    -9452);
  35         TEST_EQUAL_LOG(ngram_prob(model, "huggins", "david", NULL), -831);
  36
  37         /* Un-apply weights. */
  38         ngram_model_apply_weights(model, 1.0, 1.0, 1.0);
  39         TEST_EQUAL_LOG(ngram_score(model, "daines", "huggins", "david", NULL),
  40                        -9452);
  41         TEST_EQUAL_LOG(ngram_score(model, "huggins", "david", NULL), -831);
  42         /* Recover original score. */
  43         TEST_EQUAL_LOG(ngram_prob(model, "daines", "huggins", "david", NULL),
  44                        -9452);
  45
  46         /* Pre-weighting, this should give the "raw" score. */
  47         TEST_EQUAL_LOG(ngram_score(model, "daines", "huggins", "david", NULL),
  48                        -9452);
  49         TEST_EQUAL_LOG(ngram_score(model, "huggins", "david", NULL), -831);
  50         /* Verify that backoff mode calculations work. */
  51         ngram_bg_score(model,
  52                        ngram_wid(model, "huggins"),
  53                        ngram_wid(model, "david"), &n_used);
  54         TEST_EQUAL(n_used, 2);
  55         ngram_bg_score(model,
  56                        ngram_wid(model, "blorglehurfle"),
  57                        ngram_wid(model, "david"), &n_used);
  58         TEST_EQUAL(n_used, 1);
  59         ngram_bg_score(model,
  60                        ngram_wid(model, "david"),
  61                        ngram_wid(model, "david"), &n_used);
  62         TEST_EQUAL(n_used, 1);
  63         ngram_tg_score(model,
  64                        ngram_wid(model, "daines"),
  65                        ngram_wid(model, "huggins"),
  66                        ngram_wid(model, "david"), &n_used);
  67         TEST_EQUAL(n_used, 3);
  68 }
  69
  70 int
  71 main(int argc, char *argv[])
  72 {
  73         logmath_t *lmath;
  74         ngram_model_t *model;
  75
  76         lmath = logmath_init(1.0001, 0, 0);
  77
  78         model = ngram_model_read(NULL, LMDIR "/100.arpa.DMP", NGRAM_DMP, lmath);
  79         run_tests(model);
  80         ngram_model_free(model);
  81
  82         model = ngram_model_read(NULL, LMDIR "/100.arpa.gz", NGRAM_ARPA, lmath);
  83         run_tests(model);
  84         ngram_model_free(model);
  85
  86         logmath_free(lmath);
  87         return 0;
  88 }