Code coverage report for api/corpus/ComputationalLinguisticsCorpus.js

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
1
1
 
1
4
4
 
4
4
4
 
 
1
 
1
 
 
 
 
 
 
2
 
 
 
 
2
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
2
2
 
2
4
4
 
 
4
4
37
 
 
37
 
 
37
33
33
33
 
4
 
 
4
4
8
 
 
8
 
8
8
8
8
 
 
 
 
 
 
2
2
2
2
 
2
2
2
2
 
 
2
2
 
 
 
 
1
 
var Corpus = require("./Corpus").Corpus;
var ComputationalLinguisticsDatum = require("./../datum/ComputationalLinguisticsDatum").ComputationalLinguisticsDatum;
 
var ComputationalLinguisticsCorpus = function ComputationalLinguisticsCorpus(options) {
  Eif (!this._fieldDBtype) {
    this._fieldDBtype = "ComputationalLinguisticsCorpus";
  }
  this.debug("In ComputationalLinguisticsCorpus ", options);
  Corpus.DEFAULT_DATUM = ComputationalLinguisticsDatum;
  Corpus.apply(this, arguments);
};
 
ComputationalLinguisticsCorpus.DEFAULT_DATUM = ComputationalLinguisticsDatum;
 
ComputationalLinguisticsCorpus.prototype = Object.create(Corpus.prototype, /** @lends ComputationalLinguisticsCorpus.prototype */ {
  constructor: {
    value: ComputationalLinguisticsCorpus
  },
 
  extractStats: {
    value: function(datalist) {
      Iif (!datalist || !datalist.docs) {
        this.warn("datalist was empty", datalist);
        return datalist;
      }
 
      var stats = {
        characters: {
          unigrams: {},
          bigrams: {}
        },
        words: {
          unigrams: {},
          bigrams: {}
        },
        tokens: {
          characters: {
            unigrams: 0,
            bigrams: 0
          },
          words: {
            unigrams: 0,
            bigrams: 0
          }
        },
        types: {
          characters: {
            unigrams: 0,
            bigrams: 0
          },
          words: {
            unigrams: 0,
            bigrams: 0
          }
        },
        ttr: {
          characters: {
            unigrams: 1,
            bigrams: 1
          },
          words: {
            unigrams: 1,
            bigrams: 1
          }
        }
      };
 
      var item;
      var self = this;
 
      datalist.docs.map(function(doc) {
        Eif (!doc.stats && typeof doc.extractStats === "function") {
          doc.extractStats();
        }
 
        Eif (doc.stats.characters) {
          for (item in doc.stats.characters.unigrams) {
            Iif (!doc.stats.characters.unigrams.hasOwnProperty(item)) {
              continue;
            }
            stats.tokens.characters.unigrams = stats.tokens.characters.unigrams + doc.stats.characters.unigrams[item];
 
            // self.debug('character ' + item + doc.stats.characters.unigrams[item]);
            if (!stats.characters.unigrams[item]) {
              stats.types.characters.unigrams = stats.types.characters.unigrams + 1;
              stats.characters.unigrams[item] = doc.stats.characters.unigrams[item];
              continue;
            }
            stats.characters.unigrams[item] = stats.characters.unigrams[item] + doc.stats.characters.unigrams[item];
          }
        }
        Eif (doc.stats.words) {
          for (item in doc.stats.words.unigrams) {
            Iif (!doc.stats.words.unigrams.hasOwnProperty(item)) {
              continue;
            }
            stats.tokens.words.unigrams = stats.tokens.words.unigrams + doc.stats.words.unigrams[item];
            // self.debug('word ' + item + doc.stats.words.unigrams[item]);
            Eif (!stats.words.unigrams[item]) {
              stats.types.words.unigrams = stats.types.words.unigrams + 1;
              stats.words.unigrams[item] = doc.stats.words.unigrams[item];
              continue;
            }
            stats.words.unigrams[item] = stats.words.unigrams[item] + doc.stats.words.unigrams[item];
          }
        }
      });
 
      Eif (stats.types.characters.unigrams) {
        self.debug("dividing " + stats.tokens.characters.unigrams + " / " + stats.types.characters.unigrams);
        stats.ttr.characters.unigrams = stats.tokens.characters.unigrams / stats.types.characters.unigrams;
        self.debug("= " + stats.ttr.characters.unigrams);
      }
      Eif (stats.types.words.unigrams) {
        self.debug("dividing " + stats.tokens.words.unigrams + " / " + stats.types.words.unigrams);
        stats.ttr.words.unigrams = stats.tokens.words.unigrams / stats.types.words.unigrams;
        self.debug("= " + stats.ttr.words.unigrams);
      }
 
      self.debug("done with stats", stats);
      return stats;
    }
  }
});
 
exports.ComputationalLinguisticsCorpus = ComputationalLinguisticsCorpus;