• Home
  • Line#
  • Scopes#
  • Navigate#
  • Raw
  • Download
1 /*---------------------------------------------------------------------------*
2  *  srec_context.h  *
3  *                                                                           *
4  *  Copyright 2007, 2008 Nuance Communciations, Inc.                               *
5  *                                                                           *
6  *  Licensed under the Apache License, Version 2.0 (the 'License');          *
7  *  you may not use this file except in compliance with the License.         *
8  *                                                                           *
9  *  You may obtain a copy of the License at                                  *
10  *      http://www.apache.org/licenses/LICENSE-2.0                           *
11  *                                                                           *
12  *  Unless required by applicable law or agreed to in writing, software      *
13  *  distributed under the License is distributed on an 'AS IS' BASIS,        *
14  *  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. *
15  *  See the License for the specific language governing permissions and      *
16  *  limitations under the License.                                           *
17  *                                                                           *
18  *---------------------------------------------------------------------------*/
19 
20 /* this file contains defines needed by the srec search component*/
21 
22 #ifndef _h_srec_context_
23 #define _h_srec_context_
24 
25 #include "srec_arb.h"
26 #include "astar.h"
27 
28 #include "portable.h"
29 
30 #include "phashtable.h"
31 
32 #define CONTEXT_FILE_FORMAT_VERSION1_ID 10001
33 #define IMAGE_FORMAT_V1   32432
34 #define IMAGE_FORMAT_V2   32439
35 #define USE_HMM_BASED_ENROLLMENT 0
36 
37 /*********************************************************************
38  *                                                                   *
39  * WordMap                                                           *
40  *                                                                   *
41  *********************************************************************/
42 
43 /* todo: for dynamic vocabs, a word should eventually be replaced
44    not so much "char* word" but "wordblock* head", Jean to merge
45    in that code later. */
46 
47 typedef struct
48 {
49   wordID num_words;
50   wordID num_slots;
51   wordID max_words;
52   wordID num_base_words;      /* before any additions */
53   /* ptr32* words; c55 ?? */
54   char** words;               /* size max_words */
55 
56   char* chars;                /* FOUR_BYTE_PTR(char*, chars, dummy1); */
57   asr_int32_t max_chars;
58   char* next_chars;  /* FOUR_BYTE_PTR(char*, next_chars, dummy2); */
59   char* next_base_chars;      /* before any additions */
60   PHashTable *wordIDForWord;
61 }
62 wordmap;
63 
64 /*********************************************************************
65  *                                                                   *
66  * FST                                                               *
67  *                                                                   *
68  *********************************************************************/
69 
70 typedef struct srec_fsm_entry_point_t
71 {
72   nodeID node_index;
73   nodeID* node_for_lpcp;        /* size num_lpcps */
74 }
75 srec_fsm_entry_point;
76 
77 /**
78  * srec_fsm_exit_point_t holds information about a particular slot within
79  * the fst, so that we don't need to recalculate it each time.
80  */
81 typedef struct srec_fsm_exit_point_t
82 {
83   nodeID from_node_index;  /* from node, there can be multiple arcs leaving here */
84   arcID arc_index;         /* arc on which the "class" rests */
85   nodeID wbto_node_index;  /* node index after the .wb ilabel */
86 }
87 srec_fsm_exit_point;
88 #define MAX_NUM_SLOTS 12    /* SLOTS */
89 #define IMPORTED_RULES_DELIM '.' /* SLOT MARKER */
90 
91 typedef char FSMnode_info;
92 #define NODE_INFO_UNKNOWN 0
93 #define NODE_INFO_ENDNODE 1
94 #define NODE_INFO_OPTENDN 2
95 #define NODE_INFO_REGULAR 3
96 #define NODE_INFO_NUMS    4
97 
98 typedef struct srec_context
99 {
100   asr_uint32_t modelid;  /* modelid at compilation time, or 0 for unknown */
101   int grmtyp;            /* GrammarType */
102 
103   FSMarc* FSMarc_list;   /* allocation base */
104   arcID num_arcs;        /* number of arcs actually used */
105   arcID FSMarc_list_len; /* number of arcs allocated     */
106   arcID num_base_arcs;   /* number of arcs before additions */
107   arcID FSMarc_freelist; /* head of the free list */
108 
109   FSMnode* FSMnode_list;
110   nodeID num_nodes;
111   nodeID FSMnode_list_len;
112   nodeID num_base_nodes;
113   nodeID FSMnode_freelist;
114   FSMnode_info* FSMnode_info_list; /* todo: change this to an ary of 2bit els*/
115 
116   costdata wrapup_cost;        /* cost of going from optend nodes to endnode */
117   costdata wtw_average;        /* cost of going from optend nodes to endnode */
118 
119   nodeID start_node;
120   nodeID end_node;
121 
122   asr_int16_t num_fsm_exit_points;  /* one per rule import */
123   srec_fsm_exit_point fsm_exit_points[MAX_NUM_SLOTS];
124   /* caching for add word, because FST_AddWordToSlot() is often sequentially
125 	 on the same slot */
126   wordID addWordCaching_lastslot_num;
127   LCHAR* addWordCaching_lastslot_name;
128   ESR_BOOL addWordCaching_lastslot_needs_post_silence;
129   wordID addWordCaching_lastslot_ifsm_exit_point;
130 
131   wordID beg_silence_word;
132   wordID end_silence_word;
133   wordID hack_silence_word;
134 
135   /* aux */
136   wordmap *ilabels;           /* input arc labels */
137   wordmap *olabels;           /* word labels */
138   srec_arbdata *allotree;          /* for addword, knows hmm to state conversion */
139 
140   /* word graph, for a-star */
141   arc_token* arc_token_list;
142   arcID arc_token_list_len;
143   arc_token* arc_token_freelist;
144   arc_token* arc_token_insert_start;
145 
146   /* search capabilities, return error if adding words beyond this! */
147   nodeID max_searchable_nodes;
148   arcID max_searchable_arcs;
149 
150   /* these are pointers to data owned by others, made part of this
151      structure for completeness of information needed by an active
152      search, for banked memory models, we may want these to be a "copy" */
153   asr_int16_t hmm_ilabel_offset;        /* offset for ilabels to hmm */
154   HMMInfo* hmm_info_for_ilabel;   /* ilabel to state conversion */
155   featdata* _unused_avg_state_durations;  /* average durations */
156 
157   /* says whether a grammar has been prepared FST_Prepare()
158      a Grammar must be prepared before it is used in a recognition */
159   asr_int16_t whether_prepared;
160 }
161 srec_context;
162 
163 
164 /*********************************************************************
165  *                                                                   *
166  * Functions                                                         *
167  *                                                                   *
168  *********************************************************************/
169 
170 #define FST_SUCCESS_ON_OLD_WORD 2
171 #define FST_CONTINUE   1
172 #define FST_SUCCESS 0
173 #define FST_FAILED_ON_INVALID_ARGS -2
174 #define FST_FAILED_ON_MEMORY -3
175 #define FST_FAILED_ON_HOMONYM -4
176 #define FST_FAILED_ON_HOMOGRAPH -5
177 #define FST_FAILED_INTERNAL -6
178 /* #define FST_NEWWORD    2 // implies success */
179 
180 #define NUM_ITEMLIST_HDRWDS    4
181 enum GrammarType { GrammarTypeUnknown = 0, GrammarTypeBNF = 1, GrammarTypeItemList = 2 };
182 
183 #ifdef __cplusplus
184 extern "C"
185 {
186 #endif
187 
188   /* FST type functions */
189   int FST_AttachArbdata(srec_context* fst, srec_arbdata* allophone_tree);
190   int FST_DumpGraph(srec_context* fst, PFile* fp);
191   int FST_DumpWordMap(PFile* fp, wordmap* wmap);
192   int FST_DumpReverseWordGraph(srec_context* context, PFile* fp);
193 
194   int FST_AddWordToGrammar(srec_context* fst,
195                            const char* slot,
196                            const char* word,
197                            const char* pron, const int cost);
198   int FST_ResetGrammar(srec_context* fst);
199 
200   int FST_PrepareContext(srec_context* fst);
201   int FST_IsVoiceEnrollment(srec_context* context);
202   int FST_LoadContext(const char* synbase, srec_context** pcontext, int num_words_to_add);
203   void FST_UnloadContext(srec_context* context);
204 
205   int FST_LoadWordMap(wordmap** pwmap, int num_words_to_add, PFile* fp);
206   int FST_UnloadWordMap(wordmap** pwmap);
207   int FST_LoadGraph(srec_context* pfst, wordmap* imap, wordmap* omap,
208                     int num_words_to_add, PFile* fp);
209   int FST_UnloadGraph(srec_context* pfst);
210 
211 #if defined(DO_ALLOW_V1_G2G_FILES)
212   int FST_DumpContextAsImageV1(srec_context* context, PFile* fp);
213 #endif
214   int FST_DumpContextAsImageV2(srec_context* context, PFile* fp);
215   int FST_LoadContextFromImage(srec_context** pcontext, PFile* fp);
216 
217   int FST_CheckPath(srec_context* context, const char* transcription,
218                     char* literal, size_t max_literal_len);
219 #define FST_GetNodeInfo(cn,nd) (cn->FSMnode_info_list[nd])
220 
221   /* wordmap functions */
222   int wordmap_whether_in_rule(wordmap* wmap, wordID word, wordID rule);
223   wordID wordmap_find_index(wordmap* wmap, const char* word);
224   wordID wordmap_find_index_in_rule(wordmap* wmap, const char* word, wordID rule);
225   wordID wordmap_find_rule_index(wordmap* wmap, const char* rule);
226   int wordmap_create(wordmap** pwmap, int num_chars, int num_words, int num_words_to_add);
227   int wordmap_destroy(wordmap** pwmap);
228   wordID wordmap_add_word(wordmap* wmap, const char* word);
229   void wordmap_reset(wordmap* wmap);
230   void wordmap_setbase(wordmap* wmap);
231   void wordmap_ceiling(wordmap* wmap);
232   wordID wordmap_add_word_in_rule(wordmap* wmap, const char* word, wordID rule);
233 
234   /* utils */
235   asr_int32_t atoi_with_check(const char* buf, asr_int32_t mymax);
236   arc_token_lnk get_first_arc_leaving_node(arc_token* arc_token_list, arcID num_arcs, nodeID node);
237   ESR_ReturnCode deserializeWordMapV2(wordmap **pwordmap, PFile* fp);
238   ESR_ReturnCode serializeWordMapV2(wordmap *wordmap, PFile* fp);
239   int FST_GetGrammarType(srec_context* context);
240 
241 #ifdef __cplusplus
242 }
243 #endif
244 
245 
246 #endif
247