27 #if defined(_WIN32) && !defined(__GNUC__)
28 # define strtok_r(str, delim, saveptr) strtok_s(str, delim, saveptr)
33 static const char kAmbigDelimiters[] =
"\t ";
34 static const char kIllegalMsg[] =
"Illegal ambiguity specification on line %d\n";
35 static const char kIllegalUnicharMsg[] =
"Illegal unichar %s in ambiguity specification\n";
52 for (
unsigned i = 0; i < unicharset.
size(); ++i) {
53 replace_ambigs_.push_back(
nullptr);
54 dang_ambigs_.push_back(
nullptr);
55 one_to_one_definite_ambigs_.push_back(
nullptr);
56 if (use_ambigs_for_adaption) {
57 ambigs_for_adaption_.push_back(
nullptr);
58 reverse_ambigs_for_adaption_.push_back(
nullptr);
73 int debug_level,
bool use_ambigs_for_adaption,
77 tprintf(
"Reading ambiguities\n");
80 int test_ambig_part_size;
81 int replacement_ambig_part_size;
85 char *buffer =
new char[kBufferSize];
93 ASSERT_HOST(ambig_file->
FGets(buffer, kBufferSize) !=
nullptr && buffer[0] !=
'\0');
95 version =
static_cast<int>(strtol(buffer + 1,
nullptr, 10));
100 while (ambig_file->
FGets(buffer, kBufferSize) !=
nullptr) {
102 if (debug_level > 2) {
103 tprintf(
"read line %s\n", buffer);
106 if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set, buffer,
107 &test_ambig_part_size, test_unichar_ids, &replacement_ambig_part_size,
108 replacement_string, &type)) {
113 if (!InsertIntoTable((type ==
REPLACE_AMBIG) ? replace_ambigs_ : dang_ambigs_,
114 test_ambig_part_size, test_unichar_ids, replacement_ambig_part_size,
115 replacement_string, type, ambig_spec, unicharset)) {
120 if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 && type ==
DEFINITE_AMBIG) {
121 if (one_to_one_definite_ambigs_[test_unichar_ids[0]] ==
nullptr) {
122 one_to_one_definite_ambigs_[test_unichar_ids[0]] =
new UnicharIdVector();
124 one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(ambig_spec->correct_ngram_id);
127 if (use_ambigs_for_adaption) {
128 std::vector<UNICHAR_ID> encoding;
131 if (unicharset->
encode_string(replacement_string,
true, &encoding,
nullptr,
nullptr)) {
132 for (
int i = 0; i < test_ambig_part_size; ++i) {
133 if (ambigs_for_adaption_[test_unichar_ids[i]] ==
nullptr) {
136 adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
137 for (
int id_to_insert : encoding) {
143 j < adaption_ambigs_entry->size() && (*adaption_ambigs_entry)[j] > id_to_insert;
147 if (j < adaption_ambigs_entry->size()) {
148 if ((*adaption_ambigs_entry)[j] != id_to_insert) {
149 adaption_ambigs_entry->insert(adaption_ambigs_entry->begin() + j, id_to_insert);
152 adaption_ambigs_entry->push_back(id_to_insert);
162 if (use_ambigs_for_adaption) {
163 for (
size_t i = 0; i < ambigs_for_adaption_.size(); ++i) {
164 adaption_ambigs_entry = ambigs_for_adaption_[i];
165 if (adaption_ambigs_entry ==
nullptr) {
168 for (
size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
169 UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
170 if (reverse_ambigs_for_adaption_[ambig_id] ==
nullptr) {
173 reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
179 if (debug_level > 1) {
180 for (
int tbl = 0; tbl < 2; ++tbl) {
182 for (
size_t i = 0; i < print_table.size(); ++i) {
183 AmbigSpec_LIST *lst = print_table[i];
184 if (lst ==
nullptr) {
188 tprintf(
"%s Ambiguities for %s:\n", (tbl == 0) ?
"Replaceable" :
"Dangerous",
191 AmbigSpec_IT lst_it(lst);
192 for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
201 if (use_ambigs_for_adaption) {
202 for (
int vec_id = 0; vec_id < 2; ++vec_id) {
203 const std::vector<UnicharIdVector *> &vec =
204 (vec_id == 0) ? ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
205 for (
size_t i = 0; i < vec.size(); ++i) {
206 adaption_ambigs_entry = vec[i];
207 if (adaption_ambigs_entry !=
nullptr) {
208 tprintf(
"%sAmbigs for adaption for %s:\n", (vec_id == 0) ?
"" :
"Reverse ",
210 for (
size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
221 bool UnicharAmbigs::ParseAmbiguityLine(
int line_num,
int version,
int debug_level,
223 int *test_ambig_part_size,
UNICHAR_ID *test_unichar_ids,
224 int *replacement_ambig_part_size,
char *replacement_string,
228 std::string input(buffer);
229 std::vector<std::string> fields =
split(input,
' ');
230 if (fields.size() != 3) {
232 tprintf(kIllegalMsg, line_num);
237 std::vector<UNICHAR_ID> unichars;
238 if (!unicharset.
encode_string(fields[0].c_str(),
true, &unichars,
nullptr,
nullptr)) {
241 *test_ambig_part_size = unichars.size();
244 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
249 for (
size_t i = 0; i < unichars.size(); ++i) {
250 test_unichar_ids[i] = unichars[i];
252 test_unichar_ids[unichars.size()] = INVALID_UNICHAR_ID;
254 if (!unicharset.
encode_string(fields[1].c_str(),
true, &unichars,
nullptr,
nullptr)) {
257 *replacement_ambig_part_size = unichars.size();
260 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
264 if (sscanf(fields[2].c_str(),
"%d", type) != 1) {
266 tprintf(kIllegalMsg, line_num);
276 if (!(token = strtok_r(buffer, kAmbigDelimiters, &next_token)) ||
277 !sscanf(token,
"%d", test_ambig_part_size) || *test_ambig_part_size <= 0) {
279 tprintf(kIllegalMsg, line_num);
285 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
289 for (i = 0; i < *test_ambig_part_size; ++i) {
290 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token))) {
295 tprintf(kIllegalUnicharMsg, token);
301 test_unichar_ids[i] = INVALID_UNICHAR_ID;
303 if (i != *test_ambig_part_size || !(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)) ||
304 !sscanf(token,
"%d", replacement_ambig_part_size) || *replacement_ambig_part_size <= 0) {
306 tprintf(kIllegalMsg, line_num);
312 tprintf(
"Too many unichars in ambiguity on line %d\n", line_num);
316 replacement_string[0] =
'\0';
317 for (i = 0; i < *replacement_ambig_part_size; ++i) {
318 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token))) {
321 strcat(replacement_string, token);
324 tprintf(kIllegalUnicharMsg, token);
329 if (i != *replacement_ambig_part_size) {
331 tprintf(kIllegalMsg, line_num);
345 if (!(token = strtok_r(
nullptr, kAmbigDelimiters, &next_token)) || !sscanf(token,
"%d", type)) {
347 tprintf(kIllegalMsg, line_num);
356 UNICHAR_ID *test_unichar_ids,
int replacement_ambig_part_size,
357 const char *replacement_string,
int type, AmbigSpec *ambig_spec,
358 UNICHARSET *unicharset) {
359 ambig_spec->type =
static_cast<AmbigType>(type);
360 if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 &&
361 unicharset->to_lower(test_unichar_ids[0]) ==
362 unicharset->to_lower(unicharset->unichar_to_id(replacement_string))) {
366 ambig_spec->wrong_ngram_size =
381 ambig_spec->correct_ngram_id = unicharset->unichar_to_id(replacement_string);
382 if (replacement_ambig_part_size > 1) {
383 unicharset->set_isngram(ambig_spec->correct_ngram_id,
true);
387 for (i = 0; i < test_ambig_part_size; ++i) {
389 if (test_ambig_part_size == 1) {
390 unichar_id = ambig_spec->correct_ngram_id;
392 std::string frag_str =
395 unichar_id = unicharset->unichar_to_id(frag_str.c_str());
397 ambig_spec->correct_fragments[i] = unichar_id;
399 ambig_spec->correct_fragments[i] = INVALID_UNICHAR_ID;
403 if (table[test_unichar_ids[0]] ==
nullptr) {
404 table[test_unichar_ids[0]] =
new AmbigSpec_LIST();
const std::vector< std::string > split(const std::string &s, char c)
const char kUniversalAmbigsFile[]
std::vector< AmbigSpec_LIST * > UnicharAmbigsVector
void tprintf(const char *format,...)
void chomp_string(char *str)
const int ksizeofUniversalAmbigsFile
const int kMaxAmbigStringSize
std::vector< UNICHAR_ID > UnicharIdVector
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
static int copy(const UNICHAR_ID src[], UNICHAR_ID dst[])
UNICHAR_ID correct_ngram_id
UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE+1]
static int compare_ambig_specs(const void *spec1, const void *spec2)
UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE+1]
void LoadUniversal(const UNICHARSET &encoder_set, UNICHARSET *unicharset)
void InitUnicharAmbigs(const UNICHARSET &unicharset, bool use_ambigs_for_adaption)
void LoadUnicharAmbigs(const UNICHARSET &encoder_set, TFile *ambigs_file, int debug_level, bool use_ambigs_for_adaption, UNICHARSET *unicharset)
char * FGets(char *buffer, int buffer_size)
std::string to_string() const
bool encode_string(const char *str, bool give_up_on_failure, std::vector< UNICHAR_ID > *encoding, std::vector< char > *lengths, unsigned *encoded_length) const
bool contains_unichar(const char *const unichar_repr) const
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
std::string debug_str(UNICHAR_ID id) const
static FILE * Open(const std::string &filename, const std::string &mode)