tesseract  5.0.0
ambigs.cpp
Go to the documentation of this file.
1 // File: ambigs.cpp
3 // Description: Functions for dealing with ambiguities
4 // (training and recognition).
5 // Author: Daria Antonova
6 //
7 // (C) Copyright 2008, Google Inc.
8 // Licensed under the Apache License, Version 2.0 (the "License");
9 // you may not use this file except in compliance with the License.
10 // You may obtain a copy of the License at
11 // http://www.apache.org/licenses/LICENSE-2.0
12 // Unless required by applicable law or agreed to in writing, software
13 // distributed under the License is distributed on an "AS IS" BASIS,
14 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
15 // See the License for the specific language governing permissions and
16 // limitations under the License.
17 //
19 
20 #include "ambigs.h"
21 
22 #include "helpers.h"
23 #include "universalambigs.h"
24 
25 #include <cstdio>
26 
27 #if defined(_WIN32) && !defined(__GNUC__)
28 # define strtok_r(str, delim, saveptr) strtok_s(str, delim, saveptr)
29 #endif /* _WIN32 && !__GNUC__ */
30 
31 namespace tesseract {
32 
33 static const char kAmbigDelimiters[] = "\t ";
34 static const char kIllegalMsg[] = "Illegal ambiguity specification on line %d\n";
35 static const char kIllegalUnicharMsg[] = "Illegal unichar %s in ambiguity specification\n";
36 
37 // Maximum line size:
38 // 10 for sizes of ambigs, tabs, abmig type and newline
39 // UNICHAR_LEN * (MAX_AMBIG_SIZE + 1) for each part of the ambig
41 
43  wrong_ngram[0] = INVALID_UNICHAR_ID;
44  correct_fragments[0] = INVALID_UNICHAR_ID;
45  correct_ngram_id = INVALID_UNICHAR_ID;
46  type = NOT_AMBIG;
47  wrong_ngram_size = 0;
48 }
49 
50 // Initializes the ambigs by adding a nullptr pointer to each table.
51 void UnicharAmbigs::InitUnicharAmbigs(const UNICHARSET &unicharset, bool use_ambigs_for_adaption) {
52  for (unsigned i = 0; i < unicharset.size(); ++i) {
53  replace_ambigs_.push_back(nullptr);
54  dang_ambigs_.push_back(nullptr);
55  one_to_one_definite_ambigs_.push_back(nullptr);
56  if (use_ambigs_for_adaption) {
57  ambigs_for_adaption_.push_back(nullptr);
58  reverse_ambigs_for_adaption_.push_back(nullptr);
59  }
60  }
61 }
62 
63 // Loads the universal ambigs that are useful for any language.
64 void UnicharAmbigs::LoadUniversal(const UNICHARSET &encoder_set, UNICHARSET *unicharset) {
65  TFile file;
67  return;
68  }
69  LoadUnicharAmbigs(encoder_set, &file, 0, false, unicharset);
70 }
71 
72 void UnicharAmbigs::LoadUnicharAmbigs(const UNICHARSET &encoder_set, TFile *ambig_file,
73  int debug_level, bool use_ambigs_for_adaption,
74  UNICHARSET *unicharset) {
75  UnicharIdVector *adaption_ambigs_entry;
76  if (debug_level) {
77  tprintf("Reading ambiguities\n");
78  }
79 
80  int test_ambig_part_size;
81  int replacement_ambig_part_size;
82  // The space for buffer is allocated on the heap to avoid
83  // GCC frame size warning.
84  const int kBufferSize = 10 + 2 * kMaxAmbigStringSize;
85  char *buffer = new char[kBufferSize];
86  char replacement_string[kMaxAmbigStringSize];
87  UNICHAR_ID test_unichar_ids[MAX_AMBIG_SIZE + 1];
88  int line_num = 0;
89  int type = NOT_AMBIG;
90 
91  // Determine the version of the ambigs file.
92  int version = 0;
93  ASSERT_HOST(ambig_file->FGets(buffer, kBufferSize) != nullptr && buffer[0] != '\0');
94  if (*buffer == 'v') {
95  version = static_cast<int>(strtol(buffer + 1, nullptr, 10));
96  ++line_num;
97  } else {
98  ambig_file->Rewind();
99  }
100  while (ambig_file->FGets(buffer, kBufferSize) != nullptr) {
101  chomp_string(buffer);
102  if (debug_level > 2) {
103  tprintf("read line %s\n", buffer);
104  }
105  ++line_num;
106  if (!ParseAmbiguityLine(line_num, version, debug_level, encoder_set, buffer,
107  &test_ambig_part_size, test_unichar_ids, &replacement_ambig_part_size,
108  replacement_string, &type)) {
109  continue;
110  }
111  // Construct AmbigSpec and add it to the appropriate AmbigSpec_LIST.
112  auto *ambig_spec = new AmbigSpec();
113  if (!InsertIntoTable((type == REPLACE_AMBIG) ? replace_ambigs_ : dang_ambigs_,
114  test_ambig_part_size, test_unichar_ids, replacement_ambig_part_size,
115  replacement_string, type, ambig_spec, unicharset)) {
116  continue;
117  }
118 
119  // Update one_to_one_definite_ambigs_.
120  if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 && type == DEFINITE_AMBIG) {
121  if (one_to_one_definite_ambigs_[test_unichar_ids[0]] == nullptr) {
122  one_to_one_definite_ambigs_[test_unichar_ids[0]] = new UnicharIdVector();
123  }
124  one_to_one_definite_ambigs_[test_unichar_ids[0]]->push_back(ambig_spec->correct_ngram_id);
125  }
126  // Update ambigs_for_adaption_.
127  if (use_ambigs_for_adaption) {
128  std::vector<UNICHAR_ID> encoding;
129  // Silently ignore invalid strings, as before, so it is safe to use a
130  // universal ambigs file.
131  if (unicharset->encode_string(replacement_string, true, &encoding, nullptr, nullptr)) {
132  for (int i = 0; i < test_ambig_part_size; ++i) {
133  if (ambigs_for_adaption_[test_unichar_ids[i]] == nullptr) {
134  ambigs_for_adaption_[test_unichar_ids[i]] = new UnicharIdVector();
135  }
136  adaption_ambigs_entry = ambigs_for_adaption_[test_unichar_ids[i]];
137  for (int id_to_insert : encoding) {
138  ASSERT_HOST(id_to_insert != INVALID_UNICHAR_ID);
139  // Add the new unichar id to adaption_ambigs_entry (only if the
140  // vector does not already contain it) keeping it in sorted order.
141  size_t j;
142  for (j = 0;
143  j < adaption_ambigs_entry->size() && (*adaption_ambigs_entry)[j] > id_to_insert;
144  ++j) {
145  ;
146  }
147  if (j < adaption_ambigs_entry->size()) {
148  if ((*adaption_ambigs_entry)[j] != id_to_insert) {
149  adaption_ambigs_entry->insert(adaption_ambigs_entry->begin() + j, id_to_insert);
150  }
151  } else {
152  adaption_ambigs_entry->push_back(id_to_insert);
153  }
154  }
155  }
156  }
157  }
158  }
159  delete[] buffer;
160 
161  // Fill in reverse_ambigs_for_adaption from ambigs_for_adaption vector.
162  if (use_ambigs_for_adaption) {
163  for (size_t i = 0; i < ambigs_for_adaption_.size(); ++i) {
164  adaption_ambigs_entry = ambigs_for_adaption_[i];
165  if (adaption_ambigs_entry == nullptr) {
166  continue;
167  }
168  for (size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
169  UNICHAR_ID ambig_id = (*adaption_ambigs_entry)[j];
170  if (reverse_ambigs_for_adaption_[ambig_id] == nullptr) {
171  reverse_ambigs_for_adaption_[ambig_id] = new UnicharIdVector();
172  }
173  reverse_ambigs_for_adaption_[ambig_id]->push_back(i);
174  }
175  }
176  }
177 
178  // Print what was read from the input file.
179  if (debug_level > 1) {
180  for (int tbl = 0; tbl < 2; ++tbl) {
181  const UnicharAmbigsVector &print_table = (tbl == 0) ? replace_ambigs_ : dang_ambigs_;
182  for (size_t i = 0; i < print_table.size(); ++i) {
183  AmbigSpec_LIST *lst = print_table[i];
184  if (lst == nullptr) {
185  continue;
186  }
187  if (!lst->empty()) {
188  tprintf("%s Ambiguities for %s:\n", (tbl == 0) ? "Replaceable" : "Dangerous",
189  unicharset->debug_str(i).c_str());
190  }
191  AmbigSpec_IT lst_it(lst);
192  for (lst_it.mark_cycle_pt(); !lst_it.cycled_list(); lst_it.forward()) {
193  AmbigSpec *ambig_spec = lst_it.data();
194  tprintf("wrong_ngram:");
195  UnicharIdArrayUtils::print(ambig_spec->wrong_ngram, *unicharset);
196  tprintf("correct_fragments:");
197  UnicharIdArrayUtils::print(ambig_spec->correct_fragments, *unicharset);
198  }
199  }
200  }
201  if (use_ambigs_for_adaption) {
202  for (int vec_id = 0; vec_id < 2; ++vec_id) {
203  const std::vector<UnicharIdVector *> &vec =
204  (vec_id == 0) ? ambigs_for_adaption_ : reverse_ambigs_for_adaption_;
205  for (size_t i = 0; i < vec.size(); ++i) {
206  adaption_ambigs_entry = vec[i];
207  if (adaption_ambigs_entry != nullptr) {
208  tprintf("%sAmbigs for adaption for %s:\n", (vec_id == 0) ? "" : "Reverse ",
209  unicharset->debug_str(i).c_str());
210  for (size_t j = 0; j < adaption_ambigs_entry->size(); ++j) {
211  tprintf("%s ", unicharset->debug_str((*adaption_ambigs_entry)[j]).c_str());
212  }
213  tprintf("\n");
214  }
215  }
216  }
217  }
218  }
219 }
220 
221 bool UnicharAmbigs::ParseAmbiguityLine(int line_num, int version, int debug_level,
222  const UNICHARSET &unicharset, char *buffer,
223  int *test_ambig_part_size, UNICHAR_ID *test_unichar_ids,
224  int *replacement_ambig_part_size, char *replacement_string,
225  int *type) {
226  if (version > 1) {
227  // Simpler format is just wrong-string correct-string type\n.
228  std::string input(buffer);
229  std::vector<std::string> fields = split(input, ' ');
230  if (fields.size() != 3) {
231  if (debug_level) {
232  tprintf(kIllegalMsg, line_num);
233  }
234  return false;
235  }
236  // Encode wrong-string.
237  std::vector<UNICHAR_ID> unichars;
238  if (!unicharset.encode_string(fields[0].c_str(), true, &unichars, nullptr, nullptr)) {
239  return false;
240  }
241  *test_ambig_part_size = unichars.size();
242  if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
243  if (debug_level) {
244  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
245  }
246  return false;
247  }
248  // Copy encoded string to output.
249  for (size_t i = 0; i < unichars.size(); ++i) {
250  test_unichar_ids[i] = unichars[i];
251  }
252  test_unichar_ids[unichars.size()] = INVALID_UNICHAR_ID;
253  // Encode replacement-string to check validity.
254  if (!unicharset.encode_string(fields[1].c_str(), true, &unichars, nullptr, nullptr)) {
255  return false;
256  }
257  *replacement_ambig_part_size = unichars.size();
258  if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
259  if (debug_level) {
260  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
261  }
262  return false;
263  }
264  if (sscanf(fields[2].c_str(), "%d", type) != 1) {
265  if (debug_level) {
266  tprintf(kIllegalMsg, line_num);
267  }
268  return false;
269  }
270  snprintf(replacement_string, kMaxAmbigStringSize, "%s", fields[1].c_str());
271  return true;
272  }
273  int i;
274  char *token;
275  char *next_token;
276  if (!(token = strtok_r(buffer, kAmbigDelimiters, &next_token)) ||
277  !sscanf(token, "%d", test_ambig_part_size) || *test_ambig_part_size <= 0) {
278  if (debug_level) {
279  tprintf(kIllegalMsg, line_num);
280  }
281  return false;
282  }
283  if (*test_ambig_part_size > MAX_AMBIG_SIZE) {
284  if (debug_level) {
285  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
286  }
287  return false;
288  }
289  for (i = 0; i < *test_ambig_part_size; ++i) {
290  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) {
291  break;
292  }
293  if (!unicharset.contains_unichar(token)) {
294  if (debug_level) {
295  tprintf(kIllegalUnicharMsg, token);
296  }
297  break;
298  }
299  test_unichar_ids[i] = unicharset.unichar_to_id(token);
300  }
301  test_unichar_ids[i] = INVALID_UNICHAR_ID;
302 
303  if (i != *test_ambig_part_size || !(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) ||
304  !sscanf(token, "%d", replacement_ambig_part_size) || *replacement_ambig_part_size <= 0) {
305  if (debug_level) {
306  tprintf(kIllegalMsg, line_num);
307  }
308  return false;
309  }
310  if (*replacement_ambig_part_size > MAX_AMBIG_SIZE) {
311  if (debug_level) {
312  tprintf("Too many unichars in ambiguity on line %d\n", line_num);
313  }
314  return false;
315  }
316  replacement_string[0] = '\0';
317  for (i = 0; i < *replacement_ambig_part_size; ++i) {
318  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token))) {
319  break;
320  }
321  strcat(replacement_string, token);
322  if (!unicharset.contains_unichar(token)) {
323  if (debug_level) {
324  tprintf(kIllegalUnicharMsg, token);
325  }
326  break;
327  }
328  }
329  if (i != *replacement_ambig_part_size) {
330  if (debug_level) {
331  tprintf(kIllegalMsg, line_num);
332  }
333  return false;
334  }
335  if (version > 0) {
336  // The next field being true indicates that the ambiguity should
337  // always be substituted (e.g. '' should always be changed to ").
338  // For such "certain" n -> m ambigs tesseract will insert character
339  // fragments for the n pieces in the unicharset. AmbigsFound()
340  // will then replace the incorrect ngram with the character
341  // fragments of the correct character (or ngram if m > 1).
342  // Note that if m > 1, an ngram will be inserted into the
343  // modified word, not the individual unigrams. Tesseract
344  // has limited support for ngram unichar (e.g. dawg permuter).
345  if (!(token = strtok_r(nullptr, kAmbigDelimiters, &next_token)) || !sscanf(token, "%d", type)) {
346  if (debug_level) {
347  tprintf(kIllegalMsg, line_num);
348  }
349  return false;
350  }
351  }
352  return true;
353 }
354 
355 bool UnicharAmbigs::InsertIntoTable(UnicharAmbigsVector &table, int test_ambig_part_size,
356  UNICHAR_ID *test_unichar_ids, int replacement_ambig_part_size,
357  const char *replacement_string, int type, AmbigSpec *ambig_spec,
358  UNICHARSET *unicharset) {
359  ambig_spec->type = static_cast<AmbigType>(type);
360  if (test_ambig_part_size == 1 && replacement_ambig_part_size == 1 &&
361  unicharset->to_lower(test_unichar_ids[0]) ==
362  unicharset->to_lower(unicharset->unichar_to_id(replacement_string))) {
363  ambig_spec->type = CASE_AMBIG;
364  }
365 
366  ambig_spec->wrong_ngram_size =
367  UnicharIdArrayUtils::copy(test_unichar_ids, ambig_spec->wrong_ngram);
368 
369  // Since we need to maintain a constant number of unichar positions in
370  // order to construct ambig_blob_choices vector in NoDangerousAmbig(), for
371  // each n->m ambiguity we will have to place n character fragments of the
372  // correct ngram into the corresponding positions in the vector (e.g. given
373  // "vvvvw" and vvvv->ww we will place v and |ww|0|4 into position 0, v and
374  // |ww|1|4 into position 1 and so on. The correct ngram is reconstructed
375  // from fragments by dawg_permute_and_select().
376 
377  // Insert the corresponding correct ngram into the unicharset.
378  // Unicharset code assumes that the "base" ngram is inserted into
379  // the unicharset before fragments of this ngram are inserted.
380  unicharset->unichar_insert(replacement_string, OldUncleanUnichars::kTrue);
381  ambig_spec->correct_ngram_id = unicharset->unichar_to_id(replacement_string);
382  if (replacement_ambig_part_size > 1) {
383  unicharset->set_isngram(ambig_spec->correct_ngram_id, true);
384  }
385  // Add the corresponding fragments of the wrong ngram to unicharset.
386  int i;
387  for (i = 0; i < test_ambig_part_size; ++i) {
388  UNICHAR_ID unichar_id;
389  if (test_ambig_part_size == 1) {
390  unichar_id = ambig_spec->correct_ngram_id;
391  } else {
392  std::string frag_str =
393  CHAR_FRAGMENT::to_string(replacement_string, i, test_ambig_part_size, false);
394  unicharset->unichar_insert(frag_str.c_str(), OldUncleanUnichars::kTrue);
395  unichar_id = unicharset->unichar_to_id(frag_str.c_str());
396  }
397  ambig_spec->correct_fragments[i] = unichar_id;
398  }
399  ambig_spec->correct_fragments[i] = INVALID_UNICHAR_ID;
400 
401  // Add AmbigSpec for this ambiguity to the corresponding AmbigSpec_LIST.
402  // Keep AmbigSpec_LISTs sorted by AmbigSpec.wrong_ngram.
403  if (table[test_unichar_ids[0]] == nullptr) {
404  table[test_unichar_ids[0]] = new AmbigSpec_LIST();
405  }
406  if (table[test_unichar_ids[0]]->add_sorted(AmbigSpec::compare_ambig_specs, true, ambig_spec)) {
407  return true;
408  }
409  delete ambig_spec;
410  return false;
411 }
412 
413 } // namespace tesseract
#define UNICHAR_LEN
Definition: unichar.h:33
#define MAX_AMBIG_SIZE
Definition: ambigs.h:34
#define ASSERT_HOST(x)
Definition: errcode.h:59
const std::vector< std::string > split(const std::string &s, char c)
Definition: helpers.h:41
const char kUniversalAmbigsFile[]
std::vector< AmbigSpec_LIST * > UnicharAmbigsVector
Definition: ambigs.h:140
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
void chomp_string(char *str)
Definition: helpers.h:89
const int ksizeofUniversalAmbigsFile
int UNICHAR_ID
Definition: unichar.h:36
AmbigType
Definition: ambigs.h:40
@ CASE_AMBIG
Definition: ambigs.h:45
@ DEFINITE_AMBIG
Definition: ambigs.h:43
@ REPLACE_AMBIG
Definition: ambigs.h:42
@ NOT_AMBIG
Definition: ambigs.h:41
const int kMaxAmbigStringSize
Definition: ambigs.cpp:40
std::vector< UNICHAR_ID > UnicharIdVector
Definition: ambigs.h:38
static void print(const UNICHAR_ID array[], const UNICHARSET &unicharset)
Definition: ambigs.h:93
static int copy(const UNICHAR_ID src[], UNICHAR_ID dst[])
Definition: ambigs.h:83
UNICHAR_ID correct_ngram_id
Definition: ambigs.h:132
UNICHAR_ID wrong_ngram[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:130
static int compare_ambig_specs(const void *spec1, const void *spec2)
Definition: ambigs.h:120
UNICHAR_ID correct_fragments[MAX_AMBIG_SIZE+1]
Definition: ambigs.h:131
AmbigType type
Definition: ambigs.h:133
void LoadUniversal(const UNICHARSET &encoder_set, UNICHARSET *unicharset)
Definition: ambigs.cpp:64
void InitUnicharAmbigs(const UNICHARSET &unicharset, bool use_ambigs_for_adaption)
Definition: ambigs.cpp:51
void LoadUnicharAmbigs(const UNICHARSET &encoder_set, TFile *ambigs_file, int debug_level, bool use_ambigs_for_adaption, UNICHARSET *unicharset)
Definition: ambigs.cpp:72
char * FGets(char *buffer, int buffer_size)
Definition: serialis.cpp:195
std::string to_string() const
Definition: unicharset.h:91
bool encode_string(const char *str, bool give_up_on_failure, std::vector< UNICHAR_ID > *encoding, std::vector< char > *lengths, unsigned *encoded_length) const
Definition: unicharset.cpp:239
bool contains_unichar(const char *const unichar_repr) const
Definition: unicharset.cpp:695
UNICHAR_ID unichar_to_id(const char *const unichar_repr) const
Definition: unicharset.cpp:186
size_t size() const
Definition: unicharset.h:355
std::string debug_str(UNICHAR_ID id) const
Definition: unicharset.cpp:331
static FILE * Open(const std::string &filename, const std::string &mode)
Definition: fileio.cpp:41