tesseract  5.0.0
unicharset_extractor.cpp
Go to the documentation of this file.
1 // File: unicharset_extractor.cpp
3 // Description: Unicode character/ligature set extractor.
4 // Author: Thomas Kielbus
5 //
6 // (C) Copyright 2006, Google Inc.
7 // Licensed under the Apache License, Version 2.0 (the "License");
8 // you may not use this file except in compliance with the License.
9 // You may obtain a copy of the License at
10 // http://www.apache.org/licenses/LICENSE-2.0
11 // Unless required by applicable law or agreed to in writing, software
12 // distributed under the License is distributed on an "AS IS" BASIS,
13 // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 // See the License for the specific language governing permissions and
15 // limitations under the License.
16 //
18 
19 // Given a list of box files or text files on the command line, this program
20 // normalizes the text according to command-line options and generates
21 // a unicharset.
22 
23 #include <cstdlib>
24 #include "boxread.h"
25 #include "commandlineflags.h"
26 #include "commontraining.h" // CheckSharedLibraryVersion
27 #include "lang_model_helpers.h"
28 #include "normstrngs.h"
29 #include "unicharset.h"
31 
32 using namespace tesseract;
33 
34 static STRING_PARAM_FLAG(output_unicharset, "unicharset", "Output file path");
35 static INT_PARAM_FLAG(norm_mode, 1,
36  "Normalization mode: 1=Combine graphemes, "
37  "2=Split graphemes, 3=Pure unicode");
38 
39 namespace tesseract {
40 
41 // Helper normalizes and segments the given strings according to norm_mode, and
42 // adds the segmented parts to unicharset.
43 static void AddStringsToUnicharset(const std::vector<std::string> &strings, int norm_mode,
44  UNICHARSET *unicharset) {
45  for (const auto &string : strings) {
46  std::vector<std::string> normalized;
48  static_cast<GraphemeNormMode>(norm_mode),
49  /*report_errors*/ true, string.c_str(), &normalized)) {
50  for (const std::string &normed : normalized) {
51  // normed is a UTF-8 encoded string
52  if (normed.empty() || IsUTF8Whitespace(normed.c_str())) {
53  continue;
54  }
55  unicharset->unichar_insert(normed.c_str());
56  }
57  } else {
58  tprintf("Normalization failed for string '%s'\n", string.c_str());
59  }
60  }
61 }
62 
63 static int Main(int argc, char **argv) {
64  UNICHARSET unicharset;
65  // Load input files
66  for (int arg = 1; arg < argc; ++arg) {
67  std::string file_data = tesseract::ReadFile(argv[arg]);
68  if (file_data.empty()) {
69  continue;
70  }
71  std::vector<std::string> texts;
72  if (ReadMemBoxes(-1, /*skip_blanks*/ true, &file_data[0],
73  /*continue_on_failure*/ false, /*boxes*/ nullptr, &texts,
74  /*box_texts*/ nullptr, /*pages*/ nullptr)) {
75  tprintf("Extracting unicharset from box file %s\n", argv[arg]);
76  } else {
77  tprintf("Extracting unicharset from plain text file %s\n", argv[arg]);
78  texts.clear();
79  texts = split(file_data, '\n');
80  }
81  AddStringsToUnicharset(texts, FLAGS_norm_mode, &unicharset);
82  }
83  SetupBasicProperties(/*report_errors*/ true, /*decompose*/ false, &unicharset);
84  // Write unicharset file.
85  if (unicharset.save_to_file(FLAGS_output_unicharset.c_str())) {
86  tprintf("Wrote unicharset file %s\n", FLAGS_output_unicharset.c_str());
87  } else {
88  tprintf("Cannot save unicharset file %s\n", FLAGS_output_unicharset.c_str());
89  return EXIT_FAILURE;
90  }
91  return EXIT_SUCCESS;
92 }
93 
94 } // namespace tesseract
95 
96 int main(int argc, char **argv) {
97  tesseract::CheckSharedLibraryVersion();
98  if (argc > 1) {
99  tesseract::ParseCommandLineFlags(argv[0], &argc, &argv, true);
100  }
101  if (argc < 2) {
102  tprintf(
103  "Usage: %s [--output_unicharset filename] [--norm_mode mode]"
104  " box_or_text_file [...]\n",
105  argv[0]);
106  tprintf("Where mode means:\n");
107  tprintf(" 1=combine graphemes (use for Latin and other simple scripts)\n");
108  tprintf(" 2=split graphemes (use for Indic/Khmer/Myanmar)\n");
109  tprintf(" 3=pure unicode (use for Arabic/Hebrew/Thai/Tibetan)\n");
110  tprintf("Reads box or plain text files to extract the unicharset.\n");
111  return EXIT_FAILURE;
112  }
113  return tesseract::Main(argc, argv);
114 }
#define STRING_PARAM_FLAG(name, val, comment)
int main(int argc, char **argv)
const std::vector< std::string > split(const std::string &s, char c)
Definition: helpers.h:41
GraphemeNormMode
Definition: validator.h:36
bool ReadMemBoxes(int target_page, bool skip_blanks, const char *box_data, bool continue_on_failure, std::vector< TBOX > *boxes, std::vector< std::string > *texts, std::vector< std::string > *box_texts, std::vector< int > *pages)
Definition: boxread.cpp:90
void ParseCommandLineFlags(const char *usage, int *argc, char ***argv, const bool remove_flags)
void tprintf(const char *format,...)
Definition: tprintf.cpp:41
void SetupBasicProperties(bool report_errors, bool decompose, UNICHARSET *unicharset)
INT_PARAM_FLAG(debug_level, 0, "Level of Trainer debugging")
bool NormalizeCleanAndSegmentUTF8(UnicodeNormMode u_mode, OCRNorm ocr_normalize, GraphemeNormMode g_mode, bool report_errors, const char *str8, std::vector< std::string > *graphemes)
Definition: normstrngs.cpp:179
std::string ReadFile(const std::string &filename, FileReader reader)
bool IsUTF8Whitespace(const char *text)
Definition: normstrngs.cpp:233
void unichar_insert(const char *const unichar_repr, OldUncleanUnichars old_style)
Definition: unicharset.cpp:654
bool save_to_file(const char *const filename) const
Definition: unicharset.h:361