polish mult-lang doc and whl

MANIFEST.in

@@ -4,4 +4,5 @@ include README.md
 recursive-include ppocr/utils *.txt utility.py logging.py
 recursive-include ppocr/data/ *.py
 recursive-include ppocr/postprocess *.py
-recursive-include tools/infer *.py
+recursive-include tools/infer *.py
+recursive-include ppocr/utils/e2e_utils/ *.py

configs/rec/multi_language/generate_multi_language_configs.py

@@ -19,37 +19,87 @@
 logging.basicConfig(level=logging.INFO)
 
 support_list = {
- 'it':'italian', 'xi':'spanish', 'pu':'portuguese', 'ru':'russian', 'ar':'arabic',
- 'ta':'tamil', 'ug':'uyghur', 'fa':'persian', 'ur':'urdu', 'rs':'serbian latin',
- 'oc':'occitan', 'rsc':'serbian cyrillic', 'bg':'bulgarian', 'uk':'ukranian', 'be':'belarusian',
- 'te':'telugu', 'ka':'kannada', 'chinese_cht':'chinese tradition','hi':'hindi','mr':'marathi',
- 'ne':'nepali',
+ 'it': 'italian',
+ 'xi': 'spanish',
+ 'pu': 'portuguese',
+ 'ru': 'russian',
+ 'ar': 'arabic',
+ 'ta': 'tamil',
+ 'ug': 'uyghur',
+ 'fa': 'persian',
+ 'ur': 'urdu',
+ 'rs': 'serbian latin',
+ 'oc': 'occitan',
+ 'rsc': 'serbian cyrillic',
+ 'bg': 'bulgarian',
+ 'uk': 'ukranian',
+ 'be': 'belarusian',
+ 'te': 'telugu',
+ 'ka': 'kannada',
+ 'chinese_cht': 'chinese tradition',
+ 'hi': 'hindi',
+ 'mr': 'marathi',
+ 'ne': 'nepali',
 }
-assert(
- os.path.isfile("./rec_multi_language_lite_train.yml")
- ),"Loss basic configuration file rec_multi_language_lite_train.yml.\
+
+latin_lang = [
+ 'af', 'az', 'bs', 'cs', 'cy', 'da', 'de', 'es', 'et', 'fr', 'ga', 'hr',
+ 'hu', 'id', 'is', 'it', 'ku', 'la', 'lt', 'lv', 'mi', 'ms', 'mt', 'nl',
+ 'no', 'oc', 'pi', 'pl', 'pt', 'ro', 'rs_latin', 'sk', 'sl', 'sq', 'sv',
+ 'sw', 'tl', 'tr', 'uz', 'vi', 'latin'
+]
+arabic_lang = ['ar', 'fa', 'ug', 'ur']
+cyrillic_lang = [
+ 'ru', 'rs_cyrillic', 'be', 'bg', 'uk', 'mn', 'abq', 'ady', 'kbd', 'ava',
+ 'dar', 'inh', 'che', 'lbe', 'lez', 'tab', 'cyrillic'
+]
+devanagari_lang = [
+ 'hi', 'mr', 'ne', 'bh', 'mai', 'ang', 'bho', 'mah', 'sck', 'new', 'gom',
+ 'sa', 'bgc', 'devanagari'
+]
+multi_lang = latin_lang + arabic_lang + cyrillic_lang + devanagari_lang
+
+assert (os.path.isfile("./rec_multi_language_lite_train.yml")
+ ), "Loss basic configuration file rec_multi_language_lite_train.yml.\
 You can download it from \
 https://github.com/PaddlePaddle/PaddleOCR/tree/dygraph/configs/rec/multi_language/"
-
-global_config = yaml.load(open("./rec_multi_language_lite_train.yml", 'rb'), Loader=yaml.Loader)
+
+global_config = yaml.load(
+ open("./rec_multi_language_lite_train.yml", 'rb'), Loader=yaml.Loader)
 project_path = os.path.abspath(os.path.join(os.getcwd(), "../../../"))
 
+
 class ArgsParser(ArgumentParser):
  def __init__(self):
  super(ArgsParser, self).__init__(
  formatter_class=RawDescriptionHelpFormatter)
  self.add_argument(
  "-o", "--opt", nargs='+', help="set configuration options")
  self.add_argument(
- "-l", "--language", nargs='+', help="set language type, support {}".format(support_list))
+ "-l",
+ "--language",
+ nargs='+',
+ help="set language type, support {}".format(support_list))
  self.add_argument(
- "--train",type=str,help="you can use this command to change the train dataset default path")
+ "--train",
+ type=str,
+ help="you can use this command to change the train dataset default path"
+ )
  self.add_argument(
- "--val",type=str,help="you can use this command to change the eval dataset default path")
+ "--val",
+ type=str,
+ help="you can use this command to change the eval dataset default path"
+ )
  self.add_argument(
- "--dict",type=str,help="you can use this command to change the dictionary default path")
+ "--dict",
+ type=str,
+ help="you can use this command to change the dictionary default path"
+ )
  self.add_argument(
- "--data_dir",type=str,help="you can use this command to change the dataset default root path")
+ "--data_dir",
+ type=str,
+ help="you can use this command to change the dataset default root path"
+ )
 
  def parse_args(self, argv=None):
  args = super(ArgsParser, self).parse_args(argv)
@@ -68,21 +118,38 @@ def _parse_opt(self, opts):
  return config
 
  def _set_language(self, type):
- assert(type),"please use -l or --language to choose language type"
+ print("type:", type)
+ lang = type[0]
+ assert (type), "please use -l or --language to choose language type"
  assert(
- type[0] in support_list.keys()
+ lang in support_list.keys() or lang in multi_lang
  ),"the sub_keys(-l or --language) can only be one of support list: \n{},\nbut get: {}, " \
- "please check your running command".format(support_list, type)
- global_config['Global']['character_dict_path'] = 'ppocr/utils/dict/{}_dict.txt'.format(type[0])
- global_config['Global']['save_model_dir'] = './output/rec_{}_lite'.format(type[0])
- global_config['Train']['dataset']['label_file_list'] = ["train_data/{}_train.txt".format(type[0])]
- global_config['Eval']['dataset']['label_file_list'] = ["train_data/{}_val.txt".format(type[0])]
- global_config['Global']['character_type'] = type[0]
- assert(
- os.path.isfile(os.path.join(project_path,global_config['Global']['character_dict_path']))
- ),"Loss default dictionary file {}_dict.txt.You can download it from \
-https://github.com/PaddlePaddle/PaddleOCR/tree/dygraph/ppocr/utils/dict/".format(type[0])
- return type[0]
+ "please check your running command".format(multi_lang, type)
+ if lang in latin_lang:
+ lang = "latin"
+ elif lang in arabic_lang:
+ lang = "arabic"
+ elif lang in cyrillic_lang:
+ lang = "cyrillic"
+ elif lang in devanagari_lang:
+ lang = "devanagari"
+ global_config['Global'][
+ 'character_dict_path'] = 'ppocr/utils/dict/{}_dict.txt'.format(lang)
+ global_config['Global'][
+ 'save_model_dir'] = './output/rec_{}_lite'.format(lang)
+ global_config['Train']['dataset'][
+ 'label_file_list'] = ["train_data/{}_train.txt".format(lang)]
+ global_config['Eval']['dataset'][
+ 'label_file_list'] = ["train_data/{}_val.txt".format(lang)]
+ global_config['Global']['character_type'] = lang
+ assert (
+ os.path.isfile(
+ os.path.join(project_path, global_config['Global'][
+ 'character_dict_path']))
+ ), "Loss default dictionary file {}_dict.txt.You can download it from \
+https://github.com/PaddlePaddle/PaddleOCR/tree/dygraph/ppocr/utils/dict/".format(
+ lang)
+ return lang
 
 
 def merge_config(config):
@@ -110,43 +177,51 @@ def merge_config(config):
  cur[sub_key] = value
  else:
  cur = cur[sub_key]
-
+
+
 def loss_file(path):
- assert(
- os.path.exists(path)
- ),"There is no such file:{},Please do not forget to put in the specified file".format(path)
+ assert (
+ os.path.exists(path)
+ ), "There is no such file:{},Please do not forget to put in the specified file".format(
+ path)
+
 
-
 if __name__ == '__main__':
  FLAGS = ArgsParser().parse_args()
  merge_config(FLAGS.opt)
  save_file_path = 'rec_{}_lite_train.yml'.format(FLAGS.language)
  if os.path.isfile(save_file_path):
  os.remove(save_file_path)
- 
+
  if FLAGS.train:
  global_config['Train']['dataset']['label_file_list'] = [FLAGS.train]
- train_label_path = os.path.join(project_path,FLAGS.train)
+ train_label_path = os.path.join(project_path, FLAGS.train)
  loss_file(train_label_path)
  if FLAGS.val:
  global_config['Eval']['dataset']['label_file_list'] = [FLAGS.val]
- eval_label_path = os.path.join(project_path,FLAGS.val)
+ eval_label_path = os.path.join(project_path, FLAGS.val)
  loss_file(eval_label_path)
  if FLAGS.dict:
  global_config['Global']['character_dict_path'] = FLAGS.dict
- dict_path = os.path.join(project_path,FLAGS.dict)
+ dict_path = os.path.join(project_path, FLAGS.dict)
  loss_file(dict_path)
  if FLAGS.data_dir:
  global_config['Eval']['dataset']['data_dir'] = FLAGS.data_dir
  global_config['Train']['dataset']['data_dir'] = FLAGS.data_dir
- data_dir = os.path.join(project_path,FLAGS.data_dir)
+ data_dir = os.path.join(project_path, FLAGS.data_dir)
  loss_file(data_dir)
- 
+
  with open(save_file_path, 'w') as f:
- yaml.dump(dict(global_config), f, default_flow_style=False, sort_keys=False)
+ yaml.dump(
+ dict(global_config), f, default_flow_style=False, sort_keys=False)
  logging.info("Project path is :{}".format(project_path))
- logging.info("Train list path set to :{}".format(global_config['Train']['dataset']['label_file_list'][0]))
- logging.info("Eval list path set to :{}".format(global_config['Eval']['dataset']['label_file_list'][0]))
- logging.info("Dataset root path set to :{}".format(global_config['Eval']['dataset']['data_dir']))
- logging.info("Dict path set to :{}".format(global_config['Global']['character_dict_path']))
- logging.info("Config file set to :configs/rec/multi_language/{}".format(save_file_path))
+ logging.info("Train list path set to :{}".format(global_config['Train'][
+ 'dataset']['label_file_list'][0]))
+ logging.info("Eval list path set to :{}".format(global_config['Eval'][
+ 'dataset']['label_file_list'][0]))
+ logging.info("Dataset root path set to :{}".format(global_config['Eval'][
+ 'dataset']['data_dir']))
+ logging.info("Dict path set to :{}".format(global_config['Global'][
+ 'character_dict_path']))
+ logging.info("Config file set to :configs/rec/multi_language/{}".
+ format(save_file_path))

configs/rec/multi_language/rec_arabic_lite_train.yml

@@ -0,0 +1,111 @@
+Global:
+ use_gpu: true
+ epoch_num: 500
+ log_smooth_window: 20
+ print_batch_step: 10
+ save_model_dir: ./output/rec_arabic_lite
+ save_epoch_step: 3
+ eval_batch_step:
+ - 0
+ - 2000
+ cal_metric_during_train: true
+ pretrained_model: null
+ checkpoints: null
+ save_inference_dir: null
+ use_visualdl: false
+ infer_img: null
+ character_dict_path: ppocr/utils/dict/arabic_dict.txt
+ character_type: arabic
+ max_text_length: 25
+ infer_mode: false
+ use_space_char: true
+Optimizer:
+ name: Adam
+ beta1: 0.9
+ beta2: 0.999
+ lr:
+ name: Cosine
+ learning_rate: 0.001
+ regularizer:
+ name: L2
+ factor: 1.0e-05
+Architecture:
+ model_type: rec
+ algorithm: CRNN
+ Transform: null
+ Backbone:
+ name: MobileNetV3
+ scale: 0.5
+ model_name: small
+ small_stride:
+ - 1
+ - 2
+ - 2
+ - 2
+ Neck:
+ name: SequenceEncoder
+ encoder_type: rnn
+ hidden_size: 48
+ Head:
+ name: CTCHead
+ fc_decay: 1.0e-05
+Loss:
+ name: CTCLoss
+PostProcess:
+ name: CTCLabelDecode
+Metric:
+ name: RecMetric
+ main_indicator: acc
+Train:
+ dataset:
+ name: SimpleDataSet
+ data_dir: train_data/
+ label_file_list:
+ - train_data/arabic_train.txt
+ transforms:
+ - DecodeImage:
+ img_mode: BGR
+ channel_first: false
+ - RecAug: null
+ - CTCLabelEncode: null
+ - RecResizeImg:
+ image_shape:
+ - 3
+ - 32
+ - 320
+ - KeepKeys:
+ keep_keys:
+ - image
+ - label
+ - length
+ loader:
+ shuffle: true
+ batch_size_per_card: 256
+ drop_last: true
+ num_workers: 8
+Eval:
+ dataset:
+ name: SimpleDataSet
+ data_dir: train_data/
+ label_file_list:
+ - train_data/arabic_val.txt
+ transforms:
+ - DecodeImage:
+ img_mode: BGR
+ channel_first: false
+ - CTCLabelEncode: null
+ - RecResizeImg:
+ image_shape:
+ - 3
+ - 32
+ - 320
+ - KeepKeys:
+ keep_keys:
+ - image
+ - label
+ - length
+ loader:
+ shuffle: false
+ drop_last: false
+ batch_size_per_card: 256
+ num_workers: 8