diff --git a/installer/install_model_token.py b/installer/install_model_token.py new file mode 100644 index 000000000..e3ae31075 --- /dev/null +++ b/installer/install_model_token.py @@ -0,0 +1,25 @@ +# coding=utf-8 +import json +import os.path +from transformers import BertTokenizer + +prefix_dir = "./model" +model_config = [ + { + 'download_params': { + 'cache_dir': os.path.join(prefix_dir, 'base/hub'), + 'pretrained_model_name_or_path': 'bert-base-cased' + }, + 'download_function': BertTokenizer.from_pretrained + }, +] + + +def install(): + for model in model_config: + print(json.dumps(model.get('download_params'))) + model.get('download_function')(**model.get('download_params')) + + +if __name__ == '__main__': + install()