-
Notifications
You must be signed in to change notification settings - Fork 1
/
at_generator.py
356 lines (336 loc) · 17.1 KB
/
at_generator.py
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
312
313
314
315
316
317
318
319
320
321
322
323
324
325
326
327
328
329
330
331
332
333
334
335
336
337
338
339
340
341
342
343
344
345
346
347
348
349
350
351
352
353
354
355
356
from at_prep import *
if __name__ == '__main__':
#original_emb()
'''
https://github.com/Embedding/Chinese-Word-Vectors
'''
#word2vector('word', 'char', '001')
#word2vector('word', 'mix', '002')
#word2vector('word+ngram', 'char', '003')
#word2vector('word+ngram', 'mix', '004')
#word2vector('word+char', 'char', '005')
#word2vector('word+char', 'mix', '006')
#word2vector('word+char+ngram', 'char', '007')
#word2vector('word+char+ngram', 'mix', '008')
#word2vector('tw_word', 'char', '181')
#word2vector('tw_word', 'mix', '182')
#word2vector('tw_ngram_1', 'char', '183')
#word2vector('tw_ngram_1', 'mix', '184')
#word2vector('tw_ngram_2', 'char', '185')
#word2vector('tw_ngram_2', 'mix', '186')
#word2vector('tw_ngram_3', 'char', '187')
#word2vector('tw_ngram_3', 'mix', '188')
#word2vector('tw_char_1', 'char', '189')
#word2vector('tw_char_1', 'mix', '190')
#word2vector('tw_char_2', 'char', '191')
#word2vector('tw_char_2', 'mix', '192')
#word2vector('tw_char_3', 'char', '193')
#word2vector('tw_char_3', 'mix', '194')
#word2vector('tw_position_1', 'char', '195')
#word2vector('tw_position_1', 'mix', '196')
#word2vector('tw_position_2', 'char', '197')
#word2vector('tw_position_2', 'mix', '198')
#word2vector('cw_word', 'char', '199')
#word2vector('cw_word', 'mix', '200')
#word2vector('cw_ngram_1', 'char', '201')
#word2vector('cw_ngram_1', 'mix', '202')
#word2vector('cw_ngram_2', 'char', '203')
#word2vector('cw_ngram_2', 'mix', '204')
#word2vector('cw_ngram_3', 'char', '205')
#word2vector('cw_ngram_3', 'mix', '206')
#word2vector('cw_char_1', 'char', '207')
#word2vector('cw_char_1', 'mix', '208')
#word2vector('cw_char_2', 'char', '209')
#word2vector('cw_char_2', 'mix', '210')
#word2vector('cw_char_3', 'char', '211')
#word2vector('cw_char_3', 'mix', '212')
#word2vector('cw_position_1', 'char', '213')
#word2vector('cw_position_1', 'mix', '214')
#word2vector('cw_position_2', 'char', '215')
#word2vector('cw_position_2', 'mix', '216')
#word2vector('word_baidu', 'char', '217')
#word2vector('word_baidu', 'mix', '218')
#word2vector('word+ngram_baidu', 'char', '219')
#word2vector('word+ngram_baidu', 'mix', '220')
#word2vector('word+char_baidu', 'char', '221')
#word2vector('word+char_baidu', 'mix', '222')
#word2vector('word+char+ngram_baidu', 'char', '223')
#word2vector('word+char+ngram_baidu', 'mix', '224')
#word2vector('word_wiki', 'char', '225')
#word2vector('word_wiki', 'mix', '226')
#word2vector('word+ngram_wiki', 'char', '227')
#word2vector('word+ngram_wiki', 'mix', '228')
#word2vector('word+char_wiki', 'char', '229')
#word2vector('word+char_wiki', 'mix', '230')
#word2vector('word+char+ngram_wiki', 'char', '231')
#word2vector('word+char+ngram_wiki', 'mix', '232')
#word2vector('word_renmin', 'char', '233')
#word2vector('word_renmin', 'mix', '234')
#word2vector('word+ngram_renmin', 'char', '235')
#word2vector('word+ngram_renmin', 'mix', '236')
#word2vector('word+char_renmin', 'char', '237')
#word2vector('word+char_renmin', 'mix', '238')
#word2vector('word+char+ngram_renmin', 'char', '239')
#word2vector('word+char+ngram_renmin', 'mix', '240')
#word2vector('word_sogou', 'char', '241')
#word2vector('word_sogou', 'mix', '242')
#word2vector('word+ngram_sogou', 'char', '243')
#word2vector('word+ngram_sogou', 'mix', '244')
#word2vector('word+char_sogou', 'char', '245')
#word2vector('word+char_sogou', 'mix', '246')
#word2vector('word+char+ngram_sogou', 'char', '247')
#word2vector('word+char+ngram_sogou', 'mix', '248')
#word2vector('word_financial', 'char', '249')
#word2vector('word_financial', 'mix', '250')
#word2vector('word+ngram_financial', 'char', '251')
#word2vector('word+ngram_financial', 'mix', '252')
#word2vector('word+char_financial', 'char', '253')
#word2vector('word+char_financial', 'mix', '254')
#word2vector('word+char+ngram_financial', 'char', '255')
#word2vector('word+char+ngram_financial', 'mix', '256')
#word2vector('word_zhihu', 'char', '257')
#word2vector('word_zhihu', 'mix', '258')
#word2vector('word+ngram_zhihu', 'char', '259')
#word2vector('word+ngram_zhihu', 'mix', '260')
#word2vector('word+char_zhihu', 'char', '261')
#word2vector('word+char_zhihu', 'mix', '262')
#word2vector('word+char+ngram_zhihu', 'char', '263')
#word2vector('word+char+ngram_zhihu', 'mix', '264')
#word2vector('word_weibo', 'char', '265')
#word2vector('word_weibo', 'mix', '266')
#word2vector('word+ngram_weibo', 'char', '267')
#word2vector('word+ngram_weibo', 'mix', '268')
#word2vector('word+char_weibo', 'char', '269')
#word2vector('word+char_weibo', 'mix', '270')
#word2vector('word+char+ngram_weibo', 'char', '271')
#word2vector('word+char+ngram_weibo', 'mix', '272')
#word2vector('word_literature', 'char', '273')
#word2vector('word_literature', 'mix', '274')
#word2vector('word+ngram_literature', 'char', '275')
#word2vector('word+ngram_literature', 'mix', '276')
#word2vector('word+char_literature', 'char', '277')
#word2vector('word+char_literature', 'mix', '278')
#word2vector('word+char+ngram_literature', 'char', '279')
#word2vector('word+char+ngram_literature', 'mix', '280')
#word2vector('word_sikuquanshu', 'char', '281')
#word2vector('word+ngram_sikuquanshu', 'char', '282')
'''
https://ai.tencent.com/ailab/nlp/en/embedding.html
'''
#word2vector('v010-d200', 'char', '009')
#word2vector('v010-d200', 'mix', '010')
#word2vector('v020-d200-small', 'char', '011')
#word2vector('v020-d200-small', 'mix', '012')
#word2vector('v020-d200-large', 'char', '013')
#word2vector('v020-d200-large', 'mix', '014')
#word2vector('v020-d100-small', 'char', '015')
#word2vector('v020-d100-small', 'mix', '016')
#word2vector('v020-d100-large', 'char', '017')
#word2vector('v020-d100-large', 'mix', '018')
'''
https://huggingface.co/clue
'''
#plm_pt_emb('clue/roberta_chinese_pair_tiny', '019') #312
#plm_pt_emb('clue/roberta_chinese_pair_large', '020')
#plm_pt_emb('clue/roberta_chinese_large', '021')
#plm_pt_emb('clue/roberta_chinese_clue_tiny', '022')
#plm_pt_emb('clue/roberta_chinese_clue_large', '023')
#plm_pt_emb('clue/roberta_chinese_base', '024')
#plm_pt_emb('clue/roberta_chinese_3L768_clue_tiny', '025')
#plm_pt_emb('clue/roberta_chinese_3L312_clue_tiny', '026')
#plm_pt_emb('clue/xlnet_chinese_large', '027')
#plm_pt_emb('clue/albert_chinese_tiny', '028')
#plm_pt_emb('clue/albert_chinese_small', '029')
'''
https://github.com/CLUEbenchmark/ELECTRA
'''
#plm_tf_emb("clue/electra_tiny", "030", model="electra") #312
'''
https://github.com/ZhuiyiTechnology
'''
#plm_tf_emb("ZhuiyiTechnology/SimBERT-base", "031")
#plm_tf_emb("ZhuiyiTechnology/SimBERT-small", "032")
#plm_tf_emb("ZhuiyiTechnology/SimBERT-tiny", "033")
#plm_tf_emb("ZhuiyiTechnology/RoBERTa-tiny", "034")
#plm_tf_emb("ZhuiyiTechnology/RoBERTa-small", "035")
#plm_tf_emb("ZhuiyiTechnology/RoBERTa+-tiny", "036")
#plm_tf_emb("ZhuiyiTechnology/RoBERTa+-small", "037")
#plm_tf_emb("ZhuiyiTechnology/WoBERT", "038")
#plm_tf_emb("ZhuiyiTechnology/WoNEZHA", "039", model="nezha")
#plm_tf_emb("ZhuiyiTechnology/WoBERT+", "040")
#plm_tf_emb("ZhuiyiTechnology/GAU", "041")
#plm_tf_emb("ZhuiyiTechnology/T5-small", "042", model="mt5.1.1")
#plm_tf_emb("ZhuiyiTechnology/T5-base", "043", model="mt5.1.1")
#plm_tf_emb("ZhuiyiTechnology/mT5-small", "044", model="mt5.1.1")
#plm_tf_emb("ZhuiyiTechnology/mT5-base", "045", model="mt5.1.1")
#plm_tf_emb("ZhuiyiTechnology/GPT-LCCC-nezha", "046", model="nezha")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-small", "047", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer", "048", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-char-small", "049", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-char", "050", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-GPT", "051", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-sim-char-small", "052", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-sim-char", "053", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-sim-char-small-ft", "054", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-sim-char-ft", "055", model="roformer")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-v2-char-small", "056", model="roformer_v2")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-v2-char-base", "057", model="roformer_v2")
#plm_tf_emb("ZhuiyiTechnology/RoFormer-v2-char-large", "058", model="roformer_v2")
'''
https://huggingface.co/hfl
'''
#plm_pt_emb('hfl/chinese-pert-base-mrc', '059') #768
#plm_pt_emb('hfl/chinese-pert-large-mrc', '060') #1024
#plm_pt_emb('hfl/chinese-roberta-wwm-ext-large', '061') #1024
#plm_pt_emb('hfl/chinese-roberta-wwm-ext', '062') #768
#plm_pt_emb('hfl/chinese-pert-large', '063') #1024
#plm_pt_emb('hfl/chinese-pert-base', '064') #768
#plm_pt_emb('hfl/cino-small-v2', '065') #768
#plm_pt_emb('hfl/cino-large-v2', '066') #1024
#plm_pt_emb('hfl/cino-base-v2', '067') #768
#plm_pt_emb('hfl/cino-large', '068') #1024
#plm_pt_emb('hfl/chinese-legal-electra-large-generator', '069') #256
#plm_pt_emb('hfl/chinese-legal-electra-base-generator', '070') #192
#plm_pt_emb('hfl/chinese-legal-electra-small-generator', '071') #64
#plm_pt_emb('hfl/rbtl3', '072') #1024
#plm_pt_emb('hfl/rbt6', '073') #768
#plm_pt_emb('hfl/rbt4', '074') #768
#plm_pt_emb('hfl/rbt3', '075') #768
#plm_pt_emb('hfl/chinese-macbert-large', '076') #1024
#plm_pt_emb('hfl/chinese-macbert-base', '077') #768
#plm_pt_emb('hfl/chinese-bert-wwm', '078') #768
#plm_pt_emb('hfl/chinese-bert-wwm-ext', '079') #768
#plm_pt_emb('hfl/chinese-xlnet-mid', '080') #768
#plm_pt_emb('hfl/chinese-xlnet-base', '081') #768
#plm_pt_emb('hfl/chinese-electra-large-discriminator', '082') #1024
#plm_pt_emb('hfl/chinese-electra-large-generator', '083') #256
#plm_pt_emb('hfl/chinese-electra-base-discriminator', '084') #768
#plm_pt_emb('hfl/chinese-electra-base-generator', '085') #192
#plm_pt_emb('hfl/chinese-electra-small-ex-discriminator', '086') #256
#plm_pt_emb('hfl/chinese-electra-small-ex-generator', '087') #64
#plm_pt_emb('hfl/chinese-electra-small-discriminator', '088') #256
#plm_pt_emb('hfl/chinese-electra-small-generator', '089') #64
#plm_pt_emb('hfl/chinese-electra-180g-large-discriminator', '090') #1024
#plm_pt_emb('hfl/chinese-electra-180g-large-generator', '091') #256
#plm_pt_emb('hfl/chinese-electra-180g-base-generator', '092') #192
#plm_pt_emb('hfl/chinese-electra-180g-base-discriminator', '093') #768
#plm_pt_emb('hfl/chinese-electra-180g-small-ex-discriminator', '094') #256
#plm_pt_emb('hfl/chinese-electra-180g-small-ex-generator', '095') #64
#plm_pt_emb('hfl/chinese-electra-180g-small-generator', '096') #64
#plm_pt_emb('hfl/chinese-electra-180g-small-discriminator', '097') #256
#plm_pt_emb('hfl/chinese-legal-electra-small-discriminator', '098') #256
#plm_pt_emb('hfl/chinese-legal-electra-large-discriminator', '099') #1024
'''
https://huggingface.co/Yaxin/ernie_1.0_skep_large_ch
'''
#plm_pt_emb('Yaxin/ernie_1.0_skep_large_ch', '100') #1024
'''
https://huggingface.co/nghuyong
'''
#plm_pt_emb('nghuyong/ernie-health-zh', '101') #768
#plm_pt_emb('nghuyong/ernie-gram-zh', '102') #768
#plm_pt_emb('nghuyong/ernie-1.0', '103') #768
'''
https://github.com/brightmart
'''
#plm_tf_emb("brightmart/RoBERTa-L6", '104') #768
#plm_tf_emb("brightmart/RoBERTa-L12", '105') #768
#plm_tf_emb("brightmart/RoBERTa-Large", '106') #1024
'''
https://huggingface.co/voidful
'''
#plm_pt_emb("voidful/albert_chinese_xxlarge", '107') #4096
#plm_pt_emb("voidful/albert_chinese_xlarge", '108') #2048
#plm_pt_emb("voidful/albert_chinese_large", '109') #1024
#plm_pt_emb("voidful/albert_chinese_base", '110') #768
#plm_pt_emb("voidful/albert_chinese_small", '111') #384
#plm_pt_emb("voidful/albert_chinese_tiny", '112') #312
#plm_pt_emb('voidful/dpr-ctx_encoder-bert-base-multilingual', '113') #768
#plm_pt_emb('voidful/dpr-question_encoder-bert-base-multilingual', '114') #768
'''
https://github.com/imcaspar/gpt2-ml
'''
#plm_tf_emb("GPT2/1.5B", '115', model="gpt2_ml") #1536
'''
https://github.com/TsinghuaAI/CPM-1-Generate
'''
#plm_tf_emb("GPT2/2.6B", '116', model="gpt2") #2560
'''
https://huggingface.co/uer
'''
#plm_pt_emb("uer/bart-large-chinese-cluecorpussmall", '117') #768
#plm_pt_emb("uer/bart-base-chinese-cluecorpussmall", '118') #768
#plm_pt_emb("uer/pegasus-large-chinese-cluecorpussmall", '119') #768
#plm_pt_emb("uer/pegasus-base-chinese-cluecorpussmall", '120') #768
#plm_pt_emb("uer/sbert-base-chinese-nli", '121') #768
#plm_pt_emb("uer/albert-large-chinese-cluecorpussmall", '122') #1024
#plm_pt_emb("uer/albert-base-chinese-cluecorpussmall", '123') #768
#plm_pt_emb("uer/roberta-base-finetuned-chinanews-chinese", '124') #768
#plm_pt_emb("uer/roberta-base-finetuned-ifeng-chinese", '125') #768
#plm_pt_emb("uer/roberta-base-finetuned-dianping-chinese", '126') #768
#plm_pt_emb("uer/roberta-base-finetuned-jd-binary-chinese", '127') #768
#plm_pt_emb("uer/roberta-base-finetuned-jd-full-chinese", '128') #768
#plm_pt_emb("uer/roberta-base-finetuned-cluener2020-chinese", '129') #768
#plm_pt_emb("uer/roberta-base-chinese-extractive-qa", '130') #768
#plm_pt_emb("uer/t5-v1_1-small-chinese-cluecorpussmall", '131') #768
#plm_pt_emb("uer/t5-base-chinese-cluecorpussmall", '132') #768
#plm_pt_emb("uer/t5-small-chinese-cluecorpussmall", '133') #768
#plm_pt_emb("uer/gpt2-chinese-lyric", '134') #768
#plm_pt_emb("uer/gpt2-chinese-couplet", '135') #768
#plm_pt_emb("uer/gpt2-chinese-poem", '136') #768
#plm_pt_emb("uer/gpt2-chinese-ancient", '137') #768
#plm_pt_emb("uer/gpt2-chinese-cluecorpussmall", '138') #768
#plm_pt_emb("uer/gpt2-distil-chinese-cluecorpussmall", '139') #768
#plm_pt_emb("uer/roberta-base-word-chinese-cluecorpussmall", '140') #768
#plm_pt_emb("uer/roberta-medium-word-chinese-cluecorpussmall", '141') #512
#plm_pt_emb("uer/roberta-small-word-chinese-cluecorpussmall", '142') #512
#plm_pt_emb("uer/roberta-mini-word-chinese-cluecorpussmall", '143') #256
#plm_pt_emb("uer/roberta-tiny-word-chinese-cluecorpussmall", '144') #128
#plm_pt_emb("uer/chinese_roberta_L-12_H-768", '145') #768
#plm_pt_emb("uer/chinese_roberta_L-12_H-512", '146') #512
#plm_pt_emb("uer/chinese_roberta_L-12_H-256", '147') #256
#plm_pt_emb("uer/chinese_roberta_L-12_H-128", '148') #128
#plm_pt_emb("uer/chinese_roberta_L-10_H-768", '149') #768
#plm_pt_emb("uer/chinese_roberta_L-10_H-512", '150') #512
#plm_pt_emb("uer/chinese_roberta_L-10_H-256", '151') #256
#plm_pt_emb("uer/chinese_roberta_L-10_H-128", '152') #128
#plm_pt_emb("uer/chinese_roberta_L-8_H-768", '153') #768
#plm_pt_emb("uer/chinese_roberta_L-8_H-512", '154') #512
#plm_pt_emb("uer/chinese_roberta_L-8_H-256", '155') #256
#plm_pt_emb("uer/chinese_roberta_L-8_H-128", '156') #128
#plm_pt_emb("uer/chinese_roberta_L-6_H-768", '157') #768
#plm_pt_emb("uer/chinese_roberta_L-6_H-512", '158') #512
#plm_pt_emb("uer/chinese_roberta_L-6_H-256", '159') #256
#plm_pt_emb("uer/chinese_roberta_L-6_H-128", '160') #128
#plm_pt_emb("uer/chinese_roberta_L-4_H-768", '161') #768
#plm_pt_emb("uer/chinese_roberta_L-4_H-512", '162') #512
#plm_pt_emb("uer/chinese_roberta_L-4_H-256", '163') #256
#plm_pt_emb("uer/chinese_roberta_L-4_H-128", '164') #128
#plm_pt_emb("uer/chinese_roberta_L-2_H-768", '165') #768
#plm_pt_emb("uer/chinese_roberta_L-2_H-512", '166') #512
#plm_pt_emb("uer/chinese_roberta_L-2_H-256", '167') #256
#plm_pt_emb("uer/chinese_roberta_L-2_H-128", '168') #128
#plm_pt_emb("uer/bart-chinese-6-960-cluecorpussmall", '169') #768
#plm_pt_emb("uer/bart-chinese-4-768-cluecorpussmall", '170') #768
'''
https://huggingface.co/xlm-roberta-base
https://huggingface.co/xlm-roberta-large
'''
#plm_pt_emb("xlm-roberta-base", '171') #768
#plm_pt_emb("xlm-roberta-large", '172') #1024
'''
https://huggingface.co/google/rembert
https://github.com/google-research/bert
https://github.com/google-research/ALBERT
'''
#plm_pt_emb("google/rembert", '173') #1152
#plm_tf_emb("google/BERT-base", '174') #768
#plm_tf_emb("google/BERT-base-multi", '175') #768
#plm_tf_emb("google/ALBERT-base", '176', model="albert_unshared") #768
#plm_tf_emb("google/ALBERT-large", '177', model="albert_unshared") #1024
#plm_tf_emb("google/ALBERT-xlarge", '178', model="albert_unshared") #2048
#plm_tf_emb("google/ALBERT-xxlarge", '179', model="albert_unshared") #4096
'''
https://github.com/bojone/labse
'''
#plm_tf_emb("google/labse", '180') #768