Hugging Face's logo

riccorl
/
reader-relik-cie-small

Feature Extraction
Transformers
PyTorch
relik-reader
custom_code
Model card Files
xet
 Community
riccorl commited on
Commit
33492c0
·
1 Parent(s): c7774f0

Upload tokenizer

Browse files
Files changed (5) hide show
  1. added_tokens.json +49 -0
  2. special_tokens_map.json +57 -0
  3. spm.model +3 -0
  4. tokenizer.json +0 -0
  5. tokenizer_config.json +65 -0
added_tokens.json ADDED
@@ -0,0 +1,49 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "--NME--": 128021,
3
+ "[E-0]": 128022,
4
+ "[E-10]": 128032,
5
+ "[E-11]": 128033,
6
+ "[E-12]": 128034,
7
+ "[E-13]": 128035,
8
+ "[E-14]": 128036,
9
+ "[E-15]": 128037,
10
+ "[E-16]": 128038,
11
+ "[E-17]": 128039,
12
+ "[E-18]": 128040,
13
+ "[E-19]": 128041,
14
+ "[E-1]": 128023,
15
+ "[E-20]": 128042,
16
+ "[E-21]": 128043,
17
+ "[E-22]": 128044,
18
+ "[E-23]": 128045,
19
+ "[E-24]": 128046,
20
+ "[E-2]": 128024,
21
+ "[E-3]": 128025,
22
+ "[E-4]": 128026,
23
+ "[E-5]": 128027,
24
+ "[E-6]": 128028,
25
+ "[E-7]": 128029,
26
+ "[E-8]": 128030,
27
+ "[E-9]": 128031,
28
+ "[MASK]": 128000,
29
+ "[R-0]": 128001,
30
+ "[R-10]": 128011,
31
+ "[R-11]": 128012,
32
+ "[R-12]": 128013,
33
+ "[R-13]": 128014,
34
+ "[R-14]": 128015,
35
+ "[R-15]": 128016,
36
+ "[R-16]": 128017,
37
+ "[R-17]": 128018,
38
+ "[R-18]": 128019,
39
+ "[R-19]": 128020,
40
+ "[R-1]": 128002,
41
+ "[R-2]": 128003,
42
+ "[R-3]": 128004,
43
+ "[R-4]": 128005,
44
+ "[R-5]": 128006,
45
+ "[R-6]": 128007,
46
+ "[R-7]": 128008,
47
+ "[R-8]": 128009,
48
+ "[R-9]": 128010
49
+ }
special_tokens_map.json ADDED
@@ -0,0 +1,57 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "additional_special_tokens": [
3
+ "[R-0]",
4
+ "[R-1]",
5
+ "[R-2]",
6
+ "[R-3]",
7
+ "[R-4]",
8
+ "[R-5]",
9
+ "[R-6]",
10
+ "[R-7]",
11
+ "[R-8]",
12
+ "[R-9]",
13
+ "[R-10]",
14
+ "[R-11]",
15
+ "[R-12]",
16
+ "[R-13]",
17
+ "[R-14]",
18
+ "[R-15]",
19
+ "[R-16]",
20
+ "[R-17]",
21
+ "[R-18]",
22
+ "[R-19]",
23
+ "--NME--",
24
+ "[E-0]",
25
+ "[E-1]",
26
+ "[E-2]",
27
+ "[E-3]",
28
+ "[E-4]",
29
+ "[E-5]",
30
+ "[E-6]",
31
+ "[E-7]",
32
+ "[E-8]",
33
+ "[E-9]",
34
+ "[E-10]",
35
+ "[E-11]",
36
+ "[E-12]",
37
+ "[E-13]",
38
+ "[E-14]",
39
+ "[E-15]",
40
+ "[E-16]",
41
+ "[E-17]",
42
+ "[E-18]",
43
+ "[E-19]",
44
+ "[E-20]",
45
+ "[E-21]",
46
+ "[E-22]",
47
+ "[E-23]",
48
+ "[E-24]"
49
+ ],
50
+ "bos_token": "[CLS]",
51
+ "cls_token": "[CLS]",
52
+ "eos_token": "[SEP]",
53
+ "mask_token": "[MASK]",
54
+ "pad_token": "[PAD]",
55
+ "sep_token": "[SEP]",
56
+ "unk_token": "[UNK]"
57
+ }
spm.model ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:c679fbf93643d19aab7ee10c0b99e460bdbc02fedf34b92b05af343b4af586fd
3
+ size 2464616
tokenizer.json ADDED
The diff for this file is too large to render. See raw diff
 
tokenizer_config.json ADDED
@@ -0,0 +1,65 @@
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
 
1
+ {
2
+ "add_prefix_space": true,
3
+ "additional_special_tokens": [
4
+ "[R-0]",
5
+ "[R-1]",
6
+ "[R-2]",
7
+ "[R-3]",
8
+ "[R-4]",
9
+ "[R-5]",
10
+ "[R-6]",
11
+ "[R-7]",
12
+ "[R-8]",
13
+ "[R-9]",
14
+ "[R-10]",
15
+ "[R-11]",
16
+ "[R-12]",
17
+ "[R-13]",
18
+ "[R-14]",
19
+ "[R-15]",
20
+ "[R-16]",
21
+ "[R-17]",
22
+ "[R-18]",
23
+ "[R-19]",
24
+ "--NME--",
25
+ "[E-0]",
26
+ "[E-1]",
27
+ "[E-2]",
28
+ "[E-3]",
29
+ "[E-4]",
30
+ "[E-5]",
31
+ "[E-6]",
32
+ "[E-7]",
33
+ "[E-8]",
34
+ "[E-9]",
35
+ "[E-10]",
36
+ "[E-11]",
37
+ "[E-12]",
38
+ "[E-13]",
39
+ "[E-14]",
40
+ "[E-15]",
41
+ "[E-16]",
42
+ "[E-17]",
43
+ "[E-18]",
44
+ "[E-19]",
45
+ "[E-20]",
46
+ "[E-21]",
47
+ "[E-22]",
48
+ "[E-23]",
49
+ "[E-24]"
50
+ ],
51
+ "bos_token": "[CLS]",
52
+ "clean_up_tokenization_spaces": true,
53
+ "cls_token": "[CLS]",
54
+ "do_lower_case": false,
55
+ "eos_token": "[SEP]",
56
+ "mask_token": "[MASK]",
57
+ "model_max_length": 1000000000000000019884624838656,
58
+ "pad_token": "[PAD]",
59
+ "sep_token": "[SEP]",
60
+ "sp_model_kwargs": {},
61
+ "split_by_punct": false,
62
+ "tokenizer_class": "DebertaV2Tokenizer",
63
+ "unk_token": "[UNK]",
64
+ "vocab_type": "spm"
65
+ }