d-matrix
commited on
Commit
•
1a457fd
1
Parent(s):
c5f8250
Update README.md
Browse files
README.md
CHANGED
@@ -72,7 +72,7 @@ pipe = pipeline(
|
|
72 |
# device_map="auto", # enabling model parallel on multi-GPU nodes
|
73 |
)
|
74 |
|
75 |
-
results = pipe.
|
76 |
```
|
77 |
|
78 |
### Evaluation results
|
@@ -80,26 +80,26 @@ results = pipe.evaluate(metric="d-matrix/perplexity", dataset="wikitext", datase
|
|
80 |
- `perplexity` on `penn_treebank`
|
81 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
82 |
:-- | --: | --:
|
83 |
-
`distilgpt2` | 63.
|
84 |
-
`gpt2` | 35.
|
85 |
-
`gpt2-medium` | 27.
|
86 |
-
`gpt2-large` | 23.
|
87 |
-
`gpt2-xl` | 21.
|
88 |
|
89 |
- `perplexity` on `wikitext2`
|
90 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
91 |
:-- | --: | --:
|
92 |
-
`distilgpt2` | 46.
|
93 |
-
`gpt2` | 29.
|
94 |
-
`gpt2-medium` | 21.
|
95 |
-
`gpt2-large` | 19.
|
96 |
-
`gpt2-xl` | 17.
|
97 |
|
98 |
- `perplexity` on `wikitext103`
|
99 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
100 |
:-- | --: | --:
|
101 |
-
`distilgpt2` | 46.
|
102 |
-
`gpt2` | 29.
|
103 |
-
`gpt2-medium` | 21.
|
104 |
-
`gpt2-large` | 19.
|
105 |
-
`gpt2-xl` | 17.
|
|
|
72 |
# device_map="auto", # enabling model parallel on multi-GPU nodes
|
73 |
)
|
74 |
|
75 |
+
results = pipe.eval(metric="d-matrix/dmx_perplexity", dataset="wikitext", dataset_version="wikitext-2-raw-v1")
|
76 |
```
|
77 |
|
78 |
### Evaluation results
|
|
|
80 |
- `perplexity` on `penn_treebank`
|
81 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
82 |
:-- | --: | --:
|
83 |
+
`distilgpt2` | 63.46 | 64.13
|
84 |
+
`gpt2` | 35.77 | 35.93
|
85 |
+
`gpt2-medium` | 27.06 | 27.10
|
86 |
+
`gpt2-large` | 23.03 | 23.04
|
87 |
+
`gpt2-xl` | 21.01 | 21.02
|
88 |
|
89 |
- `perplexity` on `wikitext2`
|
90 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
91 |
:-- | --: | --:
|
92 |
+
`distilgpt2` | 46.06 | 46.44
|
93 |
+
`gpt2` | 29.94 | 30.08
|
94 |
+
`gpt2-medium` | 21.71 | 21.73
|
95 |
+
`gpt2-large` | 19.42| 19.43
|
96 |
+
`gpt2-xl` | 17.40| 17.40
|
97 |
|
98 |
- `perplexity` on `wikitext103`
|
99 |
Revision \ Configuration | **`BASELINE`** | **`BASIC`**
|
100 |
:-- | --: | --:
|
101 |
+
`distilgpt2` | 46.06 | 46.44
|
102 |
+
`gpt2` | 29.94 |30.08
|
103 |
+
`gpt2-medium` | 21.71 | 21.73
|
104 |
+
`gpt2-large` | 19.43 | 19.43
|
105 |
+
`gpt2-xl` | 17.40 | 17.40
|