Hugging Face's logo

RedHatAI
/
Meta-Llama-3.1-8B-Instruct-quantized.w4a16

Text Generation
Transformers
Safetensors
llama
facebook
meta
llama-3
int4
vllm
chat
neuralmagic
llmcompressor
conversational
4-bit precision
gptq
compressed-tensors
text-generation-inference
4-bit precision
Model card Files
xet
Community
4
alexmarques commited on
Commit
45a8720
·
verified ·
1 Parent(s): 8ecfb5a

Update README.md

Browse files
Files changed (1) hide show
  1. README.md +7 -7
README.md CHANGED
@@ -233,7 +233,7 @@ The results were obtained using the following commands:
233
  ```
234
  lm_eval \
235
  --model vllm \
236
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
237
  --tasks mmlu_llama_3.1_instruct \
238
  --fewshot_as_multiturn \
239
  --apply_chat_template \
@@ -245,7 +245,7 @@ lm_eval \
245
  ```
246
  lm_eval \
247
  --model vllm \
248
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=4064,max_gen_toks=1024,tensor_parallel_size=1 \
249
  --tasks mmlu_cot_0shot_llama_3.1_instruct \
250
  --apply_chat_template \
251
  --num_fewshot 0 \
@@ -256,7 +256,7 @@ lm_eval \
256
  ```
257
  lm_eval \
258
  --model vllm \
259
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=3940,max_gen_toks=100,tensor_parallel_size=1 \
260
  --tasks arc_challenge_llama_3.1_instruct \
261
  --apply_chat_template \
262
  --num_fewshot 0 \
@@ -267,7 +267,7 @@ lm_eval \
267
  ```
268
  lm_eval \
269
  --model vllm \
270
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=4096,max_gen_toks=1024,tensor_parallel_size=1 \
271
  --tasks gsm8k_cot_llama_3.1_instruct \
272
  --fewshot_as_multiturn \
273
  --apply_chat_template \
@@ -279,7 +279,7 @@ lm_eval \
279
  ```
280
  lm_eval \
281
  --model vllm \
282
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
283
  --tasks hellaswag \
284
  --num_fewshot 10 \
285
  --batch_size auto
@@ -289,7 +289,7 @@ lm_eval \
289
  ```
290
  lm_eval \
291
  --model vllm \
292
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
293
  --tasks winogrande \
294
  --num_fewshot 5 \
295
  --batch_size auto
@@ -299,7 +299,7 @@ lm_eval \
299
  ```
300
  lm_eval \
301
  --model vllm \
302
- --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w8a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
303
  --tasks truthfulqa \
304
  --num_fewshot 0 \
305
  --batch_size auto
 
233
  ```
234
  lm_eval \
235
  --model vllm \
236
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=3850,max_gen_toks=10,tensor_parallel_size=1 \
237
  --tasks mmlu_llama_3.1_instruct \
238
  --fewshot_as_multiturn \
239
  --apply_chat_template \
 
245
  ```
246
  lm_eval \
247
  --model vllm \
248
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=4064,max_gen_toks=1024,tensor_parallel_size=1 \
249
  --tasks mmlu_cot_0shot_llama_3.1_instruct \
250
  --apply_chat_template \
251
  --num_fewshot 0 \
 
256
  ```
257
  lm_eval \
258
  --model vllm \
259
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=3940,max_gen_toks=100,tensor_parallel_size=1 \
260
  --tasks arc_challenge_llama_3.1_instruct \
261
  --apply_chat_template \
262
  --num_fewshot 0 \
 
267
  ```
268
  lm_eval \
269
  --model vllm \
270
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=4096,max_gen_toks=1024,tensor_parallel_size=1 \
271
  --tasks gsm8k_cot_llama_3.1_instruct \
272
  --fewshot_as_multiturn \
273
  --apply_chat_template \
 
279
  ```
280
  lm_eval \
281
  --model vllm \
282
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
283
  --tasks hellaswag \
284
  --num_fewshot 10 \
285
  --batch_size auto
 
289
  ```
290
  lm_eval \
291
  --model vllm \
292
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
293
  --tasks winogrande \
294
  --num_fewshot 5 \
295
  --batch_size auto
 
299
  ```
300
  lm_eval \
301
  --model vllm \
302
+ --model_args pretrained="neuralmagic/Meta-Llama-3.1-8B-Instruct-quantized.w4a16",dtype=auto,add_bos_token=True,max_model_len=4096,tensor_parallel_size=1 \
303
  --tasks truthfulqa \
304
  --num_fewshot 0 \
305
  --batch_size auto