Upload MixtralForCausalLM

config.json

@@ -0,0 +1,46 @@
+{
+ "_name_or_path": "HuggingFaceH4/zephyr-orpo-141b-A35b-v0.1",
+ "architectures": [
+ "MixtralForCausalLM"
+ ],
+ "attention_dropout": 0.0,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "hidden_act": "silu",
+ "hidden_size": 6144,
+ "initializer_range": 0.02,
+ "intermediate_size": 16384,
+ "max_position_embeddings": 65536,
+ "model_type": "mixtral",
+ "num_attention_heads": 48,
+ "num_experts_per_tok": 2,
+ "num_hidden_layers": 56,
+ "num_key_value_heads": 8,
+ "num_local_experts": 8,
+ "output_router_logits": false,
+ "quantization_config": {
+ "_load_in_4bit": true,
+ "_load_in_8bit": false,
+ "bnb_4bit_compute_dtype": "float32",
+ "bnb_4bit_quant_storage": "uint8",
+ "bnb_4bit_quant_type": "fp4",
+ "bnb_4bit_use_double_quant": false,
+ "llm_int8_enable_fp32_cpu_offload": false,
+ "llm_int8_has_fp16_weight": false,
+ "llm_int8_skip_modules": null,
+ "llm_int8_threshold": 6.0,
+ "load_in_4bit": true,
+ "load_in_8bit": false,
+ "quant_method": "bitsandbytes"
+ },
+ "rms_norm_eps": 1e-05,
+ "rope_theta": 1000000,
+ "router_aux_loss_coef": 0.001,
+ "router_jitter_noise": 0.0,
+ "sliding_window": null,
+ "tie_word_embeddings": false,
+ "torch_dtype": "float16",
+ "transformers_version": "4.40.0.dev0",
+ "use_cache": true,
+ "vocab_size": 32000
+}

generation_config.json

@@ -0,0 +1,6 @@
+{
+ "_from_model_config": true,
+ "bos_token_id": 1,
+ "eos_token_id": 2,
+ "transformers_version": "4.40.0.dev0"
+}

model-00001-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2fb849d8c50c706bd53a5d3f846a22c1c5400aa5d5986dd41a1e2d6692019115
+size 4951626618

model-00002-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:38ffc7c25d66675bf64f51a8cd494b4ae86ee1e9c92c1a3fb46c91f51f4291ad
+size 4961824784

model-00003-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0a473dd813bef1a762afef2b484ade87cf285c18510b271e43c47597e717987c
+size 4954801840

model-00004-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:09af1d3bb8c947c7419c91ca79ff47ade0d6ccb1afe934ad6f13852fea19a632
+size 4961825192

model-00005-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:da2872347bb2ef118c87c0f614993a4d4d1f9948e2e10c516c330a6a689fce0b
+size 4954802256

model-00006-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0c6179b62993674bcf85b2f69d1b938dec3202e69d4423277a0f5c20cd83c2f4
+size 4961825192

model-00007-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fa529c2f6b145fb52a2554ac7b67e7d1c0a4bf9066ac25d4a5ac301e8d84adf6
+size 4954802256

model-00008-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:fc9ded0880967c3c4fbb11f5d9eb0416f9c76264ea20614bd75abf5ac94d70d0
+size 4961825192

model-00009-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5dcac14937afd8744a2404177ad3fc0400af8a89f82f424dcdf4e547404d2c7d
+size 4954802256

model-00010-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0adbb11ef8413619d47953d8f79015fe4e6707f3652881fe53c28afd756db4ee
+size 4961825192

model-00011-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:804e7a905a6826b21b252fc8341b5a1de0942778ce6eecaa8c41c9186dea029f
+size 4954802256

model-00012-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7846742154eaf7ecbcdf540ecf93d3ca7af0073f4d6b9ff8cc574be646a6a33d
+size 4961825192

model-00013-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:701f2d9375c7faa32c07a11547414fa15c95fecfb796999a4b497ff51a2e06f2
+size 4954802256

model-00014-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6b0731d0ef5165ec76c7a473602272662fb638ca74f3359a42cc5ab830aea6d1
+size 4990163459

model-00015-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:062db6e6546c37e3a45485c0abcd62f273b14d61c69aaab466f9fe2043f09b1c
+size 4983087791

model-00016-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f66c9aab5928deb8a5725e665fa11e1013cc4b67baed9e08659405071c0e7cb7
+size 4848614772

model-00017-of-00017.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e110af919cb6ea1a47ae33dea0ee3e597b6ed9bd851632a34aea547bf3d4a269
+size 393216128