Add files using upload-large-folder tool

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+tokenizer.json filter=lfs diff=lfs merge=lfs -text

README.md

@@ -0,0 +1,55 @@
+## DFloat11 Compressed Model: `google/gemma-3-12b-it`
+
+This is a **losslessly compressed** version of [`google/gemma-3-12b-it`](https://huggingface.co/google/gemma-3-12b-it) using our custom **DFloat11** format. The model retains **100% output fidelity**: its outputs are **bit-for-bit identical** to the original BF16 model, while reducing GPU memory consumption by approximately **30%**.
+
+### 🔍 How It Works
+
+DFloat11 compresses model weights using **Huffman coding** of exponent bits, paired with **hardware-aware algorithmic designs** for fast on-GPU decompression. During inference, the model weights remain compressed in GPU memory and are **decompressed on-the-fly by a custom CUDA kernel for matrix multiplications**.
+
+This means:
+
+* Only weights are compressed; the architecture, tokenizer, and activations remain unchanged.
+* Decompression is constant-time per forward pass and does **not scale with batch size**, making performance more efficient at larger batch sizes.
+* DFloat11 is **much faster than CPU-offloading**, enabling more efficient inference under resource constraints.
+* Inference at **batch size 1** is \~2× slower than original BF16 on GPU, but the **gap narrows significantly** with larger batches.
+
+### 🔧 How to Use
+
+1. **Install the DFloat11 pip package**
+   *(Installs the CUDA kernel automatically; select based on your CUDA version)*:
+
+   ```bash
+   pip install dfloat11[cuda12]  # or pip install dfloat11[cuda11]
+   ```
+
+2. To use the DFloat11 model, run the following example code in Python:
+
+   ```python
+   import torch
+   from dfloat11 import DFloat11Model
+   from transformers import AutoTokenizer
+
+   model_id = "DFloat11/gemma-3-12b-it-DF11"
+
+   model = DFloat11Model.from_pretrained(model_id, device_map="auto")
+
+   tokenizer = AutoTokenizer.from_pretrained(model_id)
+   tokenizer.pad_token = tokenizer.eos_token
+
+   prompt = "What is a binary tree?"
+   inputs = tokenizer(prompt, return_tensors="pt", padding=True).to(model.device)
+
+   with torch.no_grad():
+      output = model.generate(
+         **inputs,
+         max_new_tokens=256,
+         do_sample=True,
+      )
+
+   print(tokenizer.batch_decode(output, skip_special_tokens=True))
+   ```
+
+### 📄 Learn More
+
+* **Paper**: [70% Size, 100% Accuracy: Lossless LLM Compression for Efficient GPU Inference via Dynamic-Length Float](https://arxiv.org/abs/2504.11651)
+* **GitHub**: [https://github.com/LeanModels/DFloat11](https://github.com/LeanModels/DFloat11)

added_tokens.json

@@ -0,0 +1,3 @@
+{
+  "<image_soft_token>": 262144
+}

config.json

@@ -0,0 +1,80 @@
+{
+  "architectures": [
+    "Gemma3ForConditionalGeneration"
+  ],
+  "boi_token_index": 255999,
+  "dfloat11_config": {
+    "bytes_per_thread": 8,
+    "pattern_dict": {
+      "language_model.model.layers.\\d+": [
+        "self_attn.q_proj",
+        "self_attn.k_proj",
+        "self_attn.v_proj",
+        "self_attn.o_proj",
+        "mlp.gate_proj",
+        "mlp.up_proj",
+        "mlp.down_proj"
+      ]
+    },
+    "threads_per_block": [
+      512
+    ],
+    "version": "0.2.0"
+  },
+  "eoi_token_index": 256000,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "image_token_index": 262144,
+  "initializer_range": 0.02,
+  "mm_tokens_per_image": 256,
+  "model_type": "gemma3",
+  "text_config": {
+    "attention_bias": false,
+    "attention_dropout": 0.0,
+    "attn_logit_softcapping": null,
+    "cache_implementation": "hybrid",
+    "final_logit_softcapping": null,
+    "head_dim": 256,
+    "hidden_activation": "gelu_pytorch_tanh",
+    "hidden_size": 3840,
+    "initializer_range": 0.02,
+    "intermediate_size": 15360,
+    "max_position_embeddings": 131072,
+    "model_type": "gemma3_text",
+    "num_attention_heads": 16,
+    "num_hidden_layers": 48,
+    "num_key_value_heads": 8,
+    "query_pre_attn_scalar": 256,
+    "rms_norm_eps": 1e-06,
+    "rope_local_base_freq": 10000.0,
+    "rope_scaling": {
+      "factor": 8.0,
+      "rope_type": "linear"
+    },
+    "rope_theta": 1000000.0,
+    "sliding_window": 1024,
+    "sliding_window_pattern": 6,
+    "torch_dtype": "bfloat16",
+    "use_cache": true,
+    "vocab_size": 262208
+  },
+  "torch_dtype": "bfloat16",
+  "transformers_version": "4.51.3",
+  "vision_config": {
+    "attention_dropout": 0.0,
+    "hidden_act": "gelu_pytorch_tanh",
+    "hidden_size": 1152,
+    "image_size": 896,
+    "intermediate_size": 4304,
+    "layer_norm_eps": 1e-06,
+    "model_type": "siglip_vision_model",
+    "num_attention_heads": 16,
+    "num_channels": 3,
+    "num_hidden_layers": 27,
+    "patch_size": 14,
+    "torch_dtype": "bfloat16",
+    "vision_use_head": false
+  }
+}

generation_config.json

@@ -0,0 +1,13 @@
+{
+  "bos_token_id": 2,
+  "cache_implementation": "hybrid",
+  "do_sample": true,
+  "eos_token_id": [
+    1,
+    106
+  ],
+  "pad_token_id": 0,
+  "top_k": 64,
+  "top_p": 0.95,
+  "transformers_version": "4.51.3"
+}

language_model_model_layers_0.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:bc88329baf47821701bf8c2f55c30f6692dbc98e3d3963229de8436f3b1eaf81
+size 309043732

language_model_model_layers_1.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef6f89a6f1ed9c032b00013a6a8d9389f3d64641f2e0f9f71be4496324004b6a
+size 308178971

language_model_model_layers_10.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:191acaf8badf708c3c22f00c907b953b82514ea66d2f0688b187f5d39509bd8c
+size 308313955

language_model_model_layers_11.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:afe8d12b2752a7e6ea163619621a6ab0827903e7d1085e7b5073743daeb83701
+size 309660226

language_model_model_layers_12.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:58a61a38819cea4855197be71e84f4b462645e1e17f9995e9431ccb082f9dc0f
+size 308202764

language_model_model_layers_13.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c20d7ad333950fc68acd6ed4fa96af754202a916413cc295028e6422de953c5f
+size 308571043

language_model_model_layers_14.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f1231de9f219c56edeb1dc4e08a3ff7f101d501dd9f073c12abdc6ba01dd4921
+size 309759886

language_model_model_layers_15.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5feeab9c975b61285d14aa3c94fef256ccb718229f38a8a4a22cebbf67c9915f
+size 306200829

language_model_model_layers_16.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:b29949db5766914681ef00674fc69b466fa4e784eb5614cdba60047f9c5ff73e
+size 308803604

language_model_model_layers_17.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2b11746d7ffc9827eb26e15528dd59b46735502fc2566396d68e93478387a562
+size 310854271

language_model_model_layers_18.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:100cc300ccb3510597d37efbdfd09cce7e942b2d9f60208962b5d77aef95188a
+size 307043456

language_model_model_layers_19.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:efd133ac06476335a819e3ca68c28c986a63091ea980a59af385871775512cf2
+size 309141905

language_model_model_layers_2.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c748c6af9dd0f2c7975614239e1b3f7c496fb4f60f8d2496d0bb79b3ca6cee4d
+size 310689893

language_model_model_layers_20.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2f7febeec78f5bc19dd7cc9acbeb6a545cfd4c05598eec4d9c72636632b795ed
+size 310197464

language_model_model_layers_21.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:28adb49efe6d83214147d55d011e1ecded20845d3abc0b2f72487b0ef626f36f
+size 308253507

language_model_model_layers_22.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:0b94dcbfc608dfd6b221656d772599a5619f5c89d8a73f18f6cd7d7de4f56fbb
+size 310294703

language_model_model_layers_23.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9d09bd5cbe44374871a71d1dc0737b2cb3148ddb3a5ddd5e09ea57fb165afd87
+size 307888324

language_model_model_layers_24.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1e33f686faf1142e8e37f1389e61151cedcd58e0615da1c353574b3b7b744b49
+size 307857428

language_model_model_layers_25.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:39f55e8513d317b152330652acccee1c6b126d104e164fb91db84fa6cbb4e1d7
+size 310049613

language_model_model_layers_26.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:e035bde07b44513f1c6fc8aa2b9f2d842d3e230f2ec337c4a4e4089b4513d61b
+size 307092777

language_model_model_layers_27.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:90982f32a2abfe5f758b3441892640cf0483f163264e6713d3263172e7d95855
+size 310598983

language_model_model_layers_28.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:51aa7a1fa605aa87604889dc34d3c6f36e915a7b802016d5450843164c449687
+size 309682857

language_model_model_layers_29.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:de43467d7377a632d6d26a54f2f3fbe747f18e7dcc6cb4bff16059ae0c91a3a1
+size 308732554

language_model_model_layers_3.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2d725e816791b11abcb8a8076cd0b8a478f005c60b894a24f0510976c33c290f
+size 308736069

language_model_model_layers_30.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:36d58ff3536f3978de6af84144a1c17d785e837e2cb1b0bf8075a16a0a194561
+size 306054645

language_model_model_layers_31.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:9de9c7aa280db8fd3b95fcc8b261eb51b3d54c67cd9098ce2582543ab3391f29
+size 307220975

language_model_model_layers_32.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5d219a0477af5cae96db0e30ef544b8c2613af6d1acbab88ba88a3c518128c90
+size 308811767

language_model_model_layers_33.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c78eef7af4c3e22e934d04ec9c2544f5aaffd7c94c1a70162d7987a54ba4f7d9
+size 307286576

language_model_model_layers_34.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:801b4f050c014eb34e43bf4e0b6b408c5afb41fdd1fa6b1ae4b1a77091f6b667
+size 305771890

language_model_model_layers_35.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:46cd06d89bd00d7588896d735fc2a9d270ec8b4e6aadae1378389153142219f9
+size 307380704

language_model_model_layers_36.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:a836836edb92ade56f7e885de996c555723283a5f88337f94c2819158e809bf7
+size 309392920

language_model_model_layers_37.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:56669ca11f439229773df22d6bbae8da6ce32b94d007c95845a46198b8934ff8
+size 310307579

language_model_model_layers_38.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:6df64cd88e5bcd7a028cefb80410362e49c39bb404de1171abf91c0b653b7b43
+size 306513619

language_model_model_layers_39.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:953766561d4e8c62b6367b5824ac53c994e1cca5dcaddce6db3399712f37aa99
+size 305547626

language_model_model_layers_4.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:5f6597057475c0b918b79dadc10dbee83aa0bf3372e14dd547150c2edfda9fd2
+size 310046470

language_model_model_layers_40.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:63bebbb50fe8d16e83ffa4eed569ac4ebb501e079f7e46166929254c251a4bcf
+size 308688045

language_model_model_layers_41.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:c84b3e55d72b239faa1f09db72c9b23fcae4f3e8d64674d8ded40f32afeebc04
+size 307034822

language_model_model_layers_42.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:025225329195820f1827816c7b6ed272815e693eb21bd4fd94adbddf9f6770c1
+size 307103825

language_model_model_layers_43.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:788836d3b478bd4ec72cd6f97554c63fa65ebdf65569543e6cf0bbf2afa1b927
+size 309165050

language_model_model_layers_44.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:f78f25d3d5d6c58738d30a273935ae3bfb5e6fe492eb68ab8d9dcc73bc7d39fc
+size 308256564

language_model_model_layers_45.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:7704c2f4b891d9a1661836a5d8596af443a7e590cd290d3368239b9992ff1961
+size 306293554

language_model_model_layers_46.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:d71ae674ff3a9812225be86d18262c1f070edc2e8a6345ecd8039dc631064d5f
+size 310311366

language_model_model_layers_47.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:94f8221e4b3480067f782bcacb8ec20f238340b35349608b837027596c8d6811
+size 308462750

language_model_model_layers_5.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:748ad365f2c1fa3bd404e900e24ab0ba3c127470e46e0c972b3f0a75da8be72f
+size 307805814

language_model_model_layers_6.safetensors

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:1b89d35ef2fe4144d2f531eb316b9d71046bb17dac59b45079a7286f8d5ba8c6
+size 310581406