0.0.18

.gitattributes

@@ -33,3 +33,5 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+*.so filter=lfs diff=lfs merge=lfs -text
+*.whl filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1 +1,2 @@
 apps.json
+kaggle.json

Dockerfile

@@ -1,13 +1,13 @@
-# build with: docker build . --tag sctg/roco-idefics3:0.0.15 --tag sctg/roco-idefics3:latest --push
+# build with: docker build . --tag sctg/roco-idefics3:0.0.18 --tag sctg/roco-idefics3:latest --push
 # run with 
-# docker run --gpus all --user=42420:42420 -p 7000-8000:7000-8000 -p 8080:8080 -e HF_TOKEN=hf_TOKEN -it sctg/roco-idefics3:0.0.15 bash -i /start.sh sleep infinity
-# docker run --gpus all --user=42420:42420 -p 7000-8000:7000-8000 -p 8080:8080 -it sctg/roco-idefics3:0.0.15 bash -i /start.sh python /learn.py hf_...
+# docker run --gpus all --user=42420:42420 -p 7000-8000:7000-8000 -p 8080:8080 -e WANDB_API_KEY=wdkfjzfjz -e HF_TOKEN=hf_TOKEN -it sctg/roco-idefics3:0.0.18 bash -i /start.sh sleep infinity
+# docker run --gpus all --user=42420:42420 -p 7000-8000:7000-8000 -p 8080:8080 -it sctg/roco-idefics3:0.0.18 bash -i /start.sh python /learn.py hf_...
 FROM cloudflare/cloudflared:latest as cloudflared
-FROM nvidia/cuda:12.6.2-devel-ubuntu22.04
+FROM nvidia/cuda:12.6.2-cudnn-devel-ubuntu22.04
 ARG NODE_MAJOR="20"
 RUN /usr/sbin/addgroup --gid 42420 ovh
 RUN /usr/sbin/useradd -u 42420 --gid 42420 -m -d /workspace -s /bin/bash ovh
-RUN apt update -y && apt-get install -y curl git git-lfs screen sudo \
+RUN apt update -y && apt-get install -y cmake curl git git-lfs screen sudo \
       && mkdir -p /etc/apt/keyrings \
       && curl -fsSL https://deb.nodesource.com/gpgkey/nodesource-repo.gpg.key | gpg --dearmor -o /etc/apt/keyrings/nodesource.gpg \
       && echo "deb [signed-by=/etc/apt/keyrings/nodesource.gpg] https://deb.nodesource.com/node_$NODE_MAJOR.x nodistro main" | tee /etc/apt/sources.list.d/nodesource.list \
@@ -22,12 +22,14 @@ RUN echo "ovh ALL=(ALL) NOPASSWD: ALL" >> /etc/sudoers
 COPY --from=cloudflared /usr/local/bin/cloudflared /usr/local/bin/cloudflared
 USER 42420
 RUN curl -L https://repo.anaconda.com/miniconda/Miniconda3-latest-Linux-x86_64.sh > /workspace/miniconda.sh
+COPY bitsandbytes-0.45.0-cp312-cp312-linux_x86_64.whl /tmp/bitsandbytes-0.45.0-cp312-cp312-linux_x86_64.whl
 RUN /bin/bash /workspace/miniconda.sh -b -p /workspace/.miniconda3
 RUN . /workspace/.miniconda3/bin/activate && conda init --all
 RUN . /workspace/.miniconda3/bin/activate \
     && pip install -U "safetensors>=0.4.5" \
     && pip install -U tensorflow \
-    && pip install -U 'https://github.com/bitsandbytes-foundation/bitsandbytes/releases/download/continuous-release_multi-backend-refactor/bitsandbytes-0.44.1.dev0-py3-none-manylinux_2_24_x86_64.whl' \
+    && pip install -U tf-keras \
+    && pip install -U /tmp/bitsandbytes-0.45.0-cp312-cp312-linux_x86_64.whl \
     && pip install -U git+https://github.com/huggingface/transformers.git\
     && pip install huggingface_hub[cli] accelerate datasets peft\
     && pip install -U Pillow \
@@ -39,12 +41,18 @@ RUN . /workspace/.miniconda3/bin/activate \
     && pip install unsloth\
     && pip install gradio \
     && pip uninstall unsloth -y \
-    && pip install --upgrade --no-cache-dir --no-deps git+https://github.com/unslothai/unsloth.git
+    && pip install --upgrade --no-cache-dir --no-deps git+https://github.com/unslothai/unsloth.git \
+    && pip install wandb
+    
+COPY --chmod=777 libbitsandbytes_cuda124.so /workspace/.miniconda3/lib/python3.12/site-packages/bitsandbytes/libbitsandbytes_cuda124.so
+COPY --chmod=777 libbitsandbytes_cuda126.so /workspace/.miniconda3/lib/python3.12/site-packages/bitsandbytes/libbitsandbytes_cuda126.so
+RUN rm /tmp/bitsandbytes-0.45.0-cp312-cp312-linux_x86_64.whl
 
 RUN . /workspace/.miniconda3/bin/activate && conda install -y jupyter
 RUN rm -f /workspace/miniconda.sh
 RUN mkdir -p /workspace/.config/github-copilot/
 COPY apps.json.enc /workspace/.config/github-copilot/apps.json.enc
+COPY kaggle.json.enc /workspace/.config/kaggle.json.enc
 COPY CFTOKEN.enc /workspace/.config/CFTOKEN.enc
 # Mandatory to run the jobs in rootless mode
 # USER root

ROCO-idefics3.ipynb

@@ -11,58 +11,6 @@
     "The fine-tuning process stores the model checkpoints on a regular basis. Re run the notebook from the last checkpoint to continue the fine-tuning process."
    ]
   },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "2uGjdGkTI78H"
-   },
-   "source": [
-    "## Try to mount Google Drive"
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": 1,
-   "metadata": {
-    "executionInfo": {
-     "elapsed": 2,
-     "status": "aborted",
-     "timestamp": 1730998196191,
-     "user": {
-      "displayName": "Ronan Le Meillat",
-      "userId": "09161391957806824350"
-     },
-     "user_tz": -60
-    },
-    "id": "F-zJG-uPIy3d"
-   },
-   "outputs": [
-    {
-     "ename": "Exception",
-     "evalue": "You are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive",
-     "output_type": "error",
-     "traceback": [
-      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m                       Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[1], line 2\u001b[0m\n\u001b[1;32m      1\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m----> 2\u001b[0m   \u001b[38;5;28;01mimport\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcolab\u001b[39;00m\n\u001b[1;32m      3\u001b[0m   \u001b[38;5;28;01mfrom\u001b[39;00m \u001b[38;5;21;01mgoogle\u001b[39;00m\u001b[38;5;21;01m.\u001b[39;00m\u001b[38;5;21;01mcolab\u001b[39;00m \u001b[38;5;28;01mimport\u001b[39;00m drive\n",
-      "\u001b[0;31mModuleNotFoundError\u001b[0m: No module named 'google'",
-      "\nDuring handling of the above exception, another exception occurred:\n",
-      "\u001b[0;31mException\u001b[0m                                 Traceback (most recent call last)",
-      "Cell \u001b[0;32mIn[1], line 7\u001b[0m\n\u001b[1;32m      4\u001b[0m   drive\u001b[38;5;241m.\u001b[39mmount(\u001b[38;5;124m'\u001b[39m\u001b[38;5;124m/content/drive\u001b[39m\u001b[38;5;124m'\u001b[39m)\n\u001b[1;32m      6\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mModuleNotFoundError\u001b[39;00m:\n\u001b[0;32m----> 7\u001b[0m   \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mYou are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n",
-      "\u001b[0;31mException\u001b[0m: You are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive"
-     ]
-    }
-   ],
-   "source": [
-    "try:\n",
-    "  import google.colab\n",
-    "  from google.colab import drive\n",
-    "  drive.mount('/content/drive')\n",
-    "  \n",
-    "except ModuleNotFoundError:\n",
-    "  raise Exception(\"You are not running this code in Google Colab. Please use Google Colab if you would like to save the model to Google Drive\")"
-   ]
-  },
   {
    "cell_type": "markdown",
    "metadata": {},
@@ -72,7 +20,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 2,
+   "execution_count": 9,
    "metadata": {
     "executionInfo": {
      "elapsed": 1459,
@@ -91,140 +39,128 @@
     "dataset_id = \"eltorio/ROCOv2-radiology\"\n",
     "prompt= \"You are an expert radiologist certified with over 15 years of experience in diagnostic imaging, describe this image\"\n",
     "source_model_id = \"HuggingFaceM4/Idefics3-8B-Llama3\"\n",
-    "destination_model_id = \"eltorio/IDEFICS3_ROCOv2\"\n",
-    "# if Google Drive is mounted, the model will be saved in a folder called IDEFICS3_ROCO in the root of your Google Drive\n",
-    "# else the model will be saved in the current working directory in a folder called IDEFICS3_ROCO\n",
-    "if 'drive' in globals():\n",
-    "    output_dir = \"/content/drive/MyDrive/IDEFICS3_ROCOv2\"\n",
-    "else:\n",
-    "    output_dir = \"IDEFICS3_ROCOv2\""
+    "hugging_face_user = \"eltorio\"\n",
+    "destination_model = \"IDEFICS3_ROCOv2\"\n",
+    "destination_model_id = f\"{hugging_face_user}/{destination_model}\"\n",
+    "output_dir = \"IDEFICS3_ROCOv2\""
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "## Login on Hugging Face"
+    "### Log in Kaggle"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 2,
    "metadata": {},
    "outputs": [],
    "source": [
-    "!git config --global credential.helper store\n",
-    "%pip install huggingface_hub"
+    "import os\n",
+    "import json\n",
+    "if not os.path.exists('/kaggle/.kaggle/kaggle.json'):\n",
+    "    try:\n",
+    "        from kaggle_secrets import UserSecretsClient\n",
+    "        user_secrets = UserSecretsClient()\n",
+    "        KAGGLE_JSON = user_secrets.get_secret(\"KAGGLE_JSON\")\n",
+    "    except:\n",
+    "        KAGGLE_JSON = os.getenv(\"KAGGLE_JSON\")\n",
+    "\n",
+    "    kaggle_dir = os.path.expanduser(\"~/.kaggle\")\n",
+    "    kaggle_file = os.path.join(kaggle_dir, \"kaggle.json\")\n",
+    "\n",
+    "    os.makedirs(kaggle_dir, exist_ok=True)\n",
+    "\n",
+    "    with open(kaggle_file, 'w') as file:\n",
+    "        json.dump(KAGGLE_JSON, file)"
+   ]
+  },
+  {
+   "cell_type": "markdown",
+   "metadata": {},
+   "source": [
+    "### Login WandB"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 10,
    "metadata": {},
    "outputs": [
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Hugging Face token found in environment variable\n"
-     ]
-    },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.\n"
+      "Logged in to W&B\n"
      ]
     }
    ],
    "source": [
-    "from huggingface_hub import login\n",
-    "import os\n",
-    "\n",
-    "HF_TOKEN = \"hf_C…………\"\n",
-    "\n",
-    "if os.environ.get('HF_TOKEN') is not None:\n",
-    "  HF_TOKEN = os.environ.get('HF_TOKEN')\n",
-    "  print(f\"Hugging Face token found in environment variable\")\n",
+    "import wandb\n",
     "try:\n",
-    "  import google.colab\n",
-    "  from google.colab import userdata\n",
-    "  if (userdata.get('HF_TOKEN') is not None) and (HF_TOKEN == \"\"):\n",
-    "    HF_TOKEN = userdata.get('HF_TOKEN')\n",
-    "  else:\n",
-    "    raise ValueError(\"Please set your Hugging Face token in the user data panel, or pass it as an environment variable\")\n",
-    "except ModuleNotFoundError:\n",
-    "  if HF_TOKEN is None:\n",
-    "    raise ValueError(\"Please set your Hugging Face token in the user data panel, or pass it as an environment variable\")\n",
+    "  from kaggle_secrets import UserSecretsClient\n",
+    "  user_secrets = UserSecretsClient()\n",
+    "  WANDB_API_KEY = user_secrets.get_secret(\"WANDB_API_KEY\")\n",
+    "  os.environ[\"WANDB_API_KEY\"] = WANDB_API_KEY\n",
+    "except:\n",
+    "  if os.getenv(\"WANDB_API_KEY\") is None:\n",
+    "    os.environ[\"WANDB_API_KEY\"] = input(\"Enter your W&B API key: \")\n",
     "\n",
-    "login(\n",
-    "  token=HF_TOKEN,\n",
-    "  add_to_git_credential=True\n",
-    ")"
+    "if not wandb.login():\n",
+    "  raise Exception(\"Can't login to W&B\")\n",
+    "else:\n",
+    "  print(\"Logged in to W&B\")\n",
+    "  os.environ[\"WANDB_PROJECT\"]=destination_model"
    ]
   },
   {
    "cell_type": "markdown",
    "metadata": {},
    "source": [
-    "##### Optionally clone the model repository"
+    "## Login on Hugging Face"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 3,
+   "execution_count": 11,
    "metadata": {},
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "Note: Environment variable`HF_TOKEN` is set and is the current active token independently from the token you've just configured.\n"
+     ]
+    },
     {
      "name": "stdout",
      "output_type": "stream",
      "text": [
-      "Cloning into 'IDEFICS3_ROCOv2'...\n",
-      "remote: Enumerating objects: 3, done.\u001b[K\n",
-      "remote: Total 3 (delta 0), reused 0 (delta 0), pack-reused 3 (from 1)\u001b[K\n",
-      "Unpacking objects: 100% (3/3), 1.05 KiB | 1.05 MiB/s, done.\n"
+      "Login with hf_C************usZNcSKWwmtCiuUAgIxFZ\n"
      ]
     }
    ],
    "source": [
-    "# clone Hugging Face model repository\n",
-    "!git clone https://huggingface.co/$destination_model_id $output_dir"
-   ]
-  },
-  {
-   "cell_type": "markdown",
-   "metadata": {
-    "id": "s8QBFDGDAMtD"
-   },
-   "source": [
-    "### Step 1: Install libraries and dependencies."
-   ]
-  },
-  {
-   "cell_type": "code",
-   "execution_count": null,
-   "metadata": {
-    "colab": {
-     "base_uri": "https://localhost:8080/"
-    },
-    "executionInfo": {
-     "elapsed": 33422,
-     "status": "ok",
-     "timestamp": 1730997063833,
-     "user": {
-      "displayName": "Ronan Le Meillat",
-      "userId": "09161391957806824350"
-     },
-     "user_tz": -60
-    },
-    "id": "eLNGOnQtAMtE",
-    "outputId": "65d09e49-f24c-47f3-d7d7-c1a0a0237290"
-   },
-   "outputs": [],
-   "source": [
-    "%pip install -q git+https://github.com/huggingface/transformers.git\n",
-    "%pip install -q accelerate datasets peft\n",
-    "%pip install -q bitsandbytes"
+    "from huggingface_hub import login\n",
+    "import os\n",
+    "\n",
+    "try:\n",
+    "  from kaggle_secrets import UserSecretsClient\n",
+    "  user_secrets = UserSecretsClient()\n",
+    "  HF_TOKEN = user_secrets.get_secret(\"HF_TOKEN\")\n",
+    "  os.environ[\"HF_TOKEN\"] = HF_TOKEN\n",
+    "except:\n",
+    "  if not os.getenv(\"HF_TOKEN\"):\n",
+    "    raise ValueError(\"You need to set the HF_TOKEN environment variable.\")\n",
+    "  HF_TOKEN = os.getenv(\"HF_TOKEN\")\n",
+    "\n",
+    "print(f\"Login with {HF_TOKEN[:4]}{'*'*12}{HF_TOKEN[16:]}\")\n",
+    "login(\n",
+    "  token=HF_TOKEN,\n",
+    "  add_to_git_credential=False\n",
+    ")"
    ]
   },
   {
@@ -233,12 +169,12 @@
     "id": "IjLCnQVEAMtE"
    },
    "source": [
-    "### Step 2: Retrieve the dataset from Hugging Face."
+    "### Step 1: Retrieve the dataset from Hugging Face."
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 4,
+   "execution_count": 12,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -659,7 +595,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c681920ed6e24c35981eda639b1c4458",
+       "model_id": "046c24f13d4f45d7badb65f20c79e973",
        "version_major": 2,
        "version_minor": 0
       },
@@ -673,7 +609,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "c5c781c8755e4ee79f5c972bb786ddda",
+       "model_id": "2e3a624b105e406e98e7fdf1fbfcc6f9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -687,7 +623,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "4ece62cf670f441181bce789dbabb38d",
+       "model_id": "fea5a84e8b884b6d84e4ab42e9b86130",
        "version_major": 2,
        "version_minor": 0
       },
@@ -701,7 +637,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "368b3734b6094270874bc9ed05220b43",
+       "model_id": "bbe912b7f3bf448d8edcb96df599f42a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -715,7 +651,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e002292c8f73473ba28b353594d047a0",
+       "model_id": "5f70bd3f04b0473f940f1c0a66ed412b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -729,7 +665,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "b6ec67734a7644c49c6d222db885d9fa",
+       "model_id": "db4ef432b48540afb5c3750e5ac50408",
        "version_major": 2,
        "version_minor": 0
       },
@@ -743,7 +679,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d04c94230afb42c4b5488d1517c63fd6",
+       "model_id": "9deda09dddc847e38093acfe95719461",
        "version_major": 2,
        "version_minor": 0
       },
@@ -757,7 +693,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0fd069cace5146b8be5b44a89dae32dd",
+       "model_id": "2557fa8a477d4bdbbc6c63b9b4b19f77",
        "version_major": 2,
        "version_minor": 0
       },
@@ -771,7 +707,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e23e8d0ecb394513907c6ecb4a30783d",
+       "model_id": "12ddf4a453a04a60846b2520b9bdc5e4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -785,7 +721,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6d0a484c03274c628585bfd3938dd5dd",
+       "model_id": "835145ef5d21469f97d1475258b398dc",
        "version_major": 2,
        "version_minor": 0
       },
@@ -799,7 +735,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8a9d84a4ba4c4a9d8762845b466159dc",
+       "model_id": "ba78f1edad1749459454ae149982f0a4",
        "version_major": 2,
        "version_minor": 0
       },
@@ -813,7 +749,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ac978b853b8f4ce48b2240adad35813c",
+       "model_id": "d2098ffc181c437dba020d594d0a6fed",
        "version_major": 2,
        "version_minor": 0
       },
@@ -827,7 +763,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "dc39d2085a8748269438f3f518a936fa",
+       "model_id": "bb4710de5e754e119d55bbea6be576ab",
        "version_major": 2,
        "version_minor": 0
       },
@@ -841,7 +777,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "efd5dd9b5acf4a00b9fad256b239fe25",
+       "model_id": "1b8ada1887d648139c2c089f575cbbc5",
        "version_major": 2,
        "version_minor": 0
       },
@@ -855,7 +791,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "0ee7a5cdf9fc44fb95a02a215db064b1",
+       "model_id": "b290432fd1074435a3283fcefc46e6f0",
        "version_major": 2,
        "version_minor": 0
       },
@@ -869,7 +805,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d2bc061d040440d48bd1e068e7d0e754",
+       "model_id": "9f04b5bd31ad4e3ca12555b70959bdd8",
        "version_major": 2,
        "version_minor": 0
       },
@@ -883,7 +819,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "290a81df45914de6865a1fd538746d8b",
+       "model_id": "3d02778907134930b869fc0d602bf04a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -897,7 +833,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "e15f56550122473eaf0b9d2c33defc7a",
+       "model_id": "e242a3f79e4a471ca7221764bb3bc8c8",
        "version_major": 2,
        "version_minor": 0
       },
@@ -911,7 +847,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "219eeafe2fde471d9c524a8f03884678",
+       "model_id": "4af61d3296d344e09f202eb471ed5cf2",
        "version_major": 2,
        "version_minor": 0
       },
@@ -925,7 +861,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "afb3a6d52a6e46268803195d3f7e0e8e",
+       "model_id": "60c368d7fbc54ac28d1b387beea698bf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -939,7 +875,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fe952f2809c24da48e85d56b8de828d9",
+       "model_id": "9c29d19326724a50a3e782ff98d2177f",
        "version_major": 2,
        "version_minor": 0
       },
@@ -953,7 +889,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "09e7e2783b464c6197bebcbe45364e65",
+       "model_id": "16cfe87e688045d18bcd788c07f63292",
        "version_major": 2,
        "version_minor": 0
       },
@@ -967,7 +903,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f2142baf4df04c2e8fac7fda5cc4a4b4",
+       "model_id": "f339fd44a8694e769f6a95d96b0b11c9",
        "version_major": 2,
        "version_minor": 0
       },
@@ -981,7 +917,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ce16d5ccf5a244279e29b26f6b9f159f",
+       "model_id": "dea023c08e31443ab8f06f99aa36a3cf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -995,7 +931,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d53b88d0ad5049e493578571a0d33740",
+       "model_id": "da6fabb9e9ed468b95b140bdc250e667",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1009,7 +945,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "785e664dba3a40edb3858acccf6a07b0",
+       "model_id": "9ca865fa0cff4ab6bf896bd9e359f4ca",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1023,7 +959,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ab300b08c57247d5a972d0d77acfddf4",
+       "model_id": "5b5271ad34af47c3b5d1ca0c7da31632",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1037,7 +973,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "00ab18dd02f34aa3a658b456d8bfe390",
+       "model_id": "21577d910fa14b8ab32c00c0d4e31f9a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1051,7 +987,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "421d90d57c8e44a48479eef2eb40a479",
+       "model_id": "97e0316cd6c0432797fcce4c07c3391b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1065,7 +1001,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "60abcaa9df3d43a99ef1e07e9b7fbe11",
+       "model_id": "4b1deeac66544256b4df8f95baf21b4a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1079,7 +1015,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "79af8edddbfe4543a00e47d5f697866d",
+       "model_id": "257056bdde0445669fe615aaf91f5b13",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1093,7 +1029,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "9d14e22b823d4c9e85b17e4dbd57fec6",
+       "model_id": "12dabb10d7dd4cc080783ff2bbd7fe6f",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1107,7 +1043,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ee5ba070e76b4854aadfac59b0237fbf",
+       "model_id": "f631c0696721488691b50bfb35e26157",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1121,7 +1057,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8ce96862fb234f6ea335071fc8114574",
+       "model_id": "8a3cc5c12bfa40bb94427b7f84b2607b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1135,7 +1071,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "dd5860de108f49e5bcc609bb11a646df",
+       "model_id": "753b01971b4c4b818addcbee07fd5ebd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1149,7 +1085,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "bef0c258e4764f0ab9406467f01752a9",
+       "model_id": "fe9a979c6ce7437dbd58c3b2ed84ce56",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1163,7 +1099,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6c4d53689dad43dd8db780522139f599",
+       "model_id": "bfa00301efbc45d6a776f62b5ee4112b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1177,7 +1113,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "d50075ea0fcd4a34ac88ed121b1e90bf",
+       "model_id": "218ceb240e45427392547e041f045680",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1191,7 +1127,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a4e15de7a1c54400a1ebff02d4caa657",
+       "model_id": "004aa618d2a24aa5966ddaa9afd96001",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1205,7 +1141,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "41fc8cb29962483e81fa8a640f633d46",
+       "model_id": "6a9acf4704f543b593753609da89240c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1219,7 +1155,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "388b29f8655b4fee87db053d591bd72d",
+       "model_id": "39a57445dbcc4335842dadcfcbf5a4ef",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1233,7 +1169,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6c75919875544db98ee2c64215ecff97",
+       "model_id": "4ea49e27202a416ab608cea8cb46cd5c",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1247,7 +1183,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "a4e8b6ecaf734f33a842f60681e23108",
+       "model_id": "14f2007ef3264c7faa55cfcc9effd0e6",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1261,7 +1197,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "7e497cfcd6f04a94aa0ac40d6df938b1",
+       "model_id": "3a3b653ca45e4757a4e3b81c3ab43b87",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1275,7 +1211,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "56cde5f680f643be887d6f9f804676c9",
+       "model_id": "e753298cf52a40b5b3a7052146fd2cfb",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1289,7 +1225,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "fbcf378be86c40cdbf606a2446d1a252",
+       "model_id": "c05e1aa2995c489c84f5bdb09cef3ccd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1303,7 +1239,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "963491aad1f945cfb5811d7239880b54",
+       "model_id": "9a81639720f34fedbca3c19b42c7a9ae",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1334,7 +1270,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 5,
+   "execution_count": 13,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -1362,7 +1298,7 @@
        " 'cui': ['C0037005']}"
       ]
      },
-     "execution_count": 5,
+     "execution_count": 13,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1373,7 +1309,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 6,
+   "execution_count": 14,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1402,7 +1338,7 @@
        "<PIL.PngImagePlugin.PngImageFile image mode=RGB size=1684x2294>"
       ]
      },
-     "execution_count": 6,
+     "execution_count": 14,
      "metadata": {},
      "output_type": "execute_result"
     }
@@ -1422,7 +1358,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 7,
+   "execution_count": 34,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1465,7 +1401,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "f8115bd39ceb47678208bc6dc80a179a",
+       "model_id": "4a7333b9c40e4c9abface9d175b637f7",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1476,10 +1412,22 @@
      "metadata": {},
      "output_type": "display_data"
     },
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "2024-12-05 18:36:11.299502: E external/local_xla/xla/stream_executor/cuda/cuda_fft.cc:477] Unable to register cuFFT factory: Attempting to register factory for plugin cuFFT when one has already been registered\n",
+      "WARNING: All log messages before absl::InitializeLog() is called are written to STDERR\n",
+      "E0000 00:00:1733423771.360739     225 cuda_dnn.cc:8310] Unable to register cuDNN factory: Attempting to register factory for plugin cuDNN when one has already been registered\n",
+      "E0000 00:00:1733423771.378961     225 cuda_blas.cc:1418] Unable to register cuBLAS factory: Attempting to register factory for plugin cuBLAS when one has already been registered\n",
+      "2024-12-05 18:36:11.524136: I tensorflow/core/platform/cpu_feature_guard.cc:210] This TensorFlow binary is optimized to use available CPU instructions in performance-critical operations.\n",
+      "To enable the following instructions: AVX2 FMA, in other operations, rebuild TensorFlow with the appropriate compiler flags.\n"
+     ]
+    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "195ef910bea946cbac27ae80d12ab37d",
+       "model_id": "599872df212a40dba475d0647320f467",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1493,7 +1441,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "ee41668130414ec29bd670a4c00ea9dd",
+       "model_id": "45acf55c884a491bac43dd85ce80df36",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1507,7 +1455,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "cae512d46a20437cb2b9054e6d796129",
+       "model_id": "91a3839713f146889271126a3f2e6b23",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1521,7 +1469,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "15b4f6ea701b40f984b45da569b8fd50",
+       "model_id": "303deff5af0940a18e0f19c68bbd9b6b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1535,7 +1483,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "711ead0dc9ba4017a5755a1cc111d8c3",
+       "model_id": "45ed4ff534e444fab1780873ca453c1a",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1549,7 +1497,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "009a9836643d4a1285ba017cb6dee9fb",
+       "model_id": "94c98750b760459386f52c153724d603",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1560,17 +1508,10 @@
      "metadata": {},
      "output_type": "display_data"
     },
-    {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "`low_cpu_mem_usage` was None, now default to True since model is quantized.\n"
-     ]
-    },
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "1078bb163ca14f4cb242b3afb81b6a70",
+       "model_id": "8337ac6e5e6a4db4982669553b39b8ba",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1584,7 +1525,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "813c7b8d26224b07893f652c1ab25acf",
+       "model_id": "c0e7ab2db08a499bbea8bec1c17857dd",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1598,7 +1539,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "8f312307a8c34e9ebac5a4006cb75b15",
+       "model_id": "71bc274e67de44b5bf14c7113b893db7",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1612,7 +1553,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "6805d6a27b4b404594f644d8289e3e0c",
+       "model_id": "e344829a0ae94e209fc9bc43d794510e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1626,7 +1567,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "dfa588033e244ba0aef4875d4dca0087",
+       "model_id": "da3f4d75fff14d739605a8e9b4aed5cf",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1640,7 +1581,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "de1b8cc27c5744b4b34b48e7a1fb7a00",
+       "model_id": "8e594662af3946f38388ae9d6a22337b",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1654,7 +1595,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "79b23b4c9373457fb28a1cdcc1b23277",
+       "model_id": "7396830f193d4958ae9c2fba9e59bce5",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1668,7 +1609,7 @@
     {
      "data": {
       "application/vnd.jupyter.widget-view+json": {
-       "model_id": "02c68a9944a44dcc882dcaa7722dfa9b",
+       "model_id": "64bc001093534c2fa1d0e2207105419e",
        "version_major": 2,
        "version_minor": 0
       },
@@ -1678,16 +1619,27 @@
      },
      "metadata": {},
      "output_type": "display_data"
+    },
+    {
+     "name": "stdout",
+     "output_type": "stream",
+     "text": [
+      "g++ (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0\n",
+      "Copyright (C) 2021 Free Software Foundation, Inc.\n",
+      "This is free software; see the source for copying conditions.  There is NO\n",
+      "warranty; not even for MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.\n",
+      "\n"
+     ]
     }
    ],
    "source": [
     "import torch\n",
-    "from peft import LoraConfig\n",
+    "from peft import LoraConfig,get_peft_model\n",
     "from transformers import AutoProcessor, BitsAndBytesConfig, Idefics3ForConditionalGeneration\n",
     "\n",
-    "DEVICE = \"cuda:0\"\n",
-    "USE_LORA = False\n",
-    "USE_QLORA = True\n",
+    "DEVICE = \"cuda\"\n",
+    "USE_LORA = True\n",
+    "USE_QLORA = False\n",
     "\n",
     "processor = AutoProcessor.from_pretrained(\n",
     "    source_model_id,\n",
@@ -1715,8 +1667,8 @@
     "        torch_dtype=torch.float16,\n",
     "        quantization_config=bnb_config if USE_QLORA else None,\n",
     "    )\n",
-    "    model.add_adapter(lora_config)\n",
-    "    model.enable_adapters()\n",
+    "    model = get_peft_model(model, lora_config)\n",
+    "    model = model.to(DEVICE)\n",
     "else:\n",
     "    model = Idefics3ForConditionalGeneration.from_pretrained(\n",
     "        source_model_id,\n",
@@ -1736,7 +1688,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 8,
+   "execution_count": 35,
    "metadata": {
     "executionInfo": {
      "elapsed": 426,
@@ -1812,7 +1764,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 9,
+   "execution_count": null,
    "metadata": {
     "executionInfo": {
      "elapsed": 1008,
@@ -1841,7 +1793,7 @@
     "    gradient_accumulation_steps = 8,\n",
     "    dataloader_pin_memory = False,\n",
     "    save_total_limit = 3,\n",
-    "    eval_strategy = \"steps\",\n",
+    "    eval_strategy = \"epoch\",\n",
     "    save_strategy = \"steps\",\n",
     "    eval_steps = 100,\n",
     "    save_steps = 10, # checkpoint each 10 steps\n",
@@ -1851,14 +1803,15 @@
     "    push_to_hub = True,\n",
     "    label_names = [\"labels\"],\n",
     "    load_best_model_at_end = False,\n",
-    "    report_to = \"none\",\n",
+    "    report_to = \"wandb\",\n",
     "    optim = \"paged_adamw_8bit\",\n",
+    "    run_name = destination_model,\n",
     ")"
    ]
   },
   {
    "cell_type": "code",
-   "execution_count": 10,
+   "execution_count": 37,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/"
@@ -1898,7 +1851,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": null,
+   "execution_count": 38,
    "metadata": {
     "colab": {
      "base_uri": "https://localhost:8080/",
@@ -1908,26 +1861,17 @@
     "outputId": "ebb15160-f56e-4899-e608-b0d5fd0ba117"
    },
    "outputs": [
+    {
+     "name": "stderr",
+     "output_type": "stream",
+     "text": [
+      "\u001b[34m\u001b[1mwandb\u001b[0m: \u001b[33mWARNING\u001b[0m The `run_name` is currently set to the same value as `TrainingArguments.output_dir`. If this was not intended, please specify a different run name by setting the `TrainingArguments.run_name` parameter.\n"
+     ]
+    },
     {
      "data": {
       "text/html": [
-       "\n",
-       "    <div>\n",
-       "      \n",
-       "      <progress value='92' max='11241' style='width:300px; height:20px; vertical-align: middle;'></progress>\n",
-       "      [   92/11241 9:38:32 < 1194:28:54, 0.00 it/s, Epoch 0.02/3]\n",
-       "    </div>\n",
-       "    <table border=\"1\" class=\"dataframe\">\n",
-       "  <thead>\n",
-       " <tr style=\"text-align: left;\">\n",
-       "      <th>Step</th>\n",
-       "      <th>Training Loss</th>\n",
-       "      <th>Validation Loss</th>\n",
-       "    </tr>\n",
-       "  </thead>\n",
-       "  <tbody>\n",
-       "  </tbody>\n",
-       "</table><p>"
+       "Changes to your `wandb` environment variables will be ignored because your `wandb` session has already started. For more information on how to modify your settings with `wandb.init()` arguments, please refer to <a href='https://wandb.me/wandb-init' target=\"_blank\">the W&B docs</a>."
       ],
       "text/plain": [
        "<IPython.core.display.HTML object>"
@@ -1937,27 +1881,102 @@
      "output_type": "display_data"
     },
     {
-     "name": "stderr",
-     "output_type": "stream",
-     "text": [
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n",
-      "/workspace/.miniconda3/lib/python3.12/site-packages/transformers/integrations/peft.py:434: FutureWarning: The `active_adapter` method is deprecated and will be removed in a future version.\n",
-      "  warnings.warn(\n"
+     "data": {
+      "text/html": [
+       "Tracking run with wandb version 0.19.0"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Run data is saved locally in <code>/workspace/wandb/run-20241205_190533-9ckd0brc</code>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       "Syncing run <strong><a href='https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2/runs/9ckd0brc' target=\"_blank\">IDEFICS3_ROCOv2</a></strong> to <a href='https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2' target=\"_blank\">Weights & Biases</a> (<a href='https://wandb.me/developer-guide' target=\"_blank\">docs</a>)<br/>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View project at <a href='https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2' target=\"_blank\">https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "data": {
+      "text/html": [
+       " View run at <a href='https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2/runs/9ckd0brc' target=\"_blank\">https://wandb.ai/aeltorio-ac-lille/IDEFICS3_ROCOv2/runs/9ckd0brc</a>"
+      ],
+      "text/plain": [
+       "<IPython.core.display.HTML object>"
+      ]
+     },
+     "metadata": {},
+     "output_type": "display_data"
+    },
+    {
+     "ename": "KeyboardInterrupt",
+     "evalue": "",
+     "output_type": "error",
+     "traceback": [
+      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m                         Traceback (most recent call last)",
+      "Cell \u001b[0;32mIn[38], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m trainer\u001b[38;5;241m.\u001b[39mtrain()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/trainer.py:2157\u001b[0m, in \u001b[0;36mTrainer.train\u001b[0;34m(self, resume_from_checkpoint, trial, ignore_keys_for_eval, **kwargs)\u001b[0m\n\u001b[1;32m   2154\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[1;32m   2155\u001b[0m     \u001b[38;5;66;03m# Disable progress bars when uploading models during checkpoints to avoid polluting stdout\u001b[39;00m\n\u001b[1;32m   2156\u001b[0m     hf_hub_utils\u001b[38;5;241m.\u001b[39mdisable_progress_bars()\n\u001b[0;32m-> 2157\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m inner_training_loop(\n\u001b[1;32m   2158\u001b[0m         args\u001b[38;5;241m=\u001b[39margs,\n\u001b[1;32m   2159\u001b[0m         resume_from_checkpoint\u001b[38;5;241m=\u001b[39mresume_from_checkpoint,\n\u001b[1;32m   2160\u001b[0m         trial\u001b[38;5;241m=\u001b[39mtrial,\n\u001b[1;32m   2161\u001b[0m         ignore_keys_for_eval\u001b[38;5;241m=\u001b[39mignore_keys_for_eval,\n\u001b[1;32m   2162\u001b[0m     )\n\u001b[1;32m   2163\u001b[0m \u001b[38;5;28;01mfinally\u001b[39;00m:\n\u001b[1;32m   2164\u001b[0m     hf_hub_utils\u001b[38;5;241m.\u001b[39menable_progress_bars()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/accelerate/utils/memory.py:158\u001b[0m, in \u001b[0;36mfind_executable_batch_size.<locals>.decorator\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    156\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m \u001b[38;5;167;01mRuntimeError\u001b[39;00m(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mNo executable batch size found, reached zero.\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    157\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 158\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m function(batch_size, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m    159\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m \u001b[38;5;167;01mException\u001b[39;00m \u001b[38;5;28;01mas\u001b[39;00m e:\n\u001b[1;32m    160\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m should_reduce_batch_size(e):\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/trainer.py:2524\u001b[0m, in \u001b[0;36mTrainer._inner_training_loop\u001b[0;34m(self, batch_size, args, resume_from_checkpoint, trial, ignore_keys_for_eval)\u001b[0m\n\u001b[1;32m   2518\u001b[0m context \u001b[38;5;241m=\u001b[39m (\n\u001b[1;32m   2519\u001b[0m     functools\u001b[38;5;241m.\u001b[39mpartial(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39maccelerator\u001b[38;5;241m.\u001b[39mno_sync, model\u001b[38;5;241m=\u001b[39mmodel)\n\u001b[1;32m   2520\u001b[0m     \u001b[38;5;28;01mif\u001b[39;00m i \u001b[38;5;241m!=\u001b[39m \u001b[38;5;28mlen\u001b[39m(batch_samples) \u001b[38;5;241m-\u001b[39m \u001b[38;5;241m1\u001b[39m\n\u001b[1;32m   2521\u001b[0m     \u001b[38;5;28;01melse\u001b[39;00m contextlib\u001b[38;5;241m.\u001b[39mnullcontext\n\u001b[1;32m   2522\u001b[0m )\n\u001b[1;32m   2523\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m context():\n\u001b[0;32m-> 2524\u001b[0m     tr_loss_step \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtraining_step(model, inputs, num_items_in_batch)\n\u001b[1;32m   2526\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   2527\u001b[0m     args\u001b[38;5;241m.\u001b[39mlogging_nan_inf_filter\n\u001b[1;32m   2528\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m is_torch_xla_available()\n\u001b[1;32m   2529\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m (torch\u001b[38;5;241m.\u001b[39misnan(tr_loss_step) \u001b[38;5;129;01mor\u001b[39;00m torch\u001b[38;5;241m.\u001b[39misinf(tr_loss_step))\n\u001b[1;32m   2530\u001b[0m ):\n\u001b[1;32m   2531\u001b[0m     \u001b[38;5;66;03m# if loss is nan or inf simply add the average of previous logged losses\u001b[39;00m\n\u001b[1;32m   2532\u001b[0m     tr_loss \u001b[38;5;241m=\u001b[39m tr_loss \u001b[38;5;241m+\u001b[39m tr_loss \u001b[38;5;241m/\u001b[39m (\u001b[38;5;241m1\u001b[39m \u001b[38;5;241m+\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m-\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_globalstep_last_logged)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/trainer.py:3654\u001b[0m, in \u001b[0;36mTrainer.training_step\u001b[0;34m(self, model, inputs, num_items_in_batch)\u001b[0m\n\u001b[1;32m   3651\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m loss_mb\u001b[38;5;241m.\u001b[39mreduce_mean()\u001b[38;5;241m.\u001b[39mdetach()\u001b[38;5;241m.\u001b[39mto(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mdevice)\n\u001b[1;32m   3653\u001b[0m \u001b[38;5;28;01mwith\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss_context_manager():\n\u001b[0;32m-> 3654\u001b[0m     loss \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcompute_loss(model, inputs, num_items_in_batch\u001b[38;5;241m=\u001b[39mnum_items_in_batch)\n\u001b[1;32m   3656\u001b[0m \u001b[38;5;28;01mdel\u001b[39;00m inputs\n\u001b[1;32m   3657\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m (\n\u001b[1;32m   3658\u001b[0m     \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mtorch_empty_cache_steps \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   3659\u001b[0m     \u001b[38;5;129;01mand\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mstate\u001b[38;5;241m.\u001b[39mglobal_step \u001b[38;5;241m%\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mtorch_empty_cache_steps \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m\n\u001b[1;32m   3660\u001b[0m ):\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/trainer.py:3708\u001b[0m, in \u001b[0;36mTrainer.compute_loss\u001b[0;34m(self, model, inputs, return_outputs, num_items_in_batch)\u001b[0m\n\u001b[1;32m   3706\u001b[0m         loss_kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mnum_items_in_batch\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m num_items_in_batch\n\u001b[1;32m   3707\u001b[0m     inputs \u001b[38;5;241m=\u001b[39m {\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minputs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mloss_kwargs}\n\u001b[0;32m-> 3708\u001b[0m outputs \u001b[38;5;241m=\u001b[39m model(\u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39minputs)\n\u001b[1;32m   3709\u001b[0m \u001b[38;5;66;03m# Save past state if it exists\u001b[39;00m\n\u001b[1;32m   3710\u001b[0m \u001b[38;5;66;03m# TODO: this needs to be fixed and made cleaner later.\u001b[39;00m\n\u001b[1;32m   3711\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39margs\u001b[38;5;241m.\u001b[39mpast_index \u001b[38;5;241m>\u001b[39m\u001b[38;5;241m=\u001b[39m \u001b[38;5;241m0\u001b[39m:\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/accelerate/utils/operations.py:823\u001b[0m, in \u001b[0;36mconvert_outputs_to_fp32.<locals>.forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    822\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 823\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m model_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/accelerate/utils/operations.py:811\u001b[0m, in \u001b[0;36mConvertOutputsToFp32.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 811\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m convert_to_fp32(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/amp/autocast_mode.py:44\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 44\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/accelerate/utils/operations.py:823\u001b[0m, in \u001b[0;36mconvert_outputs_to_fp32.<locals>.forward\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m    822\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mforward\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 823\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m model_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/accelerate/utils/operations.py:811\u001b[0m, in \u001b[0;36mConvertOutputsToFp32.__call__\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m    810\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21m__call__\u001b[39m(\u001b[38;5;28mself\u001b[39m, \u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[0;32m--> 811\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m convert_to_fp32(\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel_forward(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs))\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/amp/autocast_mode.py:44\u001b[0m, in \u001b[0;36mautocast_decorator.<locals>.decorate_autocast\u001b[0;34m(*args, **kwargs)\u001b[0m\n\u001b[1;32m     41\u001b[0m \u001b[38;5;129m@functools\u001b[39m\u001b[38;5;241m.\u001b[39mwraps(func)\n\u001b[1;32m     42\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mdecorate_autocast\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs):\n\u001b[1;32m     43\u001b[0m     \u001b[38;5;28;01mwith\u001b[39;00m autocast_instance:\n\u001b[0;32m---> 44\u001b[0m         \u001b[38;5;28;01mreturn\u001b[39;00m func(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/idefics3/modeling_idefics3.py:1196\u001b[0m, in \u001b[0;36mIdefics3ForConditionalGeneration.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, pixel_values, pixel_attention_mask, image_hidden_states, labels, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1193\u001b[0m return_dict \u001b[38;5;241m=\u001b[39m return_dict \u001b[38;5;28;01mif\u001b[39;00m return_dict \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;28;01melse\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mconfig\u001b[38;5;241m.\u001b[39muse_return_dict\n\u001b[1;32m   1195\u001b[0m \u001b[38;5;66;03m# decoder outputs consists of (dec_features, layer_state, dec_hidden, dec_attn)\u001b[39;00m\n\u001b[0;32m-> 1196\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mmodel(\n\u001b[1;32m   1197\u001b[0m     input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[1;32m   1198\u001b[0m     attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[1;32m   1199\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[1;32m   1200\u001b[0m     past_key_values\u001b[38;5;241m=\u001b[39mpast_key_values,\n\u001b[1;32m   1201\u001b[0m     inputs_embeds\u001b[38;5;241m=\u001b[39minputs_embeds,\n\u001b[1;32m   1202\u001b[0m     pixel_values\u001b[38;5;241m=\u001b[39mpixel_values,\n\u001b[1;32m   1203\u001b[0m     pixel_attention_mask\u001b[38;5;241m=\u001b[39mpixel_attention_mask,\n\u001b[1;32m   1204\u001b[0m     image_hidden_states\u001b[38;5;241m=\u001b[39mimage_hidden_states,\n\u001b[1;32m   1205\u001b[0m     use_cache\u001b[38;5;241m=\u001b[39muse_cache,\n\u001b[1;32m   1206\u001b[0m     output_attentions\u001b[38;5;241m=\u001b[39moutput_attentions,\n\u001b[1;32m   1207\u001b[0m     output_hidden_states\u001b[38;5;241m=\u001b[39moutput_hidden_states,\n\u001b[1;32m   1208\u001b[0m     return_dict\u001b[38;5;241m=\u001b[39mreturn_dict,\n\u001b[1;32m   1209\u001b[0m )\n\u001b[1;32m   1211\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m   1212\u001b[0m logits \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mlm_head(hidden_states)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/idefics3/modeling_idefics3.py:1020\u001b[0m, in \u001b[0;36mIdefics3Model.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, pixel_values, pixel_attention_mask, image_hidden_states, use_cache, output_attentions, output_hidden_states, return_dict)\u001b[0m\n\u001b[1;32m   1011\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m past_seen_tokens \u001b[38;5;241m==\u001b[39m \u001b[38;5;241m0\u001b[39m \u001b[38;5;129;01mand\u001b[39;00m inputs_embeds \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m \u001b[38;5;129;01mand\u001b[39;00m image_hidden_states \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m   1012\u001b[0m     \u001b[38;5;66;03m# When we generate, we don't want to replace the potential image_token_id that we generated by images\u001b[39;00m\n\u001b[1;32m   1013\u001b[0m     \u001b[38;5;66;03m# that simply don't exist\u001b[39;00m\n\u001b[1;32m   1014\u001b[0m     inputs_embeds \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minputs_merger(\n\u001b[1;32m   1015\u001b[0m         input_ids\u001b[38;5;241m=\u001b[39minput_ids,\n\u001b[1;32m   1016\u001b[0m         inputs_embeds\u001b[38;5;241m=\u001b[39minputs_embeds,\n\u001b[1;32m   1017\u001b[0m         image_hidden_states\u001b[38;5;241m=\u001b[39mimage_hidden_states,\n\u001b[1;32m   1018\u001b[0m     )\n\u001b[0;32m-> 1020\u001b[0m outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mtext_model(\n\u001b[1;32m   1021\u001b[0m     inputs_embeds\u001b[38;5;241m=\u001b[39minputs_embeds,\n\u001b[1;32m   1022\u001b[0m     attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[1;32m   1023\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[1;32m   1024\u001b[0m     past_key_values\u001b[38;5;241m=\u001b[39mpast_key_values,\n\u001b[1;32m   1025\u001b[0m     use_cache\u001b[38;5;241m=\u001b[39muse_cache,\n\u001b[1;32m   1026\u001b[0m     output_attentions\u001b[38;5;241m=\u001b[39moutput_attentions,\n\u001b[1;32m   1027\u001b[0m     output_hidden_states\u001b[38;5;241m=\u001b[39moutput_hidden_states,\n\u001b[1;32m   1028\u001b[0m     return_dict\u001b[38;5;241m=\u001b[39mreturn_dict,\n\u001b[1;32m   1029\u001b[0m )\n\u001b[1;32m   1031\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m return_dict:\n\u001b[1;32m   1032\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mtuple\u001b[39m(v \u001b[38;5;28;01mfor\u001b[39;00m v \u001b[38;5;129;01min\u001b[39;00m [\u001b[38;5;241m*\u001b[39moutputs, image_hidden_states] \u001b[38;5;28;01mif\u001b[39;00m v \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/llama/modeling_llama.py:913\u001b[0m, in \u001b[0;36mLlamaModel.forward\u001b[0;34m(self, input_ids, attention_mask, position_ids, past_key_values, inputs_embeds, use_cache, output_attentions, output_hidden_states, return_dict, cache_position, **flash_attn_kwargs)\u001b[0m\n\u001b[1;32m    901\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_gradient_checkpointing_func(\n\u001b[1;32m    902\u001b[0m         decoder_layer\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__call__\u001b[39m,\n\u001b[1;32m    903\u001b[0m         hidden_states,\n\u001b[0;32m   (...)\u001b[0m\n\u001b[1;32m    910\u001b[0m         position_embeddings,\n\u001b[1;32m    911\u001b[0m     )\n\u001b[1;32m    912\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m--> 913\u001b[0m     layer_outputs \u001b[38;5;241m=\u001b[39m decoder_layer(\n\u001b[1;32m    914\u001b[0m         hidden_states,\n\u001b[1;32m    915\u001b[0m         attention_mask\u001b[38;5;241m=\u001b[39mcausal_mask,\n\u001b[1;32m    916\u001b[0m         position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[1;32m    917\u001b[0m         past_key_value\u001b[38;5;241m=\u001b[39mpast_key_values,\n\u001b[1;32m    918\u001b[0m         output_attentions\u001b[38;5;241m=\u001b[39moutput_attentions,\n\u001b[1;32m    919\u001b[0m         use_cache\u001b[38;5;241m=\u001b[39muse_cache,\n\u001b[1;32m    920\u001b[0m         cache_position\u001b[38;5;241m=\u001b[39mcache_position,\n\u001b[1;32m    921\u001b[0m         position_embeddings\u001b[38;5;241m=\u001b[39mposition_embeddings,\n\u001b[1;32m    922\u001b[0m         \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mflash_attn_kwargs,\n\u001b[1;32m    923\u001b[0m     )\n\u001b[1;32m    925\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m layer_outputs[\u001b[38;5;241m0\u001b[39m]\n\u001b[1;32m    927\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m use_cache:\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/llama/modeling_llama.py:640\u001b[0m, in \u001b[0;36mLlamaDecoderLayer.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, position_embeddings, **kwargs)\u001b[0m\n\u001b[1;32m    637\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39minput_layernorm(hidden_states)\n\u001b[1;32m    639\u001b[0m \u001b[38;5;66;03m# Self Attention\u001b[39;00m\n\u001b[0;32m--> 640\u001b[0m hidden_states, self_attn_weights, present_key_value \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mself_attn(\n\u001b[1;32m    641\u001b[0m     hidden_states\u001b[38;5;241m=\u001b[39mhidden_states,\n\u001b[1;32m    642\u001b[0m     attention_mask\u001b[38;5;241m=\u001b[39mattention_mask,\n\u001b[1;32m    643\u001b[0m     position_ids\u001b[38;5;241m=\u001b[39mposition_ids,\n\u001b[1;32m    644\u001b[0m     past_key_value\u001b[38;5;241m=\u001b[39mpast_key_value,\n\u001b[1;32m    645\u001b[0m     output_attentions\u001b[38;5;241m=\u001b[39moutput_attentions,\n\u001b[1;32m    646\u001b[0m     use_cache\u001b[38;5;241m=\u001b[39muse_cache,\n\u001b[1;32m    647\u001b[0m     cache_position\u001b[38;5;241m=\u001b[39mcache_position,\n\u001b[1;32m    648\u001b[0m     position_embeddings\u001b[38;5;241m=\u001b[39mposition_embeddings,\n\u001b[1;32m    649\u001b[0m     \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs,\n\u001b[1;32m    650\u001b[0m )\n\u001b[1;32m    651\u001b[0m hidden_states \u001b[38;5;241m=\u001b[39m residual \u001b[38;5;241m+\u001b[39m hidden_states\n\u001b[1;32m    653\u001b[0m \u001b[38;5;66;03m# Fully Connected\u001b[39;00m\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1736\u001b[0m, in \u001b[0;36mModule._wrapped_call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1734\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_compiled_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)  \u001b[38;5;66;03m# type: ignore[misc]\u001b[39;00m\n\u001b[1;32m   1735\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[0;32m-> 1736\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_call_impl(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/torch/nn/modules/module.py:1747\u001b[0m, in \u001b[0;36mModule._call_impl\u001b[0;34m(self, *args, **kwargs)\u001b[0m\n\u001b[1;32m   1742\u001b[0m \u001b[38;5;66;03m# If we don't have any hooks, we want to skip the rest of the logic in\u001b[39;00m\n\u001b[1;32m   1743\u001b[0m \u001b[38;5;66;03m# this function, and just call forward.\u001b[39;00m\n\u001b[1;32m   1744\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m (\u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_forward_pre_hooks\n\u001b[1;32m   1745\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_backward_pre_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_backward_hooks\n\u001b[1;32m   1746\u001b[0m         \u001b[38;5;129;01mor\u001b[39;00m _global_forward_hooks \u001b[38;5;129;01mor\u001b[39;00m _global_forward_pre_hooks):\n\u001b[0;32m-> 1747\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m forward_call(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[1;32m   1749\u001b[0m result \u001b[38;5;241m=\u001b[39m \u001b[38;5;28;01mNone\u001b[39;00m\n\u001b[1;32m   1750\u001b[0m called_always_called_hooks \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mset\u001b[39m()\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/llama/modeling_llama.py:541\u001b[0m, in \u001b[0;36mLlamaSdpaAttention.forward\u001b[0;34m(self, hidden_states, attention_mask, position_ids, past_key_value, output_attentions, use_cache, cache_position, position_embeddings, **kwargs)\u001b[0m\n\u001b[1;32m    539\u001b[0m \u001b[38;5;28;01melse\u001b[39;00m:\n\u001b[1;32m    540\u001b[0m     cos, sin \u001b[38;5;241m=\u001b[39m position_embeddings\n\u001b[0;32m--> 541\u001b[0m query_states, key_states \u001b[38;5;241m=\u001b[39m apply_rotary_pos_emb(query_states, key_states, cos, sin)\n\u001b[1;32m    543\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m past_key_value \u001b[38;5;129;01mis\u001b[39;00m \u001b[38;5;129;01mnot\u001b[39;00m \u001b[38;5;28;01mNone\u001b[39;00m:\n\u001b[1;32m    544\u001b[0m     \u001b[38;5;66;03m# sin and cos are specific to RoPE models; cache_position needed for the static cache\u001b[39;00m\n\u001b[1;32m    545\u001b[0m     cache_kwargs \u001b[38;5;241m=\u001b[39m {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124msin\u001b[39m\u001b[38;5;124m\"\u001b[39m: sin, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcos\u001b[39m\u001b[38;5;124m\"\u001b[39m: cos, \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mcache_position\u001b[39m\u001b[38;5;124m\"\u001b[39m: cache_position}\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/llama/modeling_llama.py:226\u001b[0m, in \u001b[0;36mapply_rotary_pos_emb\u001b[0;34m(q, k, cos, sin, position_ids, unsqueeze_dim)\u001b[0m\n\u001b[1;32m    224\u001b[0m sin \u001b[38;5;241m=\u001b[39m sin\u001b[38;5;241m.\u001b[39munsqueeze(unsqueeze_dim)\n\u001b[1;32m    225\u001b[0m q_embed \u001b[38;5;241m=\u001b[39m (q \u001b[38;5;241m*\u001b[39m cos) \u001b[38;5;241m+\u001b[39m (rotate_half(q) \u001b[38;5;241m*\u001b[39m sin)\n\u001b[0;32m--> 226\u001b[0m k_embed \u001b[38;5;241m=\u001b[39m (k \u001b[38;5;241m*\u001b[39m cos) \u001b[38;5;241m+\u001b[39m (rotate_half(k) \u001b[38;5;241m*\u001b[39m sin)\n\u001b[1;32m    227\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m q_embed, k_embed\n",
+      "File \u001b[0;32m~/.miniconda3/lib/python3.12/site-packages/transformers/models/llama/modeling_llama.py:196\u001b[0m, in \u001b[0;36mrotate_half\u001b[0;34m(x)\u001b[0m\n\u001b[1;32m    192\u001b[0m         kwargs[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mrope_type\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mdynamic\u001b[39m\u001b[38;5;124m\"\u001b[39m\n\u001b[1;32m    193\u001b[0m         \u001b[38;5;28msuper\u001b[39m()\u001b[38;5;241m.\u001b[39m\u001b[38;5;21m__init__\u001b[39m(\u001b[38;5;241m*\u001b[39margs, \u001b[38;5;241m*\u001b[39m\u001b[38;5;241m*\u001b[39mkwargs)\n\u001b[0;32m--> 196\u001b[0m \u001b[38;5;28;01mdef\u001b[39;00m \u001b[38;5;21mrotate_half\u001b[39m(x):\n\u001b[1;32m    197\u001b[0m \u001b[38;5;250m    \u001b[39m\u001b[38;5;124;03m\"\"\"Rotates half the hidden dims of the input.\"\"\"\u001b[39;00m\n\u001b[1;32m    198\u001b[0m     x1 \u001b[38;5;241m=\u001b[39m x[\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m\u001b[38;5;241m.\u001b[39m, : x\u001b[38;5;241m.\u001b[39mshape[\u001b[38;5;241m-\u001b[39m\u001b[38;5;241m1\u001b[39m] \u001b[38;5;241m/\u001b[39m\u001b[38;5;241m/\u001b[39m \u001b[38;5;241m2\u001b[39m]\n",
+      "\u001b[0;31mKeyboardInterrupt\u001b[0m: "
      ]
     }
    ],
@@ -1967,7 +1986,7 @@
   },
   {
    "cell_type": "code",
-   "execution_count": 11,
+   "execution_count": null,
    "metadata": {},
    "outputs": [
     {
@@ -2005,7 +2024,7 @@
     }
    ],
    "source": [
-    "model = Idefics3ForConditionalGeneration.from_pretrained(source_model_id , torch_dtype=torch.bfloat16).to(DEVICE)\n",
+    "model = Idefics3ForConditionalGeneration.from_pretrained(source_model_id , torch_dtype=torch.float16).to(DEVICE)\n",
     "model.load_adapter(destination_model_id, device_map=\"auto\")"
    ]
   },

bitsandbytes-0.45.0-cp312-cp312-linux_x86_64.whl

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:8904b74767c5a060b0a8fbcaa8f47017a49e005a3e9e17dad2ca170a85f3f99a
+size 10850559

kaggle.json.enc

@@ -0,0 +1,2 @@
+U2FsdGVkX1+BifIqzv3coYZFjOTt55X+lVyUHpD9N9V77Vrnsh4R8wt3OkUoA6Vf
+IY7T8RsNfr7Salx0RFUk/m3QTfHmLKXtVwsBzilKZJY=

libbitsandbytes_cuda124.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:41c9698af520efd7212d164e394aadb42d45b5fdee0b36ed31ae25422ac29c30
+size 34242400

libbitsandbytes_cuda126.so

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:2525070006f24fdfc5221e2e5bf63f85417298403ea89d80dbbf255884122990
+size 34536168

start.sh

@@ -37,6 +37,14 @@ if [ -f /workspace/.config/CFTOKEN.enc ]; then
   /usr/local/bin/cloudflared --pidfile /tmp/cf.pid  --autoupdate-freq 24h0m0s tunnel run --token $CFTOKEN &
 fi
 
+# Decode kaggle.json.enc file
+# encoded with: openssl aes-256-cbc -base64 -md sha256 -pass pass:"$HF_TOKEN" -in kaggle.json -out kaggle.json.enc
+# decode with: openssl aes-256-cbc -a -d -md sha256 -pass pass:"$HF_TOKEN" -in kaggle.json.enc -out kaggle.json
+if [ -f /workspace/.config/kaggle.json.enc ]; then
+  mkdir -p $HOME/.kaggle
+  openssl aes-256-cbc -a -d -md sha256 -pass pass:"$HF_TOKEN" -in /workspace/.config/kaggle.json.enc -out /workspace/.kaggle/kaggle.json
+fi
+
 git clone https://huggingface.co/eltorio/IDEFICS3_ROCOv2
 git config --global user.email "[email protected]"
 git config --global user.name "[email protected]"