Upload folder using huggingface_hub

.gitattributes

@@ -33,3 +33,4 @@ saved_model/**/* filter=lfs diff=lfs merge=lfs -text
 *.zip filter=lfs diff=lfs merge=lfs -text
 *.zst filter=lfs diff=lfs merge=lfs -text
 *tfevents* filter=lfs diff=lfs merge=lfs -text
+wandb/run-20241011_110114-3d93c0ng/run-3d93c0ng.wandb filter=lfs diff=lfs merge=lfs -text

arguments.yaml

@@ -0,0 +1,49 @@
+data_cfgs:
+  eval_data_files: null
+  eval_datasets: null
+  eval_optional_args: []
+  eval_size: null
+  eval_split: null
+  eval_subset: null
+  eval_template: null
+  train_data_files: llf_t2i_30k_tokenized.pt
+  train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
+  train_optional_args: []
+  train_size: null
+  train_split: train
+  train_subset: null
+  train_template: AA_textfeedback
+logger_cfgs:
+  cache_dir: null
+  log_project: align-anything
+  log_run_name: sft
+  log_type: wandb
+  output_dir: ../outputs/sft_tf_cham_1011_t2i
+  save_interval: 400.0
+model_cfgs:
+  model_max_length: 4096
+  model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
+  trust_remote_code: true
+special_tokens: null
+train_cfgs:
+  adam_betas:
+  - 0.9
+  - 0.95
+  adam_epsilon: 1.0e-08
+  bf16: true
+  ds_cfgs: ds_z3_config.json
+  epochs: 3.0
+  eval_interval: 1000
+  eval_strategy: steps
+  fp16: false
+  freeze_language_model: false
+  gradient_accumulation_steps: 2.0
+  gradient_checkpointing: true
+  learning_rate: 1.0e-06
+  lr_scheduler_type: cosine
+  lr_warmup_ratio: 0.03
+  max_grad_norm: 1.0
+  per_device_eval_batch_size: 4.0
+  per_device_train_batch_size: 4.0
+  seed: 42
+  weight_decay: 0.0

environ.txt

@@ -0,0 +1,56 @@
+CONDA_DEFAULT_ENV=hantao_stable
+CONDA_EXE=/data/align-anything/miniconda3/bin/conda
+CONDA_PREFIX=/data/align-anything/miniconda3/envs/hantao_stable
+CONDA_PREFIX_1=/data/align-anything/miniconda3
+CONDA_PROMPT_MODIFIER=(hantao_stable) 
+CONDA_PYTHON_EXE=/data/align-anything/miniconda3/bin/python
+CONDA_SHLVL=2
+CRASHDIR=/etc/ShellCrash
+CROSS_RANK=0
+CROSS_SIZE=1
+CUDA_MODULE_LOADING=LAZY
+CUDA_VISIBLE_DEVICES=0,1,2,3,4,5,6,7
+DBUS_SESSION_BUS_ADDRESS=unix:path=/run/user/2000/bus
+HOME=/home/align-anything
+LANG=en_US.UTF-8
+LD_LIBRARY_PATH=/data/align-anything/miniconda3/envs/hantao_stable/lib/python3.11/site-packages/cv2/../../lib64:
+LESSCLOSE=/usr/bin/lesspipe %s %s
+LESSOPEN=| /usr/bin/lesspipe %s
+LOCAL_RANK=0
+LOCAL_SIZE=8
+LOGLEVEL=WARNING
+LOGNAME=align-anything
+LS_COLORS=rs=0:di=01;34:ln=01;36:mh=00:pi=40;33:so=01;35:do=01;35:bd=40;33;01:cd=40;33;01:or=40;31;01:mi=00:su=37;41:sg=30;43:ca=30;41:tw=30;42:ow=34;42:st=37;44:ex=01;32:*.tar=01;31:*.tgz=01;31:*.arc=01;31:*.arj=01;31:*.taz=01;31:*.lha=01;31:*.lz4=01;31:*.lzh=01;31:*.lzma=01;31:*.tlz=01;31:*.txz=01;31:*.tzo=01;31:*.t7z=01;31:*.zip=01;31:*.z=01;31:*.dz=01;31:*.gz=01;31:*.lrz=01;31:*.lz=01;31:*.lzo=01;31:*.xz=01;31:*.zst=01;31:*.tzst=01;31:*.bz2=01;31:*.bz=01;31:*.tbz=01;31:*.tbz2=01;31:*.tz=01;31:*.deb=01;31:*.rpm=01;31:*.jar=01;31:*.war=01;31:*.ear=01;31:*.sar=01;31:*.rar=01;31:*.alz=01;31:*.ace=01;31:*.zoo=01;31:*.cpio=01;31:*.7z=01;31:*.rz=01;31:*.cab=01;31:*.wim=01;31:*.swm=01;31:*.dwm=01;31:*.esd=01;31:*.jpg=01;35:*.jpeg=01;35:*.mjpg=01;35:*.mjpeg=01;35:*.gif=01;35:*.bmp=01;35:*.pbm=01;35:*.pgm=01;35:*.ppm=01;35:*.tga=01;35:*.xbm=01;35:*.xpm=01;35:*.tif=01;35:*.tiff=01;35:*.png=01;35:*.svg=01;35:*.svgz=01;35:*.mng=01;35:*.pcx=01;35:*.mov=01;35:*.mpg=01;35:*.mpeg=01;35:*.m2v=01;35:*.mkv=01;35:*.webm=01;35:*.ogm=01;35:*.mp4=01;35:*.m4v=01;35:*.mp4v=01;35:*.vob=01;35:*.qt=01;35:*.nuv=01;35:*.wmv=01;35:*.asf=01;35:*.rm=01;35:*.rmvb=01;35:*.flc=01;35:*.avi=01;35:*.fli=01;35:*.flv=01;35:*.gl=01;35:*.dl=01;35:*.xcf=01;35:*.xwd=01;35:*.yuv=01;35:*.cgm=01;35:*.emf=01;35:*.ogv=01;35:*.ogx=01;35:*.aac=00;36:*.au=00;36:*.flac=00;36:*.m4a=00;36:*.mid=00;36:*.midi=00;36:*.mka=00;36:*.mp3=00;36:*.mpc=00;36:*.ogg=00;36:*.ra=00;36:*.wav=00;36:*.oga=00;36:*.opus=00;36:*.spx=00;36:*.xspf=00;36:
+MASTER_ADDR=127.0.0.1
+MASTER_PORT=36635
+MOTD_SHOWN=pam
+OLDPWD=/data/align-anything/hantao/align-anything/projects/text_image_to_text_image
+PATH=/data/align-anything/miniconda3/envs/hantao_stable/bin:/data/align-anything/miniconda3/condabin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/usr/games:/usr/local/games:/snap/bin
+PWD=/data/align-anything/hantao/align-anything/scripts
+PYGAME_HIDE_SUPPORT_PROMPT=1
+PYTHONHASHSEED=42
+PYTHONPATH=/data/align-anything/hantao/align-anything
+QT_QPA_FONTDIR=/data/align-anything/miniconda3/envs/hantao_stable/lib/python3.11/site-packages/cv2/qt/fonts
+QT_QPA_PLATFORM_PLUGIN_PATH=/data/align-anything/miniconda3/envs/hantao_stable/lib/python3.11/site-packages/cv2/qt/plugins
+RANK=0
+SHELL=/bin/bash
+SHLVL=3
+SSH_CLIENT=111.205.233.209 27514 30600
+SSH_CONNECTION=111.205.233.209 27514 10.10.212.196 30600
+SSH_TTY=/dev/pts/0
+TERM=screen
+TMUX=/tmp/tmux-2000/default,262430,0
+TMUX_PANE=%0
+TRITON_CACHE_DIR=/home/align-anything/cache/triton
+USER=align-anything
+WANDB_API_KEY=7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33
+WANDB_SERVICE=2-288511-tcp-localhost-43055
+WORLD_SIZE=8
+XDG_DATA_DIRS=/usr/local/share:/usr/share:/var/lib/snapd/desktop
+XDG_RUNTIME_DIR=/run/user/2000
+XDG_SESSION_CLASS=user
+XDG_SESSION_ID=73
+XDG_SESSION_TYPE=tty
+_=/data/align-anything/miniconda3/envs/hantao_stable/bin/deepspeed
+_CE_CONDA=
+_CE_M=

preprocessor_config.json

@@ -0,0 +1,28 @@
+{
+  "crop_size": {
+    "height": 512,
+    "width": 512
+  },
+  "do_center_crop": true,
+  "do_convert_rgb": true,
+  "do_normalize": true,
+  "do_rescale": true,
+  "do_resize": true,
+  "image_mean": [
+    1.0,
+    1.0,
+    1.0
+  ],
+  "image_processor_type": "ChameleonImageProcessor",
+  "image_std": [
+    1.0,
+    1.0,
+    1.0
+  ],
+  "processor_class": "ChameleonProcessor",
+  "resample": 1,
+  "rescale_factor": 0.0078,
+  "size": {
+    "shortest_edge": 512
+  }
+}

processor_config.json

@@ -0,0 +1,5 @@
+{
+  "image_seq_length": 1024,
+  "image_token": "<image>",
+  "processor_class": "ChameleonProcessor"
+}

pytorch_model.bin

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:ef0911850aab3af817ec3978b359ca0d1766977ad4f92aa6042ef865d0a3427c
+size 14086364170

script.sh

@@ -0,0 +1,49 @@
+#!/usr/bin/env bash
+#
+# Copyright 2024 PKU-Alignment Team. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+# ==============================================================================
+
+export TRITON_CACHE_DIR="/home/align-anything/cache/triton"
+# You can replace it with a local model path
+MODEL_NAME_OR_PATH="/data/align-anything/hantao/models/0917_cham_ti2t_sft"
+# You can replace it with a local dataset path
+TRAIN_DATASETS="/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs"
+TRAIN_DATA_FILES="llf_t2i_30k_tokenized.pt"
+# You can replace it with a new path
+OUTPUT_DIR="../outputs/sft_tf_cham_1011_t2i"
+# For wandb online logging
+
+export WANDB_API_KEY="7e2dcc0c310ebcb7cdcafd5e9320d6be55cf1a33"
+# Source the setup script
+source ./setup.sh
+
+# sleep 30m
+# Execute deepspeed command
+deepspeed \
+	--master_port ${MASTER_PORT} \
+	--module align_anything.trainers.text_image_to_text_image.sft \
+	--model_name_or_path ${MODEL_NAME_OR_PATH} \
+	--train_datasets ${TRAIN_DATASETS} \
+	--train_data_files ${TRAIN_DATA_FILES} \
+	--output_dir ${OUTPUT_DIR} \
+	--train_template AA_textfeedback \
+	--train_split train \
+	--per_device_train_batch_size 4 \
+	--per_device_eval_batch_size 4 \
+	--gradient_accumulation_steps 2 \
+	--save_interval 400 \
+	--learning_rate 1e-6 \
+	--epochs 3 \
+	--lr_scheduler_type cosine

special_tokens_map.json

@@ -0,0 +1,37 @@
+{
+  "bos_token": {
+    "content": "<s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "eos_token": {
+    "content": "</s>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "pad_token": {
+    "content": "<pad>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "sep_token": {
+    "content": "<reserved08706>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  },
+  "unk_token": {
+    "content": "<unk>",
+    "lstrip": false,
+    "normalized": false,
+    "rstrip": false,
+    "single_word": false
+  }
+}

wandb/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2024-10-11T11:01:14.862375198Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-11T11:01:14.862403778Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug-core.log"}
+{"time":"2024-10-11T11:01:16.865952606Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-11T11:01:16.898816628Z","level":"INFO","msg":"created new stream","id":"3d93c0ng"}
+{"time":"2024-10-11T11:01:16.898858601Z","level":"INFO","msg":"stream: started","id":"3d93c0ng"}
+{"time":"2024-10-11T11:01:16.89889158Z","level":"INFO","msg":"sender: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:16.898885588Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:16.898892314Z","level":"INFO","msg":"handler: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:35.432206788Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-11T11:01:35.435879471Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-11T15:22:01.75767486Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-11T15:22:01.784588333Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-11T15:22:02.248095445Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-10-11T15:22:02.248136399Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-10-11T15:22:06.762613059Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"}
+{"time":"2024-10-11T15:22:19.959863097Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-11T15:22:21.795072514Z","level":"INFO","msg":"stream: closing","id":"3d93c0ng"}
+{"time":"2024-10-11T15:22:21.795099978Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.795126052Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.795146106Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.796861672Z","level":"INFO","msg":"stream: closed","id":"3d93c0ng"}

wandb/debug.log

@@ -0,0 +1,33 @@
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Configure stats pid to 288511
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-11 11:01:14,845 WARNING MainThread:288511 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug.log
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug-internal.log
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():617] calling init triggers
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_30k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1011_t2i', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():667] starting backend
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():671] sending inform_init request
+2024-10-11 11:01:14,855 INFO    MainThread:288511 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-11 11:01:14,856 INFO    MainThread:288511 [wandb_init.py:init():684] backend started and connected
+2024-10-11 11:01:14,867 INFO    MainThread:288511 [wandb_init.py:init():779] updated telemetry
+2024-10-11 11:01:14,945 INFO    MainThread:288511 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-11 11:01:35,420 INFO    MainThread:288511 [wandb_init.py:init():863] starting run threads in backend
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-11 11:01:36,297 INFO    MainThread:288511 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-11 11:01:36,313 INFO    MainThread:288511 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-11 15:22:01,752 INFO    MainThread:288511 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/3d93c0ng
+2024-10-11 15:22:01,754 INFO    MainThread:288511 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
+2024-10-11 15:22:01,756 INFO    MainThread:288511 [wandb_run.py:_restore():2410] restore
+2024-10-11 15:22:01,756 INFO    MainThread:288511 [wandb_run.py:_restore():2416] restore done
+2024-10-11 15:22:21,763 INFO    MainThread:288511 [wandb_run.py:_footer_history_summary_info():4049] rendering history
+2024-10-11 15:22:21,764 INFO    MainThread:288511 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
+2024-10-11 15:22:21,791 INFO    MainThread:288511 [wandb_run.py:_footer_sync_info():4008] logging synced files

wandb/run-20241011_110114-3d93c0ng/files/config.yaml

@@ -0,0 +1,96 @@
+_wandb:
+    value:
+        cli_version: 0.18.3
+        m: []
+        python_version: 3.11.10
+        t:
+            "1":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 55
+                - 71
+                - 83
+                - 98
+                - 105
+            "2":
+                - 1
+                - 11
+                - 41
+                - 49
+                - 51
+                - 55
+                - 71
+                - 83
+                - 98
+                - 105
+            "3":
+                - 2
+                - 13
+                - 16
+                - 23
+                - 55
+                - 61
+            "4": 3.11.10
+            "5": 0.18.3
+            "6": 4.46.0.dev0
+            "8":
+                - 5
+            "12": 0.18.3
+            "13": linux-x86_64
+data_cfgs:
+    value:
+        eval_data_files: null
+        eval_datasets: null
+        eval_optional_args: []
+        eval_size: null
+        eval_split: null
+        eval_subset: null
+        eval_template: null
+        train_data_files: llf_t2i_30k_tokenized.pt
+        train_datasets: /data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs
+        train_optional_args: []
+        train_size: null
+        train_split: train
+        train_subset: null
+        train_template: AA_textfeedback
+logger_cfgs:
+    value:
+        cache_dir: null
+        log_project: align-anything
+        log_run_name: sft
+        log_type: wandb
+        output_dir: ../outputs/sft_tf_cham_1011_t2i
+        save_interval: 400
+model_cfgs:
+    value:
+        model_max_length: 4096
+        model_name_or_path: /data/align-anything/hantao/models/0917_cham_ti2t_sft
+        trust_remote_code: true
+special_tokens:
+    value: null
+train_cfgs:
+    value:
+        adam_betas:
+            - 0.9
+            - 0.95
+        adam_epsilon: 1e-08
+        bf16: true
+        ds_cfgs: ds_z3_config.json
+        epochs: 3
+        eval_interval: 1000
+        eval_strategy: steps
+        fp16: false
+        freeze_language_model: false
+        gradient_accumulation_steps: 2
+        gradient_checkpointing: true
+        learning_rate: 1e-06
+        lr_scheduler_type: cosine
+        lr_warmup_ratio: 0.03
+        max_grad_norm: 1
+        per_device_eval_batch_size: 4
+        per_device_train_batch_size: 4
+        seed: 42
+        weight_decay: 0

wandb/run-20241011_110114-3d93c0ng/files/requirements.txt

@@ -0,0 +1,229 @@
+align-anything==0.0.1.dev0
+pluggy==1.5.0
+yt-dlp==2024.8.6
+referencing==0.35.1
+nvitop==1.3.2
+shellingham==1.5.4
+pydantic==2.9.2
+grpcio==1.66.2
+packaging==24.1
+fastapi==0.115.0
+matplotlib==3.9.2
+typing_extensions==4.12.2
+pycountry==24.6.1
+rpds-py==0.20.0
+prometheus_client==0.21.0
+vllm==0.6.2
+pycryptodomex==3.21.0
+MarkupSafe==2.1.5
+prometheus-fastapi-instrumentator==7.0.0
+peft==0.13.1
+Brotli==1.1.0
+audioread==3.0.1
+numpy==1.26.4
+nest-asyncio==1.6.0
+nvidia-nvtx-cu12==12.1.105
+kiwisolver==1.4.7
+librosa==0.10.2.post1
+msgspec==0.18.6
+lark==1.2.2
+aiosignal==1.3.1
+args==0.1.0
+attrs==24.2.0
+regex==2024.9.11
+typer==0.12.5
+nvidia-ml-py==12.535.161
+httptools==0.6.1
+wheel==0.44.0
+pyairports==2.1.1
+threadpoolctl==3.5.0
+nvidia-curand-cu12==10.3.2.106
+yarl==1.14.0
+rich==13.9.2
+nvidia-nccl-cu12==2.20.5
+click==8.1.7
+importlib_metadata==8.5.0
+fsspec==2024.6.1
+h11==0.14.0
+uvloop==0.20.0
+deepspeed==0.15.2
+absl-py==2.1.0
+setproctitle==1.3.3
+Werkzeug==3.0.4
+nvidia-cuda-cupti-cu12==12.1.105
+fairscale==0.4.13
+wandb==0.18.3
+protobuf==3.20.3
+pytz==2024.2
+sentry-sdk==2.16.0
+opencv-python==4.6.0.66
+cfgv==3.4.0
+diskcache==5.6.3
+psutil==6.0.0
+frechet-audio-distance==0.1.2
+zipp==3.20.2
+pytorch-fid==0.3.0
+nodeenv==1.9.1
+align-anything==0.0.1.dev0
+joblib==1.4.2
+jiter==0.6.1
+interegular==0.3.3
+jsonschema==4.23.0
+numba==0.60.0
+outlines==0.0.46
+nvidia-cufft-cu12==11.0.2.54
+idna==3.10
+sympy==1.13.3
+proglog==0.1.10
+httpx==0.27.2
+gradio==5.0.1
+lazy_loader==0.4
+pip==24.2
+aiohappyeyeballs==2.4.3
+aiofiles==23.2.1
+contourpy==1.3.0
+mutagen==1.47.0
+watchfiles==0.24.0
+imageio-ffmpeg==0.5.1
+termcolor==2.5.0
+imageio==2.35.1
+gguf==0.10.0
+hpsv2==1.2.0
+pydantic_core==2.23.4
+ruff==0.6.9
+markdown-it-py==3.0.0
+torchlibrosa==0.1.0
+cycler==0.12.1
+tzdata==2024.2
+httpcore==1.0.6
+Markdown==3.7
+platformdirs==4.3.6
+transformers==4.46.0.dev0
+GitPython==3.1.43
+virtualenv==20.26.6
+propcache==0.2.0
+nvidia-cusparse-cu12==12.1.0.106
+identify==2.6.1
+webdataset==0.2.100
+datasets==3.0.1
+accelerate==1.0.0
+multiprocess==0.70.16
+wcwidth==0.2.13
+pre_commit==4.0.1
+tomlkit==0.12.0
+torchvision==0.19.0
+websockets==12.0
+pytest==7.2.0
+timm==0.6.13
+six==1.16.0
+aiohttp==3.10.9
+tiktoken==0.7.0
+ffmpy==0.4.0
+urllib3==2.2.3
+torchaudio==2.4.0
+tensorboard==2.18.0
+scikit-learn==1.5.2
+frozenlist==1.4.1
+multidict==6.1.0
+PyYAML==6.0.2
+xxhash==3.5.0
+torch==2.4.0
+sniffio==1.3.1
+soundfile==0.12.1
+mdurl==0.1.2
+python-multipart==0.0.12
+tensorboard-data-server==0.7.2
+nvidia-cudnn-cu12==9.1.0.70
+orjson==3.10.7
+tqdm==4.66.5
+ninja==1.11.1.1
+image-reward==1.5
+clint==0.5.1
+setuptools==75.1.0
+shortuuid==1.0.13
+cffi==1.17.1
+docker-pycreds==0.4.0
+distro==1.9.0
+msgpack==1.1.0
+sentencepiece==0.2.0
+jsonschema-specifications==2024.10.1
+anyio==4.6.0
+optree==0.13.0
+xformers==0.0.27.post2
+pooch==1.8.2
+pyzmq==26.2.0
+Pygments==2.18.0
+partial-json-parser==0.2.1.1.post4
+einops==0.8.0
+nvidia-cublas-cu12==12.1.3.1
+gitdb==4.0.11
+cachetools==5.5.0
+decorator==4.4.2
+uvicorn==0.31.1
+clip==0.2.0
+ftfy==6.2.3
+mistral_common==1.4.4
+safetensors==0.4.5
+charset-normalizer==3.4.0
+soxr==0.5.0.post1
+requests==2.32.3
+pyparsing==3.1.4
+smmap==5.0.1
+annotated-types==0.7.0
+nvidia-cuda-nvrtc-cu12==12.1.105
+cloudpickle==3.0.0
+openai==1.51.2
+ray==2.37.0
+nvidia-nvjitlink-cu12==12.6.77
+resampy==0.4.3
+triton==3.0.0
+huggingface-hub==0.25.2
+pydub==0.25.1
+semantic-version==2.10.0
+distlib==0.3.9
+networkx==3.3
+pandas==2.2.3
+fonttools==4.54.1
+lm-format-enforcer==0.10.6
+python-dotenv==1.0.1
+Jinja2==3.1.4
+nvidia-cuda-runtime-cu12==12.1.105
+iniconfig==2.0.0
+moviepy==1.0.3
+pyarrow==17.0.0
+filelock==3.16.1
+py-cpuinfo==9.0.0
+python-dateutil==2.9.0.post0
+certifi==2024.8.30
+braceexpand==0.1.7
+dill==0.3.8
+bitsandbytes==0.44.1
+starlette==0.38.6
+diffusers==0.30.3
+pillow==10.4.0
+scipy==1.14.1
+pytest-split==0.8.0
+nvidia-cusolver-cu12==11.4.5.107
+llvmlite==0.43.0
+gradio_client==1.4.0
+mpmath==1.3.0
+pycparser==2.22
+hjson==3.1.0
+tokenizers==0.20.1
+jaraco.context==5.3.0
+typeguard==4.3.0
+inflect==7.3.1
+importlib_metadata==8.0.0
+jaraco.collections==5.1.0
+backports.tarfile==1.2.0
+platformdirs==4.2.2
+jaraco.functools==4.0.1
+zipp==3.19.2
+autocommand==2.2.2
+typing_extensions==4.12.2
+jaraco.text==3.12.1
+importlib_resources==6.4.0
+more-itertools==10.3.0
+wheel==0.43.0
+packaging==24.1
+tomli==2.0.1

wandb/run-20241011_110114-3d93c0ng/files/wandb-metadata.json

@@ -0,0 +1,111 @@
+{
+  "os":  "Linux-5.4.0-144-generic-x86_64-with-glibc2.31",
+  "python":  "3.11.10",
+  "startedAt":  "2024-10-11T11:01:14.856438Z",
+  "args":  [
+    "--local_rank=0",
+    "--model_name_or_path",
+    "/data/align-anything/hantao/models/0917_cham_ti2t_sft",
+    "--train_datasets",
+    "/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs",
+    "--train_data_files",
+    "llf_t2i_30k_tokenized.pt",
+    "--output_dir",
+    "../outputs/sft_tf_cham_1011_t2i",
+    "--train_template",
+    "AA_textfeedback",
+    "--train_split",
+    "train",
+    "--per_device_train_batch_size",
+    "4",
+    "--per_device_eval_batch_size",
+    "4",
+    "--gradient_accumulation_steps",
+    "2",
+    "--save_interval",
+    "400",
+    "--learning_rate",
+    "1e-6",
+    "--epochs",
+    "3",
+    "--lr_scheduler_type",
+    "cosine"
+  ],
+  "program":  "-m align_anything.trainers.text_image_to_text_image.sft",
+  "git":  {
+    "remote":  "https://github.com/PKU-Alignment/align-anything.git",
+    "commit":  "6fde660afc9985323f147930eedf188a5699adc7"
+  },
+  "root":  "../outputs/sft_tf_cham_1011_t2i",
+  "host":  "lyg0196",
+  "username":  "align-anything",
+  "executable":  "/data/align-anything/miniconda3/envs/hantao_stable/bin/python",
+  "cpu_count":  64,
+  "cpu_count_logical":  128,
+  "gpu":  "[NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB, NVIDIA A100-SXM4-80GB]",
+  "gpu_count":  8,
+  "disk":  {
+    "/":  {
+      "total":  "940744544256",
+      "used":  "39174803456"
+    }
+  },
+  "memory":  {
+    "total":  "540660944896"
+  },
+  "cpu":  {
+    "count":  64,
+    "countLogical":  128
+  },
+  "gpu_nvidia":  [
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    },
+    {
+      "name":  "NVIDIA A100-SXM4-80GB",
+      "memoryTotal":  "85899345920",
+      "cudaCores":  6912,
+      "architecture":  "Ampere"
+    }
+  ],
+  "cudaVersion":  "12.4"
+}

wandb/run-20241011_110114-3d93c0ng/files/wandb-summary.json

@@ -0,0 +1 @@
+{"_runtime":15646.901212267,"_wandb":{"runtime":15646},"_step":5625,"train/step":5625,"train/loss":0.7483024001121521,"train/lr":1.3242630444998404e-12,"train/epoch":3,"_timestamp":1.7286600433933325e+09}

wandb/run-20241011_110114-3d93c0ng/logs/debug-internal.log

@@ -0,0 +1,21 @@
+{"time":"2024-10-11T11:01:14.862375198Z","level":"INFO","msg":"using version","core version":"0.18.3"}
+{"time":"2024-10-11T11:01:14.862403778Z","level":"INFO","msg":"created symlink","path":"../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug-core.log"}
+{"time":"2024-10-11T11:01:16.865952606Z","level":"ERROR","msg":"dialing: google: could not find default credentials. See https://cloud.google.com/docs/authentication/external/set-up-adc for more information"}
+{"time":"2024-10-11T11:01:16.898816628Z","level":"INFO","msg":"created new stream","id":"3d93c0ng"}
+{"time":"2024-10-11T11:01:16.898858601Z","level":"INFO","msg":"stream: started","id":"3d93c0ng"}
+{"time":"2024-10-11T11:01:16.89889158Z","level":"INFO","msg":"sender: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:16.898885588Z","level":"INFO","msg":"writer: Do: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:16.898892314Z","level":"INFO","msg":"handler: started","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T11:01:35.432206788Z","level":"INFO","msg":"wandb-core","!BADKEY":null}
+{"time":"2024-10-11T11:01:35.435879471Z","level":"INFO","msg":"Starting system monitor"}
+{"time":"2024-10-11T15:22:01.75767486Z","level":"INFO","msg":"Stopping system monitor"}
+{"time":"2024-10-11T15:22:01.784588333Z","level":"INFO","msg":"Stopped system monitor"}
+{"time":"2024-10-11T15:22:02.248095445Z","level":"WARN","msg":"No program path found, not creating job artifact. See https://docs.wandb.ai/guides/launch/create-job"}
+{"time":"2024-10-11T15:22:02.248136399Z","level":"INFO","msg":"sender: sendDefer: no job artifact to save"}
+{"time":"2024-10-11T15:22:06.762613059Z","level":"ERROR","msg":"monitor: gpu: timeout waiting for process to exit"}
+{"time":"2024-10-11T15:22:19.959863097Z","level":"INFO","msg":"fileTransfer: Close: file transfer manager closed"}
+{"time":"2024-10-11T15:22:21.795072514Z","level":"INFO","msg":"stream: closing","id":"3d93c0ng"}
+{"time":"2024-10-11T15:22:21.795099978Z","level":"INFO","msg":"handler: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.795126052Z","level":"INFO","msg":"writer: Close: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.795146106Z","level":"INFO","msg":"sender: closed","stream_id":{"value":"3d93c0ng"}}
+{"time":"2024-10-11T15:22:21.796861672Z","level":"INFO","msg":"stream: closed","id":"3d93c0ng"}

wandb/run-20241011_110114-3d93c0ng/logs/debug.log

@@ -0,0 +1,33 @@
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Current SDK version is 0.18.3
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Configure stats pid to 288511
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from /home/align-anything/.config/wandb/settings
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from /data/align-anything/hantao/align-anything/scripts/wandb/settings
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Loading settings from environment variables: {'api_key': '***REDACTED***'}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Applying setup settings: {'mode': None, '_disable_service': None}
+2024-10-11 11:01:14,845 WARNING MainThread:288511 [wandb_setup.py:_flush():79] Could not find program at -m align_anything.trainers.text_image_to_text_image.sft
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Inferring run settings from compute environment: {'program_relpath': None, 'program': '-m align_anything.trainers.text_image_to_text_image.sft'}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_setup.py:_flush():79] Applying login settings: {}
+2024-10-11 11:01:14,845 INFO    MainThread:288511 [wandb_init.py:_log_setup():532] Logging user logs to ../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug.log
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:_log_setup():533] Logging internal logs to ../outputs/sft_tf_cham_1011_t2i/wandb/run-20241011_110114-3d93c0ng/logs/debug-internal.log
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():617] calling init triggers
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():624] wandb.init called with sweep_config: {}
+config: {'train_cfgs': {'ds_cfgs': 'ds_z3_config.json', 'epochs': 3.0, 'seed': 42, 'per_device_train_batch_size': 4.0, 'per_device_eval_batch_size': 4.0, 'gradient_accumulation_steps': 2.0, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'lr_scheduler_type': 'cosine', 'lr_warmup_ratio': 0.03, 'weight_decay': 0.0, 'adam_betas': [0.9, 0.95], 'adam_epsilon': 1e-08, 'bf16': True, 'fp16': False, 'eval_strategy': 'steps', 'eval_interval': 1000, 'freeze_language_model': False, 'max_grad_norm': 1.0}, 'data_cfgs': {'train_datasets': '/data/align-anything/hantao/align-anything/projects/text_image_to_text_image/outputs', 'train_template': 'AA_textfeedback', 'train_size': None, 'train_split': 'train', 'train_subset': None, 'train_data_files': 'llf_t2i_30k_tokenized.pt', 'train_optional_args': [], 'eval_datasets': None, 'eval_template': None, 'eval_size': None, 'eval_split': None, 'eval_subset': None, 'eval_data_files': None, 'eval_optional_args': []}, 'logger_cfgs': {'log_type': 'wandb', 'log_project': 'align-anything', 'log_run_name': 'sft', 'output_dir': '../outputs/sft_tf_cham_1011_t2i', 'cache_dir': None, 'save_interval': 400.0}, 'model_cfgs': {'model_name_or_path': '/data/align-anything/hantao/models/0917_cham_ti2t_sft', 'trust_remote_code': True, 'model_max_length': 4096}, 'special_tokens': None}
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():667] starting backend
+2024-10-11 11:01:14,846 INFO    MainThread:288511 [wandb_init.py:init():671] sending inform_init request
+2024-10-11 11:01:14,855 INFO    MainThread:288511 [backend.py:_multiprocessing_setup():104] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
+2024-10-11 11:01:14,856 INFO    MainThread:288511 [wandb_init.py:init():684] backend started and connected
+2024-10-11 11:01:14,867 INFO    MainThread:288511 [wandb_init.py:init():779] updated telemetry
+2024-10-11 11:01:14,945 INFO    MainThread:288511 [wandb_init.py:init():812] communicating run to backend with 90.0 second timeout
+2024-10-11 11:01:35,420 INFO    MainThread:288511 [wandb_init.py:init():863] starting run threads in backend
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_console_start():2465] atexit reg
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_redirect():2313] redirect: wrap_raw
+2024-10-11 11:01:36,296 INFO    MainThread:288511 [wandb_run.py:_redirect():2378] Wrapping output streams.
+2024-10-11 11:01:36,297 INFO    MainThread:288511 [wandb_run.py:_redirect():2403] Redirects installed.
+2024-10-11 11:01:36,313 INFO    MainThread:288511 [wandb_init.py:init():907] run started, returning control to user process
+2024-10-11 15:22:01,752 INFO    MainThread:288511 [wandb_run.py:_finish():2164] finishing run htlou/align-anything/3d93c0ng
+2024-10-11 15:22:01,754 INFO    MainThread:288511 [wandb_run.py:_atexit_cleanup():2428] got exitcode: 0
+2024-10-11 15:22:01,756 INFO    MainThread:288511 [wandb_run.py:_restore():2410] restore
+2024-10-11 15:22:01,756 INFO    MainThread:288511 [wandb_run.py:_restore():2416] restore done
+2024-10-11 15:22:21,763 INFO    MainThread:288511 [wandb_run.py:_footer_history_summary_info():4049] rendering history
+2024-10-11 15:22:21,764 INFO    MainThread:288511 [wandb_run.py:_footer_history_summary_info():4081] rendering summary
+2024-10-11 15:22:21,791 INFO    MainThread:288511 [wandb_run.py:_footer_sync_info():4008] logging synced files

wandb/run-20241011_110114-3d93c0ng/run-3d93c0ng.wandb

@@ -0,0 +1,3 @@
+version https://git-lfs.github.com/spec/v1
+oid sha256:59aa83b0f674b38ca5abbdc3ef15f6b4d45ce27458b5a6212aab68e10d9d80c7
+size 22611669