accelerate config:
- /h/calvinyu/.cache/huggingface/accelerate/default_config.yaml

batchsize: 
1 / 20 


tensors = {}
with safe_open("pytorch_lora_weights.safetensors", framework="pt", device=0) as f:
    for k in f.keys():
        tensors[k] = f.get_tensor(k)

next:
- compute gram, fisher and AB for 1K samples on each model 
- track, add and commit all 24x3 and ancestor models
- run merge script

next:
- dare merges with ties lambda
- filter best ckpts and hyperparams
- run full eval on best ckpts then remaining (on 4 gpus?) + store images
- sequential merge with best hyperparams or derek's hyperparams
- time allows get fids from images

derek ques:
- 2 captions per domain-category for validation (from train / test split)
- all unique captions for indomain and outdomain (in full eval)

next:
- finish all merges, and upload to hub, and wipe.
- start evaluation of all merged models (CLIP and FID) together
- start sequential merging (no hyperparams), and upload to hub, and wipe. 
- start evaluation of seq merged models (CLIP and FID) 


FID evaluation:
- indomain and outdomain

"""
split: train
merge method: average
params: clip_scores_val_train_indomain and score: 32.71

best params: clip_scores_val_train_indomain and score: 32.71


-----------------
merge method: fisher
params: clip_scores_val_train_indomain and score: 32.06

best params: clip_scores_val_train_indomain and score: 32.06


-----------------
merge method: mats
params: iterations_100_clip_scores_val_train_indomain and score: 30.36
params: iterations_10_clip_scores_val_train_indomain and score: 33.71
params: iterations_20_clip_scores_val_train_indomain and score: 31.85
params: iterations_30_clip_scores_val_train_indomain and score: 32.25
params: iterations_40_clip_scores_val_train_indomain and score: 31.57
params: iterations_50_clip_scores_val_train_indomain and score: 31.8
params: iterations_60_clip_scores_val_train_indomain and score: 31.85
params: iterations_70_clip_scores_val_train_indomain and score: 30.34
params: iterations_80_clip_scores_val_train_indomain and score: 32.55
params: iterations_90_clip_scores_val_train_indomain and score: 32.47

best params: iterations_10_clip_scores_val_train_indomain and score: 33.71


-----------------
merge method: regmean
params: lambda_0.0_clip_scores_val_train_indomain and score: 32.69
params: lambda_0.1111111111111111_clip_scores_val_train_indomain and score: 32.31
params: lambda_0.2222222222222222_clip_scores_val_train_indomain and score: 32.75
params: lambda_0.3333333333333333_clip_scores_val_train_indomain and score: 32.1
params: lambda_0.4444444444444444_clip_scores_val_train_indomain and score: 33.73
params: lambda_0.5555555555555556_clip_scores_val_train_indomain and score: 31.33
params: lambda_0.6666666666666666_clip_scores_val_train_indomain and score: 32.36
params: lambda_0.7777777777777777_clip_scores_val_train_indomain and score: 32.94
params: lambda_0.8888888888888888_clip_scores_val_train_indomain and score: 32.55
params: lambda_1.0_clip_scores_val_train_indomain and score: 31.46

best params: lambda_0.4444444444444444_clip_scores_val_train_indomain and score: 33.73


-----------------
merge method: task
params: arithmetic_lambda_0.1_clip_scores_val_train_indomain and score: 30.57
params: arithmetic_lambda_0.2_clip_scores_val_train_indomain and score: 27.37
params: arithmetic_lambda_0.30000000000000004_clip_scores_val_train_indomain and score: 24.77
params: arithmetic_lambda_0.4_clip_scores_val_train_indomain and score: 23.35
params: arithmetic_lambda_0.5_clip_scores_val_train_indomain and score: 22.48
params: arithmetic_lambda_0.6_clip_scores_val_train_indomain and score: 22.84
params: arithmetic_lambda_0.7000000000000001_clip_scores_val_train_indomain and score: 21.42
params: arithmetic_lambda_0.8_clip_scores_val_train_indomain and score: 21.66
params: arithmetic_lambda_0.9_clip_scores_val_train_indomain and score: 22.34
params: arithmetic_lambda_1.0_clip_scores_val_train_indomain and score: 22.98

best params: arithmetic_lambda_0.1_clip_scores_val_train_indomain and score: 30.57


-----------------
merge method: ties
params: lambda_0.1_clip_scores_val_train_indomain and score: 32.98
params: lambda_0.2_clip_scores_val_train_indomain and score: 34.15
params: lambda_0.30000000000000004_clip_scores_val_train_indomain and score: 32.96
params: lambda_0.4_clip_scores_val_train_indomain and score: 31.53
params: lambda_0.5_clip_scores_val_train_indomain and score: 31.45
params: lambda_0.6_clip_scores_val_train_indomain and score: 31.53
params: lambda_0.7000000000000001_clip_scores_val_train_indomain and score: 30.02
params: lambda_0.8_clip_scores_val_train_indomain and score: 29.69
params: lambda_0.9_clip_scores_val_train_indomain and score: 30.82
params: lambda_1.0_clip_scores_val_train_indomain and score: 30.26

best params: lambda_0.2_clip_scores_val_train_indomain and score: 34.15
"""


"""
split: val
merge method: average
params: average and score: 33.9

best params: average and score: 33.9


-----------------
merge method: fisher
params: fisher and score: 32.62

best params: fisher and score: 32.62


-----------------
merge method: mats
params: iterations_100 and score: 30.75
params: iterations_10 and score: 32.83
params: iterations_20 and score: 32.78
params: iterations_30 and score: 30.93
params: iterations_40 and score: 31.76
params: iterations_50 and score: 32.15
params: iterations_60 and score: 31.07
params: iterations_70 and score: 32.1
params: iterations_80 and score: 31.14
params: iterations_90 and score: 32.0

best params: iterations_10 and score: 32.83


-----------------
merge method: regmean
params: lambda_0.0 and score: 32.61
params: lambda_0.1111111111111111 and score: 31.78
params: lambda_0.2222222222222222 and score: 32.48
params: lambda_0.3333333333333333 and score: 31.69
params: lambda_0.4444444444444444 and score: 32.82
params: lambda_0.5555555555555556 and score: 32.32
params: lambda_0.6666666666666666 and score: 32.24
params: lambda_0.7777777777777777 and score: 33.22
params: lambda_0.8888888888888888 and score: 31.64
params: lambda_1.0 and score: 30.54

best params: lambda_0.7777777777777777 and score: 33.22


-----------------
merge method: task
params: arithmetic_lambda_0.1 and score: 31.09
params: arithmetic_lambda_0.2 and score: 27.49
params: arithmetic_lambda_0.30000000000000004 and score: 24.81
params: arithmetic_lambda_0.4 and score: 22.52
params: arithmetic_lambda_0.5 and score: 22.7
params: arithmetic_lambda_0.6 and score: 23.04
params: arithmetic_lambda_0.7000000000000001 and score: 22.58
params: arithmetic_lambda_0.8 and score: 21.66
params: arithmetic_lambda_0.9 and score: 22.71
params: arithmetic_lambda_1.0 and score: 22.42

best params: arithmetic_lambda_0.1 and score: 31.09


-----------------
merge method: ties
params: lambda_0.1 and score: 33.19
params: lambda_0.2 and score: 33.41
params: lambda_0.30000000000000004 and score: 32.1
params: lambda_0.4 and score: 33.02
params: lambda_0.5 and score: 31.55
params: lambda_0.6 and score: 30.18
params: lambda_0.7000000000000001 and score: 30.87
params: lambda_0.8 and score: 30.4
params: lambda_0.9 and score: 30.1
params: lambda_1.0 and score: 28.35

best params: lambda_0.2 and score: 33.41
"""

"""
regmean_lambda_0.1111111111111111.safetensors
regmean_lambda_0.2222222222222222.safetensors
regmean_lambda_0.3333333333333333.safetensors
regmean_lambda_0.4444444444444444.safetensors
regmean_lambda_0.5555555555555556.safetensors
regmean_lambda_0.6666666666666666.safetensors
regmean_lambda_0.7777777777777777.safetensors
regmean_lambda_0.8888888888888888.safetensors


srun -p a40 --job-name=debug --gres=gpu:1 --ntasks=1 --ntasks-per-node=1 --cpus-per-task=8 --mem-per-cpu=5G  --qos=normal --time=8:00:00 --kill-on-bad-exit=1 --pty bash

zsh
coa infinigen
cd evalmerge
python scripts/run_val.py --split_idx 1


"""

