deepgoyal19
commited on
Commit
β’
06ece82
1
Parent(s):
873236b
End of training
Browse filesThis view is limited to 50 files because it contains too many changes. Β
See raw diff
- README.md +21 -0
- image_0.png +0 -0
- image_1.png +0 -0
- image_2.png +0 -0
- image_3.png +0 -0
- image_4.png +0 -0
- image_5.png +0 -0
- image_6.png +0 -0
- log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1 +3 -0
- log/text2image-fine-tune/1686718938.2222803/hparams.yml +49 -0
- log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1 +3 -0
- log/text2image-fine-tune/1686719590.3490849/hparams.yml +49 -0
- log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1 +3 -0
- log/text2image-fine-tune/1686719831.1635435/hparams.yml +49 -0
- log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1 +3 -0
- log/text2image-fine-tune/1686720913.1509812/hparams.yml +49 -0
- log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1 +3 -0
- log/text2image-fine-tune/1686720979.8056188/hparams.yml +49 -0
- log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1 +3 -0
- log/text2image-fine-tune/1686721041.5566664/hparams.yml +49 -0
- log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1 +3 -0
- log/text2image-fine-tune/1686738195.9854875/hparams.yml +49 -0
- log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1 +3 -0
- log/text2image-fine-tune/1686739744.541363/hparams.yml +48 -0
- log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1 +3 -0
- log/text2image-fine-tune/1686740628.4842744/hparams.yml +48 -0
- log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0 +3 -0
- log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0 +3 -0
- pytorch_lora_weights.bin +3 -0
- wandb/settings +2 -0
- wandb/wandb/debug-internal.log +449 -0
- wandb/wandb/debug.log +30 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml +191 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml +182 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png +0 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png +0 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png +0 -0
- wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +63 -0
README.md
ADDED
@@ -0,0 +1,21 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
|
2 |
+
---
|
3 |
+
license: creativeml-openrail-m
|
4 |
+
base_model: runwayml/stable-diffusion-v1-5
|
5 |
+
tags:
|
6 |
+
- stable-diffusion
|
7 |
+
- stable-diffusion-diffusers
|
8 |
+
- text-to-image
|
9 |
+
- diffusers
|
10 |
+
- lora
|
11 |
+
inference: true
|
12 |
+
---
|
13 |
+
|
14 |
+
# LoRA text2image fine-tuning - deepgoyal19/lora5_tb
|
15 |
+
These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the lambdalabs/pokemon-blip-captions dataset. You can find some example images in the following.
|
16 |
+
|
17 |
+
![img_0](./image_0.png)
|
18 |
+
![img_1](./image_1.png)
|
19 |
+
![img_2](./image_2.png)
|
20 |
+
|
21 |
+
|
image_0.png
ADDED
image_1.png
ADDED
image_2.png
ADDED
image_3.png
ADDED
image_4.png
ADDED
image_5.png
ADDED
image_6.png
ADDED
log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2
|
3 |
+
size 2769
|
log/text2image-fine-tune/1686718938.2222803/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 15
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 15
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a4a4522643499eb3044dde554a2004af16634c1fd90f19f7352288e003ec7da2
|
3 |
+
size 2770
|
log/text2image-fine-tune/1686719590.3490849/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora1_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 15
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 15
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18
|
3 |
+
size 2770
|
log/text2image-fine-tune/1686719831.1635435/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora1_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 15
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 15
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:e23009bcbd80acf0569ea70abd119452ccdb9a69edcc33f7038a928bdd939482
|
3 |
+
size 2770
|
log/text2image-fine-tune/1686720913.1509812/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora2_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e
|
3 |
+
size 2770
|
log/text2image-fine-tune/1686720979.8056188/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora2_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:dedc3b5204b74b2151540c3347d61ee8c84006b9c95481ff2724ac0911f8848a
|
3 |
+
size 2770
|
log/text2image-fine-tune/1686721041.5566664/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora2_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
49 |
+
validation_prompts: pikachu
|
log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b9ee6471aab351b1c3a319ba55daa8f7851ec66575ac7b55078914ffef40772b
|
3 |
+
size 2713
|
log/text2image-fine-tune/1686738195.9854875/hparams.yml
ADDED
@@ -0,0 +1,49 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora3_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
49 |
+
validation_prompts: null
|
log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:21bc60c8432f3fdbc26a513859f4c04a18e03a56aa6e8d29a0ee1391cc4497e3
|
3 |
+
size 2713
|
log/text2image-fine-tune/1686739744.541363/hparams.yml
ADDED
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora3_tensorboard
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ea1e87f8a32bfaae1eafe00dfabef13887acc06b904c6e1713d9d9bc42bf28cf
|
3 |
+
size 2704
|
log/text2image-fine-tune/1686740628.4842744/hparams.yml
ADDED
@@ -0,0 +1,48 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
accelerate_device: cuda:7
|
2 |
+
accelerate_local_process_index: 0
|
3 |
+
accelerate_num_processes: 1
|
4 |
+
accelerate_optimizer_step_was_skipped: false
|
5 |
+
accelerate_process_index: 0
|
6 |
+
accelerate_sync_gradients: true
|
7 |
+
accelerate_use_distributed: false
|
8 |
+
adam_beta1: 0.9
|
9 |
+
adam_beta2: 0.999
|
10 |
+
adam_epsilon: 1.0e-08
|
11 |
+
adam_weight_decay: 0.01
|
12 |
+
allow_tf32: false
|
13 |
+
checkpointing_steps: 500
|
14 |
+
checkpoints_total_limit: null
|
15 |
+
disable_tqdm: false
|
16 |
+
enable_xformers_memory_efficient_attention: true
|
17 |
+
eval_dataset_path: null
|
18 |
+
fp16_backend: auto
|
19 |
+
fp16_opt_level: O1
|
20 |
+
gradient_accumulation_steps: 1
|
21 |
+
gradient_checkpointing: true
|
22 |
+
hub_model_id: lora4_tb
|
23 |
+
hub_token: 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
|
24 |
+
input_perturbation: 0
|
25 |
+
learning_rate: 1.0e-06
|
26 |
+
logging_dir: log
|
27 |
+
lr_scheduler: constant
|
28 |
+
lr_warmup_steps: 0
|
29 |
+
max_grad_norm: 1
|
30 |
+
max_train_samples: null
|
31 |
+
max_train_steps: 60
|
32 |
+
mixed_precision: fp16
|
33 |
+
noise_offset: 1
|
34 |
+
num_train_epochs: 1
|
35 |
+
num_validation_images: 1
|
36 |
+
output_dir: /home/deepanshu/lmflowdiff/Model
|
37 |
+
push_to_hub: true
|
38 |
+
report_to: tensorboard
|
39 |
+
resume_from_checkpoint: latest
|
40 |
+
save_steps: 1000
|
41 |
+
save_total_limit: null
|
42 |
+
scale_lr: false
|
43 |
+
seed: 123
|
44 |
+
snr_gamma: null
|
45 |
+
tracker_project_name: text2image-fine-tune
|
46 |
+
train_batch_size: 1
|
47 |
+
use_8bit_adam: false
|
48 |
+
validation_epochs: 60
|
log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:56db31fb0d75f58bcda44e6a5b551b0cd8ff59e10469290b8395270cf6c752b0
|
3 |
+
size 3824602
|
log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:18eb5dfa7dd09aaa9476759ed5aefafacb327e25e96e38558425e983ad8e3e80
|
3 |
+
size 3719369
|
log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:ed04babe7fc1d215e5f8a80dbf2296872f61b8272a19e69a838fed25dc2a0cb0
|
3 |
+
size 3723222
|
log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:b2f212308d9755444b01ca57a1428f099ae463da8c42581e405453a8bcaf6018
|
3 |
+
size 3722023
|
log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:041ff8bfe52e59c866d87f9b1dd9a37aae790fe1b92441c351b9029d15aa980b
|
3 |
+
size 3724395
|
log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:972064b9bca317a5508f4f0b55085d6cb08b5cb1cde82c684a3b2f6d0ddc232f
|
3 |
+
size 3723251
|
log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4e380cb848e0d985ce497abd616466a455fb6dfe9bceb19df38f93b72751ca76
|
3 |
+
size 88
|
log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:13f3cd093d73bedd1bf3459d90058ad191c5a67223ba2ac6c12878a3625c7aae
|
3 |
+
size 88
|
log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:5f1b9adad5b002afc7c6ebddf403821b5a083ed15a0af72fabd13ba6a9722b9b
|
3 |
+
size 88
|
log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:4c4bfb33e34f3eb3dd784c897abd2f3a8957807d963fb9fc6eaa0dcd083cd46d
|
3 |
+
size 88
|
log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:87549ecae170bb23048a2ca45624d8628818bda58f8dbe4ee9064a0ff43e8941
|
3 |
+
size 1000
|
log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:7e3ab5c775ff52b79e327d006329d5d13313d9053300c4358034f65be5a52c88
|
3 |
+
size 88
|
log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:69cfc2f12779ca41a1cb3683d2c8d7734fb83d39d41098edbf6560a233310bb6
|
3 |
+
size 501625
|
log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:872ff443d210926e96e45e61da68ea1da4658c0f7eb963664cc6dd31912f43df
|
3 |
+
size 1245950
|
pytorch_lora_weights.bin
ADDED
@@ -0,0 +1,3 @@
|
|
|
|
|
|
|
|
|
1 |
+
version https://git-lfs.github.com/spec/v1
|
2 |
+
oid sha256:a667d4733a587ab1fd82d05132f013e57f474869c0d379c2d9e13858753f44ce
|
3 |
+
size 3287771
|
wandb/settings
ADDED
@@ -0,0 +1,2 @@
|
|
|
|
|
|
|
1 |
+
[default]
|
2 |
+
|
wandb/wandb/debug-internal.log
ADDED
@@ -0,0 +1,449 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2023-06-14 16:09:40,206 INFO StreamThr :3905241 [internal.py:wandb_internal():86] W&B internal server running at pid: 3905241, started at: 2023-06-14 16:09:40.205938
|
2 |
+
2023-06-14 16:09:40,208 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status
|
3 |
+
2023-06-14 16:09:40,211 INFO WriterThread:3905241 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/run-0vx95vx0.wandb
|
4 |
+
2023-06-14 16:09:40,212 DEBUG SenderThread:3905241 [sender.py:send():369] send: header
|
5 |
+
2023-06-14 16:09:40,401 DEBUG SenderThread:3905241 [sender.py:send():369] send: run
|
6 |
+
2023-06-14 16:09:41,650 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: check_version
|
7 |
+
2023-06-14 16:09:41,655 INFO SenderThread:3905241 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files
|
8 |
+
2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_start_run_threads():1100] run started: 0vx95vx0 with start time 1686758980.206642
|
9 |
+
2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
10 |
+
2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
11 |
+
2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: check_version
|
12 |
+
2023-06-14 16:09:42,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
13 |
+
2023-06-14 16:09:42,670 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: run_start
|
14 |
+
2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():31] System info init
|
15 |
+
2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():46] System info init done
|
16 |
+
2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:start():181] Starting system monitor
|
17 |
+
2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [system_monitor.py:_start():145] Starting system asset monitoring threads
|
18 |
+
2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:probe():201] Collecting system info
|
19 |
+
2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started cpu monitoring
|
20 |
+
2023-06-14 16:09:42,681 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started disk monitoring
|
21 |
+
2023-06-14 16:09:42,682 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started gpu monitoring
|
22 |
+
2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started memory monitoring
|
23 |
+
2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started network monitoring
|
24 |
+
2023-06-14 16:09:42,767 DEBUG HandlerThread:3905241 [system_info.py:probe():195] Probing system
|
25 |
+
2023-06-14 16:09:42,775 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():180] Probing git
|
26 |
+
2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():188] Probing git done
|
27 |
+
2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:probe():240] Probing system done
|
28 |
+
2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:09:42.767191', 'startedAt': '2023-06-14T16:09:40.185854', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': '[email protected]', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.130054687500001, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.533, 'min': 1500.0, 'max': 2800.0}, {'current': 2.322, 'min': 1500.0, 'max': 2800.0}, {'current': 3.465, 'min': 1500.0, 'max': 2800.0}, {'current': 2.612, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.856, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.46, 'min': 1500.0, 'max': 2800.0}, {'current': 2.348, 'min': 1500.0, 'max': 2800.0}, {'current': 2.382, 'min': 1500.0, 'max': 2800.0}, {'current': 2.313, 'min': 1500.0, 'max': 2800.0}, {'current': 3.161, 'min': 1500.0, 'max': 2800.0}, {'current': 3.215, 'min': 1500.0, 'max': 2800.0}, {'current': 3.212, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.363, 'min': 1500.0, 'max': 2800.0}, {'current': 2.491, 'min': 1500.0, 'max': 2800.0}, {'current': 3.406, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.818, 'min': 1500.0, 'max': 2800.0}, {'current': 3.23, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.023, 'min': 1500.0, 'max': 2800.0}, {'current': 3.395, 'min': 1500.0, 'max': 2800.0}, {'current': 3.115, 'min': 1500.0, 'max': 2800.0}, {'current': 2.718, 'min': 1500.0, 'max': 2800.0}, {'current': 2.756, 'min': 1500.0, 'max': 2800.0}, {'current': 3.473, 'min': 1500.0, 'max': 2800.0}, {'current': 2.108, 'min': 1500.0, 'max': 2800.0}, {'current': 2.274, 'min': 1500.0, 'max': 2800.0}, {'current': 1.617, 'min': 1500.0, 'max': 2800.0}, {'current': 2.259, 'min': 1500.0, 'max': 2800.0}, {'current': 3.017, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.003, 'min': 1500.0, 'max': 2800.0}, {'current': 3.0, 'min': 1500.0, 'max': 2800.0}, {'current': 3.007, 'min': 1500.0, 'max': 2800.0}, {'current': 3.019, 'min': 1500.0, 'max': 2800.0}, {'current': 3.49, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.965, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 3.478, 'min': 1500.0, 'max': 2800.0}, {'current': 2.802, 'min': 1500.0, 'max': 2800.0}, {'current': 2.961, 'min': 1500.0, 'max': 2800.0}, {'current': 2.113, 'min': 1500.0, 'max': 2800.0}, {'current': 2.343, 'min': 1500.0, 'max': 2800.0}, {'current': 2.286, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.243, 'min': 1500.0, 'max': 2800.0}, {'current': 2.902, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 1.992, 'min': 1500.0, 'max': 2800.0}, {'current': 2.51, 'min': 1500.0, 'max': 2800.0}, {'current': 2.986, 'min': 1500.0, 'max': 2800.0}, {'current': 3.222, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.281, 'min': 1500.0, 'max': 2800.0}, {'current': 3.046, 'min': 1500.0, 'max': 2800.0}, {'current': 2.506, 'min': 1500.0, 'max': 2800.0}, {'current': 2.536, 'min': 1500.0, 'max': 2800.0}, {'current': 2.544, 'min': 1500.0, 'max': 2800.0}, {'current': 3.285, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.29, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.757, 'min': 1500.0, 'max': 2800.0}, {'current': 2.531, 'min': 1500.0, 'max': 2800.0}, {'current': 2.388, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.93, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.483, 'min': 1500.0, 'max': 2800.0}, {'current': 3.476, 'min': 1500.0, 'max': 2800.0}, {'current': 3.48, 'min': 1500.0, 'max': 2800.0}, {'current': 3.073, 'min': 1500.0, 'max': 2800.0}, {'current': 3.03, 'min': 1500.0, 'max': 2800.0}, {'current': 2.99, 'min': 1500.0, 'max': 2800.0}, {'current': 1.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.027, 'min': 1500.0, 'max': 2800.0}, {'current': 3.477, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.488, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.479, 'min': 1500.0, 'max': 2800.0}, {'current': 3.108, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.469, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.227, 'min': 1500.0, 'max': 2800.0}, {'current': 3.491, 'min': 1500.0, 'max': 2800.0}, {'current': 1.884, 'min': 1500.0, 'max': 2800.0}, {'current': 3.424, 'min': 1500.0, 'max': 2800.0}, {'current': 3.119, 'min': 1500.0, 'max': 2800.0}, {'current': 3.238, 'min': 1500.0, 'max': 2800.0}, {'current': 3.121, 'min': 1500.0, 'max': 2800.0}, {'current': 3.095, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349178314208984}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}}
|
29 |
+
2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():211] Finished collecting system info
|
30 |
+
2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():214] Publishing system info
|
31 |
+
2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment
|
32 |
+
2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():67] Saving pip packages done
|
33 |
+
2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment
|
34 |
+
2023-06-14 16:09:43,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
|
35 |
+
2023-06-14 16:09:43,657 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt
|
36 |
+
2023-06-14 16:09:51,458 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():86] Saving conda packages done
|
37 |
+
2023-06-14 16:09:51,460 INFO HandlerThread:3905241 [system_monitor.py:probe():216] Finished publishing system info
|
38 |
+
2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
39 |
+
2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive
|
40 |
+
2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
41 |
+
2023-06-14 16:09:51,467 DEBUG SenderThread:3905241 [sender.py:send():369] send: files
|
42 |
+
2023-06-14 16:09:51,467 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now
|
43 |
+
2023-06-14 16:09:51,482 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
44 |
+
2023-06-14 16:09:51,484 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
45 |
+
2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
|
46 |
+
2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-metadata.json
|
47 |
+
2023-06-14 16:09:52,979 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/gpvyb6vu-wandb-metadata.json
|
48 |
+
2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: telemetry
|
49 |
+
2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: config
|
50 |
+
2023-06-14 16:09:53,660 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
51 |
+
2023-06-14 16:09:55,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
52 |
+
2023-06-14 16:09:55,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
53 |
+
2023-06-14 16:09:55,994 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
54 |
+
2023-06-14 16:09:56,352 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
55 |
+
2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
56 |
+
2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
57 |
+
2023-06-14 16:09:56,353 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
58 |
+
2023-06-14 16:09:56,617 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
59 |
+
2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
60 |
+
2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
61 |
+
2023-06-14 16:09:56,618 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
62 |
+
2023-06-14 16:09:56,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
63 |
+
2023-06-14 16:09:56,917 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
64 |
+
2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
65 |
+
2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
66 |
+
2023-06-14 16:09:56,918 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
67 |
+
2023-06-14 16:09:57,332 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
68 |
+
2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
69 |
+
2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
70 |
+
2023-06-14 16:09:57,334 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
71 |
+
2023-06-14 16:09:57,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
72 |
+
2023-06-14 16:09:57,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
73 |
+
2023-06-14 16:09:57,870 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
74 |
+
2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
75 |
+
2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
76 |
+
2023-06-14 16:09:57,872 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
77 |
+
2023-06-14 16:09:58,146 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
78 |
+
2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
79 |
+
2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
80 |
+
2023-06-14 16:09:58,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
81 |
+
2023-06-14 16:09:58,515 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
82 |
+
2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
83 |
+
2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
84 |
+
2023-06-14 16:09:58,517 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
85 |
+
2023-06-14 16:09:58,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
86 |
+
2023-06-14 16:09:59,135 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
87 |
+
2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
88 |
+
2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
89 |
+
2023-06-14 16:09:59,145 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
90 |
+
2023-06-14 16:09:59,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
91 |
+
2023-06-14 16:09:59,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
92 |
+
2023-06-14 16:09:59,987 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
93 |
+
2023-06-14 16:09:59,991 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
94 |
+
2023-06-14 16:09:59,992 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
95 |
+
2023-06-14 16:09:59,992 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
96 |
+
2023-06-14 16:10:00,580 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
97 |
+
2023-06-14 16:10:00,580 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
98 |
+
2023-06-14 16:10:00,581 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
99 |
+
2023-06-14 16:10:00,581 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
100 |
+
2023-06-14 16:10:00,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
101 |
+
2023-06-14 16:10:01,111 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
102 |
+
2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
103 |
+
2023-06-14 16:10:01,117 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
104 |
+
2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
105 |
+
2023-06-14 16:10:01,118 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
106 |
+
2023-06-14 16:10:01,405 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
107 |
+
2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
108 |
+
2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
109 |
+
2023-06-14 16:10:01,405 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
110 |
+
2023-06-14 16:10:01,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
111 |
+
2023-06-14 16:10:01,665 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
112 |
+
2023-06-14 16:10:01,668 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
113 |
+
2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
114 |
+
2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
115 |
+
2023-06-14 16:10:01,669 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
116 |
+
2023-06-14 16:10:01,949 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
117 |
+
2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
118 |
+
2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
119 |
+
2023-06-14 16:10:01,950 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
120 |
+
2023-06-14 16:10:02,184 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
121 |
+
2023-06-14 16:10:02,185 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
122 |
+
2023-06-14 16:10:02,186 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
123 |
+
2023-06-14 16:10:02,186 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
124 |
+
2023-06-14 16:10:02,440 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
125 |
+
2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
126 |
+
2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
127 |
+
2023-06-14 16:10:02,441 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
128 |
+
2023-06-14 16:10:02,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
129 |
+
2023-06-14 16:10:02,686 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
130 |
+
2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
131 |
+
2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
132 |
+
2023-06-14 16:10:02,688 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
133 |
+
2023-06-14 16:10:02,972 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
134 |
+
2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
135 |
+
2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
136 |
+
2023-06-14 16:10:02,973 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
137 |
+
2023-06-14 16:10:03,251 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
138 |
+
2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
139 |
+
2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
140 |
+
2023-06-14 16:10:03,252 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
141 |
+
2023-06-14 16:10:03,527 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
142 |
+
2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
143 |
+
2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
144 |
+
2023-06-14 16:10:03,528 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
145 |
+
2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
146 |
+
2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
147 |
+
2023-06-14 16:10:03,812 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
148 |
+
2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
149 |
+
2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
150 |
+
2023-06-14 16:10:03,813 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
151 |
+
2023-06-14 16:10:04,089 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
152 |
+
2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
153 |
+
2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
154 |
+
2023-06-14 16:10:04,091 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
155 |
+
2023-06-14 16:10:04,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
156 |
+
2023-06-14 16:10:04,389 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
157 |
+
2023-06-14 16:10:04,390 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
158 |
+
2023-06-14 16:10:04,390 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
159 |
+
2023-06-14 16:10:04,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
160 |
+
2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
161 |
+
2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
162 |
+
2023-06-14 16:10:04,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
163 |
+
2023-06-14 16:10:04,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
164 |
+
2023-06-14 16:10:04,855 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
165 |
+
2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
166 |
+
2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
167 |
+
2023-06-14 16:10:04,857 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
168 |
+
2023-06-14 16:10:05,084 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
169 |
+
2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
170 |
+
2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
171 |
+
2023-06-14 16:10:05,085 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
172 |
+
2023-06-14 16:10:05,317 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
173 |
+
2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
174 |
+
2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
175 |
+
2023-06-14 16:10:05,318 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
176 |
+
2023-06-14 16:10:05,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
177 |
+
2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
178 |
+
2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
179 |
+
2023-06-14 16:10:05,562 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
180 |
+
2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
181 |
+
2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
182 |
+
2023-06-14 16:10:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
183 |
+
2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
184 |
+
2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
185 |
+
2023-06-14 16:10:05,798 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
186 |
+
2023-06-14 16:10:06,053 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
187 |
+
2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
188 |
+
2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
189 |
+
2023-06-14 16:10:06,054 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
190 |
+
2023-06-14 16:10:06,364 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
191 |
+
2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
192 |
+
2023-06-14 16:10:06,365 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
193 |
+
2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
194 |
+
2023-06-14 16:10:06,366 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
195 |
+
2023-06-14 16:10:06,473 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
196 |
+
2023-06-14 16:10:06,474 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
197 |
+
2023-06-14 16:10:06,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
198 |
+
2023-06-14 16:10:06,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
199 |
+
2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
200 |
+
2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
201 |
+
2023-06-14 16:10:06,863 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
202 |
+
2023-06-14 16:10:07,012 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
203 |
+
2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
204 |
+
2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
205 |
+
2023-06-14 16:10:07,013 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
206 |
+
2023-06-14 16:10:07,314 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
207 |
+
2023-06-14 16:10:07,314 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
208 |
+
2023-06-14 16:10:07,315 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
209 |
+
2023-06-14 16:10:07,315 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
210 |
+
2023-06-14 16:10:07,559 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
211 |
+
2023-06-14 16:10:07,559 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
212 |
+
2023-06-14 16:10:07,560 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
213 |
+
2023-06-14 16:10:07,560 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
214 |
+
2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
215 |
+
2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
216 |
+
2023-06-14 16:10:07,903 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
217 |
+
2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
218 |
+
2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
219 |
+
2023-06-14 16:10:07,904 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
220 |
+
2023-06-14 16:10:08,145 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
221 |
+
2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
222 |
+
2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
223 |
+
2023-06-14 16:10:08,147 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
224 |
+
2023-06-14 16:10:08,407 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
225 |
+
2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
226 |
+
2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
227 |
+
2023-06-14 16:10:08,408 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
228 |
+
2023-06-14 16:10:08,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
229 |
+
2023-06-14 16:10:08,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
230 |
+
2023-06-14 16:10:08,637 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
231 |
+
2023-06-14 16:10:08,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
232 |
+
2023-06-14 16:10:08,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
233 |
+
2023-06-14 16:10:08,977 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
234 |
+
2023-06-14 16:10:08,977 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
235 |
+
2023-06-14 16:10:08,978 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
236 |
+
2023-06-14 16:10:08,978 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
237 |
+
2023-06-14 16:10:09,193 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
238 |
+
2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
239 |
+
2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
240 |
+
2023-06-14 16:10:09,194 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
241 |
+
2023-06-14 16:10:09,508 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
242 |
+
2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
243 |
+
2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
244 |
+
2023-06-14 16:10:09,509 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
245 |
+
2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
246 |
+
2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
247 |
+
2023-06-14 16:10:09,800 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
248 |
+
2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
249 |
+
2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
250 |
+
2023-06-14 16:10:09,802 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
251 |
+
2023-06-14 16:10:10,151 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
252 |
+
2023-06-14 16:10:10,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
253 |
+
2023-06-14 16:10:10,153 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
254 |
+
2023-06-14 16:10:10,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
255 |
+
2023-06-14 16:10:10,411 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
256 |
+
2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
257 |
+
2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
258 |
+
2023-06-14 16:10:10,412 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
259 |
+
2023-06-14 16:10:10,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
260 |
+
2023-06-14 16:10:10,822 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
261 |
+
2023-06-14 16:10:10,822 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
262 |
+
2023-06-14 16:10:10,823 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
263 |
+
2023-06-14 16:10:10,823 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
264 |
+
2023-06-14 16:10:11,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
265 |
+
2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
266 |
+
2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
267 |
+
2023-06-14 16:10:11,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
268 |
+
2023-06-14 16:10:11,485 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
269 |
+
2023-06-14 16:10:11,486 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
270 |
+
2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
271 |
+
2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
272 |
+
2023-06-14 16:10:11,743 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
273 |
+
2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
274 |
+
2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
275 |
+
2023-06-14 16:10:11,895 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
276 |
+
2023-06-14 16:10:11,895 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
277 |
+
2023-06-14 16:10:11,896 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
278 |
+
2023-06-14 16:10:11,897 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
279 |
+
2023-06-14 16:10:12,044 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
280 |
+
2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
281 |
+
2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
282 |
+
2023-06-14 16:10:12,045 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
283 |
+
2023-06-14 16:10:12,291 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
284 |
+
2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
285 |
+
2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
286 |
+
2023-06-14 16:10:12,292 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
287 |
+
2023-06-14 16:10:12,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml
|
288 |
+
2023-06-14 16:10:12,671 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
289 |
+
2023-06-14 16:10:12,672 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
290 |
+
2023-06-14 16:10:12,674 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
291 |
+
2023-06-14 16:10:12,675 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
292 |
+
2023-06-14 16:10:12,675 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
293 |
+
2023-06-14 16:10:13,059 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
294 |
+
2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
295 |
+
2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
296 |
+
2023-06-14 16:10:13,062 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
297 |
+
2023-06-14 16:10:13,638 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
298 |
+
2023-06-14 16:10:13,646 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
299 |
+
2023-06-14 16:10:13,652 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
300 |
+
2023-06-14 16:10:13,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
301 |
+
2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
302 |
+
2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
303 |
+
2023-06-14 16:10:14,246 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
304 |
+
2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
305 |
+
2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
306 |
+
2023-06-14 16:10:14,250 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
307 |
+
2023-06-14 16:10:14,676 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
308 |
+
2023-06-14 16:10:14,834 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
309 |
+
2023-06-14 16:10:14,836 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
310 |
+
2023-06-14 16:10:14,837 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
311 |
+
2023-06-14 16:10:14,837 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
312 |
+
2023-06-14 16:10:15,545 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
313 |
+
2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
314 |
+
2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
315 |
+
2023-06-14 16:10:15,550 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
316 |
+
2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
317 |
+
2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
318 |
+
2023-06-14 16:10:15,864 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
319 |
+
2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
320 |
+
2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
321 |
+
2023-06-14 16:10:15,866 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
322 |
+
2023-06-14 16:10:16,155 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
323 |
+
2023-06-14 16:10:16,156 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
324 |
+
2023-06-14 16:10:16,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
325 |
+
2023-06-14 16:10:16,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
326 |
+
2023-06-14 16:10:16,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
327 |
+
2023-06-14 16:10:17,157 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
328 |
+
2023-06-14 16:10:17,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
329 |
+
2023-06-14 16:10:19,678 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
330 |
+
2023-06-14 16:10:21,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
331 |
+
2023-06-14 16:10:21,488 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
332 |
+
2023-06-14 16:10:21,679 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
333 |
+
2023-06-14 16:10:22,324 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
334 |
+
2023-06-14 16:10:23,680 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
335 |
+
2023-06-14 16:10:25,681 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
336 |
+
2023-06-14 16:10:27,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
337 |
+
2023-06-14 16:10:27,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
338 |
+
2023-06-14 16:10:28,590 DEBUG SenderThread:3905241 [sender.py:send():369] send: files
|
339 |
+
2023-06-14 16:10:28,592 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now
|
340 |
+
2023-06-14 16:10:28,599 DEBUG SenderThread:3905241 [sender.py:send():369] send: files
|
341 |
+
2023-06-14 16:10:28,607 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_2ff5d73691ebb72c877d.png with policy now
|
342 |
+
2023-06-14 16:10:28,633 DEBUG SenderThread:3905241 [sender.py:send():369] send: files
|
343 |
+
2023-06-14 16:10:28,642 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_b3033a9fe6335a935570.png with policy now
|
344 |
+
2023-06-14 16:10:28,649 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history
|
345 |
+
2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send():369] send: history
|
346 |
+
2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record
|
347 |
+
2023-06-14 16:10:28,666 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end
|
348 |
+
2023-06-14 16:10:28,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json
|
349 |
+
2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png
|
350 |
+
2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png
|
351 |
+
2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png
|
352 |
+
2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images
|
353 |
+
2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media
|
354 |
+
2023-06-14 16:10:30,686 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/25n7cl81-media/images/validation_60_2ff5d73691ebb72c877d.png
|
355 |
+
2023-06-14 16:10:31,331 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/34c9bzlj-media/images/validation_60_e9c4a470ad900801f7de.png
|
356 |
+
2023-06-14 16:10:31,684 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
357 |
+
2023-06-14 16:10:32,001 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ew6x2jre-media/images/validation_60_2ff5d73691ebb72c877d.png
|
358 |
+
2023-06-14 16:10:32,323 INFO wandb-upload_2:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5utpuer6-media/images/validation_60_b3033a9fe6335a935570.png
|
359 |
+
2023-06-14 16:10:33,058 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
360 |
+
2023-06-14 16:10:33,739 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5zhjwo2z-media/images/validation_60_b3033a9fe6335a935570.png
|
361 |
+
2023-06-14 16:10:33,806 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ykn8lu3m-media/images/validation_60_e9c4a470ad900801f7de.png
|
362 |
+
2023-06-14 16:10:35,685 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
363 |
+
2023-06-14 16:10:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
364 |
+
2023-06-14 16:10:36,493 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
365 |
+
2023-06-14 16:10:38,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
366 |
+
2023-06-14 16:10:39,687 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
367 |
+
2023-06-14 16:10:41,688 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
368 |
+
2023-06-14 16:10:42,683 DEBUG SystemMonitor:3905241 [system_monitor.py:_start():159] Starting system metrics aggregation loop
|
369 |
+
2023-06-14 16:10:42,686 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
370 |
+
2023-06-14 16:10:43,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
371 |
+
2023-06-14 16:10:43,694 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
372 |
+
2023-06-14 16:10:45,690 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
373 |
+
2023-06-14 16:10:47,691 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
374 |
+
2023-06-14 16:10:48,724 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
375 |
+
2023-06-14 16:10:51,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
376 |
+
2023-06-14 16:10:51,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
377 |
+
2023-06-14 16:10:51,692 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
|
378 |
+
2023-06-14 16:10:54,194 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
379 |
+
2023-06-14 16:10:59,195 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
380 |
+
2023-06-14 16:11:04,196 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
381 |
+
2023-06-14 16:11:06,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
382 |
+
2023-06-14 16:11:06,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
383 |
+
2023-06-14 16:11:09,299 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
384 |
+
2023-06-14 16:11:12,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
385 |
+
2023-06-14 16:11:14,690 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
386 |
+
2023-06-14 16:11:19,700 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
387 |
+
2023-06-14 16:11:21,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
388 |
+
2023-06-14 16:11:21,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
389 |
+
2023-06-14 16:11:24,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
390 |
+
2023-06-14 16:11:29,845 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
391 |
+
2023-06-14 16:11:34,847 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
392 |
+
2023-06-14 16:11:36,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
393 |
+
2023-06-14 16:11:36,490 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
394 |
+
2023-06-14 16:11:39,904 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
395 |
+
2023-06-14 16:11:42,691 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
396 |
+
2023-06-14 16:11:45,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
397 |
+
2023-06-14 16:11:50,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
398 |
+
2023-06-14 16:11:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
399 |
+
2023-06-14 16:11:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
400 |
+
2023-06-14 16:11:55,795 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
401 |
+
2023-06-14 16:12:00,796 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
402 |
+
2023-06-14 16:12:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
403 |
+
2023-06-14 16:12:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
404 |
+
2023-06-14 16:12:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
405 |
+
2023-06-14 16:12:11,782 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
406 |
+
2023-06-14 16:12:12,693 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
407 |
+
2023-06-14 16:12:17,694 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
408 |
+
2023-06-14 16:12:21,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
409 |
+
2023-06-14 16:12:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
410 |
+
2023-06-14 16:12:22,832 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
411 |
+
2023-06-14 16:12:27,835 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
412 |
+
2023-06-14 16:12:32,833 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
413 |
+
2023-06-14 16:12:36,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
414 |
+
2023-06-14 16:12:36,492 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
415 |
+
2023-06-14 16:12:41,149 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
416 |
+
2023-06-14 16:12:42,695 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
417 |
+
2023-06-14 16:12:46,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
418 |
+
2023-06-14 16:12:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
419 |
+
2023-06-14 16:12:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
420 |
+
2023-06-14 16:12:51,947 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
421 |
+
2023-06-14 16:12:56,955 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
422 |
+
2023-06-14 16:13:01,956 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
423 |
+
2023-06-14 16:13:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
424 |
+
2023-06-14 16:13:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
425 |
+
2023-06-14 16:13:07,794 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
426 |
+
2023-06-14 16:13:12,697 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
427 |
+
2023-06-14 16:13:13,707 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
428 |
+
2023-06-14 16:13:18,708 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
429 |
+
2023-06-14 16:13:21,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
430 |
+
2023-06-14 16:13:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
431 |
+
2023-06-14 16:13:23,774 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
432 |
+
2023-06-14 16:13:28,779 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
433 |
+
2023-06-14 16:13:33,776 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
434 |
+
2023-06-14 16:13:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
435 |
+
2023-06-14 16:13:36,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
436 |
+
2023-06-14 16:13:41,492 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive
|
437 |
+
2023-06-14 16:13:42,393 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
438 |
+
2023-06-14 16:13:42,701 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
439 |
+
2023-06-14 16:13:47,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
440 |
+
2023-06-14 16:13:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
441 |
+
2023-06-14 16:13:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
442 |
+
2023-06-14 16:13:52,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
443 |
+
2023-06-14 16:13:57,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
444 |
+
2023-06-14 16:14:02,875 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
445 |
+
2023-06-14 16:14:06,494 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status
|
446 |
+
2023-06-14 16:14:06,494 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status
|
447 |
+
2023-06-14 16:14:08,110 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
448 |
+
2023-06-14 16:14:12,702 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats
|
449 |
+
2023-06-14 16:14:13,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report
|
wandb/wandb/debug.log
ADDED
@@ -0,0 +1,30 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4
|
2 |
+
2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Configure stats pid to 3904922
|
3 |
+
2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings
|
4 |
+
2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings
|
5 |
+
2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR
|
6 |
+
2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR
|
7 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'}
|
8 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False}
|
9 |
+
2023-06-14 16:09:40,199 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py
|
10 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'}
|
11 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug.log
|
12 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug-internal.log
|
13 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():547] calling init triggers
|
14 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():554] wandb.init called with sweep_config: {}
|
15 |
+
config: {}
|
16 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():596] starting backend
|
17 |
+
2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():600] setting up manager
|
18 |
+
2023-06-14 16:09:40,201 INFO MainThread:3904922 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn
|
19 |
+
2023-06-14 16:09:40,206 INFO MainThread:3904922 [wandb_init.py:init():606] backend started and connected
|
20 |
+
2023-06-14 16:09:40,210 INFO MainThread:3904922 [wandb_init.py:init():703] updated telemetry
|
21 |
+
2023-06-14 16:09:40,398 INFO MainThread:3904922 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout
|
22 |
+
2023-06-14 16:09:41,650 INFO MainThread:3904922 [wandb_run.py:_on_init():2176] communicating current version
|
23 |
+
2023-06-14 16:09:42,662 INFO MainThread:3904922 [wandb_run.py:_on_init():2185] got version response
|
24 |
+
2023-06-14 16:09:42,662 INFO MainThread:3904922 [wandb_init.py:init():787] starting run threads in backend
|
25 |
+
2023-06-14 16:09:51,480 INFO MainThread:3904922 [wandb_run.py:_console_start():2155] atexit reg
|
26 |
+
2023-06-14 16:09:51,481 INFO MainThread:3904922 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW
|
27 |
+
2023-06-14 16:09:51,482 INFO MainThread:3904922 [wandb_run.py:_redirect():2075] Wrapping output streams.
|
28 |
+
2023-06-14 16:09:51,482 INFO MainThread:3904922 [wandb_run.py:_redirect():2100] Redirects installed.
|
29 |
+
2023-06-14 16:09:51,483 INFO MainThread:3904922 [wandb_init.py:init():828] run started, returning control to user process
|
30 |
+
2023-06-14 16:09:51,483 INFO MainThread:3904922 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False}
|
wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml
ADDED
@@ -0,0 +1,191 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
name: deepanshu
|
2 |
+
channels:
|
3 |
+
- conda-forge
|
4 |
+
- defaults
|
5 |
+
dependencies:
|
6 |
+
- _libgcc_mutex=0.1=main
|
7 |
+
- _openmp_mutex=5.1=1_gnu
|
8 |
+
- asttokens=2.2.1=pyhd8ed1ab_0
|
9 |
+
- backcall=0.2.0=pyh9f0ad1d_0
|
10 |
+
- backports=1.0=pyhd8ed1ab_3
|
11 |
+
- backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0
|
12 |
+
- ca-certificates=2023.5.7=hbcca054_0
|
13 |
+
- debugpy=1.5.1=py39h295c915_0
|
14 |
+
- decorator=5.1.1=pyhd8ed1ab_0
|
15 |
+
- entrypoints=0.4=pyhd8ed1ab_0
|
16 |
+
- executing=1.2.0=pyhd8ed1ab_0
|
17 |
+
- ipykernel=6.15.0=pyh210e3f2_0
|
18 |
+
- ipython=8.13.2=pyh41d4057_0
|
19 |
+
- jedi=0.18.2=pyhd8ed1ab_0
|
20 |
+
- jupyter_client=7.0.6=pyhd8ed1ab_0
|
21 |
+
- jupyter_core=5.3.0=py39hf3d152e_0
|
22 |
+
- ld_impl_linux-64=2.38=h1181459_1
|
23 |
+
- libffi=3.4.4=h6a678d5_0
|
24 |
+
- libgcc-ng=11.2.0=h1234567_1
|
25 |
+
- libgomp=11.2.0=h1234567_1
|
26 |
+
- libsodium=1.0.18=h36c2ea0_1
|
27 |
+
- libstdcxx-ng=11.2.0=h1234567_1
|
28 |
+
- matplotlib-inline=0.1.6=pyhd8ed1ab_0
|
29 |
+
- ncurses=6.4=h6a678d5_0
|
30 |
+
- nest-asyncio=1.5.6=pyhd8ed1ab_0
|
31 |
+
- openssl=1.1.1t=h7f8727e_0
|
32 |
+
- packaging=23.1=pyhd8ed1ab_0
|
33 |
+
- parso=0.8.3=pyhd8ed1ab_0
|
34 |
+
- pexpect=4.8.0=pyh1a96a4e_2
|
35 |
+
- pickleshare=0.7.5=py_1003
|
36 |
+
- pip=23.0.1=py39h06a4308_0
|
37 |
+
- platformdirs=3.5.1=pyhd8ed1ab_0
|
38 |
+
- prompt-toolkit=3.0.38=pyha770c72_0
|
39 |
+
- prompt_toolkit=3.0.38=hd8ed1ab_0
|
40 |
+
- ptyprocess=0.7.0=pyhd3deb0d_0
|
41 |
+
- pure_eval=0.2.2=pyhd8ed1ab_0
|
42 |
+
- pygments=2.15.1=pyhd8ed1ab_0
|
43 |
+
- python=3.9.16=h7a1cb2a_2
|
44 |
+
- python-dateutil=2.8.2=pyhd8ed1ab_0
|
45 |
+
- python_abi=3.9=2_cp39
|
46 |
+
- pyzmq=19.0.2=py39hb69f2a1_2
|
47 |
+
- readline=8.2=h5eee18b_0
|
48 |
+
- setuptools=66.0.0=py39h06a4308_0
|
49 |
+
- six=1.16.0=pyh6c4a22f_0
|
50 |
+
- sqlite=3.41.2=h5eee18b_0
|
51 |
+
- stack_data=0.6.2=pyhd8ed1ab_0
|
52 |
+
- tk=8.6.12=h1ccaba5_0
|
53 |
+
- tornado=6.1=py39hb9d737c_3
|
54 |
+
- traitlets=5.9.0=pyhd8ed1ab_0
|
55 |
+
- typing_extensions=4.6.2=pyha770c72_0
|
56 |
+
- wcwidth=0.2.6=pyhd8ed1ab_0
|
57 |
+
- wheel=0.38.4=py39h06a4308_0
|
58 |
+
- xz=5.4.2=h5eee18b_0
|
59 |
+
- zeromq=4.3.4=h9c3ff4c_1
|
60 |
+
- zlib=1.2.13=h5eee18b_0
|
61 |
+
- pip:
|
62 |
+
- absl-py==1.4.0
|
63 |
+
- accelerate==0.19.0
|
64 |
+
- aiofiles==23.1.0
|
65 |
+
- aiohttp==3.8.4
|
66 |
+
- aiosignal==1.3.1
|
67 |
+
- altair==5.0.1
|
68 |
+
- anyio==3.7.0
|
69 |
+
- appdirs==1.4.4
|
70 |
+
- async-timeout==4.0.2
|
71 |
+
- attrs==23.1.0
|
72 |
+
- bitsandbytes==0.39.0
|
73 |
+
- cachetools==5.3.1
|
74 |
+
- certifi==2022.12.7
|
75 |
+
- charset-normalizer==2.1.1
|
76 |
+
- click==8.1.3
|
77 |
+
- clip==1.0
|
78 |
+
- cmake==3.25.0
|
79 |
+
- contourpy==1.0.7
|
80 |
+
- cycler==0.11.0
|
81 |
+
- datasets==2.12.0
|
82 |
+
- deepspeed==0.9.2
|
83 |
+
- diffusers==0.16.1
|
84 |
+
- dill==0.3.6
|
85 |
+
- docker-pycreds==0.4.0
|
86 |
+
- exceptiongroup==1.1.1
|
87 |
+
- fastapi==0.95.2
|
88 |
+
- ffmpy==0.3.0
|
89 |
+
- filelock==3.9.0
|
90 |
+
- fonttools==4.39.4
|
91 |
+
- frozenlist==1.3.3
|
92 |
+
- fsspec==2023.5.0
|
93 |
+
- ftfy==6.1.1
|
94 |
+
- gitdb==4.0.10
|
95 |
+
- gitpython==3.1.31
|
96 |
+
- google-auth==2.20.0
|
97 |
+
- google-auth-oauthlib==1.0.0
|
98 |
+
- gradio==3.32.0
|
99 |
+
- gradio-client==0.2.5
|
100 |
+
- grpcio==1.54.2
|
101 |
+
- h11==0.14.0
|
102 |
+
- hjson==3.1.0
|
103 |
+
- httpcore==0.17.2
|
104 |
+
- httpx==0.24.1
|
105 |
+
- huggingface-hub==0.14.1
|
106 |
+
- idna==3.4
|
107 |
+
- importlib-metadata==6.6.0
|
108 |
+
- importlib-resources==5.12.0
|
109 |
+
- jinja2==3.1.2
|
110 |
+
- jsonschema==4.17.3
|
111 |
+
- kiwisolver==1.4.4
|
112 |
+
- linkify-it-py==2.0.2
|
113 |
+
- lit==15.0.7
|
114 |
+
- markdown==3.4.3
|
115 |
+
- markdown-it-py==2.2.0
|
116 |
+
- markupsafe==2.1.2
|
117 |
+
- matplotlib==3.7.1
|
118 |
+
- mdit-py-plugins==0.3.3
|
119 |
+
- mdurl==0.1.2
|
120 |
+
- mpmath==1.2.1
|
121 |
+
- multidict==6.0.4
|
122 |
+
- multiprocess==0.70.14
|
123 |
+
- mxnet==1.9.1
|
124 |
+
- mypy-extensions==1.0.0
|
125 |
+
- natsort==8.3.1
|
126 |
+
- networkx==3.0
|
127 |
+
- ninja==1.11.1
|
128 |
+
- numpy==1.24.1
|
129 |
+
- oauthlib==3.2.2
|
130 |
+
- open-clip-torch==2.20.0
|
131 |
+
- orjson==3.8.14
|
132 |
+
- pandas==2.0.1
|
133 |
+
- pathtools==0.1.2
|
134 |
+
- pillow==9.3.0
|
135 |
+
- plotly==5.15.0
|
136 |
+
- protobuf==3.20.3
|
137 |
+
- psutil==5.9.5
|
138 |
+
- py-cpuinfo==9.0.0
|
139 |
+
- pyarrow==12.0.0
|
140 |
+
- pyasn1==0.5.0
|
141 |
+
- pyasn1-modules==0.3.0
|
142 |
+
- pydantic==1.10.8
|
143 |
+
- pydub==0.25.1
|
144 |
+
- pyparsing==3.0.9
|
145 |
+
- pyre-extensions==0.0.29
|
146 |
+
- pyrsistent==0.19.3
|
147 |
+
- python-graphviz==0.8.4
|
148 |
+
- python-multipart==0.0.6
|
149 |
+
- pytz==2023.3
|
150 |
+
- pyyaml==6.0
|
151 |
+
- regex==2023.5.5
|
152 |
+
- requests==2.28.1
|
153 |
+
- requests-oauthlib==1.3.1
|
154 |
+
- responses==0.18.0
|
155 |
+
- rsa==4.9
|
156 |
+
- safetensors==0.3.1
|
157 |
+
- scipy==1.10.1
|
158 |
+
- semantic-version==2.10.0
|
159 |
+
- sentencepiece==0.1.99
|
160 |
+
- sentry-sdk==1.25.1
|
161 |
+
- setproctitle==1.3.2
|
162 |
+
- smmap==5.0.0
|
163 |
+
- sniffio==1.3.0
|
164 |
+
- starlette==0.27.0
|
165 |
+
- sympy==1.11.1
|
166 |
+
- tenacity==8.2.2
|
167 |
+
- tensorboard==2.13.0
|
168 |
+
- tensorboard-data-server==0.7.1
|
169 |
+
- timm==0.9.2
|
170 |
+
- tokenizers==0.13.3
|
171 |
+
- toolz==0.12.0
|
172 |
+
- torch==2.0.1+cu118
|
173 |
+
- torchaudio==2.0.2+cu118
|
174 |
+
- torchvision==0.15.2+cu118
|
175 |
+
- tqdm==4.65.0
|
176 |
+
- transformers==4.29.2
|
177 |
+
- triton==2.0.0
|
178 |
+
- typing-extensions==4.4.0
|
179 |
+
- typing-inspect==0.9.0
|
180 |
+
- tzdata==2023.3
|
181 |
+
- uc-micro-py==1.0.2
|
182 |
+
- urllib3==1.26.13
|
183 |
+
- uvicorn==0.22.0
|
184 |
+
- wandb==0.15.4
|
185 |
+
- websockets==11.0.3
|
186 |
+
- werkzeug==2.3.6
|
187 |
+
- xformers==0.0.20
|
188 |
+
- xxhash==3.2.0
|
189 |
+
- yarl==1.9.2
|
190 |
+
- zipp==3.15.0
|
191 |
+
prefix: /home/deepanshu/anaconda3/envs/deepanshu
|
wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml
ADDED
@@ -0,0 +1,182 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
wandb_version: 1
|
2 |
+
|
3 |
+
_wandb:
|
4 |
+
desc: null
|
5 |
+
value:
|
6 |
+
python_version: 3.9.16
|
7 |
+
cli_version: 0.15.4
|
8 |
+
framework: huggingface
|
9 |
+
huggingface_version: 4.29.2
|
10 |
+
is_jupyter_run: false
|
11 |
+
is_kaggle_kernel: false
|
12 |
+
start_time: 1686758980.206642
|
13 |
+
t:
|
14 |
+
1:
|
15 |
+
- 1
|
16 |
+
- 11
|
17 |
+
- 41
|
18 |
+
- 49
|
19 |
+
- 51
|
20 |
+
- 55
|
21 |
+
- 71
|
22 |
+
- 83
|
23 |
+
2:
|
24 |
+
- 1
|
25 |
+
- 11
|
26 |
+
- 41
|
27 |
+
- 49
|
28 |
+
- 51
|
29 |
+
- 55
|
30 |
+
- 71
|
31 |
+
- 83
|
32 |
+
3:
|
33 |
+
- 23
|
34 |
+
4: 3.9.16
|
35 |
+
5: 0.15.4
|
36 |
+
6: 4.29.2
|
37 |
+
8:
|
38 |
+
- 5
|
39 |
+
num_validation_images:
|
40 |
+
desc: null
|
41 |
+
value: 1
|
42 |
+
validation_epochs:
|
43 |
+
desc: null
|
44 |
+
value: 60
|
45 |
+
max_train_samples:
|
46 |
+
desc: null
|
47 |
+
value: null
|
48 |
+
output_dir:
|
49 |
+
desc: null
|
50 |
+
value: /home/deepanshu/lmflowdiff/Model
|
51 |
+
seed:
|
52 |
+
desc: null
|
53 |
+
value: 123
|
54 |
+
train_batch_size:
|
55 |
+
desc: null
|
56 |
+
value: 1
|
57 |
+
num_train_epochs:
|
58 |
+
desc: null
|
59 |
+
value: 1
|
60 |
+
max_train_steps:
|
61 |
+
desc: null
|
62 |
+
value: 60
|
63 |
+
gradient_accumulation_steps:
|
64 |
+
desc: null
|
65 |
+
value: 1
|
66 |
+
gradient_checkpointing:
|
67 |
+
desc: null
|
68 |
+
value: true
|
69 |
+
learning_rate:
|
70 |
+
desc: null
|
71 |
+
value: 1.0e-06
|
72 |
+
scale_lr:
|
73 |
+
desc: null
|
74 |
+
value: false
|
75 |
+
lr_scheduler:
|
76 |
+
desc: null
|
77 |
+
value: constant
|
78 |
+
lr_warmup_steps:
|
79 |
+
desc: null
|
80 |
+
value: 0
|
81 |
+
snr_gamma:
|
82 |
+
desc: null
|
83 |
+
value: null
|
84 |
+
use_8bit_adam:
|
85 |
+
desc: null
|
86 |
+
value: false
|
87 |
+
allow_tf32:
|
88 |
+
desc: null
|
89 |
+
value: false
|
90 |
+
adam_beta1:
|
91 |
+
desc: null
|
92 |
+
value: 0.9
|
93 |
+
adam_beta2:
|
94 |
+
desc: null
|
95 |
+
value: 0.999
|
96 |
+
adam_weight_decay:
|
97 |
+
desc: null
|
98 |
+
value: 0.01
|
99 |
+
adam_epsilon:
|
100 |
+
desc: null
|
101 |
+
value: 1.0e-08
|
102 |
+
max_grad_norm:
|
103 |
+
desc: null
|
104 |
+
value: 1
|
105 |
+
push_to_hub:
|
106 |
+
desc: null
|
107 |
+
value: true
|
108 |
+
hub_token:
|
109 |
+
desc: null
|
110 |
+
value: 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
|
111 |
+
hub_model_id:
|
112 |
+
desc: null
|
113 |
+
value: lora5_tb
|
114 |
+
logging_dir:
|
115 |
+
desc: null
|
116 |
+
value: log
|
117 |
+
mixed_precision:
|
118 |
+
desc: null
|
119 |
+
value: fp16
|
120 |
+
resume_from_checkpoint:
|
121 |
+
desc: null
|
122 |
+
value: latest
|
123 |
+
fp16_backend:
|
124 |
+
desc: null
|
125 |
+
value: auto
|
126 |
+
fp16_opt_level:
|
127 |
+
desc: null
|
128 |
+
value: O1
|
129 |
+
disable_tqdm:
|
130 |
+
desc: null
|
131 |
+
value: false
|
132 |
+
save_steps:
|
133 |
+
desc: null
|
134 |
+
value: 1000
|
135 |
+
save_total_limit:
|
136 |
+
desc: null
|
137 |
+
value: null
|
138 |
+
eval_dataset_path:
|
139 |
+
desc: null
|
140 |
+
value: null
|
141 |
+
tracker_project_name:
|
142 |
+
desc: null
|
143 |
+
value: text2image-fine-tune
|
144 |
+
checkpoints_total_limit:
|
145 |
+
desc: null
|
146 |
+
value: null
|
147 |
+
enable_xformers_memory_efficient_attention:
|
148 |
+
desc: null
|
149 |
+
value: true
|
150 |
+
noise_offset:
|
151 |
+
desc: null
|
152 |
+
value: 1
|
153 |
+
input_perturbation:
|
154 |
+
desc: null
|
155 |
+
value: 0
|
156 |
+
checkpointing_steps:
|
157 |
+
desc: null
|
158 |
+
value: 500
|
159 |
+
report_to:
|
160 |
+
desc: null
|
161 |
+
value: wandb
|
162 |
+
accelerate_device:
|
163 |
+
desc: null
|
164 |
+
value: cuda:1
|
165 |
+
accelerate_local_process_index:
|
166 |
+
desc: null
|
167 |
+
value: 0
|
168 |
+
accelerate_num_processes:
|
169 |
+
desc: null
|
170 |
+
value: 1
|
171 |
+
accelerate_optimizer_step_was_skipped:
|
172 |
+
desc: null
|
173 |
+
value: false
|
174 |
+
accelerate_process_index:
|
175 |
+
desc: null
|
176 |
+
value: 0
|
177 |
+
accelerate_sync_gradients:
|
178 |
+
desc: null
|
179 |
+
value: true
|
180 |
+
accelerate_use_distributed:
|
181 |
+
desc: null
|
182 |
+
value: false
|
wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png
ADDED
wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png
ADDED
wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png
ADDED
wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log
ADDED
@@ -0,0 +1,63 @@
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
1 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - ***** Running training *****
|
2 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Num examples = 833
|
3 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Num Epochs = 1
|
4 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Instantaneous batch size per device = 1
|
5 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Total train batch size (w. parallel, distributed & accumulation) = 1
|
6 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Gradient Accumulation steps = 1
|
7 |
+
06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Total optimization steps = 60
|
8 |
+
Steps: 0%| | 0/60 [00:00<?, ?it/s]
|
9 |
+
Steps: 0%| | 0/60 [00:00<?, ?it/s]/home/deepanshu/anaconda3/envs/deepanshu/lib/python3.9/site-packages/xformers/ops/fmha/flash.py:339: UserWarning: TypedStorage is deprecated. It will be removed in the future and UntypedStorage will be the only storage class. This should only matter to you if you are using storages directly. To access UntypedStorage directly, use tensor.untyped_storage() instead of tensor.storage()
|
10 |
+
and inp.query.storage().data_ptr() == inp.key.storage().data_ptr()
|
11 |
+
|
12 |
+
|
13 |
+
|
14 |
+
|
15 |
+
|
16 |
+
|
17 |
+
|
18 |
+
|
19 |
+
|
20 |
+
Steps: 100%|βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 60/60 [00:24<00:00, 2.31it/s, lr=1e-6, step_loss=0.438]06/14/2023 16:10:16 - INFO - lmflow_aigen.models.diffusion_model - Running validation...
|
21 |
+
Generating 1 images with prompt: ['pikachu', 'bulbasaur', 'charmandar'].
|
22 |
+
{'requires_safety_checker'} was not found in config. Values will be initialized to default values.
|
23 |
+
`text_config_dict` is provided which will be used to initialize `CLIPTextConfig`. The value `text_config["id2label"]` will be overriden.
|
24 |
+
{'prediction_type'} was not found in config. Values will be initialized to default values.
|
25 |
+
{'scaling_factor'} was not found in config. Values will be initialized to default values.
|
26 |
+
Potential NSFW content was detected in one or more images. A black image will be returned instead. Try again with a different prompt and/or seed.ββ| 20/20 [00:02<00:00, 9.69it/s]
|
27 |
+
100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 20/20 [00:01<00:00, 11.02it/s]
|
28 |
+
|
29 |
+
|
30 |
+
Model weights saved in /home/deepanshu/lmflowdiff/Model/pytorch_lora_weights.binββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ | 19/20 [00:02<00:00, 8.16it/s]
|
31 |
+
events.out.tfevents.1686720913.A40-20230526-003.3802189.1: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 2.77k/2.77k [00:01<00:00, 2.69kB/s]
|
32 |
+
events.out.tfevents.1686719590.A40-20230526-003.3796603.1: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 2.77k/2.77k [00:01<00:00, 2.50kB/s]
|
33 |
+
events.out.tfevents.1686719590.A40-20230526-003.3796603.1: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 2.77k/2.77k [00:00<00:00, 3.62kB/s]
|
34 |
+
events.out.tfevents.1686719831.A40-20230526-003.3797971.1: 0%| | 0.00/2.77k [00:00<?, ?B/s]
|
35 |
+
Upload 24 LFS files: 0%| | 0/24 [00:00<?, ?it/s]
|
36 |
+
events.out.tfevents.1686720979.A40-20230526-003.3802846.1: 0%| | 0.00/2.77k [00:00<?, ?B/s]
|
37 |
+
'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
38 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=fbb4a7081c82a2d280d2dfc180fa058abbace33e5f048ae4781f58eb3e53384b&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
39 |
+
Retrying in 1s [Retry 1/5].
|
40 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
|
41 |
+
'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
42 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1b6fbebbf1a4ff923fd9f43a1d9c901d2bdc8bba7995611ddab37e2eba6b158d&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
43 |
+
'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
44 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - 'HTTPSConnectionPool(host='s3.us-east-1.amazonaws.com', port=443): Max retries exceeded with url: /lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject (Caused by ProxyError('Cannot connect to proxy.', ConnectionResetError(104, 'Connection reset by peer')))' thrown while requesting PUT https://s3.us-east-1.amazonaws.com/lfs.huggingface.co/repos/14/83/1483c9d72d68580c3f33413165e54de4e94db4449544c2145ddfa738a9fa3745/1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e?X-Amz-Algorithm=AWS4-HMAC-SHA256&X-Amz-Content-Sha256=UNSIGNED-PAYLOAD&X-Amz-Credential=AKIA4N7VTDGO27GPWFUO%2F20230614%2Fus-east-1%2Fs3%2Faws4_request&X-Amz-Date=20230614T161032Z&X-Amz-Expires=900&X-Amz-Signature=1fec8f00120f06f35e5a294485f80c94b923f8db91ef5c5e75b3247d047d742f&X-Amz-SignedHeaders=host&x-amz-storage-class=INTELLIGENT_TIERING&x-id=PutObject
|
45 |
+
Retrying in 1s [Retry 1/5].
|
46 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
|
47 |
+
Retrying in 1s [Retry 1/5].
|
48 |
+
06/14/2023 16:10:38 - WARNING - huggingface_hub.utils._http - Retrying in 1s [Retry 1/5].
|
49 |
+
events.out.tfevents.1686718938.A40-20230526-003.3793953.1: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 2.77k/2.77k [00:07<00:00, 388B/s]
|
50 |
+
events.out.tfevents.1686719831.A40-20230526-003.3797971.0: 0%| | 0.00/3.72M [00:00<?, ?B/s]
|
51 |
+
events.out.tfevents.1686724122.A40-20230526-003.3812918.0: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 88.0/88.0 [00:00<00:00, 307B/s]
|
52 |
+
events.out.tfevents.1686724122.A40-20230526-003.3812918.0: 0%| | 0.00/88.0 [00:00<?, ?B/s]
|
53 |
+
events.out.tfevents.1686719831.A40-20230526-003.3797971.1: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 2.77k/2.77k [00:09<00:00, 291B/s]
|
54 |
+
events.out.tfevents.1686720913.A40-20230526-003.3802189.0: 71%|βββββββββββββββββββββββββββββββββββββββββββββββββββββββ | 2.63M/3.72M [00:00<00:00, 10.3MB/s]
|
55 |
+
events.out.tfevents.1686737891.A40-20230526-003.3852910.0: 100%|βββββββββββββββββββββββββββββββββοΏ½οΏ½ββββββββββββββββββββββββββββββββββββββββββββββββ| 88.0/88.0 [00:00<00:00, 319B/s]
|
56 |
+
events.out.tfevents.1686737963.A40-20230526-003.3853473.0: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 88.0/88.0 [00:00<00:00, 321B/s]
|
57 |
+
events.out.tfevents.1686738195.A40-20230526-003.3854615.0: 100%|ββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 1.00k/1.00k [00:00<00:00, 3.53kB/s]
|
58 |
+
events.out.tfevents.1686737963.A40-20230526-003.3853473.0: 0%| | 0.00/88.0 [00:00<?, ?B/s]
|
59 |
+
events.out.tfevents.1686738251.A40-20230526-003.3855176.0: 0%| | 0.00/88.0 [00:00<?, ?B/s]
|
60 |
+
pytorch_lora_weights.bin: 100%|βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 3.29M/3.29M [00:00<00:00, 3.32MB/s]
|
61 |
+
events.out.tfevents.1686740628.A40-20230526-003.3863181.0: 100%|βββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββββ| 1.25M/1.25M [00:01<00:00, 898kB/s]
|
62 |
+
events.out.tfevents.1686740628.A40-20230526-003.3863181.0: 0%| | 0.00/1.25M [00:00<?, ?B/s]
|
63 |
+
|