diff --git a/README.md b/README.md new file mode 100644 index 0000000000000000000000000000000000000000..d73d0388c0eb5cfc1d2c902274558f161e242d14 --- /dev/null +++ b/README.md @@ -0,0 +1,21 @@ + +--- +license: creativeml-openrail-m +base_model: runwayml/stable-diffusion-v1-5 +tags: +- stable-diffusion +- stable-diffusion-diffusers +- text-to-image +- diffusers +- lora +inference: true +--- + +# LoRA text2image fine-tuning - deepgoyal19/lora5_tb +These are LoRA adaption weights for runwayml/stable-diffusion-v1-5. The weights were fine-tuned on the lambdalabs/pokemon-blip-captions dataset. You can find some example images in the following. + +![img_0](./image_0.png) +![img_1](./image_1.png) +![img_2](./image_2.png) + + \ No newline at end of file diff --git a/image_0.png b/image_0.png new file mode 100644 index 0000000000000000000000000000000000000000..5220d0f9dbfa747cdb356761a73beb1b1f1c96bb Binary files /dev/null and b/image_0.png differ diff --git a/image_1.png b/image_1.png new file mode 100644 index 0000000000000000000000000000000000000000..e890d225f3d8685a6ae00d16b4e60b18ce0a9c52 Binary files /dev/null and b/image_1.png differ diff --git a/image_2.png b/image_2.png new file mode 100644 index 0000000000000000000000000000000000000000..b555e4f48e7cf144860a9ac5bfc97dbd8e79c681 Binary files /dev/null and b/image_2.png differ diff --git a/image_3.png b/image_3.png new file mode 100644 index 0000000000000000000000000000000000000000..e214a4002196d3001d71e9769d2c8ecb384d0b6d Binary files /dev/null and b/image_3.png differ diff --git a/image_4.png b/image_4.png new file mode 100644 index 0000000000000000000000000000000000000000..c0309f1e66ffbe616bc61920fcdc688b1df90fd3 Binary files /dev/null and b/image_4.png differ diff --git a/image_5.png b/image_5.png new file mode 100644 index 0000000000000000000000000000000000000000..650852a9694cc5562fe42da73b6ed0c426d0da17 Binary files /dev/null and b/image_5.png differ diff --git a/image_6.png b/image_6.png new file mode 100644 index 0000000000000000000000000000000000000000..3f6a43e9b0a16854a97de984cc9bbb1691737fc7 Binary files /dev/null and b/image_6.png differ diff --git a/log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1 b/log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1 new file mode 100644 index 0000000000000000000000000000000000000000..4f3abd87981b9c4b99cab6f81c97c7d360666f9e --- /dev/null +++ b/log/text2image-fine-tune/1686718938.2207263/events.out.tfevents.1686718938.A40-20230526-003.3793953.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e7de852e73b9628508dc0879c8244a922cf643fb96bac22d053de6e2658d2fe2 +size 2769 diff --git a/log/text2image-fine-tune/1686718938.2222803/hparams.yml b/log/text2image-fine-tune/1686718938.2222803/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..96751db40ebb96d157125245b954e34c0ba69611 --- /dev/null +++ b/log/text2image-fine-tune/1686718938.2222803/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 15 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 15 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1 b/log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1 new file mode 100644 index 0000000000000000000000000000000000000000..3ade00be719b328b5016383e7eb42f58ed991c38 --- /dev/null +++ b/log/text2image-fine-tune/1686719590.3468492/events.out.tfevents.1686719590.A40-20230526-003.3796603.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a4a4522643499eb3044dde554a2004af16634c1fd90f19f7352288e003ec7da2 +size 2770 diff --git a/log/text2image-fine-tune/1686719590.3490849/hparams.yml b/log/text2image-fine-tune/1686719590.3490849/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..873480bca5308f0c8a300c2f905939cbe45cc1f4 --- /dev/null +++ b/log/text2image-fine-tune/1686719590.3490849/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora1_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 15 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 15 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1 b/log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1 new file mode 100644 index 0000000000000000000000000000000000000000..9c8863363d586332ff57af09bba6f0f490ae5deb --- /dev/null +++ b/log/text2image-fine-tune/1686719831.1614823/events.out.tfevents.1686719831.A40-20230526-003.3797971.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:22874eb1d33fdb2c198d2c90fc6eed2236df768d7beca23e82b5d452cd8f1f18 +size 2770 diff --git a/log/text2image-fine-tune/1686719831.1635435/hparams.yml b/log/text2image-fine-tune/1686719831.1635435/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..873480bca5308f0c8a300c2f905939cbe45cc1f4 --- /dev/null +++ b/log/text2image-fine-tune/1686719831.1635435/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora1_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 15 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 15 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1 b/log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1 new file mode 100644 index 0000000000000000000000000000000000000000..e896df04ff45447adf2242071f9047cfacb5a093 --- /dev/null +++ b/log/text2image-fine-tune/1686720913.148579/events.out.tfevents.1686720913.A40-20230526-003.3802189.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:e23009bcbd80acf0569ea70abd119452ccdb9a69edcc33f7038a928bdd939482 +size 2770 diff --git a/log/text2image-fine-tune/1686720913.1509812/hparams.yml b/log/text2image-fine-tune/1686720913.1509812/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..89fe58e8989d377d981c545bb8652c9f20cc87fc --- /dev/null +++ b/log/text2image-fine-tune/1686720913.1509812/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora2_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1 b/log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1 new file mode 100644 index 0000000000000000000000000000000000000000..a35f8520eb211f2b70d6e06163b5c10c9510a787 --- /dev/null +++ b/log/text2image-fine-tune/1686720979.8038175/events.out.tfevents.1686720979.A40-20230526-003.3802846.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:1d3fe323470d1dc94929ac14ae67af8b0b0dee9487b6544ee27b2178fb8d899e +size 2770 diff --git a/log/text2image-fine-tune/1686720979.8056188/hparams.yml b/log/text2image-fine-tune/1686720979.8056188/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..89fe58e8989d377d981c545bb8652c9f20cc87fc --- /dev/null +++ b/log/text2image-fine-tune/1686720979.8056188/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora2_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1 b/log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1 new file mode 100644 index 0000000000000000000000000000000000000000..251302c3395a20cfcddd07af8095e2280728eb57 --- /dev/null +++ b/log/text2image-fine-tune/1686721041.5550754/events.out.tfevents.1686721041.A40-20230526-003.3803507.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:dedc3b5204b74b2151540c3347d61ee8c84006b9c95481ff2724ac0911f8848a +size 2770 diff --git a/log/text2image-fine-tune/1686721041.5566664/hparams.yml b/log/text2image-fine-tune/1686721041.5566664/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..89fe58e8989d377d981c545bb8652c9f20cc87fc --- /dev/null +++ b/log/text2image-fine-tune/1686721041.5566664/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora2_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 +validation_prompts: pikachu diff --git a/log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1 b/log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1 new file mode 100644 index 0000000000000000000000000000000000000000..d80c964f23cf7b2287c9cee7f86753d696c29e08 --- /dev/null +++ b/log/text2image-fine-tune/1686738195.981729/events.out.tfevents.1686738195.A40-20230526-003.3854615.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b9ee6471aab351b1c3a319ba55daa8f7851ec66575ac7b55078914ffef40772b +size 2713 diff --git a/log/text2image-fine-tune/1686738195.9854875/hparams.yml b/log/text2image-fine-tune/1686738195.9854875/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..452a21a7667ce1f5373e5274d6d9246e9e36d6f9 --- /dev/null +++ b/log/text2image-fine-tune/1686738195.9854875/hparams.yml @@ -0,0 +1,49 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora3_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 +validation_prompts: null diff --git a/log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1 b/log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1 new file mode 100644 index 0000000000000000000000000000000000000000..ec1ebd1783235a6526a3e62a9596fb269bf07b7a --- /dev/null +++ b/log/text2image-fine-tune/1686739744.5394647/events.out.tfevents.1686739744.A40-20230526-003.3859977.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:21bc60c8432f3fdbc26a513859f4c04a18e03a56aa6e8d29a0ee1391cc4497e3 +size 2713 diff --git a/log/text2image-fine-tune/1686739744.541363/hparams.yml b/log/text2image-fine-tune/1686739744.541363/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..4539dbe9ca3dee0a3f8159352f168c4c5331c6f2 --- /dev/null +++ b/log/text2image-fine-tune/1686739744.541363/hparams.yml @@ -0,0 +1,48 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora3_tensorboard +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 diff --git a/log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1 b/log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1 new file mode 100644 index 0000000000000000000000000000000000000000..87e854e290fc1eb288d5018780ee5fce5a3ec12e --- /dev/null +++ b/log/text2image-fine-tune/1686740628.4824169/events.out.tfevents.1686740628.A40-20230526-003.3863181.1 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ea1e87f8a32bfaae1eafe00dfabef13887acc06b904c6e1713d9d9bc42bf28cf +size 2704 diff --git a/log/text2image-fine-tune/1686740628.4842744/hparams.yml b/log/text2image-fine-tune/1686740628.4842744/hparams.yml new file mode 100644 index 0000000000000000000000000000000000000000..f38d154b681048fe55399a3b4923a5d8a7139376 --- /dev/null +++ b/log/text2image-fine-tune/1686740628.4842744/hparams.yml @@ -0,0 +1,48 @@ +accelerate_device: cuda:7 +accelerate_local_process_index: 0 +accelerate_num_processes: 1 +accelerate_optimizer_step_was_skipped: false +accelerate_process_index: 0 +accelerate_sync_gradients: true +accelerate_use_distributed: false +adam_beta1: 0.9 +adam_beta2: 0.999 +adam_epsilon: 1.0e-08 +adam_weight_decay: 0.01 +allow_tf32: false +checkpointing_steps: 500 +checkpoints_total_limit: null +disable_tqdm: false +enable_xformers_memory_efficient_attention: true +eval_dataset_path: null +fp16_backend: auto +fp16_opt_level: O1 +gradient_accumulation_steps: 1 +gradient_checkpointing: true +hub_model_id: lora4_tb +hub_token: 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 +input_perturbation: 0 +learning_rate: 1.0e-06 +logging_dir: log +lr_scheduler: constant +lr_warmup_steps: 0 +max_grad_norm: 1 +max_train_samples: null +max_train_steps: 60 +mixed_precision: fp16 +noise_offset: 1 +num_train_epochs: 1 +num_validation_images: 1 +output_dir: /home/deepanshu/lmflowdiff/Model +push_to_hub: true +report_to: tensorboard +resume_from_checkpoint: latest +save_steps: 1000 +save_total_limit: null +scale_lr: false +seed: 123 +snr_gamma: null +tracker_project_name: text2image-fine-tune +train_batch_size: 1 +use_8bit_adam: false +validation_epochs: 60 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0 b/log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0 new file mode 100644 index 0000000000000000000000000000000000000000..5221f22ec0a65a33c9b6d296f81919596bc6c5c2 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686718938.A40-20230526-003.3793953.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:56db31fb0d75f58bcda44e6a5b551b0cd8ff59e10469290b8395270cf6c752b0 +size 3824602 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0 b/log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0 new file mode 100644 index 0000000000000000000000000000000000000000..46dae0a0d4bd9c7411dca6bb717792b97655f4a9 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686719590.A40-20230526-003.3796603.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:18eb5dfa7dd09aaa9476759ed5aefafacb327e25e96e38558425e983ad8e3e80 +size 3719369 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0 b/log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0 new file mode 100644 index 0000000000000000000000000000000000000000..ab0339b81aedd08c9e2f58f94d5c06cc43b84f27 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686719831.A40-20230526-003.3797971.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:ed04babe7fc1d215e5f8a80dbf2296872f61b8272a19e69a838fed25dc2a0cb0 +size 3723222 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0 b/log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0 new file mode 100644 index 0000000000000000000000000000000000000000..fce7cabbda03b51cd6ef2571389fc45ad0559a0d --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686720913.A40-20230526-003.3802189.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:b2f212308d9755444b01ca57a1428f099ae463da8c42581e405453a8bcaf6018 +size 3722023 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0 b/log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0 new file mode 100644 index 0000000000000000000000000000000000000000..2e121455af2c24de5faf97ea89ef3294dccc975c --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686720979.A40-20230526-003.3802846.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:041ff8bfe52e59c866d87f9b1dd9a37aae790fe1b92441c351b9029d15aa980b +size 3724395 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0 b/log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0 new file mode 100644 index 0000000000000000000000000000000000000000..817a2c0eda083b916a25dd4ba4fce3630e7abf4f --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686721041.A40-20230526-003.3803507.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:972064b9bca317a5508f4f0b55085d6cb08b5cb1cde82c684a3b2f6d0ddc232f +size 3723251 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0 b/log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0 new file mode 100644 index 0000000000000000000000000000000000000000..b9033425aea7c86ce94fd666937e05ac75ef2d5b --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686724122.A40-20230526-003.3812918.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4e380cb848e0d985ce497abd616466a455fb6dfe9bceb19df38f93b72751ca76 +size 88 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0 b/log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0 new file mode 100644 index 0000000000000000000000000000000000000000..7e82b4852b4a71c183d9c56dc6a565a9c072b5bc --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686737268.A40-20230526-003.3850576.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:13f3cd093d73bedd1bf3459d90058ad191c5a67223ba2ac6c12878a3625c7aae +size 88 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0 b/log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0 new file mode 100644 index 0000000000000000000000000000000000000000..f2ec22b8bd6e1a53ba61a52f4c139748b01c87d9 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686737891.A40-20230526-003.3852910.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:5f1b9adad5b002afc7c6ebddf403821b5a083ed15a0af72fabd13ba6a9722b9b +size 88 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0 b/log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0 new file mode 100644 index 0000000000000000000000000000000000000000..db318d3a3fff9a39cc058e46a92918e1689269b8 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686737963.A40-20230526-003.3853473.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:4c4bfb33e34f3eb3dd784c897abd2f3a8957807d963fb9fc6eaa0dcd083cd46d +size 88 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0 b/log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0 new file mode 100644 index 0000000000000000000000000000000000000000..610677e70a0f64766da8f26ec18005eeef9b1a51 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686738195.A40-20230526-003.3854615.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:87549ecae170bb23048a2ca45624d8628818bda58f8dbe4ee9064a0ff43e8941 +size 1000 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0 b/log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0 new file mode 100644 index 0000000000000000000000000000000000000000..5c6c7313d34f16be7d692f22b16e075509a76694 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686738251.A40-20230526-003.3855176.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:7e3ab5c775ff52b79e327d006329d5d13313d9053300c4358034f65be5a52c88 +size 88 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0 b/log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0 new file mode 100644 index 0000000000000000000000000000000000000000..2c9fa8deb2bdad145cc432b57fabb5194358ae3c --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686739744.A40-20230526-003.3859977.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:69cfc2f12779ca41a1cb3683d2c8d7734fb83d39d41098edbf6560a233310bb6 +size 501625 diff --git a/log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0 b/log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0 new file mode 100644 index 0000000000000000000000000000000000000000..baa764f2402e8fc9e4d5ec87a2a9622e3851fe32 --- /dev/null +++ b/log/text2image-fine-tune/events.out.tfevents.1686740628.A40-20230526-003.3863181.0 @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:872ff443d210926e96e45e61da68ea1da4658c0f7eb963664cc6dd31912f43df +size 1245950 diff --git a/pytorch_lora_weights.bin b/pytorch_lora_weights.bin new file mode 100644 index 0000000000000000000000000000000000000000..2bb6f59b6aaf5264acb2d78208000b5812ad12fd --- /dev/null +++ b/pytorch_lora_weights.bin @@ -0,0 +1,3 @@ +version https://git-lfs.github.com/spec/v1 +oid sha256:a667d4733a587ab1fd82d05132f013e57f474869c0d379c2d9e13858753f44ce +size 3287771 diff --git a/wandb/settings b/wandb/settings new file mode 100644 index 0000000000000000000000000000000000000000..8dae664872e647e4bf829a45038781fb33b43d92 --- /dev/null +++ b/wandb/settings @@ -0,0 +1,2 @@ +[default] + diff --git a/wandb/wandb/debug-internal.log b/wandb/wandb/debug-internal.log new file mode 100644 index 0000000000000000000000000000000000000000..46bf77fdcb84cf8300ae0221340de61a4511eab0 --- /dev/null +++ b/wandb/wandb/debug-internal.log @@ -0,0 +1,449 @@ +2023-06-14 16:09:40,206 INFO StreamThr :3905241 [internal.py:wandb_internal():86] W&B internal server running at pid: 3905241, started at: 2023-06-14 16:09:40.205938 +2023-06-14 16:09:40,208 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status +2023-06-14 16:09:40,211 INFO WriterThread:3905241 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/run-0vx95vx0.wandb +2023-06-14 16:09:40,212 DEBUG SenderThread:3905241 [sender.py:send():369] send: header +2023-06-14 16:09:40,401 DEBUG SenderThread:3905241 [sender.py:send():369] send: run +2023-06-14 16:09:41,650 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: check_version +2023-06-14 16:09:41,655 INFO SenderThread:3905241 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files +2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_start_run_threads():1100] run started: 0vx95vx0 with start time 1686758980.206642 +2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: check_version +2023-06-14 16:09:42,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:09:42,670 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: run_start +2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():31] System info init +2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():46] System info init done +2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:start():181] Starting system monitor +2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:probe():201] Collecting system info +2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started cpu monitoring +2023-06-14 16:09:42,681 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started disk monitoring +2023-06-14 16:09:42,682 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started gpu monitoring +2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started memory monitoring +2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started network monitoring +2023-06-14 16:09:42,767 DEBUG HandlerThread:3905241 [system_info.py:probe():195] Probing system +2023-06-14 16:09:42,775 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():180] Probing git +2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():188] Probing git done +2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:probe():240] Probing system done +2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:09:42.767191', 'startedAt': '2023-06-14T16:09:40.185854', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': 'goyaldeep178@gmail.com', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.130054687500001, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.533, 'min': 1500.0, 'max': 2800.0}, {'current': 2.322, 'min': 1500.0, 'max': 2800.0}, {'current': 3.465, 'min': 1500.0, 'max': 2800.0}, {'current': 2.612, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.856, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.46, 'min': 1500.0, 'max': 2800.0}, {'current': 2.348, 'min': 1500.0, 'max': 2800.0}, {'current': 2.382, 'min': 1500.0, 'max': 2800.0}, {'current': 2.313, 'min': 1500.0, 'max': 2800.0}, {'current': 3.161, 'min': 1500.0, 'max': 2800.0}, {'current': 3.215, 'min': 1500.0, 'max': 2800.0}, {'current': 3.212, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.363, 'min': 1500.0, 'max': 2800.0}, {'current': 2.491, 'min': 1500.0, 'max': 2800.0}, {'current': 3.406, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.818, 'min': 1500.0, 'max': 2800.0}, {'current': 3.23, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.023, 'min': 1500.0, 'max': 2800.0}, {'current': 3.395, 'min': 1500.0, 'max': 2800.0}, {'current': 3.115, 'min': 1500.0, 'max': 2800.0}, {'current': 2.718, 'min': 1500.0, 'max': 2800.0}, {'current': 2.756, 'min': 1500.0, 'max': 2800.0}, {'current': 3.473, 'min': 1500.0, 'max': 2800.0}, {'current': 2.108, 'min': 1500.0, 'max': 2800.0}, {'current': 2.274, 'min': 1500.0, 'max': 2800.0}, {'current': 1.617, 'min': 1500.0, 'max': 2800.0}, {'current': 2.259, 'min': 1500.0, 'max': 2800.0}, {'current': 3.017, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.003, 'min': 1500.0, 'max': 2800.0}, {'current': 3.0, 'min': 1500.0, 'max': 2800.0}, {'current': 3.007, 'min': 1500.0, 'max': 2800.0}, {'current': 3.019, 'min': 1500.0, 'max': 2800.0}, {'current': 3.49, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.965, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 3.478, 'min': 1500.0, 'max': 2800.0}, {'current': 2.802, 'min': 1500.0, 'max': 2800.0}, {'current': 2.961, 'min': 1500.0, 'max': 2800.0}, {'current': 2.113, 'min': 1500.0, 'max': 2800.0}, {'current': 2.343, 'min': 1500.0, 'max': 2800.0}, {'current': 2.286, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.243, 'min': 1500.0, 'max': 2800.0}, {'current': 2.902, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 1.992, 'min': 1500.0, 'max': 2800.0}, {'current': 2.51, 'min': 1500.0, 'max': 2800.0}, {'current': 2.986, 'min': 1500.0, 'max': 2800.0}, {'current': 3.222, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.281, 'min': 1500.0, 'max': 2800.0}, {'current': 3.046, 'min': 1500.0, 'max': 2800.0}, {'current': 2.506, 'min': 1500.0, 'max': 2800.0}, {'current': 2.536, 'min': 1500.0, 'max': 2800.0}, {'current': 2.544, 'min': 1500.0, 'max': 2800.0}, {'current': 3.285, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.29, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.757, 'min': 1500.0, 'max': 2800.0}, {'current': 2.531, 'min': 1500.0, 'max': 2800.0}, {'current': 2.388, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.93, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.483, 'min': 1500.0, 'max': 2800.0}, {'current': 3.476, 'min': 1500.0, 'max': 2800.0}, {'current': 3.48, 'min': 1500.0, 'max': 2800.0}, {'current': 3.073, 'min': 1500.0, 'max': 2800.0}, {'current': 3.03, 'min': 1500.0, 'max': 2800.0}, {'current': 2.99, 'min': 1500.0, 'max': 2800.0}, {'current': 1.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.027, 'min': 1500.0, 'max': 2800.0}, {'current': 3.477, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.488, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.479, 'min': 1500.0, 'max': 2800.0}, {'current': 3.108, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.469, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.227, 'min': 1500.0, 'max': 2800.0}, {'current': 3.491, 'min': 1500.0, 'max': 2800.0}, {'current': 1.884, 'min': 1500.0, 'max': 2800.0}, {'current': 3.424, 'min': 1500.0, 'max': 2800.0}, {'current': 3.119, 'min': 1500.0, 'max': 2800.0}, {'current': 3.238, 'min': 1500.0, 'max': 2800.0}, {'current': 3.121, 'min': 1500.0, 'max': 2800.0}, {'current': 3.095, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349178314208984}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}} +2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():211] Finished collecting system info +2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():214] Publishing system info +2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():67] Saving pip packages done +2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment +2023-06-14 16:09:43,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml +2023-06-14 16:09:43,657 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt +2023-06-14 16:09:51,458 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():86] Saving conda packages done +2023-06-14 16:09:51,460 INFO HandlerThread:3905241 [system_monitor.py:probe():216] Finished publishing system info +2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive +2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:09:51,467 DEBUG SenderThread:3905241 [sender.py:send():369] send: files +2023-06-14 16:09:51,467 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-06-14 16:09:51,482 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:09:51,484 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml +2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-metadata.json +2023-06-14 16:09:52,979 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/gpvyb6vu-wandb-metadata.json +2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: telemetry +2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: config +2023-06-14 16:09:53,660 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:09:55,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:09:55,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:09:55,994 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:56,352 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:56,353 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:56,617 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:56,618 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:56,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:09:56,917 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:56,918 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:57,332 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:57,334 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:57,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:09:57,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:09:57,870 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:57,872 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:58,146 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:58,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:58,515 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:58,517 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:58,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:09:59,135 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:59,145 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:09:59,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:09:59,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:09:59,987 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:09:59,991 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:09:59,992 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:09:59,992 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:00,580 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:00,580 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:00,581 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:00,581 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:00,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:01,111 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:01,117 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:01,118 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:01,405 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:01,405 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:01,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:01,665 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:01,668 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:01,669 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:01,949 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:01,950 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:02,184 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:02,185 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:02,186 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:02,186 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:02,440 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:02,441 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:02,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:02,686 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:02,688 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:02,972 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:02,973 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:03,251 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:03,252 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:03,527 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:03,528 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:03,812 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:03,813 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:04,089 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:04,091 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:04,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:04,389 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:04,390 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:04,390 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:04,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:04,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:04,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:04,855 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:04,857 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:05,084 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:05,085 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:05,317 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:05,318 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:05,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:05,562 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:05,798 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:06,053 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:06,054 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:06,364 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:06,365 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:06,366 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:06,473 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:10:06,474 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:10:06,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:06,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:06,863 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:07,012 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:07,013 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:07,314 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:07,314 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:07,315 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:07,315 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:07,559 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:07,559 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:07,560 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:07,560 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:07,903 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:07,904 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:08,145 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:08,147 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:08,407 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:08,408 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:08,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:08,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:08,637 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:08,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:08,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:08,977 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:08,977 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:08,978 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:08,978 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:09,193 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:09,194 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:09,508 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:09,509 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:09,800 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:09,802 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:10,151 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:10,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:10,153 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:10,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:10,411 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:10,412 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:10,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:10,822 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:10,822 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:10,823 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:10,823 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:11,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:11,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:11,485 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:11,486 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:11,743 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:11,895 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:11,895 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:11,896 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:11,897 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:12,044 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:12,045 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:12,291 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:12,292 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:12,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml +2023-06-14 16:10:12,671 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:12,672 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:12,674 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:12,675 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:12,675 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:13,059 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:13,062 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:13,638 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:13,646 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:13,652 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:13,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:14,246 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:14,250 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:14,676 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:14,834 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:14,836 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:14,837 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:14,837 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:15,545 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:15,550 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:15,864 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:15,866 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:16,155 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:16,156 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:16,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:16,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:16,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:17,157 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:17,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:19,678 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:21,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:10:21,488 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:10:21,679 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:22,324 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:23,680 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:25,681 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:27,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:27,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:28,590 DEBUG SenderThread:3905241 [sender.py:send():369] send: files +2023-06-14 16:10:28,592 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now +2023-06-14 16:10:28,599 DEBUG SenderThread:3905241 [sender.py:send():369] send: files +2023-06-14 16:10:28,607 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_2ff5d73691ebb72c877d.png with policy now +2023-06-14 16:10:28,633 DEBUG SenderThread:3905241 [sender.py:send():369] send: files +2023-06-14 16:10:28,642 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_b3033a9fe6335a935570.png with policy now +2023-06-14 16:10:28,649 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send():369] send: history +2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:10:28,666 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:10:28,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json +2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png +2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png +2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png +2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images +2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media +2023-06-14 16:10:30,686 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/25n7cl81-media/images/validation_60_2ff5d73691ebb72c877d.png +2023-06-14 16:10:31,331 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/34c9bzlj-media/images/validation_60_e9c4a470ad900801f7de.png +2023-06-14 16:10:31,684 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:32,001 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ew6x2jre-media/images/validation_60_2ff5d73691ebb72c877d.png +2023-06-14 16:10:32,323 INFO wandb-upload_2:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5utpuer6-media/images/validation_60_b3033a9fe6335a935570.png +2023-06-14 16:10:33,058 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:33,739 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5zhjwo2z-media/images/validation_60_b3033a9fe6335a935570.png +2023-06-14 16:10:33,806 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ykn8lu3m-media/images/validation_60_e9c4a470ad900801f7de.png +2023-06-14 16:10:35,685 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:10:36,493 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:10:38,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:39,687 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:41,688 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:42,683 DEBUG SystemMonitor:3905241 [system_monitor.py:_start():159] Starting system metrics aggregation loop +2023-06-14 16:10:42,686 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:10:43,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:43,694 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:45,690 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:47,691 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:48,724 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:51,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:10:51,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:10:51,692 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log +2023-06-14 16:10:54,194 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:10:59,195 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:04,196 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:06,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:11:06,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:11:09,299 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:12,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:11:14,690 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:19,700 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:21,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:11:21,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:11:24,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:29,845 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:34,847 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:36,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:11:36,490 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:11:39,904 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:42,691 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:11:45,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:50,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:11:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:11:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:11:55,795 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:00,796 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:12:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:12:11,782 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:12,693 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:12:17,694 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:21,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:12:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:12:22,832 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:27,835 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:32,833 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:36,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:12:36,492 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:12:41,149 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:42,695 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:12:46,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:12:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:12:51,947 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:12:56,955 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:01,956 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:13:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:13:07,794 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:12,697 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:13:13,707 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:18,708 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:21,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:13:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:13:23,774 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:28,779 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:33,776 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:13:36,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:13:41,492 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive +2023-06-14 16:13:42,393 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:42,701 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:13:47,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:13:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:13:52,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:13:57,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:14:02,875 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:14:06,494 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:14:06,494 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:14:08,110 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:14:12,702 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats +2023-06-14 16:14:13,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report diff --git a/wandb/wandb/debug.log b/wandb/wandb/debug.log new file mode 100644 index 0000000000000000000000000000000000000000..0bda709ffa8b20b9b539c77a48581ce9b72763ea --- /dev/null +++ b/wandb/wandb/debug.log @@ -0,0 +1,30 @@ +2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Current SDK version is 0.15.4 +2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Configure stats pid to 3904922 +2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/.config/wandb/settings +2023-06-14 16:09:40,187 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from /home/deepanshu/lmflowdiff/Model/wandb/wandb/settings +2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CACHE_DIR +2023-06-14 16:09:40,187 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Unknown environment variable: WANDB_CONFIG_DIR +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Loading settings from environment variables: {'root_dir': '/home/deepanshu/lmflowdiff/Model/wandb'} +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Applying setup settings: {'_disable_service': False} +2023-06-14 16:09:40,199 WARNING MainThread:3904922 [wandb_setup.py:_flush():76] Could not save program above cwd: /home/deepanshu/LMFlow-diffusion-main/examples/finetune.py +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_setup.py:_flush():76] Inferring run settings from compute environment: {'program_relpath': None, 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py'} +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:_log_setup():507] Logging user logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug.log +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:_log_setup():508] Logging internal logs to /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/logs/debug-internal.log +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():547] calling init triggers +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():554] wandb.init called with sweep_config: {} +config: {} +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():596] starting backend +2023-06-14 16:09:40,199 INFO MainThread:3904922 [wandb_init.py:init():600] setting up manager +2023-06-14 16:09:40,201 INFO MainThread:3904922 [backend.py:_multiprocessing_setup():106] multiprocessing start_methods=fork,spawn,forkserver, using: spawn +2023-06-14 16:09:40,206 INFO MainThread:3904922 [wandb_init.py:init():606] backend started and connected +2023-06-14 16:09:40,210 INFO MainThread:3904922 [wandb_init.py:init():703] updated telemetry +2023-06-14 16:09:40,398 INFO MainThread:3904922 [wandb_init.py:init():736] communicating run to backend with 60.0 second timeout +2023-06-14 16:09:41,650 INFO MainThread:3904922 [wandb_run.py:_on_init():2176] communicating current version +2023-06-14 16:09:42,662 INFO MainThread:3904922 [wandb_run.py:_on_init():2185] got version response +2023-06-14 16:09:42,662 INFO MainThread:3904922 [wandb_init.py:init():787] starting run threads in backend +2023-06-14 16:09:51,480 INFO MainThread:3904922 [wandb_run.py:_console_start():2155] atexit reg +2023-06-14 16:09:51,481 INFO MainThread:3904922 [wandb_run.py:_redirect():2010] redirect: SettingsConsole.WRAP_RAW +2023-06-14 16:09:51,482 INFO MainThread:3904922 [wandb_run.py:_redirect():2075] Wrapping output streams. +2023-06-14 16:09:51,482 INFO MainThread:3904922 [wandb_run.py:_redirect():2100] Redirects installed. +2023-06-14 16:09:51,483 INFO MainThread:3904922 [wandb_init.py:init():828] run started, returning control to user process +2023-06-14 16:09:51,483 INFO MainThread:3904922 [wandb_run.py:_config_callback():1283] config_cb None None {'num_validation_images': 1, 'validation_epochs': 60, 'max_train_samples': None, 'output_dir': '/home/deepanshu/lmflowdiff/Model', 'seed': 123, 'train_batch_size': 1, 'num_train_epochs': 1, 'max_train_steps': 60, 'gradient_accumulation_steps': 1, 'gradient_checkpointing': True, 'learning_rate': 1e-06, 'scale_lr': False, 'lr_scheduler': 'constant', 'lr_warmup_steps': 0, 'snr_gamma': None, 'use_8bit_adam': False, 'allow_tf32': False, 'adam_beta1': 0.9, 'adam_beta2': 0.999, 'adam_weight_decay': 0.01, 'adam_epsilon': 1e-08, 'max_grad_norm': 1, 'push_to_hub': True, 'hub_token': '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', 'hub_model_id': 'lora5_tb', 'logging_dir': 'log', 'mixed_precision': 'fp16', 'resume_from_checkpoint': 'latest', 'fp16_backend': 'auto', 'fp16_opt_level': 'O1', 'disable_tqdm': False, 'save_steps': 1000, 'save_total_limit': None, 'eval_dataset_path': None, 'tracker_project_name': 'text2image-fine-tune', 'checkpoints_total_limit': None, 'enable_xformers_memory_efficient_attention': True, 'noise_offset': 1, 'input_perturbation': 0, 'checkpointing_steps': 500, 'report_to': 'wandb', 'accelerate_device': 'cuda:1', 'accelerate_local_process_index': 0, 'accelerate_num_processes': 1, 'accelerate_optimizer_step_was_skipped': False, 'accelerate_process_index': 0, 'accelerate_sync_gradients': True, 'accelerate_use_distributed': False} diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml b/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml new file mode 100644 index 0000000000000000000000000000000000000000..5dfe8ccb9d3194ae9d515cef7f38752458bd785a --- /dev/null +++ b/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml @@ -0,0 +1,191 @@ +name: deepanshu +channels: + - conda-forge + - defaults +dependencies: + - _libgcc_mutex=0.1=main + - _openmp_mutex=5.1=1_gnu + - asttokens=2.2.1=pyhd8ed1ab_0 + - backcall=0.2.0=pyh9f0ad1d_0 + - backports=1.0=pyhd8ed1ab_3 + - backports.functools_lru_cache=1.6.4=pyhd8ed1ab_0 + - ca-certificates=2023.5.7=hbcca054_0 + - debugpy=1.5.1=py39h295c915_0 + - decorator=5.1.1=pyhd8ed1ab_0 + - entrypoints=0.4=pyhd8ed1ab_0 + - executing=1.2.0=pyhd8ed1ab_0 + - ipykernel=6.15.0=pyh210e3f2_0 + - ipython=8.13.2=pyh41d4057_0 + - jedi=0.18.2=pyhd8ed1ab_0 + - jupyter_client=7.0.6=pyhd8ed1ab_0 + - jupyter_core=5.3.0=py39hf3d152e_0 + - ld_impl_linux-64=2.38=h1181459_1 + - libffi=3.4.4=h6a678d5_0 + - libgcc-ng=11.2.0=h1234567_1 + - libgomp=11.2.0=h1234567_1 + - libsodium=1.0.18=h36c2ea0_1 + - libstdcxx-ng=11.2.0=h1234567_1 + - matplotlib-inline=0.1.6=pyhd8ed1ab_0 + - ncurses=6.4=h6a678d5_0 + - nest-asyncio=1.5.6=pyhd8ed1ab_0 + - openssl=1.1.1t=h7f8727e_0 + - packaging=23.1=pyhd8ed1ab_0 + - parso=0.8.3=pyhd8ed1ab_0 + - pexpect=4.8.0=pyh1a96a4e_2 + - pickleshare=0.7.5=py_1003 + - pip=23.0.1=py39h06a4308_0 + - platformdirs=3.5.1=pyhd8ed1ab_0 + - prompt-toolkit=3.0.38=pyha770c72_0 + - prompt_toolkit=3.0.38=hd8ed1ab_0 + - ptyprocess=0.7.0=pyhd3deb0d_0 + - pure_eval=0.2.2=pyhd8ed1ab_0 + - pygments=2.15.1=pyhd8ed1ab_0 + - python=3.9.16=h7a1cb2a_2 + - python-dateutil=2.8.2=pyhd8ed1ab_0 + - python_abi=3.9=2_cp39 + - pyzmq=19.0.2=py39hb69f2a1_2 + - readline=8.2=h5eee18b_0 + - setuptools=66.0.0=py39h06a4308_0 + - six=1.16.0=pyh6c4a22f_0 + - sqlite=3.41.2=h5eee18b_0 + - stack_data=0.6.2=pyhd8ed1ab_0 + - tk=8.6.12=h1ccaba5_0 + - tornado=6.1=py39hb9d737c_3 + - traitlets=5.9.0=pyhd8ed1ab_0 + - typing_extensions=4.6.2=pyha770c72_0 + - wcwidth=0.2.6=pyhd8ed1ab_0 + - wheel=0.38.4=py39h06a4308_0 + - xz=5.4.2=h5eee18b_0 + - zeromq=4.3.4=h9c3ff4c_1 + - zlib=1.2.13=h5eee18b_0 + - pip: + - absl-py==1.4.0 + - accelerate==0.19.0 + - aiofiles==23.1.0 + - aiohttp==3.8.4 + - aiosignal==1.3.1 + - altair==5.0.1 + - anyio==3.7.0 + - appdirs==1.4.4 + - async-timeout==4.0.2 + - attrs==23.1.0 + - bitsandbytes==0.39.0 + - cachetools==5.3.1 + - certifi==2022.12.7 + - charset-normalizer==2.1.1 + - click==8.1.3 + - clip==1.0 + - cmake==3.25.0 + - contourpy==1.0.7 + - cycler==0.11.0 + - datasets==2.12.0 + - deepspeed==0.9.2 + - diffusers==0.16.1 + - dill==0.3.6 + - docker-pycreds==0.4.0 + - exceptiongroup==1.1.1 + - fastapi==0.95.2 + - ffmpy==0.3.0 + - filelock==3.9.0 + - fonttools==4.39.4 + - frozenlist==1.3.3 + - fsspec==2023.5.0 + - ftfy==6.1.1 + - gitdb==4.0.10 + - gitpython==3.1.31 + - google-auth==2.20.0 + - google-auth-oauthlib==1.0.0 + - gradio==3.32.0 + - gradio-client==0.2.5 + - grpcio==1.54.2 + - h11==0.14.0 + - hjson==3.1.0 + - httpcore==0.17.2 + - httpx==0.24.1 + - huggingface-hub==0.14.1 + - idna==3.4 + - importlib-metadata==6.6.0 + - importlib-resources==5.12.0 + - jinja2==3.1.2 + - jsonschema==4.17.3 + - kiwisolver==1.4.4 + - linkify-it-py==2.0.2 + - lit==15.0.7 + - markdown==3.4.3 + - markdown-it-py==2.2.0 + - markupsafe==2.1.2 + - matplotlib==3.7.1 + - mdit-py-plugins==0.3.3 + - mdurl==0.1.2 + - mpmath==1.2.1 + - multidict==6.0.4 + - multiprocess==0.70.14 + - mxnet==1.9.1 + - mypy-extensions==1.0.0 + - natsort==8.3.1 + - networkx==3.0 + - ninja==1.11.1 + - numpy==1.24.1 + - oauthlib==3.2.2 + - open-clip-torch==2.20.0 + - orjson==3.8.14 + - pandas==2.0.1 + - pathtools==0.1.2 + - pillow==9.3.0 + - plotly==5.15.0 + - protobuf==3.20.3 + - psutil==5.9.5 + - py-cpuinfo==9.0.0 + - pyarrow==12.0.0 + - pyasn1==0.5.0 + - pyasn1-modules==0.3.0 + - pydantic==1.10.8 + - pydub==0.25.1 + - pyparsing==3.0.9 + - pyre-extensions==0.0.29 + - pyrsistent==0.19.3 + - python-graphviz==0.8.4 + - python-multipart==0.0.6 + - pytz==2023.3 + - pyyaml==6.0 + - regex==2023.5.5 + - requests==2.28.1 + - requests-oauthlib==1.3.1 + - responses==0.18.0 + - rsa==4.9 + - safetensors==0.3.1 + - scipy==1.10.1 + - semantic-version==2.10.0 + - sentencepiece==0.1.99 + - sentry-sdk==1.25.1 + - setproctitle==1.3.2 + - smmap==5.0.0 + - sniffio==1.3.0 + - starlette==0.27.0 + - sympy==1.11.1 + - tenacity==8.2.2 + - tensorboard==2.13.0 + - tensorboard-data-server==0.7.1 + - timm==0.9.2 + - tokenizers==0.13.3 + - toolz==0.12.0 + - torch==2.0.1+cu118 + - torchaudio==2.0.2+cu118 + - torchvision==0.15.2+cu118 + - tqdm==4.65.0 + - transformers==4.29.2 + - triton==2.0.0 + - typing-extensions==4.4.0 + - typing-inspect==0.9.0 + - tzdata==2023.3 + - uc-micro-py==1.0.2 + - urllib3==1.26.13 + - uvicorn==0.22.0 + - wandb==0.15.4 + - websockets==11.0.3 + - werkzeug==2.3.6 + - xformers==0.0.20 + - xxhash==3.2.0 + - yarl==1.9.2 + - zipp==3.15.0 +prefix: /home/deepanshu/anaconda3/envs/deepanshu diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml b/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml new file mode 100644 index 0000000000000000000000000000000000000000..719dc016d66e89fc70167269b09e43399a81dd7d --- /dev/null +++ b/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml @@ -0,0 +1,182 @@ +wandb_version: 1 + +_wandb: + desc: null + value: + python_version: 3.9.16 + cli_version: 0.15.4 + framework: huggingface + huggingface_version: 4.29.2 + is_jupyter_run: false + is_kaggle_kernel: false + start_time: 1686758980.206642 + t: + 1: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 2: + - 1 + - 11 + - 41 + - 49 + - 51 + - 55 + - 71 + - 83 + 3: + - 23 + 4: 3.9.16 + 5: 0.15.4 + 6: 4.29.2 + 8: + - 5 +num_validation_images: + desc: null + value: 1 +validation_epochs: + desc: null + value: 60 +max_train_samples: + desc: null + value: null +output_dir: + desc: null + value: /home/deepanshu/lmflowdiff/Model +seed: + desc: null + value: 123 +train_batch_size: + desc: null + value: 1 +num_train_epochs: + desc: null + value: 1 +max_train_steps: + desc: null + value: 60 +gradient_accumulation_steps: + desc: null + value: 1 +gradient_checkpointing: + desc: null + value: true +learning_rate: + desc: null + value: 1.0e-06 +scale_lr: + desc: null + value: false +lr_scheduler: + desc: null + value: constant +lr_warmup_steps: + desc: null + value: 0 +snr_gamma: + desc: null + value: null +use_8bit_adam: + desc: null + value: false +allow_tf32: + desc: null + value: false +adam_beta1: + desc: null + value: 0.9 +adam_beta2: + desc: null + value: 0.999 +adam_weight_decay: + desc: null + value: 0.01 +adam_epsilon: + desc: null + value: 1.0e-08 +max_grad_norm: + desc: null + value: 1 +push_to_hub: + desc: null + value: true +hub_token: + desc: null + value: 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 +hub_model_id: + desc: null + value: lora5_tb +logging_dir: + desc: null + value: log +mixed_precision: + desc: null + value: fp16 +resume_from_checkpoint: + desc: null + value: latest +fp16_backend: + desc: null + value: auto +fp16_opt_level: + desc: null + value: O1 +disable_tqdm: + desc: null + value: false +save_steps: + desc: null + value: 1000 +save_total_limit: + desc: null + value: null +eval_dataset_path: + desc: null + value: null +tracker_project_name: + desc: null + value: text2image-fine-tune +checkpoints_total_limit: + desc: null + value: null +enable_xformers_memory_efficient_attention: + desc: null + value: true +noise_offset: + desc: null + value: 1 +input_perturbation: + desc: null + value: 0 +checkpointing_steps: + desc: null + value: 500 +report_to: + desc: null + value: wandb +accelerate_device: + desc: null + value: cuda:1 +accelerate_local_process_index: + desc: null + value: 0 +accelerate_num_processes: + desc: null + value: 1 +accelerate_optimizer_step_was_skipped: + desc: null + value: false +accelerate_process_index: + desc: null + value: 0 +accelerate_sync_gradients: + desc: null + value: true +accelerate_use_distributed: + desc: null + value: false diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png new file mode 100644 index 0000000000000000000000000000000000000000..e890d225f3d8685a6ae00d16b4e60b18ce0a9c52 Binary files /dev/null and b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png differ diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png new file mode 100644 index 0000000000000000000000000000000000000000..b555e4f48e7cf144860a9ac5bfc97dbd8e79c681 Binary files /dev/null and b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png differ diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png new file mode 100644 index 0000000000000000000000000000000000000000..5220d0f9dbfa747cdb356761a73beb1b1f1c96bb Binary files /dev/null and b/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png differ diff --git a/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log b/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log new file mode 100644 index 0000000000000000000000000000000000000000..a9cb1cdf6f6311cacbe2ad02013cd7d49226c2ad --- /dev/null +++ b/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log @@ -0,0 +1,63 @@ +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - ***** Running training ***** +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Num examples = 833 +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Num Epochs = 1 +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Instantaneous batch size per device = 1 +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Total train batch size (w. parallel, distributed & accumulation) = 1 +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Gradient Accumulation steps = 1 +06/14/2023 16:09:51 - INFO - lmflow_aigen.pipeline.diffusion_finetuner - Total optimization steps = 60 +Steps: 0%| | 0/60 [00:00