diff --git "a/wandb/wandb/debug-internal.log" "b/wandb/wandb/debug-internal.log" --- "a/wandb/wandb/debug-internal.log" +++ "b/wandb/wandb/debug-internal.log" @@ -1,449 +1,354 @@ -2023-06-14 16:09:40,206 INFO StreamThr :3905241 [internal.py:wandb_internal():86] W&B internal server running at pid: 3905241, started at: 2023-06-14 16:09:40.205938 -2023-06-14 16:09:40,208 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status -2023-06-14 16:09:40,211 INFO WriterThread:3905241 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/run-0vx95vx0.wandb -2023-06-14 16:09:40,212 DEBUG SenderThread:3905241 [sender.py:send():369] send: header -2023-06-14 16:09:40,401 DEBUG SenderThread:3905241 [sender.py:send():369] send: run -2023-06-14 16:09:41,650 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: check_version -2023-06-14 16:09:41,655 INFO SenderThread:3905241 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files -2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_start_run_threads():1100] run started: 0vx95vx0 with start time 1686758980.206642 -2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:41,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:41,655 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: check_version -2023-06-14 16:09:42,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:09:42,670 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: run_start -2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():31] System info init -2023-06-14 16:09:42,678 DEBUG HandlerThread:3905241 [system_info.py:__init__():46] System info init done -2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:start():181] Starting system monitor -2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [system_monitor.py:_start():145] Starting system asset monitoring threads -2023-06-14 16:09:42,679 INFO HandlerThread:3905241 [system_monitor.py:probe():201] Collecting system info -2023-06-14 16:09:42,679 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started cpu monitoring -2023-06-14 16:09:42,681 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started disk monitoring -2023-06-14 16:09:42,682 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started gpu monitoring -2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started memory monitoring -2023-06-14 16:09:42,683 INFO SystemMonitor:3905241 [interfaces.py:start():190] Started network monitoring -2023-06-14 16:09:42,767 DEBUG HandlerThread:3905241 [system_info.py:probe():195] Probing system -2023-06-14 16:09:42,775 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():180] Probing git -2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:_probe_git():188] Probing git done -2023-06-14 16:09:42,811 DEBUG HandlerThread:3905241 [system_info.py:probe():240] Probing system done -2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:09:42.767191', 'startedAt': '2023-06-14T16:09:40.185854', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': 'goyaldeep178@gmail.com', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.130054687500001, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 2.533, 'min': 1500.0, 'max': 2800.0}, {'current': 2.322, 'min': 1500.0, 'max': 2800.0}, {'current': 3.465, 'min': 1500.0, 'max': 2800.0}, {'current': 2.612, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.856, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.46, 'min': 1500.0, 'max': 2800.0}, {'current': 2.348, 'min': 1500.0, 'max': 2800.0}, {'current': 2.382, 'min': 1500.0, 'max': 2800.0}, {'current': 2.313, 'min': 1500.0, 'max': 2800.0}, {'current': 3.161, 'min': 1500.0, 'max': 2800.0}, {'current': 3.215, 'min': 1500.0, 'max': 2800.0}, {'current': 3.212, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.363, 'min': 1500.0, 'max': 2800.0}, {'current': 2.491, 'min': 1500.0, 'max': 2800.0}, {'current': 3.406, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.818, 'min': 1500.0, 'max': 2800.0}, {'current': 3.23, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.023, 'min': 1500.0, 'max': 2800.0}, {'current': 3.395, 'min': 1500.0, 'max': 2800.0}, {'current': 3.115, 'min': 1500.0, 'max': 2800.0}, {'current': 2.718, 'min': 1500.0, 'max': 2800.0}, {'current': 2.756, 'min': 1500.0, 'max': 2800.0}, {'current': 3.473, 'min': 1500.0, 'max': 2800.0}, {'current': 2.108, 'min': 1500.0, 'max': 2800.0}, {'current': 2.274, 'min': 1500.0, 'max': 2800.0}, {'current': 1.617, 'min': 1500.0, 'max': 2800.0}, {'current': 2.259, 'min': 1500.0, 'max': 2800.0}, {'current': 3.017, 'min': 1500.0, 'max': 2800.0}, {'current': 2.779, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.003, 'min': 1500.0, 'max': 2800.0}, {'current': 3.0, 'min': 1500.0, 'max': 2800.0}, {'current': 3.007, 'min': 1500.0, 'max': 2800.0}, {'current': 3.019, 'min': 1500.0, 'max': 2800.0}, {'current': 3.49, 'min': 1500.0, 'max': 2800.0}, {'current': 2.793, 'min': 1500.0, 'max': 2800.0}, {'current': 2.965, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 3.478, 'min': 1500.0, 'max': 2800.0}, {'current': 2.802, 'min': 1500.0, 'max': 2800.0}, {'current': 2.961, 'min': 1500.0, 'max': 2800.0}, {'current': 2.113, 'min': 1500.0, 'max': 2800.0}, {'current': 2.343, 'min': 1500.0, 'max': 2800.0}, {'current': 2.286, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 3.243, 'min': 1500.0, 'max': 2800.0}, {'current': 2.902, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 3.489, 'min': 1500.0, 'max': 2800.0}, {'current': 1.992, 'min': 1500.0, 'max': 2800.0}, {'current': 2.51, 'min': 1500.0, 'max': 2800.0}, {'current': 2.986, 'min': 1500.0, 'max': 2800.0}, {'current': 3.222, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.281, 'min': 1500.0, 'max': 2800.0}, {'current': 3.046, 'min': 1500.0, 'max': 2800.0}, {'current': 2.506, 'min': 1500.0, 'max': 2800.0}, {'current': 2.536, 'min': 1500.0, 'max': 2800.0}, {'current': 2.544, 'min': 1500.0, 'max': 2800.0}, {'current': 3.285, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.29, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.757, 'min': 1500.0, 'max': 2800.0}, {'current': 2.531, 'min': 1500.0, 'max': 2800.0}, {'current': 2.388, 'min': 1500.0, 'max': 2800.0}, {'current': 3.466, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.326, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 2.774, 'min': 1500.0, 'max': 2800.0}, {'current': 3.468, 'min': 1500.0, 'max': 2800.0}, {'current': 2.93, 'min': 1500.0, 'max': 2800.0}, {'current': 3.467, 'min': 1500.0, 'max': 2800.0}, {'current': 3.483, 'min': 1500.0, 'max': 2800.0}, {'current': 3.476, 'min': 1500.0, 'max': 2800.0}, {'current': 3.48, 'min': 1500.0, 'max': 2800.0}, {'current': 3.073, 'min': 1500.0, 'max': 2800.0}, {'current': 3.03, 'min': 1500.0, 'max': 2800.0}, {'current': 2.99, 'min': 1500.0, 'max': 2800.0}, {'current': 1.912, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.484, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.027, 'min': 1500.0, 'max': 2800.0}, {'current': 3.477, 'min': 1500.0, 'max': 2800.0}, {'current': 3.486, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.488, 'min': 1500.0, 'max': 2800.0}, {'current': 3.487, 'min': 1500.0, 'max': 2800.0}, {'current': 3.479, 'min': 1500.0, 'max': 2800.0}, {'current': 3.108, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.482, 'min': 1500.0, 'max': 2800.0}, {'current': 3.469, 'min': 1500.0, 'max': 2800.0}, {'current': 3.492, 'min': 1500.0, 'max': 2800.0}, {'current': 2.8, 'min': 1500.0, 'max': 2800.0}, {'current': 3.485, 'min': 1500.0, 'max': 2800.0}, {'current': 3.227, 'min': 1500.0, 'max': 2800.0}, {'current': 3.491, 'min': 1500.0, 'max': 2800.0}, {'current': 1.884, 'min': 1500.0, 'max': 2800.0}, {'current': 3.424, 'min': 1500.0, 'max': 2800.0}, {'current': 3.119, 'min': 1500.0, 'max': 2800.0}, {'current': 3.238, 'min': 1500.0, 'max': 2800.0}, {'current': 3.121, 'min': 1500.0, 'max': 2800.0}, {'current': 3.095, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349178314208984}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}} -2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():211] Finished collecting system info -2023-06-14 16:09:42,812 INFO HandlerThread:3905241 [system_monitor.py:probe():214] Publishing system info -2023-06-14 16:09:42,812 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment -2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_pip():67] Saving pip packages done -2023-06-14 16:09:42,813 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment -2023-06-14 16:09:43,656 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml -2023-06-14 16:09:43,657 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/requirements.txt -2023-06-14 16:09:51,458 DEBUG HandlerThread:3905241 [system_info.py:_save_conda():86] Saving conda packages done -2023-06-14 16:09:51,460 INFO HandlerThread:3905241 [system_monitor.py:probe():216] Finished publishing system info -2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive -2023-06-14 16:09:51,466 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:09:51,467 DEBUG SenderThread:3905241 [sender.py:send():369] send: files -2023-06-14 16:09:51,467 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now -2023-06-14 16:09:51,482 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:09:51,484 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/conda-environment.yaml -2023-06-14 16:09:51,659 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-metadata.json -2023-06-14 16:09:52,979 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/gpvyb6vu-wandb-metadata.json -2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: telemetry -2023-06-14 16:09:53,018 DEBUG SenderThread:3905241 [sender.py:send():369] send: config -2023-06-14 16:09:53,660 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:09:55,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:09:55,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:09:55,994 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:56,352 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:56,353 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:56,353 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:56,617 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:56,617 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:56,618 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:56,661 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:09:56,917 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:56,917 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:56,918 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:57,332 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:57,333 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:57,334 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:57,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:09:57,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:09:57,870 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:57,872 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:57,872 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:58,146 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:58,152 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:58,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:58,515 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:58,516 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:58,517 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:58,662 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:09:59,135 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:59,144 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:59,145 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:09:59,663 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:09:59,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:09:59,987 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:09:59,991 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:09:59,992 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:09:59,992 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:00,580 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:00,580 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:00,581 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:00,581 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:00,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:01,111 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:01,117 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:01,117 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:01,118 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:01,405 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:01,405 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:01,405 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:01,664 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:01,665 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:01,668 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:01,669 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:01,669 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:01,949 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:01,950 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:01,950 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:02,184 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:02,185 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:02,186 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:02,186 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:02,440 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:02,441 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:02,441 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:02,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:02,686 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:02,688 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:02,688 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:02,972 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:02,973 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:02,973 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:03,251 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:03,252 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:03,252 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:03,527 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:03,528 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:03,528 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:03,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:03,812 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:03,813 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:03,813 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:04,089 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:04,090 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:04,091 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:04,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:04,389 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:04,390 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:04,390 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:04,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:04,636 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:04,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:04,666 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:04,855 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:04,856 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:04,857 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:05,084 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:05,085 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:05,085 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:05,317 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:05,318 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:05,318 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:05,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:05,562 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:05,562 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:05,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:05,797 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:05,798 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:06,053 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:06,054 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:06,054 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:06,364 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:06,365 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:06,365 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:06,366 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:06,473 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:10:06,474 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:10:06,667 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:06,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:06,862 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:06,863 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:07,012 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:07,013 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:07,013 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:07,314 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:07,314 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:07,315 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:07,315 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:07,559 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:07,559 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:07,560 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:07,560 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:07,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:07,903 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:07,904 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:07,904 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:08,145 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:08,146 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:08,147 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:08,407 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:08,408 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:08,408 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:08,636 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:08,636 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:08,637 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:08,637 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:08,668 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:08,977 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:08,977 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:08,978 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:08,978 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:09,193 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:09,194 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:09,194 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:09,508 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:09,508 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:09,509 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:09,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:09,800 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:09,801 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:09,802 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:10,151 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:10,152 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:10,153 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:10,153 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:10,411 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:10,412 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:10,412 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:10,669 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:10,822 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:10,822 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:10,823 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:10,823 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:11,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:11,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:11,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:11,485 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:11,486 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:11,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:11,743 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:11,894 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:11,895 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:11,895 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:11,896 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:11,897 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:12,044 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:12,045 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:12,045 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:12,291 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:12,292 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:12,292 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:12,670 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/config.yaml -2023-06-14 16:10:12,671 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:12,672 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:12,674 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:12,675 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:12,675 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:13,059 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:13,061 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:13,062 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:13,638 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:13,646 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:13,652 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:13,655 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:13,672 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:14,246 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:14,249 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:14,250 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:14,676 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:14,834 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:14,836 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:14,837 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:14,837 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:15,545 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:15,549 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:15,550 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:15,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:15,864 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:15,866 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:15,866 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:16,155 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:16,156 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:16,157 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:16,157 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:16,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:17,157 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:17,677 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:19,678 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:21,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:10:21,488 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:10:21,679 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:22,324 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:23,680 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:25,681 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:27,561 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:27,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:28,590 DEBUG SenderThread:3905241 [sender.py:send():369] send: files -2023-06-14 16:10:28,592 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now -2023-06-14 16:10:28,599 DEBUG SenderThread:3905241 [sender.py:send():369] send: files -2023-06-14 16:10:28,607 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_2ff5d73691ebb72c877d.png with policy now -2023-06-14 16:10:28,633 DEBUG SenderThread:3905241 [sender.py:send():369] send: files -2023-06-14 16:10:28,642 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file media/images/validation_60_b3033a9fe6335a935570.png with policy now -2023-06-14 16:10:28,649 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: partial_history -2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send():369] send: history -2023-06-14 16:10:28,664 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: summary_record -2023-06-14 16:10:28,666 INFO SenderThread:3905241 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end -2023-06-14 16:10:28,682 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/wandb-summary.json -2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_2ff5d73691ebb72c877d.png -2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_b3033a9fe6335a935570.png -2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images/validation_60_e9c4a470ad900801f7de.png -2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media/images -2023-06-14 16:10:28,683 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/media -2023-06-14 16:10:30,686 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/25n7cl81-media/images/validation_60_2ff5d73691ebb72c877d.png -2023-06-14 16:10:31,331 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/34c9bzlj-media/images/validation_60_e9c4a470ad900801f7de.png -2023-06-14 16:10:31,684 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:32,001 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ew6x2jre-media/images/validation_60_2ff5d73691ebb72c877d.png -2023-06-14 16:10:32,323 INFO wandb-upload_2:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5utpuer6-media/images/validation_60_b3033a9fe6335a935570.png -2023-06-14 16:10:33,058 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:33,739 INFO wandb-upload_1:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/5zhjwo2z-media/images/validation_60_b3033a9fe6335a935570.png -2023-06-14 16:10:33,806 INFO wandb-upload_0:3905241 [upload_job.py:push():133] Uploaded file /tmp/tmp0sitkosqwandb/ykn8lu3m-media/images/validation_60_e9c4a470ad900801f7de.png -2023-06-14 16:10:35,685 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:10:36,493 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:10:38,156 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:39,687 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:41,688 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:42,683 DEBUG SystemMonitor:3905241 [system_monitor.py:_start():159] Starting system metrics aggregation loop -2023-06-14 16:10:42,686 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:10:43,389 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:43,694 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:45,690 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:47,691 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:48,724 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:51,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:10:51,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:10:51,692 INFO Thread-12 :3905241 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_160940-0vx95vx0/files/output.log -2023-06-14 16:10:54,194 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:10:59,195 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:04,196 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:06,488 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:11:06,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:11:09,299 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:12,688 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:11:14,690 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:19,700 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:21,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:11:21,489 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:11:24,844 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:29,845 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:34,847 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:36,489 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:11:36,490 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:11:39,904 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:42,691 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:11:45,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:50,693 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:11:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:11:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:11:55,795 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:00,796 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:05,797 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:12:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:12:11,782 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:12,693 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:12:17,694 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:21,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:12:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:12:22,832 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:27,835 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:32,833 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:36,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:12:36,492 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:12:41,149 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:42,695 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:12:46,696 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:12:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:12:51,947 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:12:56,955 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:01,956 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:06,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:13:06,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:13:07,794 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:12,697 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:13:13,707 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:18,708 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:21,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:13:21,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:13:23,774 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:28,779 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:33,776 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:36,491 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:13:36,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:13:41,492 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: keepalive -2023-06-14 16:13:42,393 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:42,701 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:13:47,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:51,490 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:13:51,491 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:13:52,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:13:57,874 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:14:02,875 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:14:06,494 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: stop_status -2023-06-14 16:14:06,494 DEBUG SenderThread:3905241 [sender.py:send_request():396] send_request: stop_status -2023-06-14 16:14:08,110 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report -2023-06-14 16:14:12,702 DEBUG SenderThread:3905241 [sender.py:send():369] send: stats -2023-06-14 16:14:13,703 DEBUG HandlerThread:3905241 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:30:50,531 INFO StreamThr :3907002 [internal.py:wandb_internal():86] W&B internal server running at pid: 3907002, started at: 2023-06-14 16:30:50.530674 +2023-06-14 16:30:50,533 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status +2023-06-14 16:30:50,535 INFO WriterThread:3907002 [datastore.py:open_for_write():85] open: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/run-nlm9mp31.wandb +2023-06-14 16:30:50,536 DEBUG SenderThread:3907002 [sender.py:send():369] send: header +2023-06-14 16:30:50,645 DEBUG SenderThread:3907002 [sender.py:send():369] send: run +2023-06-14 16:30:52,766 INFO SenderThread:3907002 [dir_watcher.py:__init__():211] watching files in: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files +2023-06-14 16:30:52,767 INFO SenderThread:3907002 [sender.py:_start_run_threads():1100] run started: nlm9mp31 with start time 1686760250.529075 +2023-06-14 16:30:52,768 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:30:52,768 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:30:52,777 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: check_version +2023-06-14 16:30:52,780 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: check_version +2023-06-14 16:30:53,773 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:30:54,234 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: run_start +2023-06-14 16:30:54,244 DEBUG HandlerThread:3907002 [system_info.py:__init__():31] System info init +2023-06-14 16:30:54,244 DEBUG HandlerThread:3907002 [system_info.py:__init__():46] System info init done +2023-06-14 16:30:54,244 INFO HandlerThread:3907002 [system_monitor.py:start():181] Starting system monitor +2023-06-14 16:30:54,262 INFO SystemMonitor:3907002 [system_monitor.py:_start():145] Starting system asset monitoring threads +2023-06-14 16:30:54,262 INFO HandlerThread:3907002 [system_monitor.py:probe():201] Collecting system info +2023-06-14 16:30:54,278 INFO SystemMonitor:3907002 [interfaces.py:start():190] Started cpu monitoring +2023-06-14 16:30:54,305 INFO SystemMonitor:3907002 [interfaces.py:start():190] Started disk monitoring +2023-06-14 16:30:54,316 INFO SystemMonitor:3907002 [interfaces.py:start():190] Started gpu monitoring +2023-06-14 16:30:54,320 INFO SystemMonitor:3907002 [interfaces.py:start():190] Started memory monitoring +2023-06-14 16:30:54,321 INFO SystemMonitor:3907002 [interfaces.py:start():190] Started network monitoring +2023-06-14 16:30:54,388 DEBUG HandlerThread:3907002 [system_info.py:probe():195] Probing system +2023-06-14 16:30:54,402 DEBUG HandlerThread:3907002 [system_info.py:_probe_git():180] Probing git +2023-06-14 16:30:54,483 DEBUG HandlerThread:3907002 [system_info.py:_probe_git():188] Probing git done +2023-06-14 16:30:54,483 DEBUG HandlerThread:3907002 [system_info.py:probe():240] Probing system done +2023-06-14 16:30:54,483 DEBUG HandlerThread:3907002 [system_monitor.py:probe():210] {'os': 'Linux-5.4.0-90-generic-x86_64-with-glibc2.31', 'python': '3.9.16', 'heartbeatAt': '2023-06-14T16:30:54.388684', 'startedAt': '2023-06-14T16:30:50.522065', 'docker': None, 'cuda': None, 'args': (), 'state': 'running', 'program': '/home/deepanshu/LMFlow-diffusion-main/examples/finetune.py', 'git': {'remote': 'https://github.com/hendrydong/LMFlow-diffusion.git', 'commit': '7eb976e6588110787f15fb1ef1b6ce7c4d84ae87'}, 'email': 'goyaldeep178@gmail.com', 'root': '/home/deepanshu/lmflowdiff', 'host': 'A40-20230526-003', 'username': 'deepanshu', 'executable': '/home/deepanshu/anaconda3/envs/deepanshu/bin/python', 'cpu_count': 64, 'cpu_count_logical': 128, 'cpu_freq': {'current': 3.4042656249999994, 'min': 1500.0, 'max': 2800.0}, 'cpu_freq_per_core': [{'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.403, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.402, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.398, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.399, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.401, 'min': 1500.0, 'max': 2800.0}, {'current': 3.4, 'min': 1500.0, 'max': 2800.0}], 'disk': {'total': 218.5130844116211, 'used': 49.349273681640625}, 'gpu': 'NVIDIA A40', 'gpu_count': 8, 'gpu_devices': [{'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}, {'name': 'NVIDIA A40', 'memory_total': 51527024640}], 'memory': {'total': 755.7487335205078}} +2023-06-14 16:30:54,483 INFO HandlerThread:3907002 [system_monitor.py:probe():211] Finished collecting system info +2023-06-14 16:30:54,483 INFO HandlerThread:3907002 [system_monitor.py:probe():214] Publishing system info +2023-06-14 16:30:54,483 DEBUG HandlerThread:3907002 [system_info.py:_save_pip():51] Saving list of pip packages installed into the current environment +2023-06-14 16:30:54,484 DEBUG HandlerThread:3907002 [system_info.py:_save_pip():67] Saving pip packages done +2023-06-14 16:30:54,484 DEBUG HandlerThread:3907002 [system_info.py:_save_conda():74] Saving list of conda packages installed into the current environment +2023-06-14 16:30:54,773 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/requirements.txt +2023-06-14 16:30:54,775 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/conda-environment.yaml +2023-06-14 16:31:01,773 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/conda-environment.yaml +2023-06-14 16:31:01,862 DEBUG HandlerThread:3907002 [system_info.py:_save_conda():86] Saving conda packages done +2023-06-14 16:31:01,863 INFO HandlerThread:3907002 [system_monitor.py:probe():216] Finished publishing system info +2023-06-14 16:31:01,867 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:01,867 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: keepalive +2023-06-14 16:31:01,867 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:01,868 DEBUG SenderThread:3907002 [sender.py:send():369] send: files +2023-06-14 16:31:01,868 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-metadata.json with policy now +2023-06-14 16:31:01,873 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:31:01,873 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:31:02,774 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-metadata.json +2023-06-14 16:31:03,057 DEBUG SenderThread:3907002 [sender.py:send():369] send: telemetry +2023-06-14 16:31:03,057 DEBUG SenderThread:3907002 [sender.py:send():369] send: config +2023-06-14 16:31:03,418 INFO wandb-upload_0:3907002 [upload_job.py:push():133] Uploaded file /tmp/tmpr4sjdcacwandb/fpgf4rpk-wandb-metadata.json +2023-06-14 16:31:03,775 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:05,776 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:06,490 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:06,694 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:07,021 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:07,022 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:07,023 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:07,023 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:07,319 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:07,320 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:07,321 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:07,321 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:07,638 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:07,639 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:07,639 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:07,640 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:07,777 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:07,778 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:07,959 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:07,960 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:07,960 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:07,960 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:08,340 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:08,341 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:08,341 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:08,342 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:08,641 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:08,642 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:08,642 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:08,643 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:08,778 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:08,944 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:08,944 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:08,944 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:08,944 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:09,206 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:09,209 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:09,209 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:09,210 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:09,551 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:09,552 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:09,552 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:09,552 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:09,778 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:09,779 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:09,788 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:09,788 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:09,789 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:09,789 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:10,059 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:10,059 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:10,060 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:10,060 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:10,336 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:10,336 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:10,337 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:10,337 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:10,596 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:10,597 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:10,597 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:10,597 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:10,779 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:10,864 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:10,865 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:10,865 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:10,866 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:11,156 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:11,157 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:11,158 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:11,158 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:11,389 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:11,390 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:11,390 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:11,391 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:11,695 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:11,695 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:11,696 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:11,696 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:11,696 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:11,779 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:11,779 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:11,957 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:11,958 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:11,958 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:11,958 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:12,214 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:12,214 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:12,214 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:12,215 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:12,459 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:12,460 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:12,460 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:12,461 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:12,779 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:12,815 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:12,816 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:12,816 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:12,817 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:13,140 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:13,140 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:13,141 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:13,141 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:13,431 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:13,432 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:13,432 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:13,432 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:13,742 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:13,743 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:13,743 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:13,744 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:13,786 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:13,786 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:14,010 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:14,011 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:14,011 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:14,011 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:14,336 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:14,337 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:14,337 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:14,338 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:14,594 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:14,595 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:14,595 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:14,596 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:14,780 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:14,933 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:14,934 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:14,935 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:14,935 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:15,202 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:15,203 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:15,203 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:15,203 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:15,565 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:15,567 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:15,567 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:15,567 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:15,781 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:15,781 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:15,887 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:15,890 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:15,891 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:15,891 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:16,181 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:16,182 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:16,182 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:16,182 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:16,423 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:16,424 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:16,424 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:16,425 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:16,658 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:16,659 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:16,659 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:16,659 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:16,781 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:16,873 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:31:16,873 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:31:16,967 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:17,199 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:17,266 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:17,266 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:17,266 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:17,268 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:17,269 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:17,269 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:17,269 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:17,460 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:17,461 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:17,461 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:17,461 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:17,714 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:17,721 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:17,722 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:17,722 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:17,782 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:17,782 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:18,032 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:18,034 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:18,035 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:18,035 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:18,293 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:18,294 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:18,294 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:18,295 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:18,615 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:18,616 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:18,617 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:18,617 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:18,782 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:19,072 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:19,074 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:19,074 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:19,075 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:19,649 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:19,650 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:19,650 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:19,651 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:19,783 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:19,783 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:19,942 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:19,943 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:19,944 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:19,944 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:20,378 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:20,379 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:20,379 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:20,380 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:20,783 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:20,857 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:20,859 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:20,859 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:20,860 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:21,473 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:21,474 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:21,475 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:21,475 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:21,789 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:21,791 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:22,168 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:22,170 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:22,170 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:22,172 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:22,536 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:22,538 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:22,789 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:23,005 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:23,239 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:23,383 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,383 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,384 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:23,384 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,384 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,384 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:23,384 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,384 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,384 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:23,471 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:23,472 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,472 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,472 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:23,688 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:23,689 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,689 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,690 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:23,790 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:23,790 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:23,790 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/config.yaml +2023-06-14 16:31:23,911 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:23,912 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:23,912 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:23,912 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:24,155 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:24,156 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:24,156 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:24,156 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:24,406 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:24,407 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:24,407 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:24,407 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:24,754 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:24,755 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:24,756 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:24,756 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:24,790 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:25,131 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:25,132 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:25,133 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:25,133 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:25,646 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:25,650 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:25,651 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:25,652 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:25,794 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:25,796 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:27,794 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:29,158 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:29,795 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:31,796 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:31,880 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: stop_status +2023-06-14 16:31:31,881 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: stop_status +2023-06-14 16:31:33,797 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:34,333 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:35,797 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:37,798 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:39,140 DEBUG SenderThread:3907002 [sender.py:send():369] send: files +2023-06-14 16:31:39,140 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_e9c4a470ad900801f7de.png with policy now +2023-06-14 16:31:39,148 DEBUG SenderThread:3907002 [sender.py:send():369] send: files +2023-06-14 16:31:39,148 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_7673b7ce51f7ba8dbeaa.png with policy now +2023-06-14 16:31:39,149 DEBUG SenderThread:3907002 [sender.py:send():369] send: files +2023-06-14 16:31:39,149 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file media/images/validation_60_657ebe0b72dbaae9b6f3.png with policy now +2023-06-14 16:31:39,167 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: partial_history +2023-06-14 16:31:39,168 DEBUG SenderThread:3907002 [sender.py:send():369] send: history +2023-06-14 16:31:39,168 DEBUG SenderThread:3907002 [sender.py:send_request():396] send_request: summary_record +2023-06-14 16:31:39,168 INFO SenderThread:3907002 [sender.py:_save_file():1354] saving file wandb-summary.json with policy end +2023-06-14 16:31:39,799 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/wandb-summary.json +2023-06-14 16:31:39,799 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:39,800 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_7673b7ce51f7ba8dbeaa.png +2023-06-14 16:31:39,800 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_657ebe0b72dbaae9b6f3.png +2023-06-14 16:31:39,800 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images/validation_60_e9c4a470ad900801f7de.png +2023-06-14 16:31:39,800 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media/images +2023-06-14 16:31:39,800 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_created():272] file/dir created: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/media +2023-06-14 16:31:40,295 DEBUG HandlerThread:3907002 [handler.py:handle_request():144] handle_request: status_report +2023-06-14 16:31:41,801 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log +2023-06-14 16:31:43,801 INFO Thread-12 :3907002 [dir_watcher.py:_on_file_modified():289] file/dir modified: /home/deepanshu/lmflowdiff/Model/wandb/wandb/run-20230614_163050-nlm9mp31/files/output.log