DALLE2-PyTorch / decoder /2B /decoder_config.json
Veldrovive's picture
Upload decoder/2B/decoder_config.json with huggingface_hub
c081c3e
raw
history blame
3.12 kB
{
"seed": 1,
"decoder": {
"unets": [
{
"dim": 512,
"cond_dim": 512,
"image_embed_dim": 768,
"text_embed_dim": 768,
"cond_on_text_encodings": false,
"channels": 3,
"dim_mults": [1, 2, 3, 4],
"self_attn": [false, true, true, true],
"num_resnet_blocks": 4,
"attn_heads": 8,
"attn_dim_head": 64,
"sparse_attn": true,
"memory_efficient": true
}
],
"image_sizes": [64],
"channels": 3,
"timesteps": 1000,
"loss_type": "l2",
"beta_schedule": ["cosine"],
"learned_variance": false
},
"data": {
"webdataset_base_url": "pipe:aws s3 cp --quiet s3://s-datasets/laion-aesthetic/data/laion2B-en-aesthetic/{}.tar -",
"img_embeddings_url": "s3://s-datasets/laion-aesthetic/ordered_embeddings/",
"num_workers": 6,
"batch_size": 1,
"start_shard": 0,
"end_shard": 5247,
"shard_width": 5,
"index_width": 4,
"splits": {
"train": 0.75,
"val": 0.15,
"test": 0.1
},
"shuffle_train": false,
"resample_train": true,
"preprocessing": {
"RandomResizedCrop": {
"size": [64, 64],
"scale": [0.75, 1.0],
"ratio": [1.0, 1.0]
},
"ToTensor": true
}
},
"train": {
"epochs": 1000,
"lr":1.2e-4,
"wd": 0.0,
"warmup_steps": 1000,
"max_grad_norm": 0.5,
"save_every_n_samples": 545400,
"n_sample_images": 10,
"device": "cuda:0",
"epoch_samples": 2100000,
"validation_samples": 100000,
"use_ema": true,
"ema_beta": 0.99,
"save_all": false,
"save_latest": true,
"save_best": true,
"unet_training_mask": [true],
"find_unused_parameters": true
},
"evaluate": {
"n_evaluation_samples": 10,
"FID": {
"feature": 64
},
"LPIPS": {
"net_type": "vgg",
"reduction": "mean"
}
},
"tracker": {
"data_path": "/fsx/aidan/new/multinode/experiments/deepspeed/.tracker-data",
"overwrite_data_path": true,
"log": {
"log_type": "console",
"wandb_entity": "Veldrovive",
"wandb_project": "dalle2_train_decoder",
"wandb_resume": false,
"auto_resume": true,
"verbose": true
},
"load": {
"load_from": null,
"only_auto_resume": true,
"file_path": "/fsx/aidan/new/multinode/experiments/deepspeed/models/checkpoints/latest.pth"
},
"save": [{
"save_to": "local",
"save_latest_to": "/fsx/aidan/new/multinode/experiments/deepspeed/models/checkpoints/latest.pth",
"save_type": "checkpoint"
}]
}
}