Hugging Face's logo

ghidav
/
gpt2-sae-attn-tl

Model card Files Community
Davide Ghilardi commited on
Commit
54777f8
•
1 Parent(s): 8c1164c

Update

Browse files
This view is limited to 50 files because it contains too many changes.   See raw diff
Files changed (50) hide show
  1. .DS_Store +0 -0
  2. layer_0/.DS_Store +0 -0
  3. layer_0/{final_61440000/cfg.json → cfg.json} +0 -0
  4. layer_0/final_61440000/h3i20ez6/final_61440000/cfg.json +0 -1
  5. layer_0/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  6. layer_0/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  7. layer_1/.DS_Store +0 -0
  8. layer_1/{final_61440000/cfg.json → cfg.json} +0 -0
  9. layer_0/final_61440000/h3i20ez6/final_61440000/sparsity.safetensors → layer_1/sae_weights.safetensors +2 -2
  10. layer_1/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  11. layer_10/.DS_Store +0 -0
  12. layer_10/{final_61440000/cfg.json → cfg.json} +0 -0
  13. layer_10/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  14. layer_10/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  15. layer_11/.DS_Store +0 -0
  16. layer_11/{final_61440000/cfg.json → cfg.json} +0 -0
  17. layer_11/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  18. layer_11/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  19. layer_2/.DS_Store +0 -0
  20. layer_2/{final_61440000/cfg.json → cfg.json} +0 -0
  21. layer_2/sae_weights.safetensors +3 -0
  22. layer_2/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  23. layer_3/.DS_Store +0 -0
  24. layer_3/{final_61440000/cfg.json → cfg.json} +0 -0
  25. layer_3/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  26. layer_3/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  27. layer_4/{final_61440000/cfg.json → cfg.json} +0 -0
  28. layer_4/sae_weights.safetensors +3 -0
  29. layer_4/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  30. layer_5/.DS_Store +0 -0
  31. layer_5/{final_61440000/cfg.json → cfg.json} +0 -0
  32. layer_5/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  33. layer_5/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  34. layer_6/{final_61440000/cfg.json → cfg.json} +0 -0
  35. layer_6/sae_weights.safetensors +3 -0
  36. layer_6/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  37. layer_7/{final_61440000/cfg.json → cfg.json} +0 -0
  38. layer_7/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  39. layer_7/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  40. layer_8/.DS_Store +0 -0
  41. layer_8/{final_61440000/cfg.json → cfg.json} +0 -0
  42. layer_8/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} +0 -0
  43. layer_8/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  44. layer_9/{final_61440000/cfg.json → cfg.json} +0 -0
  45. layer_9/sae_weights.safetensors +3 -0
  46. layer_9/{final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
  47. transfer_layer_1/.DS_Store +0 -0
  48. transfer_layer_1/{byp42y0x/final_61440000/cfg.json → cfg.json} +0 -0
  49. transfer_layer_1/sae_weights.safetensors +3 -0
  50. transfer_layer_1/{byp42y0x/final_61440000/sparsity.safetensors → sparsity.safetensors} +0 -0
.DS_Store ADDED
Binary file (12.3 kB). View file
 
layer_0/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_0/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_0/final_61440000/h3i20ez6/final_61440000/cfg.json DELETED
@@ -1 +0,0 @@
1
- {"model_name": "gpt2", "model_class_name": "HookedTransformer", "hook_point": "blocks.1.hook_attn_out", "hook_point_eval": "blocks.{layer}.attn.pattern", "hook_point_layer": 1, "hook_point_head_index": null, "dataset_path": "apollo-research/Skylion007-openwebtext-tokenizer-gpt2", "streaming": true, "is_dataset_tokenized": true, "context_size": 128, "use_cached_activations": false, "cached_activations_path": null, "d_in": 768, "d_sae": 49152, "b_dec_init_method": "zeros", "expansion_factor": 64, "activation_fn": "relu", "normalize_sae_decoder": false, "noise_scale": 0.0, "from_pretrained_path": null, "apply_b_dec_to_input": false, "decoder_orthogonal_init": false, "decoder_heuristic_init": true, "init_encoder_as_decoder_transpose": true, "n_batches_in_buffer": 128, "training_tokens": 61440000, "finetuning_tokens": 0, "store_batch_size_prompts": 32, "train_batch_size_tokens": 4096, "normalize_activations": false, "device": "cuda", "act_store_device": "cuda", "seed": 42, "dtype": "torch.float32", "prepend_bos": true, "autocast": true, "autocast_lm": false, "compile_llm": true, "llm_compilation_mode": null, "compile_sae": true, "sae_compilation_mode": null, "adam_beta1": 0.9, "adam_beta2": 0.999, "mse_loss_normalization": null, "l1_coefficient": 1, "lp_norm": 1.0, "scale_sparsity_penalty_by_decoder_norm": true, "l1_warm_up_steps": 1500, "lr": 5e-05, "lr_scheduler_name": "constant", "lr_warm_up_steps": 0, "lr_end": 5e-06, "lr_decay_steps": 0, "n_restart_cycles": 1, "finetuning_method": null, "use_ghost_grads": false, "feature_sampling_window": 1000, "dead_feature_window": 1000, "dead_feature_threshold": 0.0001, "n_eval_batches": 10, "eval_batch_size_prompts": 2, "log_to_wandb": true, "log_activations_store_to_wandb": false, "log_optimizer_state_to_wandb": false, "wandb_project": "transfer-learning", "wandb_id": null, "run_name": "49152-L1-1-LR-5e-05-Tokens-6.144e+07", "wandb_entity": null, "wandb_log_frequency": 50, "eval_every_n_wandb_logs": 10, "resume": false, "n_checkpoints": 0, "checkpoint_path": "sae-transfer-learning/first-attn-transfer-gpt2/layer_0/final_61440000//h3i20ez6", "verbose": true, "model_kwargs": {}, "model_from_pretrained_kwargs": {}, "sae_lens_version": "2.1.3", "sae_lens_training_version": "2.1.3", "tokens_per_buffer": 67108864}
 
 
layer_0/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_0/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_1/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_1/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_0/final_61440000/h3i20ez6/final_61440000/sparsity.safetensors → layer_1/sae_weights.safetensors RENAMED
@@ -1,3 +1,3 @@
1
  version https://git-lfs.github.com/spec/v1
2
- oid sha256:bb5168849c80c6bbd7b93c4560ed3535bf3c7645d3d7660895ae2fae52f3e05a
3
- size 196688
 
1
  version https://git-lfs.github.com/spec/v1
2
+ oid sha256:d4bb3f9dee4c51028caea05ee89229ac6d09d9a547ff4d63149fb12ae1fc4ab2
3
+ size 302386584
layer_1/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_10/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_10/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_10/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_10/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_11/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_11/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_11/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_11/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_2/.DS_Store ADDED
Binary file (8.2 kB). View file
 
layer_2/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_2/sae_weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:fbd28492e369a117b916b8f2f837fd0f1d53df4fb7c5e54461f74bc0706001f3
3
+ size 302386584
layer_2/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_3/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_3/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_3/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_3/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_4/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_4/sae_weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:6151e5eec3f6fd79fd511dc3a39a3b4fd81fc1eb6822e5fb4f18e15d0455618b
3
+ size 302386584
layer_4/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_5/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_5/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_5/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_5/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_6/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_6/sae_weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:5df19c6feaec8e86aef3a86d3dd68b68383d21a36b163b91e6fe460cc5627d4f
3
+ size 302386584
layer_6/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_7/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_7/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_7/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_8/.DS_Store ADDED
Binary file (6.15 kB). View file
 
layer_8/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_8/{final_61440000/sae_weights.safetensors → sae_weights.safetensors} RENAMED
File without changes
layer_8/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
layer_9/{final_61440000/cfg.json → cfg.json} RENAMED
File without changes
layer_9/sae_weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:3d81cedbe8320dbff36e5a2ddb4afb90125eda6e47a9e727628cccc7bce1df9d
3
+ size 302386584
layer_9/{final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes
transfer_layer_1/.DS_Store ADDED
Binary file (6.15 kB). View file
 
transfer_layer_1/{byp42y0x/final_61440000/cfg.json → cfg.json} RENAMED
File without changes
transfer_layer_1/sae_weights.safetensors ADDED
@@ -0,0 +1,3 @@
 
 
 
 
1
+ version https://git-lfs.github.com/spec/v1
2
+ oid sha256:ab04bf79785bc1b42b496c5402016bf50179d744357ee30932614145778911ec
3
+ size 302386584
transfer_layer_1/{byp42y0x/final_61440000/sparsity.safetensors → sparsity.safetensors} RENAMED
File without changes