Upload absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.17.hook_resid_post_eval_results.json with huggingface_hub

absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.17.hook_resid_post_eval_results.json

@@ -0,0 +1,275 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50,
+    "min_GT_probe_f1": 0.6,
+    "min_feats_for_eval": 20
+  },
+  "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6",
+  "datetime_epoch_millis": 1745731323773,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.09190284658267972,
+      "mean_full_absorption_score": 0.07636835724538374,
+      "mean_num_split_features": 1.3076923076923077,
+      "std_dev_absorption_fraction_score": 0.14571296351546223,
+      "std_dev_full_absorption_score": 0.13893384441642123,
+      "std_dev_num_split_features": 0.6176879969195301
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.31956071410316983,
+      "full_absorption_rate": 0.2877503852080123,
+      "num_full_absorption": 747,
+      "num_probe_true_positives": 2596,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.04449745477504374,
+      "full_absorption_rate": 0.023646546359676415,
+      "num_full_absorption": 38,
+      "num_probe_true_positives": 1607,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.589654239145153,
+      "full_absorption_rate": 0.5625226531351939,
+      "num_full_absorption": 1552,
+      "num_probe_true_positives": 2759,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.001339060755086657,
+      "full_absorption_rate": 0.0041841004184100415,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 1673,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.061662389404888496,
+      "full_absorption_rate": 0.056214865708931916,
+      "num_full_absorption": 90,
+      "num_probe_true_positives": 1601,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.011349852176804556,
+      "full_absorption_rate": 0.004273504273504274,
+      "num_full_absorption": 5,
+      "num_probe_true_positives": 1170,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.0870451819893666,
+      "full_absorption_rate": 0.0546875,
+      "num_full_absorption": 63,
+      "num_probe_true_positives": 1152,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.0381254421552879,
+      "full_absorption_rate": 0.014395393474088292,
+      "num_full_absorption": 15,
+      "num_probe_true_positives": 1042,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.00724038034143389,
+      "full_absorption_rate": 0.012330456226880395,
+      "num_full_absorption": 20,
+      "num_probe_true_positives": 1622,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0060390200128819125,
+      "full_absorption_rate": 0.0022675736961451248,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 441,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.014336384996811618,
+      "full_absorption_rate": 0.01088646967340591,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 643,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.029794771860448033,
+      "full_absorption_rate": 0.022764227642276424,
+      "num_full_absorption": 28,
+      "num_probe_true_positives": 1230,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.16111525953558284,
+      "full_absorption_rate": 0.1074792243767313,
+      "num_full_absorption": 194,
+      "num_probe_true_positives": 1805,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.09544210944678017,
+      "full_absorption_rate": 0.0640096618357488,
+      "num_full_absorption": 53,
+      "num_probe_true_positives": 828,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.3065232988881943,
+      "full_absorption_rate": 0.2209188660801564,
+      "num_full_absorption": 226,
+      "num_probe_true_positives": 1023,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.00870559114530204,
+      "full_absorption_rate": 0.003036876355748373,
+      "num_full_absorption": 7,
+      "num_probe_true_positives": 2305,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.002457081995669703,
+      "full_absorption_rate": 0.0055248618784530384,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 181,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.0009763400503572069,
+      "full_absorption_rate": 0.001851851851851852,
+      "num_full_absorption": 3,
+      "num_probe_true_positives": 1620,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.37384705903782756,
+      "full_absorption_rate": 0.3992857142857143,
+      "num_full_absorption": 1118,
+      "num_probe_true_positives": 2800,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.002104050395839018,
+      "full_absorption_rate": 0.002335084646818447,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 1713,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.07293505307107637,
+      "full_absorption_rate": 0.03595206391478029,
+      "num_full_absorption": 27,
+      "num_probe_true_positives": 751,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.015528941369840081,
+      "full_absorption_rate": 0.013530135301353014,
+      "num_full_absorption": 11,
+      "num_probe_true_positives": 813,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.03684412623353439,
+      "full_absorption_rate": 0.029275808936825885,
+      "num_full_absorption": 19,
+      "num_probe_true_positives": 649,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.007447853326184163,
+      "full_absorption_rate": 0.010638297872340425,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 94,
+      "num_split_features": 3
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.09283834515963053,
+      "full_absorption_rate": 0.027972027972027972,
+      "num_full_absorption": 4,
+      "num_probe_true_positives": 143,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.0020640097774781985,
+      "full_absorption_rate": 0.00784313725490196,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 255,
+      "num_split_features": 2
+    }
+  ],
+  "sae_bench_commit_hash": "Unknown",
+  "sae_lens_id": "blocks.17.hook_resid_post",
+  "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
+  "sae_lens_version": "5.9.1",
+  "sae_cfg_dict": {
+    "architecture": "jumprelu",
+    "d_in": 2304,
+    "d_sae": 32768,
+    "activation_fn_str": "relu",
+    "apply_b_dec_to_input": true,
+    "finetuning_scaling_factor": false,
+    "context_size": 1024,
+    "model_name": "gemma-2-2b",
+    "hook_name": "blocks.17.hook_resid_post",
+    "hook_layer": 17,
+    "hook_head_index": null,
+    "prepend_bos": true,
+    "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
+    "dataset_trust_remote_code": true,
+    "normalize_activations": "none",
+    "dtype": "torch.bfloat16",
+    "device": "cuda",
+    "sae_lens_training_version": "5.5.1",
+    "activation_fn_kwargs": {
+      "k": 40
+    },
+    "neuronpedia_id": null,
+    "model_from_pretrained_kwargs": {
+      "center_writing_weights": false
+    },
+    "seqpos_slice": [
+      null
+    ]
+  },
+  "eval_result_unstructured": null
+}