Upload absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.20.hook_resid_post_eval_results.json with huggingface_hub

absorption/gemma-2-2b-res-snap-matryoshka-dc_blocks.20.hook_resid_post_eval_results.json

@@ -0,0 +1,275 @@
+{
+  "eval_type_id": "absorption_first_letter",
+  "eval_config": {
+    "model_name": "gemma-2-2b",
+    "random_seed": 42,
+    "f1_jump_threshold": 0.03,
+    "max_k_value": 10,
+    "prompt_template": "{word} has the first letter:",
+    "prompt_token_pos": -6,
+    "llm_batch_size": 32,
+    "llm_dtype": "bfloat16",
+    "k_sparse_probe_l1_decay": 0.01,
+    "k_sparse_probe_batch_size": 4096,
+    "k_sparse_probe_num_epochs": 50,
+    "min_GT_probe_f1": 0.6,
+    "min_feats_for_eval": 20
+  },
+  "eval_id": "7dac3d47-5282-48db-a705-23f393c4c7d6",
+  "datetime_epoch_millis": 1745736003376,
+  "eval_result_metrics": {
+    "mean": {
+      "mean_absorption_fraction_score": 0.1513608256492037,
+      "mean_full_absorption_score": 0.13129228410758353,
+      "mean_num_split_features": 1.2307692307692308,
+      "std_dev_absorption_fraction_score": 0.16159229196694166,
+      "std_dev_full_absorption_score": 0.148270149627756,
+      "std_dev_num_split_features": 0.6516251872168421
+    }
+  },
+  "eval_result_details": [
+    {
+      "first_letter": "a",
+      "mean_absorption_fraction": 0.016958900029211927,
+      "full_absorption_rate": 0.023575638506876228,
+      "num_full_absorption": 60,
+      "num_probe_true_positives": 2545,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "b",
+      "mean_absorption_fraction": 0.1765498221332393,
+      "full_absorption_rate": 0.14348370927318296,
+      "num_full_absorption": 229,
+      "num_probe_true_positives": 1596,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "c",
+      "mean_absorption_fraction": 0.5158276350810271,
+      "full_absorption_rate": 0.5103092783505154,
+      "num_full_absorption": 1386,
+      "num_probe_true_positives": 2716,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "d",
+      "mean_absorption_fraction": 0.20701322750745715,
+      "full_absorption_rate": 0.18970861748295104,
+      "num_full_absorption": 306,
+      "num_probe_true_positives": 1613,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "e",
+      "mean_absorption_fraction": 0.2150298532854617,
+      "full_absorption_rate": 0.20758234928527036,
+      "num_full_absorption": 334,
+      "num_probe_true_positives": 1609,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "f",
+      "mean_absorption_fraction": 0.01854203613011102,
+      "full_absorption_rate": 0.01991701244813278,
+      "num_full_absorption": 24,
+      "num_probe_true_positives": 1205,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "g",
+      "mean_absorption_fraction": 0.16931561384861052,
+      "full_absorption_rate": 0.13501742160278746,
+      "num_full_absorption": 155,
+      "num_probe_true_positives": 1148,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "h",
+      "mean_absorption_fraction": 0.004459278887566089,
+      "full_absorption_rate": 0.008255933952528379,
+      "num_full_absorption": 8,
+      "num_probe_true_positives": 969,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "i",
+      "mean_absorption_fraction": 0.00397421354453482,
+      "full_absorption_rate": 0.024140012070006035,
+      "num_full_absorption": 40,
+      "num_probe_true_positives": 1657,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "j",
+      "mean_absorption_fraction": 0.0,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 415,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "k",
+      "mean_absorption_fraction": 0.014600428445242805,
+      "full_absorption_rate": 0.013782542113323124,
+      "num_full_absorption": 9,
+      "num_probe_true_positives": 653,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "l",
+      "mean_absorption_fraction": 0.016262146737710842,
+      "full_absorption_rate": 0.010699588477366255,
+      "num_full_absorption": 13,
+      "num_probe_true_positives": 1215,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "m",
+      "mean_absorption_fraction": 0.24084381799061855,
+      "full_absorption_rate": 0.2106430155210643,
+      "num_full_absorption": 380,
+      "num_probe_true_positives": 1804,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "n",
+      "mean_absorption_fraction": 0.07249241787307215,
+      "full_absorption_rate": 0.06773399014778325,
+      "num_full_absorption": 55,
+      "num_probe_true_positives": 812,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "o",
+      "mean_absorption_fraction": 0.05881773573224856,
+      "full_absorption_rate": 0.0687984496124031,
+      "num_full_absorption": 71,
+      "num_probe_true_positives": 1032,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "p",
+      "mean_absorption_fraction": 0.45164800176784026,
+      "full_absorption_rate": 0.4316330904414916,
+      "num_full_absorption": 1007,
+      "num_probe_true_positives": 2333,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "q",
+      "mean_absorption_fraction": 0.005802329637485148,
+      "full_absorption_rate": 0.0,
+      "num_full_absorption": 0,
+      "num_probe_true_positives": 199,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "r",
+      "mean_absorption_fraction": 0.3809332372028721,
+      "full_absorption_rate": 0.383587786259542,
+      "num_full_absorption": 603,
+      "num_probe_true_positives": 1572,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "s",
+      "mean_absorption_fraction": 0.43549007504044557,
+      "full_absorption_rate": 0.3720014321518081,
+      "num_full_absorption": 1039,
+      "num_probe_true_positives": 2793,
+      "num_split_features": 4
+    },
+    {
+      "first_letter": "t",
+      "mean_absorption_fraction": 0.25248348026527134,
+      "full_absorption_rate": 0.18623232944068838,
+      "num_full_absorption": 303,
+      "num_probe_true_positives": 1627,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "u",
+      "mean_absorption_fraction": 0.16627379052469685,
+      "full_absorption_rate": 0.13009198423127463,
+      "num_full_absorption": 99,
+      "num_probe_true_positives": 761,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "v",
+      "mean_absorption_fraction": 0.07822742762843062,
+      "full_absorption_rate": 0.061519903498190594,
+      "num_full_absorption": 51,
+      "num_probe_true_positives": 829,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "w",
+      "mean_absorption_fraction": 0.014538354406545962,
+      "full_absorption_rate": 0.00906344410876133,
+      "num_full_absorption": 6,
+      "num_probe_true_positives": 662,
+      "num_split_features": 1
+    },
+    {
+      "first_letter": "x",
+      "mean_absorption_fraction": 0.3565302847898297,
+      "full_absorption_rate": 0.19090909090909092,
+      "num_full_absorption": 21,
+      "num_probe_true_positives": 110,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "y",
+      "mean_absorption_fraction": 0.03175459501072523,
+      "full_absorption_rate": 0.005780346820809248,
+      "num_full_absorption": 1,
+      "num_probe_true_positives": 173,
+      "num_split_features": 2
+    },
+    {
+      "first_letter": "z",
+      "mean_absorption_fraction": 0.03101276337904054,
+      "full_absorption_rate": 0.0091324200913242,
+      "num_full_absorption": 2,
+      "num_probe_true_positives": 219,
+      "num_split_features": 1
+    }
+  ],
+  "sae_bench_commit_hash": "Unknown",
+  "sae_lens_id": "blocks.20.hook_resid_post",
+  "sae_lens_release_id": "gemma-2-2b-res-snap-matryoshka-dc",
+  "sae_lens_version": "5.9.1",
+  "sae_cfg_dict": {
+    "architecture": "jumprelu",
+    "d_in": 2304,
+    "d_sae": 32768,
+    "activation_fn_str": "relu",
+    "apply_b_dec_to_input": true,
+    "finetuning_scaling_factor": false,
+    "context_size": 1024,
+    "model_name": "gemma-2-2b",
+    "hook_name": "blocks.20.hook_resid_post",
+    "hook_layer": 20,
+    "hook_head_index": null,
+    "prepend_bos": true,
+    "dataset_path": "chanind/pile-uncopyrighted-gemma-1024-abbrv-1B",
+    "dataset_trust_remote_code": true,
+    "normalize_activations": "none",
+    "dtype": "torch.bfloat16",
+    "device": "cuda",
+    "sae_lens_training_version": "5.5.1",
+    "activation_fn_kwargs": {
+      "k": 40
+    },
+    "neuronpedia_id": null,
+    "model_from_pretrained_kwargs": {
+      "center_writing_weights": false
+    },
+    "seqpos_slice": [
+      null
+    ]
+  },
+  "eval_result_unstructured": null
+}