Do not create our own query language after all.

examples/Model use.lynxkite.json

@@ -129,7 +129,8 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               }
             },
             "other": {},
@@ -171,10 +172,7 @@
               "name": "table_name",
               "type": {
                 "format": "dropdown",
-                "metadata_path": [
-                  "*",
-                  "dataframes"
-                ]
+                "metadata_query": "[].dataframes[].keys(@)[]"
               }
             },
             {
@@ -393,21 +391,24 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               }
             },
             "other": {},
@@ -593,25 +594,29 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               }
             },
             "other": {
               "model": {
+                "key": "model",
                 "model": {
                   "input_output_names": {
                     "Input__tensor_1_output": "X",
@@ -671,12 +676,7 @@
               "name": "model_name",
               "type": {
                 "format": "dropdown",
-                "metadata_filter_key": "type",
-                "metadata_filter_value": "model",
-                "metadata_path": [
-                  "*",
-                  "other"
-                ]
+                "metadata_query": "[].other.*[] | [?type == 'model'].key"
               }
             },
             {
@@ -842,30 +842,35 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               },
               "training": {
                 "columns": [
                   "training_loss"
-                ]
+                ],
+                "key": "training"
               }
             },
             "other": {
               "model": {
+                "key": "model",
                 "model": {
                   "input_output_names": {
                     "Input__tensor_1_output": "X",
@@ -925,12 +930,7 @@
               "name": "model_name",
               "type": {
                 "format": "dropdown",
-                "metadata_filter_key": "type",
-                "metadata_filter_value": "model",
-                "metadata_path": [
-                  "*",
-                  "other"
-                ]
+                "metadata_query": "[].other.*[] | [?type == 'model'].key"
               }
             },
             {
@@ -1606,30 +1606,35 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               },
               "training": {
                 "columns": [
                   "training_loss"
-                ]
+                ],
+                "key": "training"
               }
             },
             "other": {
               "model": {
+                "key": "model",
                 "model": {
                   "input_output_names": {
                     "Input__tensor_1_output": "X",
@@ -2702,7 +2707,8 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
@@ -2710,23 +2716,27 @@
                   "pred",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               },
               "training": {
                 "columns": [
                   "training_loss"
-                ]
+                ],
+                "key": "training"
               }
             },
             "other": {
               "model": {
+                "key": "model",
                 "model": {
                   "input_output_names": {
                     "Input__tensor_1_output": "X",
@@ -2807,184 +2817,184 @@
             {
               "data": [
                 [
-                  12.80282211303711,
-                  10.424864768981934,
+                  1.1259804964065552,
+                  5.3163533210754395,
                   "",
-                  -0.6019015312194824
+                  9.33173656463623
                 ],
                 [
-                  8.988066673278809,
-                  11.807311058044434,
+                  0.8733923435211182,
+                  0.5778636932373047,
                   "",
-                  8.764083862304688
+                  19.34723663330078
                 ],
                 [
-                  9.591389656066895,
-                  11.901337623596191,
+                  1.067949891090393,
+                  1.2907801866531372,
                   "",
-                  6.310827255249023
+                  20.190479278564453
                 ],
                 [
-                  8.783502578735352,
-                  12.252634048461914,
+                  1.4413745403289795,
+                  0.5289837718009949,
                   "",
-                  6.943178653717041
+                  19.06584930419922
                 ],
                 [
-                  10.00415325164795,
-                  12.100346565246582,
+                  0.6194189190864563,
+                  1.265136480331421,
                   "",
-                  5.549333572387695
+                  20.362045288085938
                 ],
                 [
-                  11.892269134521484,
-                  9.91622543334961,
+                  1.2515668869018555,
+                  4.7913312911987305,
                   "",
-                  0.22873035073280334
+                  8.057446479797363
                 ],
                 [
-                  12.981367111206055,
-                  9.589776992797852,
+                  -0.20645830035209656,
+                  4.6087422370910645,
                   "",
-                  -1.744539499282837
+                  10.770417213439941
                 ],
                 [
-                  11.813135147094727,
-                  10.878338813781738,
+                  0.8678598999977112,
+                  3.810476541519165,
                   "",
-                  1.3811211585998535
+                  8.457158088684082
                 ],
                 [
-                  11.561697006225586,
-                  10.171477317810059,
+                  1.6636368036270142,
+                  4.523313522338867,
                   "",
-                  0.3501940667629242
+                  8.167683601379395
                 ],
                 [
-                  8.242599487304688,
-                  12.206793785095215,
+                  1.5554659366607666,
+                  0.131289541721344,
                   "",
-                  7.688523292541504
+                  18.83956527709961
                 ],
                 [
-                  11.373790740966797,
-                  11.141427993774414,
+                  0.660535454750061,
+                  3.5511350631713867,
                   "",
-                  1.8117276430130005
+                  8.917879104614258
                 ],
                 [
-                  10.96745777130127,
-                  11.495363235473633,
+                  0.289438933134079,
+                  2.8580000400543213,
                   "",
-                  2.7511894702911377
+                  9.801031112670898
                 ],
                 [
-                  10.69228458404541,
-                  12.281194686889648,
+                  0.3500458598136902,
+                  2.201747417449951,
                   "",
-                  4.513139247894287
+                  21.30710792541504
                 ],
                 [
-                  9.344225883483887,
-                  11.59525203704834,
+                  0.6661970019340515,
+                  0.7411483526229858,
                   "",
-                  6.08269739151001
+                  19.81875991821289
                 ],
                 [
-                  12.43300724029541,
-                  9.454752922058105,
+                  -0.028679972514510155,
+                  4.847508430480957,
                   "",
-                  -1.2523200511932373
+                  10.324771881103516
                 ],
                 [
-                  8.347390174865723,
-                  12.715335845947266,
+                  0.9988114833831787,
+                  -0.31374749541282654,
                   "",
-                  7.751235008239746
+                  18.43217658996582
                 ],
                 [
-                  7.955441474914551,
-                  12.755577087402344,
+                  1.2578866481781006,
+                  -0.49167218804359436,
                   "",
-                  7.392812252044678
+                  17.98540687561035
                 ],
                 [
-                  12.56313419342041,
-                  10.382201194763184,
+                  0.7829321026802063,
+                  5.131852626800537,
                   "",
-                  -0.8537407517433167
+                  7.504575252532959
                 ],
                 [
-                  8.534817695617676,
-                  11.72046947479248,
+                  1.079374074935913,
+                  0.2569352090358734,
                   "",
-                  9.050346374511719
+                  19.47058868408203
                 ],
                 [
-                  10.44416618347168,
-                  12.523679733276367,
+                  0.7425185441970825,
+                  1.9031448364257812,
                   "",
-                  4.971646308898926
+                  20.669689178466797
                 ],
                 [
-                  8.51686954498291,
-                  13.201717376708984,
+                  1.1184741258621216,
+                  -0.8501556515693665,
                   "",
-                  8.017154693603516
+                  17.83696174621582
                 ],
                 [
-                  8.511856079101562,
-                  13.668107032775879,
+                  1.5970081090927124,
+                  -1.0676954984664917,
                   "",
-                  8.3856782913208
+                  17.155776977539062
                 ],
                 [
-                  11.167283058166504,
-                  11.889444351196289,
+                  0.6614332795143127,
+                  2.833474636077881,
                   "",
-                  3.279087543487549
+                  9.3106689453125
                 ],
                 [
-                  8.22509479522705,
-                  13.500997543334961,
+                  1.811506986618042,
+                  -0.7138973474502563,
                   "",
-                  8.311565399169922
+                  17.416305541992188
                 ],
                 [
-                  11.921534538269043,
-                  10.35403823852539,
+                  1.3115981817245483,
+                  4.23833703994751,
                   "",
-                  0.5600303411483765
+                  8.467390060424805
                 ],
                 [
-                  12.259814262390137,
-                  10.091246604919434,
+                  1.5368468761444092,
+                  5.071412563323975,
                   "",
-                  -0.18147911131381989
+                  7.690001010894775
                 ],
                 [
-                  12.283517837524414,
-                  10.76547908782959,
+                  0.9306067228317261,
+                  4.39690637588501,
                   "",
-                  0.947300374507904
+                  8.620335578918457
                 ],
                 [
-                  12.700486183166504,
-                  9.822050094604492,
+                  0.30367591977119446,
+                  4.658701419830322,
                   "",
-                  -1.4780642986297607
+                  10.502456665039062
                 ],
                 [
-                  10.082621574401855,
-                  12.619409561157227,
+                  0.4010639190673828,
+                  1.674473762512207,
                   "",
-                  5.293985366821289
+                  20.892684936523438
                 ],
                 [
-                  7.820052623748779,
-                  13.126693725585938,
+                  0.6275453567504883,
+                  -0.7735381126403809,
                   "",
-                  7.530407428741455
+                  18.215309143066406
                 ]
               ],
               "symbolSize": 25.65378780242026,
@@ -3014,8 +3024,8 @@
                 "#FDE725"
               ]
             },
-            "max": 9.050346374511719,
-            "min": -1.744539499282837,
+            "max": 21.30710792541504,
+            "min": 7.504575252532959,
             "right": 10,
             "top": "center"
           },
@@ -3038,7 +3048,8 @@
                 "columns": [
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df"
               },
               "df_test": {
                 "columns": [
@@ -3046,23 +3057,27 @@
                   "pred",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_test"
               },
               "df_train": {
                 "columns": [
                   "index",
                   "x",
                   "y"
-                ]
+                ],
+                "key": "df_train"
               },
               "training": {
                 "columns": [
                   "training_loss"
-                ]
+                ],
+                "key": "training"
               }
             },
             "other": {
               "model": {
+                "key": "model",
                 "model": {
                   "input_output_names": {
                     "Input__tensor_1_output": "X",
@@ -3108,21 +3123,24 @@
               "default": "nodes",
               "name": "table_name",
               "type": {
-                "type": "<class 'str'>"
+                "format": "dropdown",
+                "metadata_query": "[].dataframes[].keys(@)[]"
               }
             },
             {
               "default": "",
               "name": "vector_column",
               "type": {
-                "type": "<class 'str'>"
+                "format": "dropdown",
+                "metadata_query": "[].dataframes[].<table_name>.columns[]"
               }
             },
             {
               "default": "",
               "name": "label_column",
               "type": {
-                "type": "<class 'str'>"
+                "format": "dropdown",
+                "metadata_query": "[].dataframes[].<table_name>.columns[]"
               }
             },
             {

lynxkite-app/web/package-lock.json

@@ -23,6 +23,7 @@
         "daisyui": "^4.12.20",
         "echarts": "^5.5.1",
         "fuse.js": "^7.0.0",
+        "jmespath": "^0.16.0",
         "json-schema-to-typescript": "^15.0.3",
         "monaco-editor": "^0.52.2",
         "react": "^18.3.1",
@@ -40,6 +41,7 @@
       "devDependencies": {
         "@playwright/test": "^1.50.1",
         "@tailwindcss/typography": "^0.5.16",
+        "@types/jmespath": "^0.15.2",
         "@types/node": "^22.13.1",
         "@types/react": "^18.3.14",
         "@types/react-dom": "^18.3.2",
@@ -1894,6 +1896,13 @@
         "@types/unist": "*"
       }
     },
+    "node_modules/@types/jmespath": {
+      "version": "0.15.2",
+      "resolved": "https://registry.npmjs.org/@types/jmespath/-/jmespath-0.15.2.tgz",
+      "integrity": "sha512-pegh49FtNsC389Flyo9y8AfkVIZn9MMPE9yJrO9svhq6Fks2MwymULWjZqySuxmctd3ZH4/n7Mr98D+1Qo5vGA==",
+      "dev": true,
+      "license": "MIT"
+    },
     "node_modules/@types/json-schema": {
       "version": "7.0.15",
       "resolved": "https://registry.npmjs.org/@types/json-schema/-/json-schema-7.0.15.tgz",
@@ -3667,6 +3676,15 @@
         "jiti": "bin/jiti.js"
       }
     },
+    "node_modules/jmespath": {
+      "version": "0.16.0",
+      "resolved": "https://registry.npmjs.org/jmespath/-/jmespath-0.16.0.tgz",
+      "integrity": "sha512-9FzQjJ7MATs1tSpnco1K6ayiYE3figslrXA72G2HQ/n76RzvYlofyi5QM+iX4YRs/pu3yzxlVQSST23+dMDknw==",
+      "license": "Apache-2.0",
+      "engines": {
+        "node": ">= 0.6.0"
+      }
+    },
     "node_modules/js-tokens": {
       "version": "4.0.0",
       "resolved": "https://registry.npmjs.org/js-tokens/-/js-tokens-4.0.0.tgz",

lynxkite-app/web/package.json

@@ -25,6 +25,7 @@
     "daisyui": "^4.12.20",
     "echarts": "^5.5.1",
     "fuse.js": "^7.0.0",
+    "jmespath": "^0.16.0",
     "json-schema-to-typescript": "^15.0.3",
     "monaco-editor": "^0.52.2",
     "react": "^18.3.1",
@@ -42,6 +43,7 @@
   "devDependencies": {
     "@playwright/test": "^1.50.1",
     "@tailwindcss/typography": "^0.5.16",
+    "@types/jmespath": "^0.15.2",
     "@types/node": "^22.13.1",
     "@types/react": "^18.3.14",
     "@types/react-dom": "^18.3.2",

lynxkite-app/web/src/workspace/nodes/NodeParameter.tsx

@@ -1,3 +1,4 @@
+import jmespath from "jmespath";
 // @ts-ignore
 import ArrowsHorizontal from "~icons/tabler/arrows-horizontal.jsx";
 // @ts-ignore
@@ -133,55 +134,14 @@ export default function NodeParameter({ name, value, meta, data, setParam }: Nod
   );
 }
 
-// We have a little "language" for describing which part of the input_metadata
-// to use in the dropdown.
 function getDropDownValues(data: any, meta: any): string[] {
   const metadata = data.input_metadata.value;
-  const { metadata_path, metadata_filter_key, metadata_filter_value } = meta.type;
-  // Starting from the root element of the input_metadata, we follow the
-  // metadata_path to find the items.
-  let o = [metadata];
-  for (const path of metadata_path) {
-    o = o.flatMap((x: any) => {
-      if (x === undefined || x === null) {
-        return [];
-      }
-      // We have a path step, so x must be an object or an array.
-      // For arrays we pick an element by index or the whole array if the path is "*".
-      if (Array.isArray(x)) {
-        if (path === "*") {
-          return x;
-        }
-        return [x[Number.parseInt(path)]];
-      }
-      // For objects we pick a value by key or all values if the path is "*".
-      if (path === "*") {
-        return Object.values(x);
-      }
-      return [x[path]];
-    });
+  let query = meta.type.metadata_query;
+  // Substitute parameters in the query.
+  for (const p in data.params) {
+    query = query.replace(`<${p}>`, data.params[p]);
   }
-  // Now we transform the list of matched items into a list of strings.
-  o = o.flatMap((x: any) => {
-    if (x === undefined || x === null) {
-      return [];
-    }
-    if (Array.isArray(x)) {
-      return x;
-    }
-    if (typeof x === "object") {
-      if (metadata_filter_key && metadata_filter_value) {
-        const keys = [];
-        for (const key in x) {
-          if (x[key][metadata_filter_key] === metadata_filter_value) {
-            keys.push(key);
-          }
-        }
-        return keys;
-      }
-      return Object.keys(x);
-    }
-    return [x];
-  });
-  return ["", ...o];
+  const res = ["", ...jmespath.search(metadata, query)];
+  res.sort();
+  return res;
 }

lynxkite-graph-analytics/src/lynxkite_graph_analytics/core.py

@@ -14,25 +14,33 @@ import typing
 
 ENV = "LynxKite Graph Analytics"
 
-TableDropdown = typing.Annotated[str, {"format": "dropdown", "metadata_path": ["*", "dataframes"]}]
+# Annotated types with format "dropdown" let you specify the available options
+# as a query on the input_metadata. These query expressions are JMESPath expressions.
+TableDropdown = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].keys(@)[]"}
+]
 NodeAttribute = typing.Annotated[
-    str, {"format": "dropdown", "metadata_path": ["*", "dataframes", "nodes", "columns"]}
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].nodes[].columns[]"}
 ]
 EdgeAttribute = typing.Annotated[
-    str, {"format": "dropdown", "metadata_path": ["*", "dataframes", "edges", "columns"]}
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].edges[].columns[]"}
+]
+OtherDropdown = typing.Annotated[
+    str, {"format": "dropdown", "metadata_query": "[].other.keys(@)[]"}
 ]
-OtherDropdown = typing.Annotated[str, {"format": "dropdown", "metadata_path": ["*", "other"]}]
 ModelDropdown = typing.Annotated[
     str,
     {
         "format": "dropdown",
-        "metadata_path": ["*", "other"],
-        "metadata_filter_key": "type",
-        "metadata_filter_value": "model",
+        "metadata_query": "[].other.*[] | [?type == 'model'].key",
     },
 ]
+# Parameter names in angle brackets, like <table_name>, will be replaced with the parameter
+# values. (This is not part of JMESPath.)
+# ColumnDropdownByTableName will list the columns of the DataFrame with the name
+# specified by the `table_name` parameter.
 ColumnDropdownByTableName = typing.Annotated[
-    str, {"format": "dropdown", "metadata_path": ["*", "dataframes", "<table_name>", "columns"]}
+    str, {"format": "dropdown", "metadata_query": "[].dataframes[].<table_name>.columns[]"}
 ]
 
 
@@ -162,12 +170,15 @@ class Bundle:
         return {
             "dataframes": {
                 name: {
+                    "key": name,
                     "columns": sorted(str(c) for c in df.columns),
                 }
                 for name, df in self.dfs.items()
             },
             "relations": [dataclasses.asdict(relation) for relation in self.relations],
-            "other": {k: getattr(v, "metadata", lambda: {})() for k, v in self.other.items()},
+            "other": {
+                k: {"key": k, **getattr(v, "metadata", lambda: {})()} for k, v in self.other.items()
+            },
         }
 
 

lynxkite-graph-analytics/src/lynxkite_graph_analytics/ml_ops.py

@@ -209,8 +209,8 @@ def view_vectors(
     bundle: core.Bundle,
     *,
     table_name: core.TableDropdown = "nodes",
-    vector_column: str = "",
-    label_column: str = "",
+    vector_column: core.ColumnDropdownByTableName = "",
+    label_column: core.ColumnDropdownByTableName = "",
     n_neighbors: int = 15,
     min_dist: float = 0.1,
     metric: UMAPMetric = UMAPMetric.euclidean,