{
  "accuracy": 0.95,
  "correct": 475,
  "total": 500,
  "safety_violations": 0,
  "per_tool": {
    "drone_look": {
      "correct": 41,
      "total": 42
    },
    "drone_mission": {
      "correct": 45,
      "total": 61
    },
    "drone_goto": {
      "correct": 43,
      "total": 46
    },
    "drone_move": {
      "correct": 69,
      "total": 69
    },
    "drone_yaw": {
      "correct": 50,
      "total": 51
    },
    "drone_flight": {
      "correct": 122,
      "total": 123
    },
    "drone_track": {
      "correct": 34,
      "total": 34
    },
    "drone_notify": {
      "correct": 24,
      "total": 26
    },
    "drone_scan": {
      "correct": 47,
      "total": 48
    }
  },
  "failures": [
    {
      "expected": "drone_notify(message=\"Alt 53.2m, bat 85%, all systems nominal\")",
      "predicted": "drone_notify(message=\"Alt 53.2m, bat 85%, all current bat 85%, all current bat 85%, all current bat 85%, all current.",
      "tool": "drone_notify",
      "user": "858,-97.121 | vision=SCANNING target=vehicle wp=0/0\n[CMD] how's the search going"
    },
    {
      "expected": "drone_mission(action=\"patrol\", waypoints=\"32.854,-97.147;32.86,-97.127;32.846,-97.116;32.847,-97.118\")",
      "predicted": "drone_mission(action=\"patrol\", waypoints=\"32.85,-97.149;32.848,-97.122;32.848,-97.122;32.84,-97.124;32.84,-97.124;3",
      "tool": "drone_mission",
      "user": " pos=32.843,-97.102 | vision=IDLE target=none wp=0/0\n[CMD] patrol these 4 points"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"person\", waypoints=\"32.855,-97.102;32.87,-97.111;32.849,-97.145;32.853,-97.117\")",
      "predicted": "drone_mission(action=\"search\", target=\"person\", waypoints=\"32.85,-97.149;32.848,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124",
      "tool": "drone_mission",
      "user": ",-97.122 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for person"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"animal\", waypoints=\"32.868,-97.127;32.857,-97.142;32.868,-97.129;32.864,-97.125\")",
      "predicted": "drone_mission(action=\"search\", target=\"animal\", waypoints=\"32.85,-97.11;32.848,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124;",
      "tool": "drone_mission",
      "user": ",-97.113 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for animal"
    },
    {
      "expected": "drone_mission(action=\"search\")",
      "predicted": "drone_scan()",
      "tool": "drone_mission",
      "user": "=5.2m/s pos=32.863,-97.132 | vision=IDLE target=none wp=0/0\n[CMD] sweep the area"
    },
    {
      "expected": "drone_notify(message=\"Alt 89.4m, bat 81%, all systems nominal\")",
      "predicted": "drone_notify(message=\"Alt 89.4m, alt 89.4m, range ~10m, tracking\")",
      "tool": "drone_notify",
      "user": "4\u00b0 spd=2.3m/s pos=32.842,-97.133 | vision=LOST target=person wp=0/0\n[CMD] report"
    },
    {
      "expected": "drone_goto(lat=32.866, lon=-97.125, on_arrival=\"scan\")",
      "predicted": "drone_goto(lat=32.853, lon=-97.123, on_arrival=\"scan\")",
      "tool": "drone_goto",
      "user": ".9m/s pos=32.855,-97.118 | vision=LOST target=any wp=0/0\n[CMD] go there and scan"
    },
    {
      "expected": "drone_goto(lat=32.852, lon=-97.138, on_arrival=\"scan\")",
      "predicted": "drone_goto(lat=32.853, lon=-97.124, on_arrival=\"scan\")",
      "tool": "drone_goto",
      "user": "/s pos=32.862,-97.123 | vision=LOST target=person wp=0/0\n[CMD] go there and scan"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"person\", waypoints=\"32.865,-97.137;32.859,-97.144\")",
      "predicted": "drone_mission(action=\"search\", target=\"person\", waypoints=\"32.85,-97.129;32.848,-97.124;32.848,-97.124;32.84,-97.124;32.84,-97.12",
      "tool": "drone_mission",
      "user": "7,-97.13 | vision=IDLE target=none wp=0/0\n[CMD] search these 2 points for person"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"vehicle\", waypoints=\"32.866,-97.134;32.846,-97.112;32.87,-97.134;32.869,-97.13\")",
      "predicted": "drone_mission(action=\"search\", target=\"vehicle\", waypoints=\"32.85,-97.149;32.84,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124;",
      "tool": "drone_mission",
      "user": "-97.107 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for vehicle"
    },
    {
      "expected": "drone_mission(action=\"search\")",
      "predicted": "drone_scan()",
      "tool": "drone_mission",
      "user": "=3.7m/s pos=32.863,-97.104 | vision=IDLE target=none wp=0/0\n[CMD] sweep the area"
    },
    {
      "expected": "drone_mission(action=\"patrol\", waypoints=\"32.859,-97.144;32.869,-97.147;32.861,-97.122;32.84,-97.126;32.856,-97.141;32.841,-97.137\")",
      "predicted": "drone_mission(action=\"patrol\", waypoints=\"32.85,-97.104;32.848,-97.124;32.84,-97.124;32.848,-97.124;32.84,-97.124;3",
      "tool": "drone_mission",
      "user": "s pos=32.84,-97.107 | vision=IDLE target=none wp=0/0\n[CMD] patrol these 6 points"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.86,-97.137;32.864,-97.116;32.863,-97.118;32.842,-97.11;32.847,-97.101\")",
      "predicted": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.85,-97.11;32.843,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124;",
      "tool": "drone_mission",
      "user": "868,-97.121 | vision=IDLE target=none wp=0/0\n[CMD] search these 5 points for any"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"person\")",
      "predicted": "drone_scan(lock_on_class=\"person\")",
      "tool": "drone_mission",
      "user": "32.864,-97.147 | vision=IDLE target=none wp=0/0\n[CMD] search the area for people"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"animal\", waypoints=\"32.859,-97.105;32.843,-97.101;32.859,-97.138;32.852,-97.143\")",
      "predicted": "drone_mission(action=\"search\", target=\"animal\", waypoints=\"32.85,-97.149;32.84,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124;",
      "tool": "drone_mission",
      "user": ",-97.128 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for animal"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"any\", pattern=\"lawnmower\")",
      "predicted": "drone_mission(action=\"search\", pattern=\"lawnmower\")",
      "tool": "drone_mission",
      "user": "4m/s pos=32.859,-97.126 | vision=IDLE target=none wp=0/0\n[CMD] broad area search"
    },
    {
      "expected": "drone_mission(action=\"search\")",
      "predicted": "drone_scan()",
      "tool": "drone_mission",
      "user": "d=0.6m/s pos=32.86,-97.139 | vision=IDLE target=none wp=0/0\n[CMD] sweep the area"
    },
    {
      "expected": "drone_yaw(degrees=360)",
      "predicted": "drone_yaw(degrees=90)",
      "tool": "drone_yaw",
      "user": "pd=0.7m/s pos=32.861,-97.126 | vision=LOST target=any wp=0/0\n[CMD] full rotation"
    },
    {
      "expected": "drone_goto(lat=32.848, lon=-97.109, on_arrival=\"photo\")",
      "predicted": "drone_goto(lat=32.853, lon=-97.123, on_arrival=\"photo\")",
      "tool": "drone_goto",
      "user": "97.103 | vision=IDLE target=none wp=0/0\n[CMD] fly to that point and take a photo"
    },
    {
      "expected": "drone_scan(lock_on_class=\"person\")",
      "predicted": "drone_scan(lock_on_class=\"vehicle\")",
      "tool": "drone_scan",
      "user": "97.132 | vision=IDLE target=none wp=0/0\n[CMD] find someone and keep eyes on them"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.848,-97.103;32.847,-97.134;32.853,-97.127;32.844,-97.147\")",
      "predicted": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.85,-97.129;32.848,-97.144;32.84,-97.124;32.84,-97.144;32.84,-97.124",
      "tool": "drone_mission",
      "user": ".86,-97.141 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for any"
    },
    {
      "expected": "drone_look(action=\"status\")",
      "predicted": "drone_look(action=\"investigate\")",
      "tool": "drone_look",
      "user": "m/s pos=32.864,-97.116 | vision=LOST target=animal wp=0/0\n[CMD] how high are you"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"vehicle\", waypoints=\"32.861,-97.111;32.847,-97.131\")",
      "predicted": "drone_mission(action=\"search\", target=\"vehicle\", waypoints=\"32.85,-97.11;32.843,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124;",
      "tool": "drone_mission",
      "user": "-97.137 | vision=IDLE target=none wp=0/0\n[CMD] search these 2 points for vehicle"
    },
    {
      "expected": "drone_flight(action=\"rth\")",
      "predicted": "drone_move(direction=\"back\", distance_m=5)",
      "tool": "drone_flight",
      "user": "\u00b0 spd=1.3m/s pos=32.862,-97.144 | vision=IDLE target=none wp=0/0\n[CMD] head back"
    },
    {
      "expected": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.858,-97.142;32.868,-97.124;32.862,-97.143;32.863,-97.105\")",
      "predicted": "drone_mission(action=\"search\", target=\"any\", waypoints=\"32.85,-97.149;32.848,-97.124;32.84,-97.124;32.84,-97.124;32.84,-97.124",
      "tool": "drone_mission",
      "user": "846,-97.115 | vision=IDLE target=none wp=0/0\n[CMD] search these 4 points for any"
    }
  ]
}