More E2E progress, print E2E paths by occurence count

trace-analysis.ipynb

@@ -825,7 +825,6 @@
     "scatter_topic(\"/autoware/engage\")\n",
     "scatter_topic(\"/planning/scenario_planning/parking/trajectory\", y=-.04, color=\"cadetblue\")\n",
     "scatter_topic(\"/planning/scenario_planning/lane_driving/trajectory\", y=-.08, color=\"darkgreen\")\n",
-    "scatter_topic(\"/initialpose2d\", y=-.12, color=\"orange\")\n",
     "\n",
     "ax.set_xlabel(\"Simulation time [s]\")\n",
     "ax.set_ylabel(\"End-to-End latency [s]\")\n",
@@ -845,6 +844,8 @@
    "outputs": [],
    "source": [
     "def critical_path(self):\n",
+    "    if not self.deps:\n",
+    "        return [self.head]\n",
     "    return [self.head, *min(map(critical_path, self.deps), key=lambda ls: ls[-1].timestamp)]\n",
     "\n",
     "\n",
@@ -1003,7 +1004,7 @@
     "ax.set_prop_cycle(cycler('color', [plt.cm.nipy_spectral(i/4) for i in range(5)]))\n",
     "\n",
     "kde = stats.gaussian_kde(timestep_mags)\n",
-    "xs = np.linspace(values.min(), values.max(), 1000)\n",
+    "xs = np.linspace(timestep_mags.min(), timestep_mags.max(), 1000)\n",
     "ax.plot(xs, kde(xs), label=\"End-to-End Latency\")\n",
     "perc = 90\n",
     "ax.axvline(np.percentile(timestep_mags, perc), label=f\"{perc}th percentile\")\n",
@@ -1209,6 +1210,102 @@
     }
    }
   },
+  {
+   "cell_type": "markdown",
+   "source": [
+    "# Find Top Critical Paths\n",
+    "\n",
+    "For each message tree, find its critical path and runner-up candidates. Then, for the whole timeseries, find the top critical paths by occurence count.\n",
+    "Critical paths are abstracted by their E2E-latency and a list of passed CBs and topics.\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%% md\n"
+    }
+   }
+  },
+  {
+   "cell_type": "code",
+   "execution_count": null,
+   "outputs": [],
+   "source": [
+    "from termcolor import colored\n",
+    "from tqdm import tqdm\n",
+    "from matching.subscriptions import sanitize\n",
+    "\n",
+    "critical_paths = {}\n",
+    "\n",
+    "for tree in tqdm(relevant_trees):\n",
+    "    crit = critical_path(tree)\n",
+    "\n",
+    "    def _owner(inst):\n",
+    "        match inst:\n",
+    "            case TrCallbackInstance(callback_obj=cb_obj):\n",
+    "                cb_obj: TrCallbackObject\n",
+    "                if cb_obj and cb_obj.callback_symbol:\n",
+    "                    sym = repr(sanitize(cb_obj.callback_symbol.symbol))\n",
+    "                else:\n",
+    "                    sym = str(cb_obj.id)\n",
+    "                return sym\n",
+    "            case TrPublishInstance(publisher=pub):\n",
+    "                pub: TrPublisher\n",
+    "                topic = pub.topic_name\n",
+    "                return topic\n",
+    "            case _:\n",
+    "                raise ValueError()\n",
+    "\n",
+    "    key = tuple(map(_owner, crit[::-1]))\n",
+    "    if key not in critical_paths:\n",
+    "        critical_paths[key] = []\n",
+    "    critical_paths[key].append(crit)\n",
+    "\n",
+    "items = list(critical_paths.items())\n",
+    "items.sort(key=lambda pair: len(pair[1]), reverse=True)\n",
+    "\n",
+    "for key, paths in items:\n",
+    "    print(f\"======== {len(paths)}x: {sum(map(lambda p: p[0].timestamp - p[-1].timestamp, paths))/len(paths)*1000:.3f}ms\")\n",
+    "    paths_durations = []\n",
+    "    for path in paths:\n",
+    "        next_inst = None\n",
+    "        durations = []\n",
+    "        for inst in path:\n",
+    "            match inst:\n",
+    "                case TrCallbackInstance(timestamp=t, duration=d):\n",
+    "                    if not next_inst:\n",
+    "                        durations.append(d)\n",
+    "                    else:\n",
+    "                        durations.append(min(d, next_inst.timestamp - t))\n",
+    "                case TrPublishInstance(timestamp=t):\n",
+    "                    if not next_inst:\n",
+    "                        durations.append(0.0)\n",
+    "                    else:\n",
+    "                        durations.append(next_inst.timestamp - t)\n",
+    "                case _:\n",
+    "                    raise ValueError()\n",
+    "            next_inst = inst\n",
+    "        paths_durations.append(durations)\n",
+    "\n",
+    "    durations = list(map(lambda l: sum(l) / len(l), zip(*paths_durations)))[::-1]\n",
+    "    assert len(durations) == len(key)\n",
+    "    perc = np.percentile(durations, [70, 90, 95, 100])\n",
+    "    colors = [\"green\", \"yellow\", \"red\", \"magenta\"]\n",
+    "    for part, duration in zip(key, durations):\n",
+    "        i = 0\n",
+    "        for j, p in enumerate(perc):\n",
+    "            if duration < p:\n",
+    "                break\n",
+    "            i = j\n",
+    "        dur_str = colored(f\"{duration * 1000 :>.3f}ms\", colors[i])\n",
+    "        print(f\"  -> {dur_str} {part}\")\n"
+   ],
+   "metadata": {
+    "collapsed": false,
+    "pycharm": {
+     "name": "#%%\n"
+    }
+   }
+  },
   {
    "cell_type": "code",
    "execution_count": null,