From 98cd6fc3db3546d8c5198c993bb47ad703058d4b Mon Sep 17 00:00:00 2001 From: Maximilian Schmeller Date: Mon, 19 Sep 2022 12:39:53 +0200 Subject: [PATCH] Fixes for random crashes * Removed papermill cell headers (prevented cell magics from working) * Changed bisect to bisect_right because of Python's name resolution acting stupid * Print exceptions when building message trees --- latency_graph/latency_graph.py | 4 ++-- trace-analysis.ipynb | 43 ++++++++++++---------------------- tracing_interop/tr_types.py | 1 - 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/latency_graph/latency_graph.py b/latency_graph/latency_graph.py index 0489226..642a762 100644 --- a/latency_graph/latency_graph.py +++ b/latency_graph/latency_graph.py @@ -1,4 +1,4 @@ -from bisect import bisect_left, bisect +from bisect import bisect_left, bisect_right from dataclasses import dataclass from itertools import combinations from multiprocessing import Pool @@ -71,7 +71,7 @@ def _get_publishing_cbs(cbs: Set[TrCallbackObject], pub: TrPublisher): cb_intervals = map(inst_runtime_interval, cb.callback_instances) for t_start, t_end in cb_intervals: i_overlap_begin = bisect_left(pub_insts, t_start, key=lambda x: x.timestamp) - i_overlap_end = bisect(pub_insts, t_end, key=lambda x: x.timestamp) + i_overlap_end = bisect_right(pub_insts, t_end, key=lambda x: x.timestamp) for i in range(i_overlap_begin, i_overlap_end): pub_cb_overlaps[i].add(cb) diff --git a/trace-analysis.ipynb b/trace-analysis.ipynb index e160352..97f3267 100644 --- a/trace-analysis.ipynb +++ b/trace-analysis.ipynb @@ -5,14 +5,13 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Importing libraries\"\n", "import os\n", "import sys\n", "import re\n", "import math\n", "import graphviz as gv\n", "from tqdm import tqdm\n", - "from bisect import bisect\n", + "from bisect import bisect_right\n", "from termcolor import colored\n", "\n", "import matplotlib.patches as mpatch\n", @@ -41,7 +40,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Parsing user settings\"\n", "##################################################\n", "# User Settings\n", "##################################################\n", @@ -167,7 +165,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Importing system-dependent libraries\"\n", "sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_read/\"))\n", "sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_analysis/\"))\n", "from tracetools_read.trace import *\n", @@ -202,7 +199,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Loading traces\"\n", "\n", "def _load_traces():\n", " file = load_file(TR_PATH)\n", @@ -234,7 +230,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Printing message counts\"\n", "\n", "for topic in sorted(topics, key=lambda t: t.name):\n", " topic: TrTopic\n", @@ -264,7 +259,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Building latency graph\"\n", "\n", "from latency_graph import latency_graph as lg\n", "\n", @@ -287,7 +281,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Plotting full data flow graph\"\n", "\n", "%%skip_if_false DFG_ENABLED\n", "%%skip_if_false DFG_PLOT\n", @@ -387,7 +380,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Plotting high-level data flow graph\"\n", "\n", "%%skip_if_false DFG_ENABLED\n", "%%skip_if_false DFG_PLOT\n", @@ -522,7 +514,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Defining tree helper functions\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "\n", @@ -544,7 +535,7 @@ " pubs = []\n", "\n", " def _pub_latest_msg_before(pub: TrPublisher, inst):\n", - " i_latest_msg = bisect(pub.instances, inst.timestamp, key=lambda x: x.timestamp) - 1\n", + " i_latest_msg = bisect_right(pub.instances, inst.timestamp, key=lambda x: x.timestamp) - 1\n", " if i_latest_msg < 0 or i_latest_msg >= len(pub.instances):\n", " return None\n", " latest_msg = pub.instances[i_latest_msg]\n", @@ -571,7 +562,7 @@ " dep_cbs = get_cb_dep_cbs(inst.callback_obj)\n", "\n", " def _cb_to_chronological_inst(cb: TrCallbackObject, inst):\n", - " i_inst_latest = bisect(cb.callback_instances, inst.timestamp, key=lambda x: x.timestamp)\n", + " i_inst_latest = bisect_right(cb.callback_instances, inst.timestamp, key=lambda x: x.timestamp)\n", "\n", " for inst_before in cb.callback_instances[i_inst_latest::-1]:\n", " if lg.inst_runtime_interval(inst_before)[-1] < inst.timestamp:\n", @@ -622,7 +613,7 @@ " cb_inst_candidates = []\n", " for cb in pub_cbs:\n", " # print(f\" > CB ({len(cb.callback_instances)} instances): {cb.callback_symbol.symbol if cb.callback_symbol else cb.id}\")\n", - " i_inst_after = bisect(cb.callback_instances, msg.timestamp, key=lambda x: x.timestamp)\n", + " i_inst_after = bisect_right(cb.callback_instances, msg.timestamp, key=lambda x: x.timestamp)\n", "\n", " for inst in cb.callback_instances[:i_inst_after]:\n", " inst_start, inst_end = lg.inst_runtime_interval(inst)\n", @@ -691,7 +682,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Building message trees\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "\n", @@ -707,16 +697,22 @@ " pubs = end_topic.publishers\n", " print(len(pubs))\n", " for pub in pubs:\n", - " msgs = pub.instances\n", - " print(len(msgs))\n", - " for msg in tqdm(msgs, desc=f\"Building message chains for topic {end_topic.name}\"):\n", + " msgs = list(pub.instances)\n", + " print(\"\\n\".join(map(str, msgs)))\n", + " for msg in msgs:\n", " msg: TrPublishInstance\n", " tree = get_dep_tree(msg)\n", " all_trees.append(tree)\n", + " print()\n", " return all_trees\n", "\n", "\n", - "trees = cached(\"trees\", build_dep_trees, [TR_PATH], not CACHING_ENABLED)" + "try:\n", + " trees = cached(\"trees\", build_dep_trees, [TR_PATH], not CACHING_ENABLED)\n", + "except Exception as e:\n", + " import traceback\n", + " print(e)\n", + " traceback.print_exc()" ], "metadata": { "collapsed": false, @@ -730,7 +726,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Parsing message stats\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "%%skip_if_false BW_ENABLED\n", @@ -793,7 +788,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Calculating E2E latencies\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "\n", @@ -867,7 +861,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Animating latency distribution over time\"\n", "\n", "#from matplotlib.animation import FuncAnimation\n", "#from IPython import display\n", @@ -920,7 +913,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Plotting E2E latencies\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_PLOT\n", @@ -974,7 +966,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Decimating irrelevant trees\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "\n", @@ -1042,7 +1033,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Plotting E2E latency breakdown\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_PLOT\n", @@ -1138,7 +1128,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Plotting E2E distribution\"\n", "\n", "\n", "%%skip_if_false E2E_ENABLED\n", @@ -1178,7 +1167,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Visualizing E2E tree\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_PLOT\n", @@ -1382,7 +1370,6 @@ "execution_count": null, "outputs": [], "source": [ - "#papermill_description=\"Finding top critical paths\"\n", "\n", "%%skip_if_false E2E_ENABLED\n", "\n", @@ -1494,4 +1481,4 @@ }, "nbformat": 4, "nbformat_minor": 2 -} \ No newline at end of file +} diff --git a/tracing_interop/tr_types.py b/tracing_interop/tr_types.py index 0f541d8..16c2fad 100644 --- a/tracing_interop/tr_types.py +++ b/tracing_interop/tr_types.py @@ -1,7 +1,6 @@ from collections import namedtuple, UserList from dataclasses import dataclass, field from typing import List, Dict, Optional, Set, TypeVar, Generic, Iterable -import bisect from tracetools_analysis.processor.ros2 import Ros2Handler