niklas/dataflow-analysis
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Fixes for random crashes

Browse source 
* Removed papermill cell headers (prevented cell magics from working)
* Changed bisect to bisect_right because of Python's name resolution acting stupid
* Print exceptions when building message trees
This commit is contained in:
Maximilian Schmeller 2022-09-19 12:39:53 +02:00
parent c8a88fc785
commit 98cd6fc3db
3 changed files with 17 additions and 31 deletions
Download patch file Download diff file Expand all files Collapse all files

4
latency_graph/latency_graph.py
View file

@ -1,4 +1,4 @@
from bisect import bisect_left, bisect from bisect import bisect_left, bisect_right
from dataclasses import dataclass from dataclasses import dataclass
from itertools import combinations from itertools import combinations
from multiprocessing import Pool from multiprocessing import Pool
@ -71,7 +71,7 @@ def _get_publishing_cbs(cbs: Set[TrCallbackObject], pub: TrPublisher):
cb_intervals = map(inst_runtime_interval, cb.callback_instances) cb_intervals = map(inst_runtime_interval, cb.callback_instances)
for t_start, t_end in cb_intervals: for t_start, t_end in cb_intervals:
i_overlap_begin = bisect_left(pub_insts, t_start, key=lambda x: x.timestamp) i_overlap_begin = bisect_left(pub_insts, t_start, key=lambda x: x.timestamp)
i_overlap_end = bisect(pub_insts, t_end, key=lambda x: x.timestamp) i_overlap_end = bisect_right(pub_insts, t_end, key=lambda x: x.timestamp)
for i in range(i_overlap_begin, i_overlap_end): for i in range(i_overlap_begin, i_overlap_end):
pub_cb_overlaps[i].add(cb) pub_cb_overlaps[i].add(cb)

41
trace-analysis.ipynb
View file

@ -5,14 +5,13 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Importing libraries\"\n",
"import os\n", "import os\n",
"import sys\n", "import sys\n",
"import re\n", "import re\n",
"import math\n", "import math\n",
"import graphviz as gv\n", "import graphviz as gv\n",
"from tqdm import tqdm\n", "from tqdm import tqdm\n",
"from bisect import bisect\n", "from bisect import bisect_right\n",
"from termcolor import colored\n", "from termcolor import colored\n",
"\n", "\n",
"import matplotlib.patches as mpatch\n", "import matplotlib.patches as mpatch\n",
@ -41,7 +40,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Parsing user settings\"\n",
"##################################################\n", "##################################################\n",
"# User Settings\n", "# User Settings\n",
"##################################################\n", "##################################################\n",
@ -167,7 +165,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Importing system-dependent libraries\"\n",
"sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_read/\"))\n", "sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_read/\"))\n",
"sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_analysis/\"))\n", "sys.path.append(os.path.join(TRACING_WS_BUILD_PATH, \"tracetools_analysis/\"))\n",
"from tracetools_read.trace import *\n", "from tracetools_read.trace import *\n",
@ -202,7 +199,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Loading traces\"\n",
"\n", "\n",
"def _load_traces():\n", "def _load_traces():\n",
" file = load_file(TR_PATH)\n", " file = load_file(TR_PATH)\n",
@ -234,7 +230,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Printing message counts\"\n",
"\n", "\n",
"for topic in sorted(topics, key=lambda t: t.name):\n", "for topic in sorted(topics, key=lambda t: t.name):\n",
" topic: TrTopic\n", " topic: TrTopic\n",
@ -264,7 +259,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Building latency graph\"\n",
"\n", "\n",
"from latency_graph import latency_graph as lg\n", "from latency_graph import latency_graph as lg\n",
"\n", "\n",
@ -287,7 +281,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Plotting full data flow graph\"\n",
"\n", "\n",
"%%skip_if_false DFG_ENABLED\n", "%%skip_if_false DFG_ENABLED\n",
"%%skip_if_false DFG_PLOT\n", "%%skip_if_false DFG_PLOT\n",
@ -387,7 +380,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Plotting high-level data flow graph\"\n",
"\n", "\n",
"%%skip_if_false DFG_ENABLED\n", "%%skip_if_false DFG_ENABLED\n",
"%%skip_if_false DFG_PLOT\n", "%%skip_if_false DFG_PLOT\n",
@ -522,7 +514,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Defining tree helper functions\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"\n", "\n",
@ -544,7 +535,7 @@
" pubs = []\n", " pubs = []\n",
"\n", "\n",
" def _pub_latest_msg_before(pub: TrPublisher, inst):\n", " def _pub_latest_msg_before(pub: TrPublisher, inst):\n",
" i_latest_msg = bisect(pub.instances, inst.timestamp, key=lambda x: x.timestamp) - 1\n", " i_latest_msg = bisect_right(pub.instances, inst.timestamp, key=lambda x: x.timestamp) - 1\n",
" if i_latest_msg < 0 or i_latest_msg >= len(pub.instances):\n", " if i_latest_msg < 0 or i_latest_msg >= len(pub.instances):\n",
" return None\n", " return None\n",
" latest_msg = pub.instances[i_latest_msg]\n", " latest_msg = pub.instances[i_latest_msg]\n",
@ -571,7 +562,7 @@
" dep_cbs = get_cb_dep_cbs(inst.callback_obj)\n", " dep_cbs = get_cb_dep_cbs(inst.callback_obj)\n",
"\n", "\n",
" def _cb_to_chronological_inst(cb: TrCallbackObject, inst):\n", " def _cb_to_chronological_inst(cb: TrCallbackObject, inst):\n",
" i_inst_latest = bisect(cb.callback_instances, inst.timestamp, key=lambda x: x.timestamp)\n", " i_inst_latest = bisect_right(cb.callback_instances, inst.timestamp, key=lambda x: x.timestamp)\n",
"\n", "\n",
" for inst_before in cb.callback_instances[i_inst_latest::-1]:\n", " for inst_before in cb.callback_instances[i_inst_latest::-1]:\n",
" if lg.inst_runtime_interval(inst_before)[-1] < inst.timestamp:\n", " if lg.inst_runtime_interval(inst_before)[-1] < inst.timestamp:\n",
@ -622,7 +613,7 @@
" cb_inst_candidates = []\n", " cb_inst_candidates = []\n",
" for cb in pub_cbs:\n", " for cb in pub_cbs:\n",
" # print(f\" > CB ({len(cb.callback_instances)} instances): {cb.callback_symbol.symbol if cb.callback_symbol else cb.id}\")\n", " # print(f\" > CB ({len(cb.callback_instances)} instances): {cb.callback_symbol.symbol if cb.callback_symbol else cb.id}\")\n",
" i_inst_after = bisect(cb.callback_instances, msg.timestamp, key=lambda x: x.timestamp)\n", " i_inst_after = bisect_right(cb.callback_instances, msg.timestamp, key=lambda x: x.timestamp)\n",
"\n", "\n",
" for inst in cb.callback_instances[:i_inst_after]:\n", " for inst in cb.callback_instances[:i_inst_after]:\n",
" inst_start, inst_end = lg.inst_runtime_interval(inst)\n", " inst_start, inst_end = lg.inst_runtime_interval(inst)\n",
@ -691,7 +682,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Building message trees\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"\n", "\n",
@ -707,16 +697,22 @@
" pubs = end_topic.publishers\n", " pubs = end_topic.publishers\n",
" print(len(pubs))\n", " print(len(pubs))\n",
" for pub in pubs:\n", " for pub in pubs:\n",
" msgs = pub.instances\n", " msgs = list(pub.instances)\n",
" print(len(msgs))\n", " print(\"\\n\".join(map(str, msgs)))\n",
" for msg in tqdm(msgs, desc=f\"Building message chains for topic {end_topic.name}\"):\n", " for msg in msgs:\n",
" msg: TrPublishInstance\n", " msg: TrPublishInstance\n",
" tree = get_dep_tree(msg)\n", " tree = get_dep_tree(msg)\n",
" all_trees.append(tree)\n", " all_trees.append(tree)\n",
" print()\n",
" return all_trees\n", " return all_trees\n",
"\n", "\n",
"\n", "\n",
"trees = cached(\"trees\", build_dep_trees, [TR_PATH], not CACHING_ENABLED)" "try:\n",
" trees = cached(\"trees\", build_dep_trees, [TR_PATH], not CACHING_ENABLED)\n",
"except Exception as e:\n",
" import traceback\n",
" print(e)\n",
" traceback.print_exc()"
], ],
"metadata": { "metadata": {
"collapsed": false, "collapsed": false,
@ -730,7 +726,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Parsing message stats\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false BW_ENABLED\n", "%%skip_if_false BW_ENABLED\n",
@ -793,7 +788,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Calculating E2E latencies\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"\n", "\n",
@ -867,7 +861,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Animating latency distribution over time\"\n",
"\n", "\n",
"#from matplotlib.animation import FuncAnimation\n", "#from matplotlib.animation import FuncAnimation\n",
"#from IPython import display\n", "#from IPython import display\n",
@ -920,7 +913,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Plotting E2E latencies\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n", "%%skip_if_false E2E_PLOT\n",
@ -974,7 +966,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Decimating irrelevant trees\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"\n", "\n",
@ -1042,7 +1033,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Plotting E2E latency breakdown\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n", "%%skip_if_false E2E_PLOT\n",
@ -1138,7 +1128,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Plotting E2E distribution\"\n",
"\n", "\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
@ -1178,7 +1167,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Visualizing E2E tree\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n", "%%skip_if_false E2E_PLOT\n",
@ -1382,7 +1370,6 @@
"execution_count": null, "execution_count": null,
"outputs": [], "outputs": [],
"source": [ "source": [
"#papermill_description=\"Finding top critical paths\"\n",
"\n", "\n",
"%%skip_if_false E2E_ENABLED\n", "%%skip_if_false E2E_ENABLED\n",
"\n", "\n",

1
tracing_interop/tr_types.py
View file

@ -1,7 +1,6 @@
from collections import namedtuple, UserList from collections import namedtuple, UserList
from dataclasses import dataclass, field from dataclasses import dataclass, field
from typing import List, Dict, Optional, Set, TypeVar, Generic, Iterable from typing import List, Dict, Optional, Set, TypeVar, Generic, Iterable
import bisect
from tracetools_analysis.processor.ros2 import Ros2Handler from tracetools_analysis.processor.ros2 import Ros2Handler