niklas/dataflow-analysis
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Added CSV output back in, fixed bugs in refactoring

Browse source
This commit is contained in:
Maximilian Schmeller 2022-10-14 00:15:33 +09:00
parent b1dc01b101
commit 65c21fb6ce
4 changed files with 120 additions and 33 deletions
Download patch file Download diff file Expand all files Collapse all files

109
trace-analysis.ipynb
View file

@ -25,6 +25,7 @@
"\n",
"from misc.utils import cached, parse_as\n",
"\n",
"# TODO: This is useless for some reason (goal is to make graphs in notebook large and hi-res)\n",
"plt.rcParams['figure.dpi'] = 300\n",
"\n",
"%matplotlib inline"
@ -134,10 +135,10 @@
"\n",
"# E2E paths are uniquely identified by a string like \"/topic/1 -> void(Node1)(args1) -> /topic/2 -> void(Node2)(args2) -> void(Node2)(args3) -> ...\".\n",
"# Certain patterns only occur in initial setup or in scenario switching and can be excluded via RegEx patterns here.\n",
"E2E_EXCLUDED_PATH_PATTERNS = [r\"NDTScanMatcher\"]\n",
"E2E_EXCLUDED_PATH_PATTERNS = [r\"NDTScanMatcher\", r\"^/parameter_events\"]\n",
"\n",
"\n",
"DDD = False\n",
"DEBUG = False\n",
"\n",
"# This code overrides the above constants with environment variables, do not edit.\n",
"for env_key, env_value in os.environ.items():\n",
@ -443,7 +444,7 @@
"source": [
"%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n",
"%%skip_if_false DDD\n",
"%%skip_if_false DEBUG\n",
"\n",
"\n",
"fig, ax = plt.subplots(num=\"e2e_plot\")\n",
@ -486,33 +487,53 @@
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false DEBUG\n",
"\n",
"import pickle\n",
"with open(\"trees.pkl\", \"rb\") as f:\n",
" trees = pickle.load(f)"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n",
"\n",
"%load_ext line_profiler\n",
"\n",
"\n",
"from message_tree.message_tree_algorithms import e2e_paths_sorted_desc\n",
"from message_tree.message_tree_plots import e2e_breakdown_type_hist\n",
"\n",
"def _map_tree_to_e2es(tree):\n",
" return e2e_paths_sorted_desc(tree, E2E_INPUT_TOPIC_PATTERNS)\n",
"trees_paths = [e2e_paths_sorted_desc(tree, E2E_INPUT_TOPIC_PATTERNS) for tree in tqdm(trees, mininterval=10.0,\n",
" desc=\"Extracting E2E paths\")]\n",
"all_paths = [p for paths in trees_paths for p in paths]\n",
"#all_e2e_items = [i for p in all_paths for i in p]"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n",
"\n",
"def ppp():\n",
" trees_paths = [e2e_paths_sorted_desc(tree, E2E_INPUT_TOPIC_PATTERNS) for tree in tqdm(trees, desc=\"Extracting E2E paths\")]\n",
" all_paths = [p for paths in trees_paths for p in paths]\n",
" all_e2e_items = [i for p in all_paths for i in p]\n",
"from message_tree.message_tree_algorithms import e2e_latency_breakdown\n",
"\n",
" return all_e2e_items\n",
"\n",
"%lprun -f e2e_paths_sorted_desc ppp()\n",
"\n",
"e2e_breakdown_type_hist(ppp())\n",
"conv_items = [i for p in tqdm(all_paths, mininterval=5.0, desc=\"Calculating E2E latency breakdowns\")\n",
" for i in e2e_latency_breakdown(p)]\n",
"e2e_breakdown_type_hist(conv_items)\n",
"\n",
"None\n"
],
@ -525,10 +546,60 @@
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false E2E_ENABLED\n",
"\n",
"from message_tree.message_tree_algorithms import aggregate_e2e_paths\n",
"\n",
"cohorts = aggregate_e2e_paths(all_paths)\n",
"cohort_pairs = [(k, v) for k, v in cohorts.items()]\n",
"cohort_pairs.sort(key=lambda kv: len(kv[1]), reverse=True)\n",
"\n",
"path_records = [{\"path\": path_key,\n",
" \"timestamp\": path[-1].timestamp,\n",
" \"e2e_latency\": path[-1].timestamp - path[0].timestamp} \\\n",
" for path_key, paths in cohort_pairs for path in paths if path]\n",
"\n",
"out_df = pd.DataFrame.from_records(path_records)\n",
"out_df.to_csv(os.path.join(OUT_PATH, \"e2e.csv\"), sep=\"\\t\", index=False)\n",
"\n",
"mode_cohort_key, mode_cohort = cohort_pairs[0]"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false E2E_PLOT\n",
"\n",
"from message_tree.message_tree_plots import e2e_breakdown_inst_stack\n",
"\n",
"\n",
"mode_cohort_breakdown = [e2e_latency_breakdown(p) for p in mode_cohort[:200]]\n",
"print(len(mode_cohort))\n",
"print(mode_cohort_key.replace(\" -> \", \"\\n -> \"))\n",
"e2e_breakdown_inst_stack(*mode_cohort_breakdown)"
],
"metadata": {
"collapsed": false
}
},
{
"cell_type": "code",
"execution_count": null,
"outputs": [],
"source": [
"%%skip_if_false E2E_ENABLED\n",
"%%skip_if_false DEBUG\n",
"\n",
"import pickle\n",
"\n",
"with open(\"trees.pkl\", \"wb\") as f:\n",
" pickle.dump(trees, f)\n"
" pickle.dump(trees, f)"
],
"metadata": {
"collapsed": false