Hierarchical latency graph, bugfixes, renamed types.py to not interfere with other python packages.
This commit is contained in:
Maximilian Schmeller
parent
5885be5974
commit
0261b8200f
10 changed files with 1022 additions and 170 deletions
|
|
@ -1,24 +1,36 @@
|
|||
import os
|
||||
import re
|
||||
from dataclasses import dataclass, field
|
||||
from typing import List, Literal, Dict, Set
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClTranslationUnit:
|
||||
dependencies: Dict[int, Set[int]]
|
||||
publications: Dict[int, Set[int]]
|
||||
nodes: Dict[int, 'ClNode']
|
||||
publishers: Dict[int, 'ClPublisher']
|
||||
subscriptions: Dict[int, 'ClSubscription']
|
||||
timers: Dict[int, 'ClTimer']
|
||||
fields: Dict[int, 'ClField']
|
||||
methods: Dict[int, 'ClMethod']
|
||||
accesses: List['ClMemberRef']
|
||||
filename: str
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.filename)
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClContext:
|
||||
translation_units: Dict[str, 'ClTranslationUnit'] = field(default_factory=dict)
|
||||
translation_units: Set['ClTranslationUnit']
|
||||
|
||||
nodes: Set['ClNode']
|
||||
publishers: Set['ClPublisher']
|
||||
subscriptions: Set['ClSubscription']
|
||||
timers: Set['ClTimer']
|
||||
|
||||
fields: Set['ClField']
|
||||
methods: Set['ClMethod']
|
||||
|
||||
accesses: List['ClMemberRef']
|
||||
|
||||
dependencies: Dict['ClMethod', Set['ClMethod']]
|
||||
publications: Dict['ClMethod', Set['ClPublisher']]
|
||||
|
||||
def __repr__(self):
|
||||
return f"ClContext({len(self.translation_units)} TUs)"
|
||||
|
||||
|
||||
@dataclass
|
||||
|
|
@ -50,15 +62,17 @@ class ClSourceRange:
|
|||
|
||||
@dataclass
|
||||
class ClNode:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
id: int
|
||||
qualified_name: str
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
field_ids: List[int] | None
|
||||
method_ids: List[int] | None
|
||||
ros_name: str | None
|
||||
ros_namespace: str | None
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.id = json_obj['id']
|
||||
self.qualified_name = json_obj['qualified_name']
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
|
|
@ -68,23 +82,43 @@ class ClNode:
|
|||
self.ros_namespace = json_obj['ros_namespace'] if 'ros_namespace' in json_obj else None
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.id)
|
||||
return hash((self.tu, self.id))
|
||||
|
||||
|
||||
@dataclass
|
||||
class ClMethod:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
id: int
|
||||
qualified_name: str
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
return_type: str | None
|
||||
parameter_types: List[str] | None
|
||||
is_lambda: bool | None
|
||||
|
||||
def __init__(self, json_obj):
|
||||
@property
|
||||
def signature(self):
|
||||
# Lambda definitions end in this suffix
|
||||
class_name = self.qualified_name.removesuffix("::(anonymous class)::operator()")
|
||||
|
||||
# If the definition is no lambda (and hence no suffix has been removed), the last part after :: is the method
|
||||
# name. Remove it to get the class name.
|
||||
if class_name == self.qualified_name:
|
||||
class_name = "::".join(class_name.split("::")[:-1])
|
||||
|
||||
if self.is_lambda:
|
||||
return f"{class_name}$lambda"
|
||||
|
||||
param_str = ','.join(self.parameter_types) if self.parameter_types is not None else ''
|
||||
return f"{self.return_type if self.return_type else ''} ({class_name})({param_str})"
|
||||
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.id = json_obj['id']
|
||||
self.qualified_name = json_obj['qualified_name']
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
self.return_type = json_obj['signature']['return_type'] if 'signature' in json_obj else None
|
||||
self.parameter_types = json_obj['signature']['parameter_types'] if 'signature' in json_obj else None
|
||||
self.is_lambda = json_obj['is_lambda'] if 'is_lambda' in json_obj else None
|
||||
|
||||
def __hash__(self):
|
||||
return hash(self.id)
|
||||
|
|
@ -92,11 +126,13 @@ class ClMethod:
|
|||
|
||||
@dataclass
|
||||
class ClField:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
id: int
|
||||
qualified_name: str
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.id = json_obj['id']
|
||||
self.qualified_name = json_obj['qualified_name']
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
|
|
@ -107,13 +143,15 @@ class ClField:
|
|||
|
||||
@dataclass
|
||||
class ClMemberRef:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
type: Literal["read", "write", "call", "arg", "pub"] | None
|
||||
member_chain: List[int]
|
||||
method_id: int | None
|
||||
node_id: int | None
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
access_type = json_obj['context']['access_type']
|
||||
if access_type == 'none':
|
||||
access_type = None
|
||||
|
|
@ -129,11 +167,13 @@ class ClMemberRef:
|
|||
|
||||
@dataclass
|
||||
class ClSubscription:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
topic: str | None
|
||||
callback_id: int | None
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.topic = json_obj['topic'] if 'topic' in json_obj else None
|
||||
self.callback_id = json_obj['callback']['id'] if 'callback' in json_obj else None
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
|
|
@ -144,14 +184,16 @@ class ClSubscription:
|
|||
|
||||
@dataclass
|
||||
class ClPublisher:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
topic: str | None
|
||||
member_id: int | None
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
|
||||
def update(self, t2: 'ClTimer'):
|
||||
return self
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.topic = json_obj['topic'] if 'topic' in json_obj else None
|
||||
self.member_id = json_obj['member']['id'] if 'member' in json_obj else None
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
|
|
@ -162,10 +204,12 @@ class ClPublisher:
|
|||
|
||||
@dataclass
|
||||
class ClTimer:
|
||||
tu: 'ClTranslationUnit' = field(repr=False)
|
||||
callback_id: int | None
|
||||
source_range: 'ClSourceRange'
|
||||
source_range: 'ClSourceRange' = field(repr=False)
|
||||
|
||||
def __init__(self, json_obj):
|
||||
def __init__(self, json_obj, tu):
|
||||
self.tu = tu
|
||||
self.callback_id = json_obj['callback']['id'] if 'callback' in json_obj else None
|
||||
self.source_range = ClSourceRange(json_obj['source_range'])
|
||||
|
||||
|
|
@ -2,14 +2,12 @@ import functools
|
|||
import json
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
from typing import Tuple, Iterable
|
||||
from typing import Iterable
|
||||
|
||||
import numpy as np
|
||||
import pandas as pd
|
||||
import termcolor
|
||||
|
||||
from clang_interop.types import ClNode, ClField, ClTimer, ClMethod, ClPublisher, ClSubscription, ClMemberRef, ClContext, \
|
||||
from clang_interop.cl_types import ClNode, ClField, ClTimer, ClMethod, ClPublisher, ClSubscription, ClMemberRef, ClContext, \
|
||||
ClTranslationUnit
|
||||
|
||||
IN_DIR = "/home/max/Projects/ma-ros2-internal-dependency-analyzer/output"
|
||||
|
|
@ -123,14 +121,14 @@ def dedup(elems):
|
|||
ret_list.append(elem)
|
||||
print(f"Fused {len(elems)} {type(elem)}s")
|
||||
|
||||
return ret_list
|
||||
return set(ret_list)
|
||||
|
||||
|
||||
def dictify(elems, key='id'):
|
||||
return {getattr(e, key): e for e in elems}
|
||||
|
||||
|
||||
def definitions_from_json(cb_dict):
|
||||
def definitions_from_json(cb_dict, tu):
|
||||
nodes = []
|
||||
pubs = []
|
||||
subs = []
|
||||
|
|
@ -141,145 +139,85 @@ def definitions_from_json(cb_dict):
|
|||
|
||||
if "nodes" in cb_dict:
|
||||
for node in cb_dict["nodes"]:
|
||||
nodes.append(ClNode(node))
|
||||
nodes.append(ClNode(node, tu))
|
||||
for field in node["fields"]:
|
||||
fields.append(ClField(field))
|
||||
fields.append(ClField(field, tu))
|
||||
for method in node["methods"]:
|
||||
methods.append(ClMethod(method))
|
||||
methods.append(ClMethod(method, tu))
|
||||
|
||||
if "publishers" in cb_dict:
|
||||
for publisher in cb_dict["publishers"]:
|
||||
pubs.append(ClPublisher(publisher))
|
||||
pubs.append(ClPublisher(publisher, tu))
|
||||
|
||||
if "subscriptions" in cb_dict:
|
||||
for subscription in cb_dict["subscriptions"]:
|
||||
subs.append(ClSubscription(subscription))
|
||||
subs.append(ClSubscription(subscription, tu))
|
||||
if "callback" in subscription:
|
||||
methods.append(ClMethod(subscription["callback"], tu))
|
||||
|
||||
if "timers" in cb_dict:
|
||||
for timer in cb_dict["timers"]:
|
||||
timers.append(ClTimer(timer))
|
||||
timers.append(ClTimer(timer, tu))
|
||||
if "callback" in timer:
|
||||
methods.append(ClMethod(timer["callback"], tu))
|
||||
|
||||
if "accesses" in cb_dict:
|
||||
for access_type in cb_dict["accesses"]:
|
||||
for access in cb_dict["accesses"][access_type]:
|
||||
accesses.append(ClMemberRef(access))
|
||||
accesses.append(ClMemberRef(access, tu))
|
||||
if "method" in access["context"]:
|
||||
methods.append(ClMethod(access["context"]["method"], tu))
|
||||
|
||||
nodes = dictify(dedup(nodes))
|
||||
pubs = dictify(dedup(pubs), key='member_id')
|
||||
subs = dictify(dedup(subs), key='callback_id')
|
||||
timers = dictify(dedup(timers), key='callback_id')
|
||||
fields = dictify(dedup(fields))
|
||||
methods = dictify(dedup(methods))
|
||||
nodes = dedup(nodes)
|
||||
pubs = dedup(pubs)
|
||||
subs = dedup(subs)
|
||||
timers = dedup(timers)
|
||||
fields = dedup(fields)
|
||||
methods = dedup(methods)
|
||||
|
||||
return nodes, pubs, subs, timers, fields, methods, accesses
|
||||
|
||||
|
||||
def highlight(substr: str, text: str):
|
||||
regex = r"(?<=\W)({substr})(?=\W)|^({substr})$"
|
||||
return re.sub(regex.format(substr=substr), termcolor.colored(r"\1\2", 'magenta', attrs=['bold']), text)
|
||||
|
||||
|
||||
def prompt_user(file: str, cb: str, idf: str, text: str) -> Tuple[str, bool, bool]:
|
||||
print('\n' * 5)
|
||||
print(f"{file.rstrip('.cpp').rstrip('.hpp')}\n->{cb}:")
|
||||
print(highlight(idf.split('::')[-1], text))
|
||||
answer = input(f"{highlight(idf, idf)}\n"
|
||||
f"write (w), read (r), both (rw), ignore future (i) exit and save (q), undo (z), skip (Enter): ")
|
||||
if answer not in ["", "r", "w", "rw", "q", "z", "i"]:
|
||||
print(f"Invalid answer '{answer}', try again.")
|
||||
answer = prompt_user(file, cb, idf, text)
|
||||
|
||||
if answer == 'i':
|
||||
ignored_idfs.add(idf)
|
||||
elif any(x in answer for x in ['r', 'w']):
|
||||
ignored_idfs.discard(idf)
|
||||
|
||||
return answer, answer == "q", answer == "z"
|
||||
|
||||
|
||||
def main(cbs):
|
||||
open_files = {}
|
||||
cb_rw_dict = {}
|
||||
|
||||
jobs = []
|
||||
|
||||
for cb_id, cb_dict in cbs.items():
|
||||
cb_rw_dict[cb_dict['qualified_name']] = {'reads': set(), 'writes': set()}
|
||||
for ref_dict in cb_dict['member_refs']:
|
||||
if ref_dict['file'] not in open_files:
|
||||
with open(ref_dict['file'], 'r') as f:
|
||||
open_files[ref_dict['file']] = f.readlines()
|
||||
|
||||
ln = ref_dict['start_line'] - 1
|
||||
text = open_files[ref_dict['file']]
|
||||
line = termcolor.colored(text[ln], None, "on_cyan")
|
||||
lines = [*text[ln - 3:ln], line, *text[ln + 1:ln + 4]]
|
||||
text = ''.join(lines)
|
||||
jobs.append((ref_dict['file'], cb_dict['qualified_name'], ref_dict['qualified_name'], text))
|
||||
|
||||
i = 0
|
||||
do_undo = False
|
||||
while i < len(jobs):
|
||||
file, cb, idf, text = jobs[i]
|
||||
|
||||
if do_undo:
|
||||
ignored_idfs.discard(idf)
|
||||
cb_rw_dict[cb]['reads'].discard(idf)
|
||||
cb_rw_dict[cb]['writes'].discard(idf)
|
||||
do_undo = False
|
||||
|
||||
if idf in ignored_idfs:
|
||||
print("Ignoring", idf)
|
||||
i += 1
|
||||
continue
|
||||
|
||||
if idf in cb_rw_dict[cb]['reads'] and idf in cb_rw_dict[cb]['writes']:
|
||||
print(f"{idf} is already written to and read from in {cb}, skipping.")
|
||||
i += 1
|
||||
continue
|
||||
|
||||
classification, answ_quit, answ_undo = prompt_user(file, cb, idf, text)
|
||||
|
||||
if answ_quit:
|
||||
del cb_rw_dict[file][cb]
|
||||
break
|
||||
elif answ_undo:
|
||||
i -= 1
|
||||
do_undo = True
|
||||
continue
|
||||
|
||||
if 'r' in classification:
|
||||
cb_rw_dict[cb]['reads'].add(idf)
|
||||
if 'w' in classification:
|
||||
cb_rw_dict[cb]['writes'].add(idf)
|
||||
if not any(x in classification for x in ['r', 'w']):
|
||||
print(f"Ignoring occurences of {idf} in cb.")
|
||||
|
||||
i += 1
|
||||
|
||||
with open("deps.json", "w") as f:
|
||||
json.dump(cb_rw_dict, f, cls=SetEncoder)
|
||||
|
||||
print("Done.")
|
||||
|
||||
|
||||
def process_clang_output(directory=IN_DIR):
|
||||
clang_context = ClContext()
|
||||
all_tus = set()
|
||||
all_nodes = set()
|
||||
all_pubs = set()
|
||||
all_subs = set()
|
||||
all_timers = set()
|
||||
all_fields = set()
|
||||
all_methods = set()
|
||||
all_accesses = []
|
||||
all_deps = {}
|
||||
all_publications = {}
|
||||
|
||||
for filename in os.listdir(IN_DIR):
|
||||
source_filename = SRC_FILE_NAME(filename)
|
||||
print(f"Processing {source_filename}")
|
||||
|
||||
with open(os.path.join(IN_DIR, filename), "r") as f:
|
||||
cb_dict = json.load(f)
|
||||
if cb_dict is None:
|
||||
print(f" [WARN ] Empty tool output detected in {filename}")
|
||||
continue
|
||||
|
||||
nodes, pubs, subs, timers, fields, methods, accesses = definitions_from_json(cb_dict)
|
||||
tu = ClTranslationUnit(source_filename)
|
||||
all_tus.add(tu)
|
||||
|
||||
nodes, pubs, subs, timers, fields, methods, accesses = definitions_from_json(cb_dict, tu)
|
||||
deps, publications = find_data_deps(accesses)
|
||||
|
||||
tu = ClTranslationUnit(deps, publications, nodes, pubs, subs, timers, fields, methods, accesses)
|
||||
clang_context.translation_units[source_filename] = tu
|
||||
all_nodes.update(nodes)
|
||||
all_pubs.update(pubs)
|
||||
all_subs.update(subs)
|
||||
all_timers.update(timers)
|
||||
all_fields.update(fields)
|
||||
all_methods.update(methods)
|
||||
all_accesses += accesses
|
||||
all_deps.update(deps)
|
||||
all_publications.update(publications)
|
||||
|
||||
clang_context = ClContext(all_tus, all_nodes, all_pubs, all_subs, all_timers, all_fields, all_methods, all_accesses,
|
||||
all_deps, all_publications)
|
||||
|
||||
return clang_context
|
||||
|
||||
|
|
|
|||
Loading…
Add table
Add a link
Reference in a new issue