niklas/tracetools_analysis
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

Merge branch 'refactor-data-model-utils' into 'master'

Browse source 
Split utils file into multiple files inside a submodule

See merge request micro-ROS/ros_tracing/tracetools_analysis!29
This commit is contained in:
Christophe Bedard 2019-11-17 21:45:08 +00:00
commit c12488ecf9
8 changed files with 261 additions and 199 deletions
Download patch file Download diff file Expand all files Collapse all files

18
README.md
View file

@ -40,24 +40,22 @@ Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, a
For example:
```python
from tracetools_analysis import loading
from tracetools_analysis import processor
from tracetools_analysis import utils
from tracetools_analysis.loading import load_file
from tracetools_analysis.processor import Processor
from tracetools_analysis.processor.cpu_time import CpuTimeHandler
from tracetools_analysis.processor.ros2 import Ros2Handler
# Load converted trace file
events = load_file('/path/to/converted/file')
events = loading.load_file('/path/to/converted/file')
# Process
ros2_handler = Ros2Handler()
cpu_handler = CpuTimeHandler()
ros2_handler = processor.Ros2Handler()
cpu_handler = processor.CpuTimeHandler()
Processor(ros2_handler, cpu_handler).process(events)
processor.Processor(ros2_handler, cpu_handler).process(events)
# Use data model utils to extract information
ros2_util = utils.RosDataModelUtil(ros2_handler.data)
cpu_util = CpuTimeDataModelUtil(cpu_handler.data)
ros2_util = utils.ros2.Ros2DataModelUtil(ros2_handler.data)
cpu_util = utils.cpu_time.CpuTimeDataModelUtil(cpu_handler.data)
callback_durations = ros2_util.get_callback_durations()
time_per_thread = cpu_util.get_time_per_thread()

6
tracetools_analysis/analysis/callback_duration.ipynb
View file

@ -55,9 +55,9 @@
"import numpy as np\n",
"import pandas as pd\n",
"\n",
"from tracetools_analysis import utils\n",
"from tracetools_analysis.loading import load_file\n",
"from tracetools_analysis.processor.ros2 import Ros2Handler"
"from tracetools_analysis.processor.ros2 import Ros2Handler\n",
"from tracetools_analysis.utils.ros2 import Ros2DataModelUtil"
]
},
{
@ -78,7 +78,7 @@
"metadata": {},
"outputs": [],
"source": [
"data_util = utils.RosDataModelUtil(handler.data)\n",
"data_util = Ros2DataModelUtil(handler.data)\n",
"\n",
"callback_symbols = data_util.get_callback_symbols()\n",
"\n",

8
tracetools_analysis/tracetools_analysis/data_model/ros.py → tracetools_analysis/tracetools_analysis/data_model/ros2.py
View file

@ -12,18 +12,18 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for ROS data model."""
"""Module for ROS 2 data model."""
import pandas as pd
from . import DataModel
class RosDataModel(DataModel):
class Ros2DataModel(DataModel):
"""
Container to model pre-processed ROS data for analysis.
Container to model pre-processed ROS 2 data for analysis.
This aims to represent the data in a ROS-aware way.
This aims to represent the data in a ROS 2-aware way.
"""
def __init__(self) -> None:

6
tracetools_analysis/tracetools_analysis/processor/ros2.py
View file

@ -20,7 +20,7 @@ from tracetools_read import get_field
from . import EventHandler
from . import EventMetadata
from ..data_model.ros import RosDataModel
from ..data_model.ros2 import Ros2DataModel
class Ros2Handler(EventHandler):
@ -70,13 +70,13 @@ class Ros2Handler(EventHandler):
**kwargs,
)
self._data_model = RosDataModel()
self._data_model = Ros2DataModel()
# Temporary buffers
self._callback_instances = {}
@property
def data(self) -> RosDataModel:
def data(self) -> Ros2DataModel:
return self._data_model
def _handle_rcl_init(

97
tracetools_analysis/tracetools_analysis/utils/__init__.py Normal file
View file

@ -0,0 +1,97 @@
# Copyright 2019 Robert Bosch GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for data model utility classes."""
from datetime import datetime as dt
from typing import List
from typing import Union
from pandas import DataFrame
from ..data_model import DataModel
class DataModelUtil():
"""
Base data model util class, which provides functions to get more info about a data model.
This class provides basic util functions.
"""
def __init__(
self,
data_model: DataModel,
) -> None:
"""
Constructor.
:param data_model: the data model
"""
self.__data = data_model
@property
def data(self) -> DataModel:
return self.__data
@staticmethod
def convert_time_columns(
original: DataFrame,
columns_ns_to_ms: Union[List[str], str] = [],
columns_ns_to_datetime: Union[List[str], str] = [],
inplace: bool = True,
) -> DataFrame:
"""
Convert time columns from nanoseconds to either milliseconds or `datetime` objects.
:param original: the original `DataFrame`
:param columns_ns_to_ms: the column(s) for which to convert ns to ms
:param columns_ns_to_datetime: the column(s) for which to convert ns to `datetime`
:param inplace: whether to convert in place or to return a copy
:return: the resulting `DataFrame`
"""
if not isinstance(columns_ns_to_ms, list):
columns_ns_to_ms = list(columns_ns_to_ms)
if not isinstance(columns_ns_to_datetime, list):
columns_ns_to_datetime = list(columns_ns_to_datetime)
df = original if inplace else original.copy()
# Convert from ns to ms
if len(columns_ns_to_ms) > 0:
df[columns_ns_to_ms] = df[columns_ns_to_ms].applymap(
lambda t: t / 1000000.0
)
# Convert from ns to ms + ms to datetime, as UTC
if len(columns_ns_to_datetime) > 0:
df[columns_ns_to_datetime] = df[columns_ns_to_datetime].applymap(
lambda t: dt.utcfromtimestamp(t / 1000000000.0)
)
return df
@staticmethod
def compute_column_difference(
df: DataFrame,
left_column: str,
right_column: str,
diff_column: str,
) -> None:
"""
Create new column with difference between two columns.
:param df: the dataframe (inplace)
:param left_column: the name of the left column
:param right_column: the name of the right column
:param diff_column: the name of the new column with differences
"""
df[diff_column] = df.apply(lambda row: row[left_column] - row[right_column], axis=1)

39
tracetools_analysis/tracetools_analysis/utils/cpu_time.py Normal file
View file

@ -0,0 +1,39 @@
# Copyright 2019 Robert Bosch GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for CPU time data model utils."""
from pandas import DataFrame
from . import DataModelUtil
from ..data_model.cpu_time import CpuTimeDataModel
class CpuTimeDataModelUtil(DataModelUtil):
"""CPU time data model utility class."""
def __init__(
self,
data_model: CpuTimeDataModel,
) -> None:
"""
Constructor.
:param data_model: the data model object to use
"""
super().__init__(data_model)
def get_time_per_thread(self) -> DataFrame:
"""Get a DataFrame of total duration for each thread."""
return self.data.times.loc[:, ['tid', 'duration']].groupby(by='tid').sum()

100
tracetools_analysis/tracetools_analysis/utils/profile.py Normal file
View file

@ -0,0 +1,100 @@
# Copyright 2019 Robert Bosch GmbH
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for profiling data model utils."""
from collections import defaultdict
from typing import Dict
from typing import List
from typing import Set
from typing import Union
from pandas import DataFrame
from . import DataModelUtil
from ..data_model.profile import ProfileDataModel
class ProfileDataModelUtil(DataModelUtil):
"""Profiling data model utility class."""
def __init__(
self,
data_model: ProfileDataModel,
) -> None:
"""
Constructor.
:param data_model: the data model object to use
"""
super().__init__(data_model)
def with_tid(
self,
tid: int,
) -> DataFrame:
return self.data.times.loc[self.data.times['tid'] == tid]
def get_tids(self) -> Set[int]:
"""Get the TIDs in the data model."""
return set(self.data.times['tid'])
def get_call_tree(
self,
tid: int,
) -> Dict[str, List[str]]:
depth_names = self.with_tid(tid)[
['depth', 'function_name', 'parent_name']
].drop_duplicates()
# print(depth_names.to_string())
tree = defaultdict(set)
for _, row in depth_names.iterrows():
depth = row['depth']
name = row['function_name']
parent = row['parent_name']
if depth == 0:
tree[name]
else:
tree[parent].add(name)
return dict(tree)
def get_function_duration_data(
self,
tid: int,
) -> List[Dict[str, Union[int, str, DataFrame]]]:
"""Get duration data for each function."""
tid_df = self.with_tid(tid)
depth_names = tid_df[['depth', 'function_name', 'parent_name']].drop_duplicates()
functions_data = []
for _, row in depth_names.iterrows():
depth = row['depth']
name = row['function_name']
parent = row['parent_name']
data = tid_df.loc[
(tid_df['depth'] == depth) &
(tid_df['function_name'] == name)
][['start_timestamp', 'duration', 'actual_duration']]
self.compute_column_difference(
data,
'duration',
'actual_duration',
'duration_difference',
)
functions_data.append({
'depth': depth,
'function_name': name,
'parent_name': parent,
'data': data,
})
return functions_data

186
tracetools_analysis/tracetools_analysis/utils.py → tracetools_analysis/tracetools_analysis/utils/ros2.py
View file

@ -1,4 +1,5 @@
# Copyright 2019 Robert Bosch GmbH
# Copyright 2019 Apex.AI, Inc.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
@ -12,198 +13,25 @@
# See the License for the specific language governing permissions and
# limitations under the License.
"""Module for data model utility classes."""
"""Module for ROS data model utils."""
from collections import defaultdict
from datetime import datetime as dt
from typing import Any
from typing import Dict
from typing import List
from typing import Mapping
from typing import Set
from typing import Union
from pandas import DataFrame
from .data_model import DataModel
from .data_model.cpu_time import CpuTimeDataModel
from .data_model.profile import ProfileDataModel
from .data_model.ros import RosDataModel
from . import DataModelUtil
from ..data_model.ros2 import Ros2DataModel
class DataModelUtil():
"""
Base data model util class, which provides functions to get more info about a data model.
This class provides basic util functions.
"""
class Ros2DataModelUtil(DataModelUtil):
"""ROS 2 data model utility class."""
def __init__(
self,
data_model: DataModel,
) -> None:
"""
Constructor.
:param data_model: the data model
"""
self.__data = data_model
@property
def data(self) -> DataModel:
return self.__data
@staticmethod
def convert_time_columns(
original: DataFrame,
columns_ns_to_ms: Union[List[str], str] = [],
columns_ns_to_datetime: Union[List[str], str] = [],
inplace: bool = True,
) -> DataFrame:
"""
Convert time columns from nanoseconds to either milliseconds or `datetime` objects.
:param original: the original `DataFrame`
:param columns_ns_to_ms: the column(s) for which to convert ns to ms
:param columns_ns_to_datetime: the column(s) for which to convert ns to `datetime`
:param inplace: whether to convert in place or to return a copy
:return: the resulting `DataFrame`
"""
if not isinstance(columns_ns_to_ms, list):
columns_ns_to_ms = list(columns_ns_to_ms)
if not isinstance(columns_ns_to_datetime, list):
columns_ns_to_datetime = list(columns_ns_to_datetime)
df = original if inplace else original.copy()
# Convert from ns to ms
if len(columns_ns_to_ms) > 0:
df[columns_ns_to_ms] = df[columns_ns_to_ms].applymap(
lambda t: t / 1000000.0
)
# Convert from ns to ms + ms to datetime, as UTC
if len(columns_ns_to_datetime) > 0:
df[columns_ns_to_datetime] = df[columns_ns_to_datetime].applymap(
lambda t: dt.utcfromtimestamp(t / 1000000000.0)
)
return df
@staticmethod
def compute_column_difference(
df: DataFrame,
left_column: str,
right_column: str,
diff_column: str,
) -> None:
"""
Create new column with difference between two columns.
:param df: the dataframe (inplace)
:param left_column: the name of the left column
:param right_column: the name of the right column
:param diff_column: the name of the new column with differences
"""
df[diff_column] = df.apply(lambda row: row[left_column] - row[right_column], axis=1)
class ProfileDataModelUtil(DataModelUtil):
"""Profiling data model utility class."""
def __init__(
self,
data_model: ProfileDataModel,
) -> None:
"""
Constructor.
:param data_model: the data model object to use
"""
super().__init__(data_model)
def with_tid(
self,
tid: int,
) -> DataFrame:
return self.data.times.loc[self.data.times['tid'] == tid]
def get_tids(self) -> Set[int]:
"""Get the TIDs in the data model."""
return set(self.data.times['tid'])
def get_call_tree(
self,
tid: int,
) -> Dict[str, List[str]]:
depth_names = self.with_tid(tid)[
['depth', 'function_name', 'parent_name']
].drop_duplicates()
# print(depth_names.to_string())
tree = defaultdict(set)
for _, row in depth_names.iterrows():
depth = row['depth']
name = row['function_name']
parent = row['parent_name']
if depth == 0:
tree[name]
else:
tree[parent].add(name)
return dict(tree)
def get_function_duration_data(
self,
tid: int,
) -> List[Dict[str, Union[int, str, DataFrame]]]:
"""Get duration data for each function."""
tid_df = self.with_tid(tid)
depth_names = tid_df[['depth', 'function_name', 'parent_name']].drop_duplicates()
functions_data = []
for _, row in depth_names.iterrows():
depth = row['depth']
name = row['function_name']
parent = row['parent_name']
data = tid_df.loc[
(tid_df['depth'] == depth) &
(tid_df['function_name'] == name)
][['start_timestamp', 'duration', 'actual_duration']]
self.compute_column_difference(
data,
'duration',
'actual_duration',
'duration_difference',
)
functions_data.append({
'depth': depth,
'function_name': name,
'parent_name': parent,
'data': data,
})
return functions_data
class CpuTimeDataModelUtil(DataModelUtil):
"""CPU time data model utility class."""
def __init__(
self,
data_model: CpuTimeDataModel,
) -> None:
"""
Constructor.
:param data_model: the data model object to use
"""
super().__init__(data_model)
def get_time_per_thread(self) -> DataFrame:
"""Get a DataFrame of total duration for each thread."""
return self.data.times.loc[:, ['tid', 'duration']].groupby(by='tid').sum()
class RosDataModelUtil(DataModelUtil):
"""ROS data model utility class."""
def __init__(
self,
data_model: RosDataModel,
data_model: Ros2DataModel,
) -> None:
"""
Constructor.