From 81c32a037aaa3eb1b4fbd39f7ebde635545a43b7 Mon Sep 17 00:00:00 2001 From: Christophe Bedard Date: Sun, 17 Nov 2019 13:24:36 -0800 Subject: [PATCH] Split utils file into multiple files inside a submodule --- README.md | 18 +- .../analysis/callback_duration.ipynb | 6 +- .../data_model/{ros.py => ros2.py} | 8 +- .../tracetools_analysis/processor/ros2.py | 6 +- .../tracetools_analysis/utils/__init__.py | 97 +++++++++ .../tracetools_analysis/utils/cpu_time.py | 39 ++++ .../tracetools_analysis/utils/profile.py | 100 ++++++++++ .../{utils.py => utils/ros2.py} | 184 +----------------- 8 files changed, 261 insertions(+), 197 deletions(-) rename tracetools_analysis/tracetools_analysis/data_model/{ros.py => ros2.py} (97%) create mode 100644 tracetools_analysis/tracetools_analysis/utils/__init__.py create mode 100644 tracetools_analysis/tracetools_analysis/utils/cpu_time.py create mode 100644 tracetools_analysis/tracetools_analysis/utils/profile.py rename tracetools_analysis/tracetools_analysis/{utils.py => utils/ros2.py} (69%) diff --git a/README.md b/README.md index 7586c88..5716d86 100644 --- a/README.md +++ b/README.md @@ -40,24 +40,22 @@ Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, a For example: ```python +from tracetools_analysis import loading +from tracetools_analysis import processor from tracetools_analysis import utils -from tracetools_analysis.loading import load_file -from tracetools_analysis.processor import Processor -from tracetools_analysis.processor.cpu_time import CpuTimeHandler -from tracetools_analysis.processor.ros2 import Ros2Handler # Load converted trace file -events = load_file('/path/to/converted/file') +events = loading.load_file('/path/to/converted/file') # Process -ros2_handler = Ros2Handler() -cpu_handler = CpuTimeHandler() +ros2_handler = processor.Ros2Handler() +cpu_handler = processor.CpuTimeHandler() -Processor(ros2_handler, cpu_handler).process(events) +processor.Processor(ros2_handler, cpu_handler).process(events) # Use data model utils to extract information -ros2_util = utils.RosDataModelUtil(ros2_handler.data) -cpu_util = CpuTimeDataModelUtil(cpu_handler.data) +ros2_util = utils.ros2.Ros2DataModelUtil(ros2_handler.data) +cpu_util = utils.cpu_time.CpuTimeDataModelUtil(cpu_handler.data) callback_durations = ros2_util.get_callback_durations() time_per_thread = cpu_util.get_time_per_thread() diff --git a/tracetools_analysis/analysis/callback_duration.ipynb b/tracetools_analysis/analysis/callback_duration.ipynb index f624140..5953137 100644 --- a/tracetools_analysis/analysis/callback_duration.ipynb +++ b/tracetools_analysis/analysis/callback_duration.ipynb @@ -55,9 +55,9 @@ "import numpy as np\n", "import pandas as pd\n", "\n", - "from tracetools_analysis import utils\n", "from tracetools_analysis.loading import load_file\n", - "from tracetools_analysis.processor.ros2 import Ros2Handler" + "from tracetools_analysis.processor.ros2 import Ros2Handler\n", + "from tracetools_analysis.utils.ros2 import Ros2DataModelUtil" ] }, { @@ -78,7 +78,7 @@ "metadata": {}, "outputs": [], "source": [ - "data_util = utils.RosDataModelUtil(handler.data)\n", + "data_util = Ros2DataModelUtil(handler.data)\n", "\n", "callback_symbols = data_util.get_callback_symbols()\n", "\n", diff --git a/tracetools_analysis/tracetools_analysis/data_model/ros.py b/tracetools_analysis/tracetools_analysis/data_model/ros2.py similarity index 97% rename from tracetools_analysis/tracetools_analysis/data_model/ros.py rename to tracetools_analysis/tracetools_analysis/data_model/ros2.py index fbf2cb6..7654254 100644 --- a/tracetools_analysis/tracetools_analysis/data_model/ros.py +++ b/tracetools_analysis/tracetools_analysis/data_model/ros2.py @@ -12,18 +12,18 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module for ROS data model.""" +"""Module for ROS 2 data model.""" import pandas as pd from . import DataModel -class RosDataModel(DataModel): +class Ros2DataModel(DataModel): """ - Container to model pre-processed ROS data for analysis. + Container to model pre-processed ROS 2 data for analysis. - This aims to represent the data in a ROS-aware way. + This aims to represent the data in a ROS 2-aware way. """ def __init__(self) -> None: diff --git a/tracetools_analysis/tracetools_analysis/processor/ros2.py b/tracetools_analysis/tracetools_analysis/processor/ros2.py index edec9e0..9daba18 100644 --- a/tracetools_analysis/tracetools_analysis/processor/ros2.py +++ b/tracetools_analysis/tracetools_analysis/processor/ros2.py @@ -20,7 +20,7 @@ from tracetools_read import get_field from . import EventHandler from . import EventMetadata -from ..data_model.ros import RosDataModel +from ..data_model.ros2 import Ros2DataModel class Ros2Handler(EventHandler): @@ -70,13 +70,13 @@ class Ros2Handler(EventHandler): **kwargs, ) - self._data_model = RosDataModel() + self._data_model = Ros2DataModel() # Temporary buffers self._callback_instances = {} @property - def data(self) -> RosDataModel: + def data(self) -> Ros2DataModel: return self._data_model def _handle_rcl_init( diff --git a/tracetools_analysis/tracetools_analysis/utils/__init__.py b/tracetools_analysis/tracetools_analysis/utils/__init__.py new file mode 100644 index 0000000..e893466 --- /dev/null +++ b/tracetools_analysis/tracetools_analysis/utils/__init__.py @@ -0,0 +1,97 @@ +# Copyright 2019 Robert Bosch GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for data model utility classes.""" + +from datetime import datetime as dt +from typing import List +from typing import Union + +from pandas import DataFrame + +from ..data_model import DataModel + + +class DataModelUtil(): + """ + Base data model util class, which provides functions to get more info about a data model. + + This class provides basic util functions. + """ + + def __init__( + self, + data_model: DataModel, + ) -> None: + """ + Constructor. + + :param data_model: the data model + """ + self.__data = data_model + + @property + def data(self) -> DataModel: + return self.__data + + @staticmethod + def convert_time_columns( + original: DataFrame, + columns_ns_to_ms: Union[List[str], str] = [], + columns_ns_to_datetime: Union[List[str], str] = [], + inplace: bool = True, + ) -> DataFrame: + """ + Convert time columns from nanoseconds to either milliseconds or `datetime` objects. + + :param original: the original `DataFrame` + :param columns_ns_to_ms: the column(s) for which to convert ns to ms + :param columns_ns_to_datetime: the column(s) for which to convert ns to `datetime` + :param inplace: whether to convert in place or to return a copy + :return: the resulting `DataFrame` + """ + if not isinstance(columns_ns_to_ms, list): + columns_ns_to_ms = list(columns_ns_to_ms) + if not isinstance(columns_ns_to_datetime, list): + columns_ns_to_datetime = list(columns_ns_to_datetime) + + df = original if inplace else original.copy() + # Convert from ns to ms + if len(columns_ns_to_ms) > 0: + df[columns_ns_to_ms] = df[columns_ns_to_ms].applymap( + lambda t: t / 1000000.0 + ) + # Convert from ns to ms + ms to datetime, as UTC + if len(columns_ns_to_datetime) > 0: + df[columns_ns_to_datetime] = df[columns_ns_to_datetime].applymap( + lambda t: dt.utcfromtimestamp(t / 1000000000.0) + ) + return df + + @staticmethod + def compute_column_difference( + df: DataFrame, + left_column: str, + right_column: str, + diff_column: str, + ) -> None: + """ + Create new column with difference between two columns. + + :param df: the dataframe (inplace) + :param left_column: the name of the left column + :param right_column: the name of the right column + :param diff_column: the name of the new column with differences + """ + df[diff_column] = df.apply(lambda row: row[left_column] - row[right_column], axis=1) diff --git a/tracetools_analysis/tracetools_analysis/utils/cpu_time.py b/tracetools_analysis/tracetools_analysis/utils/cpu_time.py new file mode 100644 index 0000000..f2c60bf --- /dev/null +++ b/tracetools_analysis/tracetools_analysis/utils/cpu_time.py @@ -0,0 +1,39 @@ +# Copyright 2019 Robert Bosch GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for CPU time data model utils.""" + +from pandas import DataFrame + +from . import DataModelUtil +from ..data_model.cpu_time import CpuTimeDataModel + + +class CpuTimeDataModelUtil(DataModelUtil): + """CPU time data model utility class.""" + + def __init__( + self, + data_model: CpuTimeDataModel, + ) -> None: + """ + Constructor. + + :param data_model: the data model object to use + """ + super().__init__(data_model) + + def get_time_per_thread(self) -> DataFrame: + """Get a DataFrame of total duration for each thread.""" + return self.data.times.loc[:, ['tid', 'duration']].groupby(by='tid').sum() diff --git a/tracetools_analysis/tracetools_analysis/utils/profile.py b/tracetools_analysis/tracetools_analysis/utils/profile.py new file mode 100644 index 0000000..34d59d9 --- /dev/null +++ b/tracetools_analysis/tracetools_analysis/utils/profile.py @@ -0,0 +1,100 @@ +# Copyright 2019 Robert Bosch GmbH +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for profiling data model utils.""" + +from collections import defaultdict +from typing import Dict +from typing import List +from typing import Set +from typing import Union + +from pandas import DataFrame + +from . import DataModelUtil +from ..data_model.profile import ProfileDataModel + + +class ProfileDataModelUtil(DataModelUtil): + """Profiling data model utility class.""" + + def __init__( + self, + data_model: ProfileDataModel, + ) -> None: + """ + Constructor. + + :param data_model: the data model object to use + """ + super().__init__(data_model) + + def with_tid( + self, + tid: int, + ) -> DataFrame: + return self.data.times.loc[self.data.times['tid'] == tid] + + def get_tids(self) -> Set[int]: + """Get the TIDs in the data model.""" + return set(self.data.times['tid']) + + def get_call_tree( + self, + tid: int, + ) -> Dict[str, List[str]]: + depth_names = self.with_tid(tid)[ + ['depth', 'function_name', 'parent_name'] + ].drop_duplicates() + # print(depth_names.to_string()) + tree = defaultdict(set) + for _, row in depth_names.iterrows(): + depth = row['depth'] + name = row['function_name'] + parent = row['parent_name'] + if depth == 0: + tree[name] + else: + tree[parent].add(name) + return dict(tree) + + def get_function_duration_data( + self, + tid: int, + ) -> List[Dict[str, Union[int, str, DataFrame]]]: + """Get duration data for each function.""" + tid_df = self.with_tid(tid) + depth_names = tid_df[['depth', 'function_name', 'parent_name']].drop_duplicates() + functions_data = [] + for _, row in depth_names.iterrows(): + depth = row['depth'] + name = row['function_name'] + parent = row['parent_name'] + data = tid_df.loc[ + (tid_df['depth'] == depth) & + (tid_df['function_name'] == name) + ][['start_timestamp', 'duration', 'actual_duration']] + self.compute_column_difference( + data, + 'duration', + 'actual_duration', + 'duration_difference', + ) + functions_data.append({ + 'depth': depth, + 'function_name': name, + 'parent_name': parent, + 'data': data, + }) + return functions_data diff --git a/tracetools_analysis/tracetools_analysis/utils.py b/tracetools_analysis/tracetools_analysis/utils/ros2.py similarity index 69% rename from tracetools_analysis/tracetools_analysis/utils.py rename to tracetools_analysis/tracetools_analysis/utils/ros2.py index c8ad1c8..012ed36 100644 --- a/tracetools_analysis/tracetools_analysis/utils.py +++ b/tracetools_analysis/tracetools_analysis/utils/ros2.py @@ -1,4 +1,5 @@ # Copyright 2019 Robert Bosch GmbH +# Copyright 2019 Apex.AI, Inc. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. @@ -12,10 +13,8 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module for data model utility classes.""" +"""Module for ROS data model utils.""" -from collections import defaultdict -from datetime import datetime as dt from typing import Any from typing import Dict from typing import List @@ -25,185 +24,16 @@ from typing import Union from pandas import DataFrame -from .data_model import DataModel -from .data_model.cpu_time import CpuTimeDataModel -from .data_model.profile import ProfileDataModel -from .data_model.ros import RosDataModel +from . import DataModelUtil +from ..data_model.ros2 import Ros2DataModel -class DataModelUtil(): - """ - Base data model util class, which provides functions to get more info about a data model. - - This class provides basic util functions. - """ +class Ros2DataModelUtil(DataModelUtil): + """ROS 2 data model utility class.""" def __init__( self, - data_model: DataModel, - ) -> None: - """ - Constructor. - - :param data_model: the data model - """ - self.__data = data_model - - @property - def data(self) -> DataModel: - return self.__data - - @staticmethod - def convert_time_columns( - original: DataFrame, - columns_ns_to_ms: Union[List[str], str] = [], - columns_ns_to_datetime: Union[List[str], str] = [], - inplace: bool = True, - ) -> DataFrame: - """ - Convert time columns from nanoseconds to either milliseconds or `datetime` objects. - - :param original: the original `DataFrame` - :param columns_ns_to_ms: the column(s) for which to convert ns to ms - :param columns_ns_to_datetime: the column(s) for which to convert ns to `datetime` - :param inplace: whether to convert in place or to return a copy - :return: the resulting `DataFrame` - """ - if not isinstance(columns_ns_to_ms, list): - columns_ns_to_ms = list(columns_ns_to_ms) - if not isinstance(columns_ns_to_datetime, list): - columns_ns_to_datetime = list(columns_ns_to_datetime) - - df = original if inplace else original.copy() - # Convert from ns to ms - if len(columns_ns_to_ms) > 0: - df[columns_ns_to_ms] = df[columns_ns_to_ms].applymap( - lambda t: t / 1000000.0 - ) - # Convert from ns to ms + ms to datetime, as UTC - if len(columns_ns_to_datetime) > 0: - df[columns_ns_to_datetime] = df[columns_ns_to_datetime].applymap( - lambda t: dt.utcfromtimestamp(t / 1000000000.0) - ) - return df - - @staticmethod - def compute_column_difference( - df: DataFrame, - left_column: str, - right_column: str, - diff_column: str, - ) -> None: - """ - Create new column with difference between two columns. - - :param df: the dataframe (inplace) - :param left_column: the name of the left column - :param right_column: the name of the right column - :param diff_column: the name of the new column with differences - """ - df[diff_column] = df.apply(lambda row: row[left_column] - row[right_column], axis=1) - - -class ProfileDataModelUtil(DataModelUtil): - """Profiling data model utility class.""" - - def __init__( - self, - data_model: ProfileDataModel, - ) -> None: - """ - Constructor. - - :param data_model: the data model object to use - """ - super().__init__(data_model) - - def with_tid( - self, - tid: int, - ) -> DataFrame: - return self.data.times.loc[self.data.times['tid'] == tid] - - def get_tids(self) -> Set[int]: - """Get the TIDs in the data model.""" - return set(self.data.times['tid']) - - def get_call_tree( - self, - tid: int, - ) -> Dict[str, List[str]]: - depth_names = self.with_tid(tid)[ - ['depth', 'function_name', 'parent_name'] - ].drop_duplicates() - # print(depth_names.to_string()) - tree = defaultdict(set) - for _, row in depth_names.iterrows(): - depth = row['depth'] - name = row['function_name'] - parent = row['parent_name'] - if depth == 0: - tree[name] - else: - tree[parent].add(name) - return dict(tree) - - def get_function_duration_data( - self, - tid: int, - ) -> List[Dict[str, Union[int, str, DataFrame]]]: - """Get duration data for each function.""" - tid_df = self.with_tid(tid) - depth_names = tid_df[['depth', 'function_name', 'parent_name']].drop_duplicates() - functions_data = [] - for _, row in depth_names.iterrows(): - depth = row['depth'] - name = row['function_name'] - parent = row['parent_name'] - data = tid_df.loc[ - (tid_df['depth'] == depth) & - (tid_df['function_name'] == name) - ][['start_timestamp', 'duration', 'actual_duration']] - self.compute_column_difference( - data, - 'duration', - 'actual_duration', - 'duration_difference', - ) - functions_data.append({ - 'depth': depth, - 'function_name': name, - 'parent_name': parent, - 'data': data, - }) - return functions_data - - -class CpuTimeDataModelUtil(DataModelUtil): - """CPU time data model utility class.""" - - def __init__( - self, - data_model: CpuTimeDataModel, - ) -> None: - """ - Constructor. - - :param data_model: the data model object to use - """ - super().__init__(data_model) - - def get_time_per_thread(self) -> DataFrame: - """Get a DataFrame of total duration for each thread.""" - return self.data.times.loc[:, ['tid', 'duration']].groupby(by='tid').sum() - - -class RosDataModelUtil(DataModelUtil): - """ROS data model utility class.""" - - def __init__( - self, - data_model: RosDataModel, + data_model: Ros2DataModel, ) -> None: """ Constructor.