diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml index 35597ae..06a0fa3 100644 --- a/.gitlab-ci.yml +++ b/.gitlab-ci.yml @@ -1,6 +1,6 @@ variables: DOCKER_DRIVER: overlay2 - PACKAGES_LIST: tracetools_analysis + PACKAGES_LIST: tracetools_analysis ros2trace_analysis base_image_id: registry.gitlab.com/micro-ros/ros_tracing/ci_base .global_artifacts: &global_artifacts diff --git a/README.md b/README.md index 9635cf1..7586c88 100644 --- a/README.md +++ b/README.md @@ -2,19 +2,71 @@ Analysis tools for [ROS 2 tracing](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing). -# Setup +## Setup To display results, install: * [Jupyter](https://jupyter.org/install) * [Bokeh](https://bokeh.pydata.org/en/latest/docs/user_guide/quickstart.html#userguide-quickstart-install) -# Use +## Trace analysis -Start Jupyter Notebook: +After generating a trace (see [`ros2_tracing`](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing#tracing)), we can analyze it to extract useful execution data. + +### Commands + +Since CTF traces (the output format of the [LTTng](https://lttng.org/) tracer) are very slow to read, we first convert them into a single file which can be read much faster. + +``` +$ ros2 trace-analysis convert /path/to/trace/directory +``` + +Then we can process it to create a data model which could be queried for analysis. + +``` +$ ros2 trace-analysis process /path/to/trace/directory +``` + +### Jupyter + +The last command will process and output the raw data models, but to actually display results, process and analyze using a Jupyter Notebook. ``` $ jupyter notebook ``` Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, and select one of the provided notebooks, or create your own! + +For example: + +```python +from tracetools_analysis import utils +from tracetools_analysis.loading import load_file +from tracetools_analysis.processor import Processor +from tracetools_analysis.processor.cpu_time import CpuTimeHandler +from tracetools_analysis.processor.ros2 import Ros2Handler + +# Load converted trace file +events = load_file('/path/to/converted/file') + +# Process +ros2_handler = Ros2Handler() +cpu_handler = CpuTimeHandler() + +Processor(ros2_handler, cpu_handler).process(events) + +# Use data model utils to extract information +ros2_util = utils.RosDataModelUtil(ros2_handler.data) +cpu_util = CpuTimeDataModelUtil(cpu_handler.data) + +callback_durations = ros2_util.get_callback_durations() +time_per_thread = cpu_util.get_time_per_thread() +# ... + +# Display, e.g. with bokeh or matplotlib +# ... +``` + +## Design + +See the [`ros2_tracing` design document](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md), especially the [*Goals and requirements*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#goals-and-requirements) and [*Analysis architecture*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#analysis-architecture) sections. diff --git a/ros2trace_analysis/.gitignore b/ros2trace_analysis/.gitignore new file mode 100644 index 0000000..eef29c1 --- /dev/null +++ b/ros2trace_analysis/.gitignore @@ -0,0 +1,3 @@ +*~ +*.pyc + diff --git a/ros2trace_analysis/package.xml b/ros2trace_analysis/package.xml new file mode 100644 index 0000000..9fa17c0 --- /dev/null +++ b/ros2trace_analysis/package.xml @@ -0,0 +1,23 @@ + + + + ros2trace_analysis + 0.1.1 + The trace analysis command for ROS 2 command line tools. + Christophe Bedard + Apache 2.0 + Christophe Bedard + + ros2cli + tracetools_analysis + + ament_copyright + ament_flake8 + ament_pep257 + ament_xmllint + python3-pytest + + + ament_python + + diff --git a/ros2trace_analysis/ros2trace_analysis/__init__.py b/ros2trace_analysis/ros2trace_analysis/__init__.py new file mode 100644 index 0000000..35ffb6e --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ros2trace_analysis/ros2trace_analysis/api/__init__.py b/ros2trace_analysis/ros2trace_analysis/api/__init__.py new file mode 100644 index 0000000..35ffb6e --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/api/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ros2trace_analysis/ros2trace_analysis/command/__init__.py b/ros2trace_analysis/ros2trace_analysis/command/__init__.py new file mode 100644 index 0000000..35ffb6e --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/command/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py b/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py new file mode 100644 index 0000000..0ae2f1f --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py @@ -0,0 +1,41 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Module for trace analysis command extension implementation.""" + +from ros2cli.command import add_subparsers +from ros2cli.command import CommandExtension +from ros2cli.verb import get_verb_extensions + + +class TraceAnalysisCommand(CommandExtension): + """Analyze traces to extract useful execution data.""" + + def add_arguments(self, parser, cli_name): + self._subparser = parser + # get verb extensions and let them add their arguments + verb_extensions = get_verb_extensions('ros2trace_analysis.verb') + add_subparsers( + parser, cli_name, '_verb', verb_extensions, required=False) + + def main(self, *, parser, args): + if not hasattr(args, '_verb'): + # in case no verb was passed + self._subparser.print_help() + return 0 + + extension = getattr(args, '_verb') + + # call the verb's main method + return extension.main(args=args) diff --git a/ros2trace_analysis/ros2trace_analysis/verb/__init__.py b/ros2trace_analysis/ros2trace_analysis/verb/__init__.py new file mode 100644 index 0000000..35ffb6e --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/verb/__init__.py @@ -0,0 +1,13 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. diff --git a/ros2trace_analysis/ros2trace_analysis/verb/convert.py b/ros2trace_analysis/ros2trace_analysis/verb/convert.py new file mode 100644 index 0000000..62ea13f --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/verb/convert.py @@ -0,0 +1,30 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ros2cli.verb import VerbExtension +from tracetools_analysis.convert import add_args +from tracetools_analysis.convert import convert + + +class ConvertVerb(VerbExtension): + """Convert trace data to a file.""" + + def add_arguments(self, parser, cli_name): + add_args(parser) + + def main(self, *, args): + return convert( + args.trace_directory, + args.output_file_name, + ) diff --git a/ros2trace_analysis/ros2trace_analysis/verb/process.py b/ros2trace_analysis/ros2trace_analysis/verb/process.py new file mode 100644 index 0000000..9d650e8 --- /dev/null +++ b/ros2trace_analysis/ros2trace_analysis/verb/process.py @@ -0,0 +1,30 @@ +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ros2cli.verb import VerbExtension +from tracetools_analysis.process import add_args +from tracetools_analysis.process import process + + +class ProcessVerb(VerbExtension): + """Process a file converted from a trace directory and output model data.""" + + def add_arguments(self, parser, cli_name): + add_args(parser) + + def main(self, *, args): + return process( + args.input_path, + args.force_conversion, + ) diff --git a/ros2trace_analysis/setup.py b/ros2trace_analysis/setup.py new file mode 100644 index 0000000..744965f --- /dev/null +++ b/ros2trace_analysis/setup.py @@ -0,0 +1,44 @@ +from setuptools import find_packages +from setuptools import setup + +package_name = 'ros2trace_analysis' + +setup( + name=package_name, + version='0.1.1', + packages=find_packages(exclude=['test']), + data_files=[ + ('share/' + package_name, ['package.xml']), + ], + install_requires=['ros2cli'], + zip_safe=True, + maintainer=( + 'Christophe Bedard' + ), + maintainer_email=( + 'christophe.bedard@apex.ai' + ), + author='Christophe Bedard', + author_email='christophe.bedard@apex.ai', + url='https://gitlab.com/micro-ROS/ros_tracing/tracetools_analysis', + keywords=[], + description='The trace analysis command for ROS 2 command line tools.', + long_description=( + 'The package provides the trace analysis ' + 'command for the ROS 2 command line tools.' + ), + license='Apache 2.0', + tests_require=['pytest'], + entry_points={ + 'ros2cli.command': [ + f'trace-analysis = {package_name}.command.trace_analysis:TraceAnalysisCommand', + ], + 'ros2cli.extension_point': [ + f'{package_name}.verb = {package_name}.verb:VerbExtension', + ], + f'{package_name}.verb': [ + f'convert = {package_name}.verb.convert:ConvertVerb', + f'process = {package_name}.verb.process:ProcessVerb', + ], + } +) diff --git a/ros2trace_analysis/test/test_copyright.py b/ros2trace_analysis/test/test_copyright.py new file mode 100644 index 0000000..cf0fae3 --- /dev/null +++ b/ros2trace_analysis/test/test_copyright.py @@ -0,0 +1,23 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_copyright.main import main +import pytest + + +@pytest.mark.copyright +@pytest.mark.linter +def test_copyright(): + rc = main(argv=['.', 'test']) + assert rc == 0, 'Found errors' diff --git a/ros2trace_analysis/test/test_flake8.py b/ros2trace_analysis/test/test_flake8.py new file mode 100644 index 0000000..eff8299 --- /dev/null +++ b/ros2trace_analysis/test/test_flake8.py @@ -0,0 +1,23 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_flake8.main import main +import pytest + + +@pytest.mark.flake8 +@pytest.mark.linter +def test_flake8(): + rc = main(argv=[]) + assert rc == 0, 'Found errors' diff --git a/ros2trace_analysis/test/test_pep257.py b/ros2trace_analysis/test/test_pep257.py new file mode 100644 index 0000000..0e38a6c --- /dev/null +++ b/ros2trace_analysis/test/test_pep257.py @@ -0,0 +1,23 @@ +# Copyright 2017 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_pep257.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.pep257 +def test_pep257(): + rc = main(argv=[]) + assert rc == 0, 'Found code style errors / warnings' diff --git a/ros2trace_analysis/test/test_xmllint.py b/ros2trace_analysis/test/test_xmllint.py new file mode 100644 index 0000000..f46285e --- /dev/null +++ b/ros2trace_analysis/test/test_xmllint.py @@ -0,0 +1,23 @@ +# Copyright 2019 Open Source Robotics Foundation, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +from ament_xmllint.main import main +import pytest + + +@pytest.mark.linter +@pytest.mark.xmllint +def test_xmllint(): + rc = main(argv=[]) + assert rc == 0, 'Found errors' diff --git a/tracetools_analysis/analysis/callback_duration.ipynb b/tracetools_analysis/analysis/callback_duration.ipynb index 9bab160..9b10a7e 100644 --- a/tracetools_analysis/analysis/callback_duration.ipynb +++ b/tracetools_analysis/analysis/callback_duration.ipynb @@ -17,8 +17,8 @@ "#\n", "# OR\n", "#\n", - "# Use the provided sample pickle file, changing the path below to:\n", - "# 'sample_data/pickle_pingpong'" + "# Use the provided sample converted trace file, changing the path below to:\n", + "# 'sample_data/converted_pingpong'" ] }, { @@ -27,8 +27,8 @@ "metadata": {}, "outputs": [], "source": [ - "pickle_path = '~/.ros/tracing/pingpong/ust/pickle'\n", - "#pickle_path = 'sample_data/pickle_pingpong'" + "converted_file_path = '~/.ros/tracing/pingpong/ust/converted'\n", + "#converted_file_path = 'sample_data/converted_pingpong'" ] }, { @@ -44,7 +44,6 @@ "sys.path.insert(0, '../')\n", "sys.path.insert(0, '../../../micro-ROS/ros_tracing/ros2_tracing/tracetools_read/')\n", "import datetime as dt\n", - "import os\n", "\n", "from bokeh.plotting import figure\n", "from bokeh.plotting import output_notebook\n", @@ -57,7 +56,7 @@ "import pandas as pd\n", "\n", "from tracetools_analysis import utils\n", - "from tracetools_analysis.loading import load_pickle\n", + "from tracetools_analysis.loading import load_file\n", "from tracetools_analysis.processor.ros2 import Ros2Handler" ] }, @@ -68,8 +67,7 @@ "outputs": [], "source": [ "# Process\n", - "pickle_path = os.path.expanduser(pickle_path)\n", - "events = load_pickle(pickle_path)\n", + "events = load_file(converted_file_path)\n", "handler = Ros2Handler.process(events)\n", "#handler.data.print_model()" ] diff --git a/tracetools_analysis/analysis/sample_data/pickle_pingpong b/tracetools_analysis/analysis/sample_data/converted_pingpong similarity index 100% rename from tracetools_analysis/analysis/sample_data/pickle_pingpong rename to tracetools_analysis/analysis/sample_data/converted_pingpong diff --git a/tracetools_analysis/test/tracetools_analysis/test_data_model_util.py b/tracetools_analysis/test/test_data_model_util.py similarity index 100% rename from tracetools_analysis/test/tracetools_analysis/test_data_model_util.py rename to tracetools_analysis/test/test_data_model_util.py diff --git a/tracetools_analysis/test/tracetools_analysis/test_dependency_solver.py b/tracetools_analysis/test/test_dependency_solver.py similarity index 100% rename from tracetools_analysis/test/tracetools_analysis/test_dependency_solver.py rename to tracetools_analysis/test/test_dependency_solver.py diff --git a/tracetools_analysis/test/test_process_command.py b/tracetools_analysis/test/test_process_command.py new file mode 100644 index 0000000..141039f --- /dev/null +++ b/tracetools_analysis/test/test_process_command.py @@ -0,0 +1,90 @@ +#!/usr/bin/env python3 +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import tempfile +import unittest + +from tracetools_analysis.process import inspect_input_path + + +class TestProcessCommand(unittest.TestCase): + + def __init__(self, *args) -> None: + super().__init__( + *args, + ) + + def setUp(self): + self.test_dir_path = tempfile.mkdtemp() + + # Create directory that contains a 'converted' file + self.with_converted_file_dir = os.path.join( + self.test_dir_path, + 'with_converted_file', + ) + os.mkdir(self.with_converted_file_dir) + self.converted_file_path = os.path.join( + self.with_converted_file_dir, + 'converted', + ) + open(self.converted_file_path, 'a').close() + self.assertTrue(os.path.exists(self.converted_file_path)) + + # Create directory that contains a file with another name that is not 'converted' + self.without_converted_file_dir = os.path.join( + self.test_dir_path, + 'without_converted_file', + ) + os.mkdir(self.without_converted_file_dir) + self.random_file_path = os.path.join( + self.without_converted_file_dir, + 'a_file', + ) + open(self.random_file_path, 'a').close() + self.assertTrue(os.path.exists(self.random_file_path)) + + def tearDown(self): + shutil.rmtree(self.test_dir_path) + + def test_inspect_input_path(self) -> None: + # Should find converted file under directory + file_path, create_file = inspect_input_path(self.with_converted_file_dir, False) + self.assertEqual(self.converted_file_path, file_path) + self.assertFalse(create_file) + # Should find it but set it to be re-created + file_path, create_file = inspect_input_path(self.with_converted_file_dir, True) + self.assertEqual(self.converted_file_path, file_path) + self.assertTrue(create_file) + + # Should fail to find converted file under directory + file_path, create_file = inspect_input_path(self.without_converted_file_dir, False) + self.assertIsNone(file_path) + self.assertIsNone(create_file) + file_path, create_file = inspect_input_path(self.without_converted_file_dir, True) + self.assertIsNone(file_path) + self.assertIsNone(create_file) + + # Should accept any file path if it exists + file_path, create_file = inspect_input_path(self.random_file_path, False) + self.assertEqual(self.random_file_path, file_path) + self.assertFalse(create_file) + # Should set it to be re-created + file_path, create_file = inspect_input_path(self.random_file_path, True) + self.assertEqual(self.random_file_path, file_path) + self.assertTrue(create_file) + + # TODO try with a trace directory diff --git a/tracetools_analysis/test/tracetools_analysis/test_processor.py b/tracetools_analysis/test/test_processor.py similarity index 100% rename from tracetools_analysis/test/tracetools_analysis/test_processor.py rename to tracetools_analysis/test/test_processor.py diff --git a/tracetools_analysis/test/tracetools_analysis/test_profile_handler.py b/tracetools_analysis/test/test_profile_handler.py similarity index 100% rename from tracetools_analysis/test/tracetools_analysis/test_profile_handler.py rename to tracetools_analysis/test/test_profile_handler.py diff --git a/tracetools_analysis/test/test_utils.py b/tracetools_analysis/test/test_utils.py new file mode 100644 index 0000000..16d0886 --- /dev/null +++ b/tracetools_analysis/test/test_utils.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +# Copyright 2019 Apex.AI, Inc. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import unittest + +from tracetools_analysis import time_diff_to_str + + +class TestUtils(unittest.TestCase): + + def __init__(self, *args) -> None: + super().__init__( + *args, + ) + + def test_time_diff_to_str(self) -> None: + self.assertEqual('11 ms', time_diff_to_str(0.0106)) + self.assertEqual('6.9 s', time_diff_to_str(6.9069)) + self.assertEqual('1 m 10 s', time_diff_to_str(69.6969)) + self.assertEqual('6 m 10 s', time_diff_to_str(369.6969)) + self.assertEqual('2 m 0 s', time_diff_to_str(120.499999999)) diff --git a/tracetools_analysis/tracetools_analysis/__init__.py b/tracetools_analysis/tracetools_analysis/__init__.py index e4deba3..94788ac 100644 --- a/tracetools_analysis/tracetools_analysis/__init__.py +++ b/tracetools_analysis/tracetools_analysis/__init__.py @@ -12,5 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Reading and interpreting of LTTng trace data.""" -__author__ = 'Luetkebohle Ingo (CR/AEX3)' +"""Tools for analysing trace data.""" + + +def time_diff_to_str( + time_diff: float, +) -> str: + """ + Format time difference as a string. + + :param time_diff: the difference between two timepoints (e.g. `time.time()`) + """ + if time_diff < 1.0: + # ms + return f'{time_diff * 1000:.0f} ms' + elif time_diff < 60.0: + # s + return f'{time_diff:.1f} s' + else: + # m s + return f'{time_diff // 60.0:.0f} m {time_diff % 60.0:.0f} s' diff --git a/tracetools_analysis/tracetools_analysis/conversion/ctf.py b/tracetools_analysis/tracetools_analysis/conversion/ctf.py index aebcbaf..3f0fb74 100644 --- a/tracetools_analysis/tracetools_analysis/conversion/ctf.py +++ b/tracetools_analysis/tracetools_analysis/conversion/ctf.py @@ -25,7 +25,7 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int: Load CTF trace, convert events, and dump to a pickle file. :param trace_directory: the trace directory - :param target: the target pickle file to write to + :param target: the target file to write to :return: the number of events written """ ctf_events = get_trace_ctf_events(trace_directory) @@ -43,15 +43,15 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int: return count_written -def convert(trace_directory: str, pickle_target_path: str) -> int: +def convert(trace_directory: str, output_file_path: str) -> int: """ Convert CTF trace to pickle file. :param trace_directory: the trace directory - :param pickle_target_path: the path to the pickle file that will be created - :return: the number of events written to the pickle file + :param output_file_path: the path to the output file that will be created + :return: the number of events written to the output file """ - with open(pickle_target_path, 'wb') as f: + with open(output_file_path, 'wb') as f: p = Pickler(f, protocol=4) count = ctf_to_pickle(trace_directory, p) diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py index b1ffa42..c2384a2 100644 --- a/tracetools_analysis/tracetools_analysis/convert.py +++ b/tracetools_analysis/tracetools_analysis/convert.py @@ -13,38 +13,64 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Entrypoint/script to convert CTF trace data to a pickle file.""" +"""Entrypoint/script to convert CTF trace data to a file.""" import argparse import os import time +from typing import Optional from tracetools_analysis.conversion import ctf +from . import time_diff_to_str + + +DEFAULT_CONVERT_FILE_NAME = 'converted' + + +def add_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + 'trace_directory', + help='the path to the main trace directory') + parser.add_argument( + '-o', '--output-file-name', dest='output_file_name', + default=DEFAULT_CONVERT_FILE_NAME, + help='the name of the output file to generate, ' + 'under $trace_directory (default: %(default)s)') + def parse_args(): parser = argparse.ArgumentParser( - description='Convert CTF trace data to a pickle file.') - parser.add_argument( - 'trace_directory', help='the path to the main CTF trace directory') - parser.add_argument( - '--pickle-path', '-p', - help='the path to the target pickle file to generate (default: $trace_directory/pickle)') - args = parser.parse_args() - if args.pickle_path is None: - args.pickle_path = os.path.join(args.trace_directory, 'pickle') - return args + description='Convert trace data to a file.') + add_args(parser) + return parser.parse_args() + + +def convert( + trace_directory: str, + output_file_name: str = DEFAULT_CONVERT_FILE_NAME, +) -> Optional[int]: + """ + Convert trace directory to a file. + + The output file will be placed under the trace directory. + + :param trace_directory: the path to the trace directory to import + :param outout_file_name: the name of the output file + """ + print(f'converting trace directory: {trace_directory}') + output_file_path = os.path.join(os.path.expanduser(trace_directory), output_file_name) + start_time = time.time() + count = ctf.convert(trace_directory, output_file_path) + time_diff = time.time() - start_time + print(f'converted {count} events in {time_diff_to_str(time_diff)}') + print(f'output written to: {output_file_path}') def main(): args = parse_args() trace_directory = args.trace_directory - pickle_target_path = args.pickle_path + output_file_name = args.output_file_name - print(f'importing trace directory: {trace_directory}') - start_time = time.time() - count = ctf.convert(trace_directory, pickle_target_path) - time_diff = time.time() - start_time - print(f'converted {count} events in {time_diff * 1000:.2f} ms') - print(f'pickle written to: {pickle_target_path}') + convert(trace_directory, output_file_name) diff --git a/tracetools_analysis/tracetools_analysis/loading/__init__.py b/tracetools_analysis/tracetools_analysis/loading/__init__.py index e79dfa4..96abc39 100644 --- a/tracetools_analysis/tracetools_analysis/loading/__init__.py +++ b/tracetools_analysis/tracetools_analysis/loading/__init__.py @@ -12,22 +12,23 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Module for pickle loading.""" +"""Module for converted trace file loading.""" +import os import pickle from typing import Dict from typing import List -def load_pickle(pickle_file_path: str) -> List[Dict]: +def load_file(file_path: str) -> List[Dict]: """ - Load pickle file containing converted trace events. + Load file containing converted trace events. - :param pickle_file_path: the path to the pickle file to load + :param file_path: the path to the converted file to load :return: the list of events read from the file """ events = [] - with open(pickle_file_path, 'rb') as f: + with open(os.path.expanduser(file_path), 'rb') as f: p = pickle.Unpickler(f) while True: try: diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py index 59cae89..d7ea399 100644 --- a/tracetools_analysis/tracetools_analysis/process.py +++ b/tracetools_analysis/tracetools_analysis/process.py @@ -13,32 +13,137 @@ # See the License for the specific language governing permissions and # limitations under the License. -"""Entrypoint/script to process events from a pickle file to build a ROS model.""" +"""Entrypoint/script to process events from a converted file to build a ROS model.""" import argparse +import os +import sys import time +from typing import Optional +from typing import Tuple -from tracetools_analysis.loading import load_pickle +from tracetools_analysis.convert import convert +from tracetools_analysis.convert import DEFAULT_CONVERT_FILE_NAME +from tracetools_analysis.loading import load_file from tracetools_analysis.processor.ros2 import Ros2Handler +from tracetools_read.trace import is_trace_directory + +from . import time_diff_to_str + + +def add_args(parser: argparse.ArgumentParser) -> None: + parser.add_argument( + 'input_path', + help='the path to a converted file to import and process, ' + 'or the path to a CTF directory to convert and process') + parser.add_argument( + '-f', '--force-conversion', dest='force_conversion', + action='store_true', default=False, + help='re-convert trace directory even if converted file is found') def parse_args(): - parser = argparse.ArgumentParser(description='Process a pickle file generated ' - 'from tracing and analyze the data.') - parser.add_argument('pickle_file', help='the pickle file to import') + parser = argparse.ArgumentParser(description='Process a file converted from a trace ' + 'directory and output model data.') + add_args(parser) return parser.parse_args() +def inspect_input_path( + input_path: str, + force_conversion: bool = False, +) -> Tuple[str, bool]: + """ + Check input path for a converted file or a trace directory. + + If the input path is a file, it uses it as a converted file. + If the input path is a directory, it checks if there is a "converted" file directly inside it, + otherwise it tries to import the path as a trace directory. + If `force_conversion` is set to `True`, even if a converted file is found, it will ask to + re-create it. + + :param input_path: the path to a converted file or trace directory + :param force_conversion: whether to re-creating converted file even if it is found + :return: + the path to a converted file (or `None` if could not find), + `True` if the given converted file should be (re-)created, `False` otherwise + """ + input_path = os.path.expanduser(input_path) + converted_file_path = None + # Check if not a file + if not os.path.isfile(input_path): + input_directory = input_path + # Might be a (trace) directory + # Check if there is a converted file under the given directory + prospective_converted_file = os.path.join(input_directory, DEFAULT_CONVERT_FILE_NAME) + if os.path.isfile(prospective_converted_file): + # Use that as the converted input file + converted_file_path = prospective_converted_file + if force_conversion: + print(f'found converted file but will re-create it: {prospective_converted_file}') + return prospective_converted_file, True + else: + print(f'found converted file: {prospective_converted_file}') + return prospective_converted_file, False + else: + # Check if it is a trace directory + # Result could be unexpected because it will look for trace directories recursively + # (e.g. '/' is a valid trace directory if there is at least one trace anywhere) + if is_trace_directory(input_directory): + # Convert trace directory first to create converted file + return prospective_converted_file, True + else: + # We cannot do anything + print( + f'cannot find either a trace directory or a converted file: {input_directory}', + file=sys.stderr) + return None, None + else: + converted_file_path = input_path + if force_conversion: + # It's a file, but re-create it anyway + print(f'found converted file but will re-create it: {converted_file_path}') + return converted_file_path, True + else: + # Simplest use-case: given path is an existing converted file + print(f'found converted file: {converted_file_path}') + return converted_file_path, False + + +def process( + input_path: str, + force_conversion: bool = False, +) -> Optional[int]: + """ + Process converted trace file. + + :param input_path: the path to a converted file or trace directory + :param force_conversion: whether to re-creating converted file even if it is found + """ + converted_file_path, create_converted_file = inspect_input_path(input_path, force_conversion) + + if converted_file_path is None: + return 1 + + # Convert trace directory to file if necessary + if create_converted_file: + input_directory = os.path.dirname(converted_file_path) + input_file_name = os.path.basename(converted_file_path) + convert(input_directory, input_file_name) + + start_time = time.time() + + events = load_file(converted_file_path) + ros2_handler = Ros2Handler.process(events) + + time_diff = time.time() - start_time + ros2_handler.data.print_model() + print(f'processed {len(events)} events in {time_diff_to_str(time_diff)}') + + def main(): args = parse_args() - pickle_filename = args.pickle_file + input_path = args.input_path + force_conversion = args.force_conversion - start_time = time.time() - - events = load_pickle(pickle_filename) - ros2_handler = Ros2Handler.process(events) - - time_diff = time.time() - start_time - print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms') - - ros2_handler.data.print_model() + process(input_path, force_conversion) diff --git a/tracetools_analysis/tracetools_analysis/processor/__init__.py b/tracetools_analysis/tracetools_analysis/processor/__init__.py index 9b440f5..b807e2d 100644 --- a/tracetools_analysis/tracetools_analysis/processor/__init__.py +++ b/tracetools_analysis/tracetools_analysis/processor/__init__.py @@ -15,6 +15,7 @@ """Base processor module.""" from collections import defaultdict +import sys from typing import Callable from typing import Dict from typing import List @@ -258,6 +259,9 @@ class Processor(): expanded_handlers = self._expand_dependencies(*handlers, **kwargs) self._handler_multimap = self._get_handler_maps(expanded_handlers) self._register_with_handlers(expanded_handlers) + self._progress_display = ProcessingProgressDisplay( + [type(handler).__name__ for handler in expanded_handlers], + ) @staticmethod def _expand_dependencies( @@ -306,8 +310,10 @@ class Processor(): :param events: the events to process """ + self._progress_display.set_work_total(len(events)) for event in events: self._process_event(event) + self._progress_display.did_work() def _process_event(self, event: DictEvent) -> None: """Process a single event.""" @@ -339,3 +345,57 @@ class Processor(): raise_if_not_found=False) metadata = EventMetadata(event_name, timestamp, cpu_id, procname, pid, tid) handler_function(event, metadata) + + +class ProcessingProgressDisplay(): + """Display processing progress periodically on stdout.""" + + def __init__( + self, + processing_elements: List[str], + ) -> None: + """ + Constructor. + + :param processing_elements: the list of elements doing processing + """ + self.__info_string = '[' + ', '.join(processing_elements) + ']' + self.__total_work = None + self.__progress_count = None + self.__rolling_count = None + self.__work_display_period = None + + def set_work_total( + self, + total: int, + ) -> None: + """ + Set the total units of work. + + :param total: the total number of units of work to do + """ + self.__total_work = total + self.__progress_count = 0 + self.__rolling_count = 0 + self.__work_display_period = total // 100 + self._update() + + def did_work( + self, + increment: int = 1, + ) -> None: + """ + Increment the amount of work done. + + :param increment: the number of units of work to add to the total + """ + # For now, let it fail if set_work_total() hasn't been called + self.__progress_count += increment + self.__rolling_count += increment + if self.__rolling_count >= self.__work_display_period: + self.__rolling_count -= self.__work_display_period + self._update() + + def _update(self) -> None: + percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work)) + sys.stdout.write(f' [{percentage:2.0f}%] {self.__info_string}\r')