Merge branch 'improve-ux' into 'master'

Improve UX

Closes #10

See merge request micro-ROS/ros_tracing/tracetools_analysis!20

.gitlab-ci.yml

@@ -1,6 +1,6 @@
 variables:
   DOCKER_DRIVER: overlay2
-  PACKAGES_LIST: tracetools_analysis
+  PACKAGES_LIST: tracetools_analysis ros2trace_analysis
   base_image_id: registry.gitlab.com/micro-ros/ros_tracing/ci_base
 
 .global_artifacts: &global_artifacts

README.md

@@ -2,19 +2,71 @@
 
 Analysis tools for [ROS 2 tracing](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing).
 
-# Setup
+## Setup
 
 To display results, install:
 
 * [Jupyter](https://jupyter.org/install)
 * [Bokeh](https://bokeh.pydata.org/en/latest/docs/user_guide/quickstart.html#userguide-quickstart-install)
 
-# Use
+## Trace analysis
 
-Start Jupyter Notebook:
+After generating a trace (see [`ros2_tracing`](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing#tracing)), we can analyze it to extract useful execution data.
+
+### Commands
+
+Since CTF traces (the output format of the [LTTng](https://lttng.org/) tracer) are very slow to read, we first convert them into a single file which can be read much faster.
+
+```
+$ ros2 trace-analysis convert /path/to/trace/directory
+```
+
+Then we can process it to create a data model which could be queried for analysis.
+
+```
+$ ros2 trace-analysis process /path/to/trace/directory
+```
+
+### Jupyter
+
+The last command will process and output the raw data models, but to actually display results, process and analyze using a Jupyter Notebook.
 
 ```
 $ jupyter notebook
 ```
 
 Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, and select one of the provided notebooks, or create your own!
+
+For example:
+
+```python
+from tracetools_analysis import utils
+from tracetools_analysis.loading import load_file
+from tracetools_analysis.processor import Processor
+from tracetools_analysis.processor.cpu_time import CpuTimeHandler
+from tracetools_analysis.processor.ros2 import Ros2Handler
+
+# Load converted trace file
+events = load_file('/path/to/converted/file')
+
+# Process
+ros2_handler = Ros2Handler()
+cpu_handler = CpuTimeHandler()
+
+Processor(ros2_handler, cpu_handler).process(events)
+
+# Use data model utils to extract information
+ros2_util = utils.RosDataModelUtil(ros2_handler.data)
+cpu_util = CpuTimeDataModelUtil(cpu_handler.data)
+
+callback_durations = ros2_util.get_callback_durations()
+time_per_thread = cpu_util.get_time_per_thread()
+# ...
+
+# Display, e.g. with bokeh or matplotlib
+# ...
+```
+
+## Design
+
+See the [`ros2_tracing` design document](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md), especially the [*Goals and requirements*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#goals-and-requirements) and [*Analysis architecture*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#analysis-architecture) sections.

ros2trace_analysis/.gitignore

@@ -0,0 +1,3 @@
+*~
+*.pyc
+

ros2trace_analysis/package.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format2.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="2">
+  <name>ros2trace_analysis</name>
+  <version>0.1.1</version>
+  <description>The trace analysis command for ROS 2 command line tools.</description>
+  <maintainer email="christophe.bedard@apex.ai">Christophe Bedard</maintainer>
+  <license>Apache 2.0</license>
+  <author email="christophe.bedard@apex.ai">Christophe Bedard</author>
+
+  <depend>ros2cli</depend>
+  <depend>tracetools_analysis</depend>
+
+  <test_depend>ament_copyright</test_depend>
+  <test_depend>ament_flake8</test_depend>
+  <test_depend>ament_pep257</test_depend>
+  <test_depend>ament_xmllint</test_depend>
+  <test_depend>python3-pytest</test_depend>
+
+  <export>
+    <build_type>ament_python</build_type>
+  </export>
+</package>

ros2trace_analysis/ros2trace_analysis/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ros2trace_analysis/ros2trace_analysis/api/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ros2trace_analysis/ros2trace_analysis/command/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py

@@ -0,0 +1,41 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for trace analysis command extension implementation."""
+
+from ros2cli.command import add_subparsers
+from ros2cli.command import CommandExtension
+from ros2cli.verb import get_verb_extensions
+
+
+class TraceAnalysisCommand(CommandExtension):
+    """Analyze traces to extract useful execution data."""
+
+    def add_arguments(self, parser, cli_name):
+        self._subparser = parser
+        # get verb extensions and let them add their arguments
+        verb_extensions = get_verb_extensions('ros2trace_analysis.verb')
+        add_subparsers(
+            parser, cli_name, '_verb', verb_extensions, required=False)
+
+    def main(self, *, parser, args):
+        if not hasattr(args, '_verb'):
+            # in case no verb was passed
+            self._subparser.print_help()
+            return 0
+
+        extension = getattr(args, '_verb')
+
+        # call the verb's main method
+        return extension.main(args=args)

ros2trace_analysis/ros2trace_analysis/verb/__init__.py

@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.

ros2trace_analysis/ros2trace_analysis/verb/convert.py

@@ -0,0 +1,30 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ros2cli.verb import VerbExtension
+from tracetools_analysis.convert import add_args
+from tracetools_analysis.convert import convert
+
+
+class ConvertVerb(VerbExtension):
+    """Convert trace data to a file."""
+
+    def add_arguments(self, parser, cli_name):
+        add_args(parser)
+
+    def main(self, *, args):
+        return convert(
+            args.trace_directory,
+            args.output_file_name,
+        )

ros2trace_analysis/ros2trace_analysis/verb/process.py

@@ -0,0 +1,30 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ros2cli.verb import VerbExtension
+from tracetools_analysis.process import add_args
+from tracetools_analysis.process import process
+
+
+class ProcessVerb(VerbExtension):
+    """Process a file converted from a trace directory and output model data."""
+
+    def add_arguments(self, parser, cli_name):
+        add_args(parser)
+
+    def main(self, *, args):
+        return process(
+            args.input_path,
+            args.force_conversion,
+        )

ros2trace_analysis/setup.py

@@ -0,0 +1,44 @@
+from setuptools import find_packages
+from setuptools import setup
+
+package_name = 'ros2trace_analysis'
+
+setup(
+    name=package_name,
+    version='0.1.1',
+    packages=find_packages(exclude=['test']),
+    data_files=[
+        ('share/' + package_name, ['package.xml']),
+    ],
+    install_requires=['ros2cli'],
+    zip_safe=True,
+    maintainer=(
+        'Christophe Bedard'
+    ),
+    maintainer_email=(
+        'christophe.bedard@apex.ai'
+    ),
+    author='Christophe Bedard',
+    author_email='christophe.bedard@apex.ai',
+    url='https://gitlab.com/micro-ROS/ros_tracing/tracetools_analysis',
+    keywords=[],
+    description='The trace analysis command for ROS 2 command line tools.',
+    long_description=(
+        'The package provides the trace analysis '
+        'command for the ROS 2 command line tools.'
+    ),
+    license='Apache 2.0',
+    tests_require=['pytest'],
+    entry_points={
+        'ros2cli.command': [
+            f'trace-analysis = {package_name}.command.trace_analysis:TraceAnalysisCommand',
+        ],
+        'ros2cli.extension_point': [
+            f'{package_name}.verb = {package_name}.verb:VerbExtension',
+        ],
+        f'{package_name}.verb': [
+            f'convert = {package_name}.verb.convert:ConvertVerb',
+            f'process = {package_name}.verb.process:ProcessVerb',
+        ],
+    }
+)

ros2trace_analysis/test/test_copyright.py

@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_copyright.main import main
+import pytest
+
+
+@pytest.mark.copyright
+@pytest.mark.linter
+def test_copyright():
+    rc = main(argv=['.', 'test'])
+    assert rc == 0, 'Found errors'

ros2trace_analysis/test/test_flake8.py

@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_flake8.main import main
+import pytest
+
+
+@pytest.mark.flake8
+@pytest.mark.linter
+def test_flake8():
+    rc = main(argv=[])
+    assert rc == 0, 'Found errors'

ros2trace_analysis/test/test_pep257.py

@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_pep257.main import main
+import pytest
+
+
+@pytest.mark.linter
+@pytest.mark.pep257
+def test_pep257():
+    rc = main(argv=[])
+    assert rc == 0, 'Found code style errors / warnings'

ros2trace_analysis/test/test_xmllint.py

@@ -0,0 +1,23 @@
+# Copyright 2019 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_xmllint.main import main
+import pytest
+
+
+@pytest.mark.linter
+@pytest.mark.xmllint
+def test_xmllint():
+    rc = main(argv=[])
+    assert rc == 0, 'Found errors'

tracetools_analysis/analysis/callback_duration.ipynb

@@ -17,8 +17,8 @@
     "#\n",
     "# OR\n",
     "#\n",
-    "# Use the provided sample pickle file, changing the path below to:\n",
+    "# Use the provided sample converted trace file, changing the path below to:\n",
-    "#    'sample_data/pickle_pingpong'"
+    "#    'sample_data/converted_pingpong'"
    ]
   },
   {
@@ -27,8 +27,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pickle_path = '~/.ros/tracing/pingpong/ust/pickle'\n",
+    "converted_file_path = '~/.ros/tracing/pingpong/ust/converted'\n",
-    "#pickle_path = 'sample_data/pickle_pingpong'"
+    "#converted_file_path = 'sample_data/converted_pingpong'"
    ]
   },
   {
@@ -44,7 +44,6 @@
     "sys.path.insert(0, '../')\n",
     "sys.path.insert(0, '../../../micro-ROS/ros_tracing/ros2_tracing/tracetools_read/')\n",
     "import datetime as dt\n",
-    "import os\n",
     "\n",
     "from bokeh.plotting import figure\n",
     "from bokeh.plotting import output_notebook\n",
@@ -57,7 +56,7 @@
     "import pandas as pd\n",
     "\n",
     "from tracetools_analysis import utils\n",
-    "from tracetools_analysis.loading import load_pickle\n",
+    "from tracetools_analysis.loading import load_file\n",
     "from tracetools_analysis.processor.ros2 import Ros2Handler"
    ]
   },
@@ -68,8 +67,7 @@
    "outputs": [],
    "source": [
     "# Process\n",
-    "pickle_path = os.path.expanduser(pickle_path)\n",
+    "events = load_file(converted_file_path)\n",
-    "events = load_pickle(pickle_path)\n",
     "handler = Ros2Handler.process(events)\n",
     "#handler.data.print_model()"
    ]

tracetools_analysis/test/test_process_command.py

@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from tracetools_analysis.process import inspect_input_path
+
+
+class TestProcessCommand(unittest.TestCase):
+
+    def __init__(self, *args) -> None:
+        super().__init__(
+            *args,
+        )
+
+    def setUp(self):
+        self.test_dir_path = tempfile.mkdtemp()
+
+        # Create directory that contains a 'converted' file
+        self.with_converted_file_dir = os.path.join(
+            self.test_dir_path,
+            'with_converted_file',
+        )
+        os.mkdir(self.with_converted_file_dir)
+        self.converted_file_path = os.path.join(
+            self.with_converted_file_dir,
+            'converted',
+        )
+        open(self.converted_file_path, 'a').close()
+        self.assertTrue(os.path.exists(self.converted_file_path))
+
+        # Create directory that contains a file with another name that is not 'converted'
+        self.without_converted_file_dir = os.path.join(
+            self.test_dir_path,
+            'without_converted_file',
+        )
+        os.mkdir(self.without_converted_file_dir)
+        self.random_file_path = os.path.join(
+            self.without_converted_file_dir,
+            'a_file',
+        )
+        open(self.random_file_path, 'a').close()
+        self.assertTrue(os.path.exists(self.random_file_path))
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir_path)
+
+    def test_inspect_input_path(self) -> None:
+        # Should find converted file under directory
+        file_path, create_file = inspect_input_path(self.with_converted_file_dir, False)
+        self.assertEqual(self.converted_file_path, file_path)
+        self.assertFalse(create_file)
+        # Should find it but set it to be re-created
+        file_path, create_file = inspect_input_path(self.with_converted_file_dir, True)
+        self.assertEqual(self.converted_file_path, file_path)
+        self.assertTrue(create_file)
+
+        # Should fail to find converted file under directory
+        file_path, create_file = inspect_input_path(self.without_converted_file_dir, False)
+        self.assertIsNone(file_path)
+        self.assertIsNone(create_file)
+        file_path, create_file = inspect_input_path(self.without_converted_file_dir, True)
+        self.assertIsNone(file_path)
+        self.assertIsNone(create_file)
+
+        # Should accept any file path if it exists
+        file_path, create_file = inspect_input_path(self.random_file_path, False)
+        self.assertEqual(self.random_file_path, file_path)
+        self.assertFalse(create_file)
+        # Should set it to be re-created
+        file_path, create_file = inspect_input_path(self.random_file_path, True)
+        self.assertEqual(self.random_file_path, file_path)
+        self.assertTrue(create_file)
+
+        # TODO try with a trace directory

tracetools_analysis/test/test_utils.py

@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from tracetools_analysis import time_diff_to_str
+
+
+class TestUtils(unittest.TestCase):
+
+    def __init__(self, *args) -> None:
+        super().__init__(
+            *args,
+        )
+
+    def test_time_diff_to_str(self) -> None:
+        self.assertEqual('11 ms', time_diff_to_str(0.0106))
+        self.assertEqual('6.9 s', time_diff_to_str(6.9069))
+        self.assertEqual('1 m 10 s', time_diff_to_str(69.6969))
+        self.assertEqual('6 m 10 s', time_diff_to_str(369.6969))
+        self.assertEqual('2 m 0 s', time_diff_to_str(120.499999999))

tracetools_analysis/tracetools_analysis/__init__.py

@@ -12,5 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Reading and interpreting of LTTng trace data."""
+"""Tools for analysing trace data."""
-__author__ = 'Luetkebohle Ingo (CR/AEX3)'
+
+
+def time_diff_to_str(
+    time_diff: float,
+) -> str:
+    """
+    Format time difference as a string.
+
+    :param time_diff: the difference between two timepoints (e.g. `time.time()`)
+    """
+    if time_diff < 1.0:
+        # ms
+        return f'{time_diff * 1000:.0f} ms'
+    elif time_diff < 60.0:
+        # s
+        return f'{time_diff:.1f} s'
+    else:
+        # m s
+        return f'{time_diff // 60.0:.0f} m {time_diff % 60.0:.0f} s'

tracetools_analysis/tracetools_analysis/conversion/ctf.py

@@ -25,7 +25,7 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int:
     Load CTF trace, convert events, and dump to a pickle file.
 
     :param trace_directory: the trace directory
-    :param target: the target pickle file to write to
+    :param target: the target file to write to
     :return: the number of events written
     """
     ctf_events = get_trace_ctf_events(trace_directory)
@@ -43,15 +43,15 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int:
     return count_written
 
 
-def convert(trace_directory: str, pickle_target_path: str) -> int:
+def convert(trace_directory: str, output_file_path: str) -> int:
     """
     Convert CTF trace to pickle file.
 
     :param trace_directory: the trace directory
-    :param pickle_target_path: the path to the pickle file that will be created
+    :param output_file_path: the path to the output file that will be created
-    :return: the number of events written to the pickle file
+    :return: the number of events written to the output file
     """
-    with open(pickle_target_path, 'wb') as f:
+    with open(output_file_path, 'wb') as f:
         p = Pickler(f, protocol=4)
         count = ctf_to_pickle(trace_directory, p)
 

tracetools_analysis/tracetools_analysis/convert.py

@@ -13,38 +13,64 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Entrypoint/script to convert CTF trace data to a pickle file."""
+"""Entrypoint/script to convert CTF trace data to a file."""
 
 import argparse
 import os
 import time
+from typing import Optional
 
 from tracetools_analysis.conversion import ctf
 
+from . import time_diff_to_str
+
+
+DEFAULT_CONVERT_FILE_NAME = 'converted'
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        'trace_directory',
+        help='the path to the main trace directory')
+    parser.add_argument(
+        '-o', '--output-file-name', dest='output_file_name',
+        default=DEFAULT_CONVERT_FILE_NAME,
+        help='the name of the output file to generate, '
+        'under $trace_directory (default: %(default)s)')
+
 
 def parse_args():
     parser = argparse.ArgumentParser(
-        description='Convert CTF trace data to a pickle file.')
+        description='Convert trace data to a file.')
-    parser.add_argument(
+    add_args(parser)
-        'trace_directory', help='the path to the main CTF trace directory')
+    return parser.parse_args()
-    parser.add_argument(
+
-        '--pickle-path', '-p',
+
-        help='the path to the target pickle file to generate (default: $trace_directory/pickle)')
+def convert(
-    args = parser.parse_args()
+    trace_directory: str,
-    if args.pickle_path is None:
+    output_file_name: str = DEFAULT_CONVERT_FILE_NAME,
-        args.pickle_path = os.path.join(args.trace_directory, 'pickle')
+) -> Optional[int]:
-    return args
+    """
+    Convert trace directory to a file.
+
+    The output file will be placed under the trace directory.
+
+    :param trace_directory: the path to the trace directory to import
+    :param outout_file_name: the name of the output file
+    """
+    print(f'converting trace directory: {trace_directory}')
+    output_file_path = os.path.join(os.path.expanduser(trace_directory), output_file_name)
+    start_time = time.time()
+    count = ctf.convert(trace_directory, output_file_path)
+    time_diff = time.time() - start_time
+    print(f'converted {count} events in {time_diff_to_str(time_diff)}')
+    print(f'output written to: {output_file_path}')
 
 
 def main():
     args = parse_args()
 
     trace_directory = args.trace_directory
-    pickle_target_path = args.pickle_path
+    output_file_name = args.output_file_name
 
-    print(f'importing trace directory: {trace_directory}')
+    convert(trace_directory, output_file_name)
-    start_time = time.time()
-    count = ctf.convert(trace_directory, pickle_target_path)
-    time_diff = time.time() - start_time
-    print(f'converted {count} events in {time_diff * 1000:.2f} ms')
-    print(f'pickle written to: {pickle_target_path}')

tracetools_analysis/tracetools_analysis/loading/__init__.py

@@ -12,22 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Module for pickle loading."""
+"""Module for converted trace file loading."""
 
+import os
 import pickle
 from typing import Dict
 from typing import List
 
 
-def load_pickle(pickle_file_path: str) -> List[Dict]:
+def load_file(file_path: str) -> List[Dict]:
     """
-    Load pickle file containing converted trace events.
+    Load file containing converted trace events.
 
-    :param pickle_file_path: the path to the pickle file to load
+    :param file_path: the path to the converted file to load
     :return: the list of events read from the file
     """
     events = []
-    with open(pickle_file_path, 'rb') as f:
+    with open(os.path.expanduser(file_path), 'rb') as f:
         p = pickle.Unpickler(f)
         while True:
             try:

tracetools_analysis/tracetools_analysis/process.py

@@ -13,32 +13,137 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Entrypoint/script to process events from a pickle file to build a ROS model."""
+"""Entrypoint/script to process events from a converted file to build a ROS model."""
 
 import argparse
+import os
+import sys
 import time
+from typing import Optional
+from typing import Tuple
 
-from tracetools_analysis.loading import load_pickle
+from tracetools_analysis.convert import convert
+from tracetools_analysis.convert import DEFAULT_CONVERT_FILE_NAME
+from tracetools_analysis.loading import load_file
 from tracetools_analysis.processor.ros2 import Ros2Handler
+from tracetools_read.trace import is_trace_directory
+
+from . import time_diff_to_str
+
+
+def add_args(parser: argparse.ArgumentParser) -> None:
+    parser.add_argument(
+        'input_path',
+        help='the path to a converted file to import and process, '
+        'or the path to a CTF directory to convert and process')
+    parser.add_argument(
+        '-f', '--force-conversion', dest='force_conversion',
+        action='store_true', default=False,
+        help='re-convert trace directory even if converted file is found')
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Process a pickle file generated '
+    parser = argparse.ArgumentParser(description='Process a file converted from a trace '
-                                                 'from tracing and analyze the data.')
+                                                 'directory and output model data.')
-    parser.add_argument('pickle_file', help='the pickle file to import')
+    add_args(parser)
     return parser.parse_args()
 
 
+def inspect_input_path(
+    input_path: str,
+    force_conversion: bool = False,
+) -> Tuple[str, bool]:
+    """
+    Check input path for a converted file or a trace directory.
+
+    If the input path is a file, it uses it as a converted file.
+    If the input path is a directory, it checks if there is a "converted" file directly inside it,
+    otherwise it tries to import the path as a trace directory.
+    If `force_conversion` is set to `True`, even if a converted file is found, it will ask to
+    re-create it.
+
+    :param input_path: the path to a converted file or trace directory
+    :param force_conversion: whether to re-creating converted file even if it is found
+    :return:
+        the path to a converted file (or `None` if could not find),
+        `True` if the given converted file should be (re-)created, `False` otherwise
+    """
+    input_path = os.path.expanduser(input_path)
+    converted_file_path = None
+    # Check if not a file
+    if not os.path.isfile(input_path):
+        input_directory = input_path
+        # Might be a (trace) directory
+        # Check if there is a converted file under the given directory
+        prospective_converted_file = os.path.join(input_directory, DEFAULT_CONVERT_FILE_NAME)
+        if os.path.isfile(prospective_converted_file):
+            # Use that as the converted input file
+            converted_file_path = prospective_converted_file
+            if force_conversion:
+                print(f'found converted file but will re-create it: {prospective_converted_file}')
+                return prospective_converted_file, True
+            else:
+                print(f'found converted file: {prospective_converted_file}')
+                return prospective_converted_file, False
+        else:
+            # Check if it is a trace directory
+            # Result could be unexpected because it will look for trace directories recursively
+            # (e.g. '/' is a valid trace directory if there is at least one trace anywhere)
+            if is_trace_directory(input_directory):
+                # Convert trace directory first to create converted file
+                return prospective_converted_file, True
+            else:
+                # We cannot do anything
+                print(
+                    f'cannot find either a trace directory or a converted file: {input_directory}',
+                    file=sys.stderr)
+                return None, None
+    else:
+        converted_file_path = input_path
+        if force_conversion:
+            # It's a file, but re-create it anyway
+            print(f'found converted file but will re-create it: {converted_file_path}')
+            return converted_file_path, True
+        else:
+            # Simplest use-case: given path is an existing converted file
+            print(f'found converted file: {converted_file_path}')
+            return converted_file_path, False
+
+
+def process(
+    input_path: str,
+    force_conversion: bool = False,
+) -> Optional[int]:
+    """
+    Process converted trace file.
+
+    :param input_path: the path to a converted file or trace directory
+    :param force_conversion: whether to re-creating converted file even if it is found
+    """
+    converted_file_path, create_converted_file = inspect_input_path(input_path, force_conversion)
+
+    if converted_file_path is None:
+        return 1
+
+    # Convert trace directory to file if necessary
+    if create_converted_file:
+        input_directory = os.path.dirname(converted_file_path)
+        input_file_name = os.path.basename(converted_file_path)
+        convert(input_directory, input_file_name)
+
+    start_time = time.time()
+
+    events = load_file(converted_file_path)
+    ros2_handler = Ros2Handler.process(events)
+
+    time_diff = time.time() - start_time
+    ros2_handler.data.print_model()
+    print(f'processed {len(events)} events in {time_diff_to_str(time_diff)}')
+
+
 def main():
     args = parse_args()
-    pickle_filename = args.pickle_file
+    input_path = args.input_path
+    force_conversion = args.force_conversion
 
-    start_time = time.time()
+    process(input_path, force_conversion)
-
-    events = load_pickle(pickle_filename)
-    ros2_handler = Ros2Handler.process(events)
-
-    time_diff = time.time() - start_time
-    print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms')
-
-    ros2_handler.data.print_model()

tracetools_analysis/tracetools_analysis/processor/__init__.py

@@ -15,6 +15,7 @@
 """Base processor module."""
 
 from collections import defaultdict
+import sys
 from typing import Callable
 from typing import Dict
 from typing import List
@@ -258,6 +259,9 @@ class Processor():
         expanded_handlers = self._expand_dependencies(*handlers, **kwargs)
         self._handler_multimap = self._get_handler_maps(expanded_handlers)
         self._register_with_handlers(expanded_handlers)
+        self._progress_display = ProcessingProgressDisplay(
+            [type(handler).__name__ for handler in expanded_handlers],
+        )
 
     @staticmethod
     def _expand_dependencies(
@@ -306,8 +310,10 @@ class Processor():
 
         :param events: the events to process
         """
+        self._progress_display.set_work_total(len(events))
         for event in events:
             self._process_event(event)
+            self._progress_display.did_work()
 
     def _process_event(self, event: DictEvent) -> None:
         """Process a single event."""
@@ -339,3 +345,57 @@ class Processor():
                     raise_if_not_found=False)
                 metadata = EventMetadata(event_name, timestamp, cpu_id, procname, pid, tid)
                 handler_function(event, metadata)
+
+
+class ProcessingProgressDisplay():
+    """Display processing progress periodically on stdout."""
+
+    def __init__(
+        self,
+        processing_elements: List[str],
+    ) -> None:
+        """
+        Constructor.
+
+        :param processing_elements: the list of elements doing processing
+        """
+        self.__info_string = '[' + ', '.join(processing_elements) + ']'
+        self.__total_work = None
+        self.__progress_count = None
+        self.__rolling_count = None
+        self.__work_display_period = None
+
+    def set_work_total(
+        self,
+        total: int,
+    ) -> None:
+        """
+        Set the total units of work.
+
+        :param total: the total number of units of work to do
+        """
+        self.__total_work = total
+        self.__progress_count = 0
+        self.__rolling_count = 0
+        self.__work_display_period = total // 100
+        self._update()
+
+    def did_work(
+        self,
+        increment: int = 1,
+    ) -> None:
+        """
+        Increment the amount of work done.
+
+        :param increment: the number of units of work to add to the total
+        """
+        # For now, let it fail if set_work_total() hasn't been called
+        self.__progress_count += increment
+        self.__rolling_count += increment
+        if self.__rolling_count >= self.__work_display_period:
+            self.__rolling_count -= self.__work_display_period
+            self._update()
+
+    def _update(self) -> None:
+        percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work))
+        sys.stdout.write(f' [{percentage:2.0f}%] {self.__info_string}\r')