From 6f7a7a6d79cab672ecf3fb401c589dec5bf8a01e Mon Sep 17 00:00:00 2001
From: Christophe Bedard <fixed-term.christophe.bourquebedard@de.bosch.com>
Date: Wed, 7 Aug 2019 14:34:28 +0200
Subject: [PATCH 01/22] Display Processor progress on stdout

---
 .../tracetools_analysis/processor/__init__.py | 34 +++++++++++++++++++
 1 file changed, 34 insertions(+)

diff --git a/tracetools_analysis/tracetools_analysis/processor/__init__.py b/tracetools_analysis/tracetools_analysis/processor/__init__.py
index 9b440f5..8a9a712 100644
--- a/tracetools_analysis/tracetools_analysis/processor/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/processor/__init__.py
@@ -15,6 +15,7 @@
 """Base processor module."""
 
 from collections import defaultdict
+import sys
 from typing import Callable
 from typing import Dict
 from typing import List
@@ -258,6 +259,9 @@ class Processor():
         expanded_handlers = self._expand_dependencies(*handlers, **kwargs)
         self._handler_multimap = self._get_handler_maps(expanded_handlers)
         self._register_with_handlers(expanded_handlers)
+        self._progress_display = ProcessingProgressDisplay(
+            [type(handler).__name__ for handler in expanded_handlers],
+        )
 
     @staticmethod
     def _expand_dependencies(
@@ -306,8 +310,10 @@ class Processor():
 
         :param events: the events to process
         """
+        self._progress_display.set_work_total(len(events))
         for event in events:
             self._process_event(event)
+            self._progress_display.did_work()
 
     def _process_event(self, event: DictEvent) -> None:
         """Process a single event."""
@@ -339,3 +345,31 @@ class Processor():
                     raise_if_not_found=False)
                 metadata = EventMetadata(event_name, timestamp, cpu_id, procname, pid, tid)
                 handler_function(event, metadata)
+
+
+class ProcessingProgressDisplay():
+
+    def __init__(
+        self,
+        processing_elements: List[str],
+    ) -> None:
+        self.__info_string = '[' + ', '.join(processing_elements) + ']'
+        self.__progress_count = 0
+        self.__total_work = 0
+        self.__work_display_period = 1
+
+    def set_work_total(
+        self,
+        total: int,
+    ) -> None:
+        self.__total_work = total
+        self.__work_display_period = int(self.__total_work / 100.0)
+
+    def did_work(
+        self,
+        increment: int = 1,
+    ) -> None:
+        self.__progress_count += increment
+        if self.__progress_count % self.__work_display_period == 0:
+            percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work))
+            sys.stdout.write(f' [{percentage:2.0f}%] {self.__info_string}\r')

From 625491d24232e22a8a3320137e32e2818e1f0f3a Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 15:11:08 -0700
Subject: [PATCH 02/22] Document ProcessingProgressDisplay

---
 .../tracetools_analysis/processor/__init__.py    | 16 ++++++++++++++++
 1 file changed, 16 insertions(+)

diff --git a/tracetools_analysis/tracetools_analysis/processor/__init__.py b/tracetools_analysis/tracetools_analysis/processor/__init__.py
index 8a9a712..d072b5e 100644
--- a/tracetools_analysis/tracetools_analysis/processor/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/processor/__init__.py
@@ -348,11 +348,17 @@ class Processor():
 
 
 class ProcessingProgressDisplay():
+    """Display processing progress periodically on stdout."""
 
     def __init__(
         self,
         processing_elements: List[str],
     ) -> None:
+        """
+        Constructor.
+
+        :param processing_elements: the list of elements doing processing
+        """
         self.__info_string = '[' + ', '.join(processing_elements) + ']'
         self.__progress_count = 0
         self.__total_work = 0
@@ -362,6 +368,11 @@ class ProcessingProgressDisplay():
         self,
         total: int,
     ) -> None:
+        """
+        Set the total units of work.
+
+        :param total: the total number of units of work to do
+        """
         self.__total_work = total
         self.__work_display_period = int(self.__total_work / 100.0)
 
@@ -369,6 +380,11 @@ class ProcessingProgressDisplay():
         self,
         increment: int = 1,
     ) -> None:
+        """
+        Increment the amount of work done.
+
+        :param increment: the number of units of work to add to the total
+        """
         self.__progress_count += increment
         if self.__progress_count % self.__work_display_period == 0:
             percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work))

From 844a215156ad3f1ef0ad1ea3eb9d20876d1be006 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 15:11:40 -0700
Subject: [PATCH 03/22] Add --debug option to print model after processing

---
 tracetools_analysis/tracetools_analysis/process.py | 9 +++++++--
 1 file changed, 7 insertions(+), 2 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 59cae89..e7ce653 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -26,12 +26,17 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Process a pickle file generated '
                                                  'from tracing and analyze the data.')
     parser.add_argument('pickle_file', help='the pickle file to import')
+    parser.add_argument(
+        '-d', '--debug',
+        action='store_true', default=False,
+        help='display debug information (e.g. resulting model)')
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
     pickle_filename = args.pickle_file
+    debug = args.debug
 
     start_time = time.time()
 
@@ -39,6 +44,6 @@ def main():
     ros2_handler = Ros2Handler.process(events)
 
     time_diff = time.time() - start_time
+    if debug:
+        ros2_handler.data.print_model()
     print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms')
-
-    ros2_handler.data.print_model()

From 36b789dc6ab6ebca9ce4e539f84417e81f4dc169 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 15:24:48 -0700
Subject: [PATCH 04/22] Remove references to "pickle" file and simply use
 "output" file

---
 .../analysis/callback_duration.ipynb          |  14 +++++++-------
 .../{pickle_pingpong => converted_pingpong}   | Bin
 .../tracetools_analysis/conversion/ctf.py     |  10 +++++-----
 .../tracetools_analysis/convert.py            |  18 +++++++++---------
 .../tracetools_analysis/loading/__init__.py   |  10 +++++-----
 .../tracetools_analysis/process.py            |  15 ++++++++-------
 6 files changed, 34 insertions(+), 33 deletions(-)
 rename tracetools_analysis/analysis/sample_data/{pickle_pingpong => converted_pingpong} (100%)

diff --git a/tracetools_analysis/analysis/callback_duration.ipynb b/tracetools_analysis/analysis/callback_duration.ipynb
index 9bab160..faac368 100644
--- a/tracetools_analysis/analysis/callback_duration.ipynb
+++ b/tracetools_analysis/analysis/callback_duration.ipynb
@@ -17,8 +17,8 @@
     "#\n",
     "# OR\n",
     "#\n",
-    "# Use the provided sample pickle file, changing the path below to:\n",
-    "#    'sample_data/pickle_pingpong'"
+    "# Use the provided sample converted trace file, changing the path below to:\n",
+    "#    'sample_data/converted_pingpong'"
    ]
   },
   {
@@ -27,8 +27,8 @@
    "metadata": {},
    "outputs": [],
    "source": [
-    "pickle_path = '~/.ros/tracing/pingpong/ust/pickle'\n",
-    "#pickle_path = 'sample_data/pickle_pingpong'"
+    "converted_file_path = '~/.ros/tracing/pingpong/ust/converted'\n",
+    "#converted_file_path = 'sample_data/converted_pingpong'"
    ]
   },
   {
@@ -57,7 +57,7 @@
     "import pandas as pd\n",
     "\n",
     "from tracetools_analysis import utils\n",
-    "from tracetools_analysis.loading import load_pickle\n",
+    "from tracetools_analysis.loading import load_file\n",
     "from tracetools_analysis.processor.ros2 import Ros2Handler"
    ]
   },
@@ -68,8 +68,8 @@
    "outputs": [],
    "source": [
     "# Process\n",
-    "pickle_path = os.path.expanduser(pickle_path)\n",
-    "events = load_pickle(pickle_path)\n",
+    "converted_file_path = os.path.expanduser(converted_file_path)\n",
+    "events = load_file(converted_file_path)\n",
     "handler = Ros2Handler.process(events)\n",
     "#handler.data.print_model()"
    ]
diff --git a/tracetools_analysis/analysis/sample_data/pickle_pingpong b/tracetools_analysis/analysis/sample_data/converted_pingpong
similarity index 100%
rename from tracetools_analysis/analysis/sample_data/pickle_pingpong
rename to tracetools_analysis/analysis/sample_data/converted_pingpong
diff --git a/tracetools_analysis/tracetools_analysis/conversion/ctf.py b/tracetools_analysis/tracetools_analysis/conversion/ctf.py
index aebcbaf..3f0fb74 100644
--- a/tracetools_analysis/tracetools_analysis/conversion/ctf.py
+++ b/tracetools_analysis/tracetools_analysis/conversion/ctf.py
@@ -25,7 +25,7 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int:
     Load CTF trace, convert events, and dump to a pickle file.
 
     :param trace_directory: the trace directory
-    :param target: the target pickle file to write to
+    :param target: the target file to write to
     :return: the number of events written
     """
     ctf_events = get_trace_ctf_events(trace_directory)
@@ -43,15 +43,15 @@ def ctf_to_pickle(trace_directory: str, target: Pickler) -> int:
     return count_written
 
 
-def convert(trace_directory: str, pickle_target_path: str) -> int:
+def convert(trace_directory: str, output_file_path: str) -> int:
     """
     Convert CTF trace to pickle file.
 
     :param trace_directory: the trace directory
-    :param pickle_target_path: the path to the pickle file that will be created
-    :return: the number of events written to the pickle file
+    :param output_file_path: the path to the output file that will be created
+    :return: the number of events written to the output file
     """
-    with open(pickle_target_path, 'wb') as f:
+    with open(output_file_path, 'wb') as f:
         p = Pickler(f, protocol=4)
         count = ctf_to_pickle(trace_directory, p)
 
diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index b1ffa42..a8f16e2 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -13,7 +13,7 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Entrypoint/script to convert CTF trace data to a pickle file."""
+"""Entrypoint/script to convert CTF trace data to a file."""
 
 import argparse
 import os
@@ -24,15 +24,15 @@ from tracetools_analysis.conversion import ctf
 
 def parse_args():
     parser = argparse.ArgumentParser(
-        description='Convert CTF trace data to a pickle file.')
+        description='Convert CTF trace data to a file.')
     parser.add_argument(
         'trace_directory', help='the path to the main CTF trace directory')
     parser.add_argument(
-        '--pickle-path', '-p',
-        help='the path to the target pickle file to generate (default: $trace_directory/pickle)')
+        '-o', '--output-file-path',
+        help='the path to the output file to generate (default: $trace_directory/converted)')
     args = parser.parse_args()
-    if args.pickle_path is None:
-        args.pickle_path = os.path.join(args.trace_directory, 'pickle')
+    if args.output_file_path is None:
+        args.output_file_path = os.path.join(args.trace_directory, 'converted')
     return args
 
 
@@ -40,11 +40,11 @@ def main():
     args = parse_args()
 
     trace_directory = args.trace_directory
-    pickle_target_path = args.pickle_path
+    output_file_path = args.output_file_path
 
     print(f'importing trace directory: {trace_directory}')
     start_time = time.time()
-    count = ctf.convert(trace_directory, pickle_target_path)
+    count = ctf.convert(trace_directory, output_file_path)
     time_diff = time.time() - start_time
     print(f'converted {count} events in {time_diff * 1000:.2f} ms')
-    print(f'pickle written to: {pickle_target_path}')
+    print(f'output written to: {output_file_path}')
diff --git a/tracetools_analysis/tracetools_analysis/loading/__init__.py b/tracetools_analysis/tracetools_analysis/loading/__init__.py
index e79dfa4..0347e7e 100644
--- a/tracetools_analysis/tracetools_analysis/loading/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/loading/__init__.py
@@ -12,22 +12,22 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Module for pickle loading."""
+"""Module for converted trace file loading."""
 
 import pickle
 from typing import Dict
 from typing import List
 
 
-def load_pickle(pickle_file_path: str) -> List[Dict]:
+def load_file(file_path: str) -> List[Dict]:
     """
-    Load pickle file containing converted trace events.
+    Load file containing converted trace events.
 
-    :param pickle_file_path: the path to the pickle file to load
+    :param file_path: the path to the converted file to load
     :return: the list of events read from the file
     """
     events = []
-    with open(pickle_file_path, 'rb') as f:
+    with open(file_path, 'rb') as f:
         p = pickle.Unpickler(f)
         while True:
             try:
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index e7ce653..70cc7e1 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -13,19 +13,20 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Entrypoint/script to process events from a pickle file to build a ROS model."""
+"""Entrypoint/script to process events from a converted file to build a ROS model."""
 
 import argparse
 import time
 
-from tracetools_analysis.loading import load_pickle
+from tracetools_analysis.loading import load_file
 from tracetools_analysis.processor.ros2 import Ros2Handler
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Process a pickle file generated '
-                                                 'from tracing and analyze the data.')
-    parser.add_argument('pickle_file', help='the pickle file to import')
+    parser = argparse.ArgumentParser(description='Process a file converted from a trace'
+                                                 'directory and analyze the data.')
+    parser.add_argument(
+        'output_file_path', help='the converted file to import')
     parser.add_argument(
         '-d', '--debug',
         action='store_true', default=False,
@@ -35,12 +36,12 @@ def parse_args():
 
 def main():
     args = parse_args()
-    pickle_filename = args.pickle_file
+    output_file_path = args.output_file_path
     debug = args.debug
 
     start_time = time.time()
 
-    events = load_pickle(pickle_filename)
+    events = load_file(output_file_path)
     ros2_handler = Ros2Handler.process(events)
 
     time_diff = time.time() - start_time

From 05caf3a4cd70ec2cb1dccb7668270b01e4502f60 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 15:36:04 -0700
Subject: [PATCH 05/22] Re-order arg flags

---
 tracetools_analysis/tracetools_analysis/convert.py | 5 +++--
 tracetools_analysis/tracetools_analysis/process.py | 5 +++--
 2 files changed, 6 insertions(+), 4 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index a8f16e2..88f33e2 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -26,9 +26,10 @@ def parse_args():
     parser = argparse.ArgumentParser(
         description='Convert CTF trace data to a file.')
     parser.add_argument(
-        'trace_directory', help='the path to the main CTF trace directory')
+        'trace_directory',
+        help='the path to the main CTF trace directory')
     parser.add_argument(
-        '-o', '--output-file-path',
+        '-o', '--output-file-path', dest='output_file_path',
         help='the path to the output file to generate (default: $trace_directory/converted)')
     args = parser.parse_args()
     if args.output_file_path is None:
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 70cc7e1..e629c29 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -26,9 +26,10 @@ def parse_args():
     parser = argparse.ArgumentParser(description='Process a file converted from a trace'
                                                  'directory and analyze the data.')
     parser.add_argument(
-        'output_file_path', help='the converted file to import')
+        'output_file_path',
+        help='the converted file to import')
     parser.add_argument(
-        '-d', '--debug',
+        '-d', '--debug', dest='debug',
         action='store_true', default=False,
         help='display debug information (e.g. resulting model)')
     return parser.parse_args()

From b26fe178e2549e0bfb6e93d5e20719c60664cc23 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 16:28:53 -0700
Subject: [PATCH 06/22] Fix modulo 0 error with progress display

---
 .../tracetools_analysis/processor/__init__.py | 24 +++++++++++++------
 1 file changed, 17 insertions(+), 7 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/processor/__init__.py b/tracetools_analysis/tracetools_analysis/processor/__init__.py
index d072b5e..b807e2d 100644
--- a/tracetools_analysis/tracetools_analysis/processor/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/processor/__init__.py
@@ -360,9 +360,10 @@ class ProcessingProgressDisplay():
         :param processing_elements: the list of elements doing processing
         """
         self.__info_string = '[' + ', '.join(processing_elements) + ']'
-        self.__progress_count = 0
-        self.__total_work = 0
-        self.__work_display_period = 1
+        self.__total_work = None
+        self.__progress_count = None
+        self.__rolling_count = None
+        self.__work_display_period = None
 
     def set_work_total(
         self,
@@ -374,7 +375,10 @@ class ProcessingProgressDisplay():
         :param total: the total number of units of work to do
         """
         self.__total_work = total
-        self.__work_display_period = int(self.__total_work / 100.0)
+        self.__progress_count = 0
+        self.__rolling_count = 0
+        self.__work_display_period = total // 100
+        self._update()
 
     def did_work(
         self,
@@ -385,7 +389,13 @@ class ProcessingProgressDisplay():
 
         :param increment: the number of units of work to add to the total
         """
+        # For now, let it fail if set_work_total() hasn't been called
         self.__progress_count += increment
-        if self.__progress_count % self.__work_display_period == 0:
-            percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work))
-            sys.stdout.write(f' [{percentage:2.0f}%] {self.__info_string}\r')
+        self.__rolling_count += increment
+        if self.__rolling_count >= self.__work_display_period:
+            self.__rolling_count -= self.__work_display_period
+            self._update()
+
+    def _update(self) -> None:
+        percentage = 100.0 * (float(self.__progress_count) / float(self.__total_work))
+        sys.stdout.write(f' [{percentage:2.0f}%] {self.__info_string}\r')

From 5f3be48c578a00e7b2aa9ca1169c0f2f8c8a8a06 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 16:31:05 -0700
Subject: [PATCH 07/22] Fix process help message

---
 tracetools_analysis/tracetools_analysis/process.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index e629c29..d82dcdb 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -23,8 +23,8 @@ from tracetools_analysis.processor.ros2 import Ros2Handler
 
 
 def parse_args():
-    parser = argparse.ArgumentParser(description='Process a file converted from a trace'
-                                                 'directory and analyze the data.')
+    parser = argparse.ArgumentParser(description='Process a file converted from a trace '
+                                                 'directory and output model data.')
     parser.add_argument(
         'output_file_path',
         help='the converted file to import')

From 7b19b9e0682462c01dcf6491aa1698a847343b28 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 16:31:49 -0700
Subject: [PATCH 08/22] Remove debug flag

---
 tracetools_analysis/tracetools_analysis/process.py | 8 +-------
 1 file changed, 1 insertion(+), 7 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index d82dcdb..687b295 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -28,17 +28,12 @@ def parse_args():
     parser.add_argument(
         'output_file_path',
         help='the converted file to import')
-    parser.add_argument(
-        '-d', '--debug', dest='debug',
-        action='store_true', default=False,
-        help='display debug information (e.g. resulting model)')
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
     output_file_path = args.output_file_path
-    debug = args.debug
 
     start_time = time.time()
 
@@ -46,6 +41,5 @@ def main():
     ros2_handler = Ros2Handler.process(events)
 
     time_diff = time.time() - start_time
-    if debug:
-        ros2_handler.data.print_model()
+    ros2_handler.data.print_model()
     print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms')

From 8a0c3a4eb4610a8a5f3d5d4a4ee66a2f0f3b4128 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 17:20:08 -0700
Subject: [PATCH 09/22] Make process command convert directory if necessary

---
 .../tracetools_analysis/convert.py            | 27 +++++++++-----
 .../tracetools_analysis/process.py            | 35 ++++++++++++++++---
 2 files changed, 50 insertions(+), 12 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index 88f33e2..74832f2 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -22,6 +22,9 @@ import time
 from tracetools_analysis.conversion import ctf
 
 
+DEFAULT_CONVERT_FILE_NAME = 'converted'
+
+
 def parse_args():
     parser = argparse.ArgumentParser(
         description='Convert CTF trace data to a file.')
@@ -30,22 +33,30 @@ def parse_args():
         help='the path to the main CTF trace directory')
     parser.add_argument(
         '-o', '--output-file-path', dest='output_file_path',
-        help='the path to the output file to generate (default: $trace_directory/converted)')
+        help='the path to the output file to generate '
+        f'(default: $trace_directory/{DEFAULT_CONVERT_FILE_NAME})')
     args = parser.parse_args()
     if args.output_file_path is None:
-        args.output_file_path = os.path.join(args.trace_directory, 'converted')
+        args.output_file_path = os.path.join(args.trace_directory, DEFAULT_CONVERT_FILE_NAME)
     return args
 
 
+def convert(
+    trace_directory: str,
+    output_file_path: str,
+) -> None:
+    print(f'importing trace directory: {trace_directory}')
+    start_time = time.time()
+    count = ctf.convert(trace_directory, output_file_path)
+    time_diff = time.time() - start_time
+    print(f'converted {count} events in {time_diff * 1000:.2f} ms')
+    print(f'output written to: {output_file_path}')
+
+
 def main():
     args = parse_args()
 
     trace_directory = args.trace_directory
     output_file_path = args.output_file_path
 
-    print(f'importing trace directory: {trace_directory}')
-    start_time = time.time()
-    count = ctf.convert(trace_directory, output_file_path)
-    time_diff = time.time() - start_time
-    print(f'converted {count} events in {time_diff * 1000:.2f} ms')
-    print(f'output written to: {output_file_path}')
+    convert(trace_directory, output_file_path)
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 687b295..4a44138 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -16,28 +16,55 @@
 """Entrypoint/script to process events from a converted file to build a ROS model."""
 
 import argparse
+import os
+import sys
 import time
 
+from tracetools_analysis.convert import convert
+from tracetools_analysis.convert import DEFAULT_CONVERT_FILE_NAME
 from tracetools_analysis.loading import load_file
 from tracetools_analysis.processor.ros2 import Ros2Handler
+from tracetools_read.trace import is_trace_directory
 
 
 def parse_args():
     parser = argparse.ArgumentParser(description='Process a file converted from a trace '
                                                  'directory and output model data.')
     parser.add_argument(
-        'output_file_path',
-        help='the converted file to import')
+        'input_path',
+        help='the path to a converted file to import, '
+        'or the path to a CTF directory to convert and process')
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
-    output_file_path = args.output_file_path
+    input_path = args.input_path
 
     start_time = time.time()
 
-    events = load_file(output_file_path)
+    # Check if not a file
+    if not os.path.isfile(input_path):
+        # Might be a trace directory
+        # Check if there is a converted file
+        prospective_converted_file = os.path.join(input_path, DEFAULT_CONVERT_FILE_NAME)
+        if os.path.isfile(prospective_converted_file):
+            # Use that as the converted input file
+            print(f'found converted file: {prospective_converted_file}')
+            input_path = prospective_converted_file
+        else:
+            # Check if it is a trace directory
+            # Result could be unexpected because it will look for trace directories recursively
+            if is_trace_directory(input_path):
+                # Convert trace directory first to create converted file
+                convert(input_path, prospective_converted_file)
+                input_path = prospective_converted_file
+            else:
+                # We cannot do anything
+                print('cannot find either a trace directory or a converted file', file=sys.stderr)
+                return 1
+
+    events = load_file(input_path)
     ros2_handler = Ros2Handler.process(events)
 
     time_diff = time.time() - start_time

From e23c2d62c14eb31faaf6ef16f15b94ac62168adc Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 18:00:25 -0700
Subject: [PATCH 10/22] Switch to output file name instead of full path

---
 .../tracetools_analysis/convert.py            | 29 ++++++++++++-------
 .../tracetools_analysis/process.py            |  8 +++--
 2 files changed, 23 insertions(+), 14 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index 74832f2..7ae73a5 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -32,20 +32,27 @@ def parse_args():
         'trace_directory',
         help='the path to the main CTF trace directory')
     parser.add_argument(
-        '-o', '--output-file-path', dest='output_file_path',
-        help='the path to the output file to generate '
-        f'(default: $trace_directory/{DEFAULT_CONVERT_FILE_NAME})')
-    args = parser.parse_args()
-    if args.output_file_path is None:
-        args.output_file_path = os.path.join(args.trace_directory, DEFAULT_CONVERT_FILE_NAME)
-    return args
+        '-o', '--output-file-name', dest='output_file_name',
+        default=DEFAULT_CONVERT_FILE_NAME,
+        help='the name of the output file to generate, '
+        'under $trace_directory (default: %(default)s)')
+    return parser.parse_args()
 
 
 def convert(
     trace_directory: str,
-    output_file_path: str,
+    output_file_name: str = DEFAULT_CONVERT_FILE_NAME,
 ) -> None:
-    print(f'importing trace directory: {trace_directory}')
+    """
+    Convert trace directory to a file.
+
+    The output file will be placed under the trace directory.
+
+    :param trace_directory: the path to the trace directory to import
+    :param outout_file_name: the name of the output file
+    """
+    print(f'converting trace directory: {trace_directory}')
+    output_file_path = os.path.join(os.path.expanduser(trace_directory), output_file_name)
     start_time = time.time()
     count = ctf.convert(trace_directory, output_file_path)
     time_diff = time.time() - start_time
@@ -57,6 +64,6 @@ def main():
     args = parse_args()
 
     trace_directory = args.trace_directory
-    output_file_path = args.output_file_path
+    output_file_name = args.output_file_name
 
-    convert(trace_directory, output_file_path)
+    convert(trace_directory, output_file_name)
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 4a44138..3bac1a1 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -32,7 +32,7 @@ def parse_args():
                                                  'directory and output model data.')
     parser.add_argument(
         'input_path',
-        help='the path to a converted file to import, '
+        help='the path to a converted file to import and process, '
         'or the path to a CTF directory to convert and process')
     return parser.parse_args()
 
@@ -57,11 +57,13 @@ def main():
             # Result could be unexpected because it will look for trace directories recursively
             if is_trace_directory(input_path):
                 # Convert trace directory first to create converted file
-                convert(input_path, prospective_converted_file)
+                convert(input_path, DEFAULT_CONVERT_FILE_NAME)
                 input_path = prospective_converted_file
             else:
                 # We cannot do anything
-                print('cannot find either a trace directory or a converted file', file=sys.stderr)
+                print(
+                    f'cannot find either a trace directory or a converted file: {input_path}',
+                    file=sys.stderr)
                 return 1
 
     events = load_file(input_path)

From 459362bd535051e56f969fb1f4ba1746b7635370 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 18:00:45 -0700
Subject: [PATCH 11/22] Make sure to expand user path

---
 tracetools_analysis/tracetools_analysis/process.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 3bac1a1..63acdd2 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -43,6 +43,7 @@ def main():
 
     start_time = time.time()
 
+    input_path = os.path.expanduser(input_path)
     # Check if not a file
     if not os.path.isfile(input_path):
         # Might be a trace directory

From 87ff5c245af9b18f38820dd41dc080bde82d7f9a Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sat, 12 Oct 2019 20:38:43 -0700
Subject: [PATCH 12/22] Add flag for process command to force re-conversion of
 trace directory

---
 .../tracetools_analysis/process.py            | 46 +++++++++++++------
 1 file changed, 33 insertions(+), 13 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 63acdd2..4da6538 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -34,40 +34,60 @@ def parse_args():
         'input_path',
         help='the path to a converted file to import and process, '
         'or the path to a CTF directory to convert and process')
+    parser.add_argument(
+        '-f', '--force-conversion', dest='force_conversion',
+        action='store_true', default=False,
+        help='re-convert trace directory even if converted file is found')
     return parser.parse_args()
 
 
 def main():
     args = parse_args()
     input_path = args.input_path
+    force_conversion = args.force_conversion
 
-    start_time = time.time()
-
+    converted_file_path = None
     input_path = os.path.expanduser(input_path)
     # Check if not a file
     if not os.path.isfile(input_path):
-        # Might be a trace directory
-        # Check if there is a converted file
-        prospective_converted_file = os.path.join(input_path, DEFAULT_CONVERT_FILE_NAME)
-        if os.path.isfile(prospective_converted_file):
+        input_directory = input_path
+        # Might be a (trace) directory
+        # Check if there is a converted file under the given directory
+        prospective_converted_file_path = os.path.join(input_directory, DEFAULT_CONVERT_FILE_NAME)
+        if os.path.isfile(prospective_converted_file_path):
             # Use that as the converted input file
-            print(f'found converted file: {prospective_converted_file}')
-            input_path = prospective_converted_file
+            converted_file_path = prospective_converted_file_path
+            if force_conversion:
+                print(f'found converted file but re-creating it: {converted_file_path}')
+                convert(input_directory, DEFAULT_CONVERT_FILE_NAME)
+            else:
+                print(f'found converted file: {converted_file_path}')
         else:
             # Check if it is a trace directory
             # Result could be unexpected because it will look for trace directories recursively
-            if is_trace_directory(input_path):
+            # (e.g. '/' is a valid trace directory if there is at least one trace anywhere)
+            if is_trace_directory(input_directory):
                 # Convert trace directory first to create converted file
-                convert(input_path, DEFAULT_CONVERT_FILE_NAME)
-                input_path = prospective_converted_file
+                convert(input_directory, DEFAULT_CONVERT_FILE_NAME)
+                converted_file_path = prospective_converted_file_path
             else:
                 # We cannot do anything
                 print(
-                    f'cannot find either a trace directory or a converted file: {input_path}',
+                    f'cannot find either a trace directory or a converted file: {input_directory}',
                     file=sys.stderr)
                 return 1
+    else:
+        converted_file_path = input_path
+        if force_conversion:
+            # It's a file, but re-create it anyway
+            print(f'found converted file but re-creating it: {converted_file_path}')
+            input_directory = os.path.dirname(converted_file_path)
+            input_file_name = os.path.basename(converted_file_path)
+            convert(input_directory, input_file_name)
 
-    events = load_file(input_path)
+    start_time = time.time()
+
+    events = load_file(converted_file_path)
     ros2_handler = Ros2Handler.process(events)
 
     time_diff = time.time() - start_time

From 329151d7d95f39b595c9ba36ea9dc3a98e894776 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 11:36:05 -0700
Subject: [PATCH 13/22] Extract processing path checking function

---
 .../tracetools_analysis/process.py            | 82 +++++++++++++++----
 1 file changed, 64 insertions(+), 18 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 4da6538..a199f4c 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -19,6 +19,7 @@ import argparse
 import os
 import sys
 import time
+from typing import Tuple
 
 from tracetools_analysis.convert import convert
 from tracetools_analysis.convert import DEFAULT_CONVERT_FILE_NAME
@@ -41,49 +42,86 @@ def parse_args():
     return parser.parse_args()
 
 
-def main():
-    args = parse_args()
-    input_path = args.input_path
-    force_conversion = args.force_conversion
+def inspect_input_path(
+    input_path: str,
+    force_conversion: bool = False,
+) -> Tuple[str, bool]:
+    """
+    Check input path for a converted file or a trace directory.
 
-    converted_file_path = None
+    If the input path is a file, it uses it as a converted file.
+    If the input path is a directory, it checks if there is a "converted" file directly inside it,
+    otherwise it tries to import the path as a trace directory.
+    If `force_conversion` is set to `True`, even if a converted file is found, it will ask to
+    re-create it.
+
+    :param input_path: the path to a converted file or trace directory
+    :param force_conversion: whether to re-creating converted file even if it is found
+    :return:
+        the path to a converted file (or `None` if could not find),
+        `True` if the given converted file should be (re-)created, `False` otherwise
+    """
     input_path = os.path.expanduser(input_path)
+    converted_file_path = None
     # Check if not a file
     if not os.path.isfile(input_path):
         input_directory = input_path
         # Might be a (trace) directory
         # Check if there is a converted file under the given directory
-        prospective_converted_file_path = os.path.join(input_directory, DEFAULT_CONVERT_FILE_NAME)
-        if os.path.isfile(prospective_converted_file_path):
+        prospective_converted_file = os.path.join(input_directory, DEFAULT_CONVERT_FILE_NAME)
+        if os.path.isfile(prospective_converted_file):
             # Use that as the converted input file
-            converted_file_path = prospective_converted_file_path
+            converted_file_path = prospective_converted_file
             if force_conversion:
-                print(f'found converted file but re-creating it: {converted_file_path}')
-                convert(input_directory, DEFAULT_CONVERT_FILE_NAME)
+                print(f'found converted file but will re-create it: {prospective_converted_file}')
+                return prospective_converted_file, True
             else:
-                print(f'found converted file: {converted_file_path}')
+                print(f'found converted file: {prospective_converted_file}')
+                return prospective_converted_file, False
         else:
             # Check if it is a trace directory
             # Result could be unexpected because it will look for trace directories recursively
             # (e.g. '/' is a valid trace directory if there is at least one trace anywhere)
             if is_trace_directory(input_directory):
                 # Convert trace directory first to create converted file
-                convert(input_directory, DEFAULT_CONVERT_FILE_NAME)
-                converted_file_path = prospective_converted_file_path
+                return prospective_converted_file, True
             else:
                 # We cannot do anything
                 print(
                     f'cannot find either a trace directory or a converted file: {input_directory}',
                     file=sys.stderr)
-                return 1
+                return None, None
     else:
         converted_file_path = input_path
         if force_conversion:
             # It's a file, but re-create it anyway
-            print(f'found converted file but re-creating it: {converted_file_path}')
-            input_directory = os.path.dirname(converted_file_path)
-            input_file_name = os.path.basename(converted_file_path)
-            convert(input_directory, input_file_name)
+            print(f'found converted file but will re-create it: {converted_file_path}')
+            return converted_file_path, True
+        else:
+            # Simplest use-case: given path is an existing converted file
+            return converted_file_path, False
+
+
+def process(
+    input_path: str,
+    force_conversion: bool = False,
+) -> None:
+    """
+    Process converted trace file.
+
+    :param input_path: the path to a converted file or trace directory
+    :param force_conversion: whether to re-creating converted file even if it is found
+    """
+    converted_file_path, create_converted_file = inspect_input_path(input_path, force_conversion)
+
+    if converted_file_path is None:
+        return 1
+
+    # Convert trace directory to file if necessary
+    if create_converted_file:
+        input_directory = os.path.dirname(converted_file_path)
+        input_file_name = os.path.basename(converted_file_path)
+        convert(input_directory, input_file_name)
 
     start_time = time.time()
 
@@ -93,3 +131,11 @@ def main():
     time_diff = time.time() - start_time
     ros2_handler.data.print_model()
     print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms')
+
+
+def main():
+    args = parse_args()
+    input_path = args.input_path
+    force_conversion = args.force_conversion
+
+    process(input_path, force_conversion)

From faf8965b7b552c09206a16109d5aedf54a9f6944 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 11:49:38 -0700
Subject: [PATCH 14/22] Move tests to test directory

---
 .../test/{tracetools_analysis => }/test_data_model_util.py        | 0
 .../test/{tracetools_analysis => }/test_dependency_solver.py      | 0
 .../test/{tracetools_analysis => }/test_processor.py              | 0
 .../test/{tracetools_analysis => }/test_profile_handler.py        | 0
 4 files changed, 0 insertions(+), 0 deletions(-)
 rename tracetools_analysis/test/{tracetools_analysis => }/test_data_model_util.py (100%)
 rename tracetools_analysis/test/{tracetools_analysis => }/test_dependency_solver.py (100%)
 rename tracetools_analysis/test/{tracetools_analysis => }/test_processor.py (100%)
 rename tracetools_analysis/test/{tracetools_analysis => }/test_profile_handler.py (100%)

diff --git a/tracetools_analysis/test/tracetools_analysis/test_data_model_util.py b/tracetools_analysis/test/test_data_model_util.py
similarity index 100%
rename from tracetools_analysis/test/tracetools_analysis/test_data_model_util.py
rename to tracetools_analysis/test/test_data_model_util.py
diff --git a/tracetools_analysis/test/tracetools_analysis/test_dependency_solver.py b/tracetools_analysis/test/test_dependency_solver.py
similarity index 100%
rename from tracetools_analysis/test/tracetools_analysis/test_dependency_solver.py
rename to tracetools_analysis/test/test_dependency_solver.py
diff --git a/tracetools_analysis/test/tracetools_analysis/test_processor.py b/tracetools_analysis/test/test_processor.py
similarity index 100%
rename from tracetools_analysis/test/tracetools_analysis/test_processor.py
rename to tracetools_analysis/test/test_processor.py
diff --git a/tracetools_analysis/test/tracetools_analysis/test_profile_handler.py b/tracetools_analysis/test/test_profile_handler.py
similarity index 100%
rename from tracetools_analysis/test/tracetools_analysis/test_profile_handler.py
rename to tracetools_analysis/test/test_profile_handler.py

From 6466c9953fc5e5e1cf6b483609ca586288a95191 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 13:00:53 -0700
Subject: [PATCH 15/22] Add test for process command and input path inspection

---
 .../test/test_process_command.py              | 90 +++++++++++++++++++
 .../tracetools_analysis/process.py            |  1 +
 2 files changed, 91 insertions(+)
 create mode 100644 tracetools_analysis/test/test_process_command.py

diff --git a/tracetools_analysis/test/test_process_command.py b/tracetools_analysis/test/test_process_command.py
new file mode 100644
index 0000000..141039f
--- /dev/null
+++ b/tracetools_analysis/test/test_process_command.py
@@ -0,0 +1,90 @@
+#!/usr/bin/env python3
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import os
+import shutil
+import tempfile
+import unittest
+
+from tracetools_analysis.process import inspect_input_path
+
+
+class TestProcessCommand(unittest.TestCase):
+
+    def __init__(self, *args) -> None:
+        super().__init__(
+            *args,
+        )
+
+    def setUp(self):
+        self.test_dir_path = tempfile.mkdtemp()
+
+        # Create directory that contains a 'converted' file
+        self.with_converted_file_dir = os.path.join(
+            self.test_dir_path,
+            'with_converted_file',
+        )
+        os.mkdir(self.with_converted_file_dir)
+        self.converted_file_path = os.path.join(
+            self.with_converted_file_dir,
+            'converted',
+        )
+        open(self.converted_file_path, 'a').close()
+        self.assertTrue(os.path.exists(self.converted_file_path))
+
+        # Create directory that contains a file with another name that is not 'converted'
+        self.without_converted_file_dir = os.path.join(
+            self.test_dir_path,
+            'without_converted_file',
+        )
+        os.mkdir(self.without_converted_file_dir)
+        self.random_file_path = os.path.join(
+            self.without_converted_file_dir,
+            'a_file',
+        )
+        open(self.random_file_path, 'a').close()
+        self.assertTrue(os.path.exists(self.random_file_path))
+
+    def tearDown(self):
+        shutil.rmtree(self.test_dir_path)
+
+    def test_inspect_input_path(self) -> None:
+        # Should find converted file under directory
+        file_path, create_file = inspect_input_path(self.with_converted_file_dir, False)
+        self.assertEqual(self.converted_file_path, file_path)
+        self.assertFalse(create_file)
+        # Should find it but set it to be re-created
+        file_path, create_file = inspect_input_path(self.with_converted_file_dir, True)
+        self.assertEqual(self.converted_file_path, file_path)
+        self.assertTrue(create_file)
+
+        # Should fail to find converted file under directory
+        file_path, create_file = inspect_input_path(self.without_converted_file_dir, False)
+        self.assertIsNone(file_path)
+        self.assertIsNone(create_file)
+        file_path, create_file = inspect_input_path(self.without_converted_file_dir, True)
+        self.assertIsNone(file_path)
+        self.assertIsNone(create_file)
+
+        # Should accept any file path if it exists
+        file_path, create_file = inspect_input_path(self.random_file_path, False)
+        self.assertEqual(self.random_file_path, file_path)
+        self.assertFalse(create_file)
+        # Should set it to be re-created
+        file_path, create_file = inspect_input_path(self.random_file_path, True)
+        self.assertEqual(self.random_file_path, file_path)
+        self.assertTrue(create_file)
+
+        # TODO try with a trace directory
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index a199f4c..90fb615 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -99,6 +99,7 @@ def inspect_input_path(
             return converted_file_path, True
         else:
             # Simplest use-case: given path is an existing converted file
+            print(f'found converted file: {converted_file_path}')
             return converted_file_path, False
 
 

From f09cd960d61fa98d267e5a666dcfde39e6aa7565 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 14:39:21 -0700
Subject: [PATCH 16/22] Add ros2trace_analysis command and process/convert
 verbs

---
 .gitlab-ci.yml                                |  2 +-
 ros2trace_analysis/.gitignore                 |  3 ++
 ros2trace_analysis/package.xml                | 23 ++++++++++
 .../ros2trace_analysis/__init__.py            | 13 ++++++
 .../ros2trace_analysis/api/__init__.py        | 13 ++++++
 .../ros2trace_analysis/command/__init__.py    | 13 ++++++
 .../command/trace_analysis.py                 | 41 +++++++++++++++++
 .../ros2trace_analysis/verb/__init__.py       | 13 ++++++
 .../ros2trace_analysis/verb/convert.py        | 30 +++++++++++++
 .../ros2trace_analysis/verb/process.py        | 30 +++++++++++++
 ros2trace_analysis/setup.py                   | 44 +++++++++++++++++++
 ros2trace_analysis/test/test_copyright.py     | 23 ++++++++++
 ros2trace_analysis/test/test_flake8.py        | 23 ++++++++++
 ros2trace_analysis/test/test_pep257.py        | 23 ++++++++++
 ros2trace_analysis/test/test_xmllint.py       | 23 ++++++++++
 .../tracetools_analysis/convert.py            | 15 ++++---
 .../tracetools_analysis/process.py            | 13 ++++--
 17 files changed, 335 insertions(+), 10 deletions(-)
 create mode 100644 ros2trace_analysis/.gitignore
 create mode 100644 ros2trace_analysis/package.xml
 create mode 100644 ros2trace_analysis/ros2trace_analysis/__init__.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/api/__init__.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/command/__init__.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/verb/__init__.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/verb/convert.py
 create mode 100644 ros2trace_analysis/ros2trace_analysis/verb/process.py
 create mode 100644 ros2trace_analysis/setup.py
 create mode 100644 ros2trace_analysis/test/test_copyright.py
 create mode 100644 ros2trace_analysis/test/test_flake8.py
 create mode 100644 ros2trace_analysis/test/test_pep257.py
 create mode 100644 ros2trace_analysis/test/test_xmllint.py

diff --git a/.gitlab-ci.yml b/.gitlab-ci.yml
index 35597ae..06a0fa3 100644
--- a/.gitlab-ci.yml
+++ b/.gitlab-ci.yml
@@ -1,6 +1,6 @@
 variables:
   DOCKER_DRIVER: overlay2
-  PACKAGES_LIST: tracetools_analysis
+  PACKAGES_LIST: tracetools_analysis ros2trace_analysis
   base_image_id: registry.gitlab.com/micro-ros/ros_tracing/ci_base
 
 .global_artifacts: &global_artifacts
diff --git a/ros2trace_analysis/.gitignore b/ros2trace_analysis/.gitignore
new file mode 100644
index 0000000..eef29c1
--- /dev/null
+++ b/ros2trace_analysis/.gitignore
@@ -0,0 +1,3 @@
+*~
+*.pyc
+
diff --git a/ros2trace_analysis/package.xml b/ros2trace_analysis/package.xml
new file mode 100644
index 0000000..9fa17c0
--- /dev/null
+++ b/ros2trace_analysis/package.xml
@@ -0,0 +1,23 @@
+<?xml version="1.0"?>
+<?xml-model href="http://download.ros.org/schema/package_format2.xsd" schematypens="http://www.w3.org/2001/XMLSchema"?>
+<package format="2">
+  <name>ros2trace_analysis</name>
+  <version>0.1.1</version>
+  <description>The trace analysis command for ROS 2 command line tools.</description>
+  <maintainer email="christophe.bedard@apex.ai">Christophe Bedard</maintainer>
+  <license>Apache 2.0</license>
+  <author email="christophe.bedard@apex.ai">Christophe Bedard</author>
+
+  <depend>ros2cli</depend>
+  <depend>tracetools_analysis</depend>
+
+  <test_depend>ament_copyright</test_depend>
+  <test_depend>ament_flake8</test_depend>
+  <test_depend>ament_pep257</test_depend>
+  <test_depend>ament_xmllint</test_depend>
+  <test_depend>python3-pytest</test_depend>
+
+  <export>
+    <build_type>ament_python</build_type>
+  </export>
+</package>
diff --git a/ros2trace_analysis/ros2trace_analysis/__init__.py b/ros2trace_analysis/ros2trace_analysis/__init__.py
new file mode 100644
index 0000000..35ffb6e
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/ros2trace_analysis/ros2trace_analysis/api/__init__.py b/ros2trace_analysis/ros2trace_analysis/api/__init__.py
new file mode 100644
index 0000000..35ffb6e
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/api/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/ros2trace_analysis/ros2trace_analysis/command/__init__.py b/ros2trace_analysis/ros2trace_analysis/command/__init__.py
new file mode 100644
index 0000000..35ffb6e
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/command/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py b/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py
new file mode 100644
index 0000000..0ae2f1f
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/command/trace_analysis.py
@@ -0,0 +1,41 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Module for trace analysis command extension implementation."""
+
+from ros2cli.command import add_subparsers
+from ros2cli.command import CommandExtension
+from ros2cli.verb import get_verb_extensions
+
+
+class TraceAnalysisCommand(CommandExtension):
+    """Analyze traces to extract useful execution data."""
+
+    def add_arguments(self, parser, cli_name):
+        self._subparser = parser
+        # get verb extensions and let them add their arguments
+        verb_extensions = get_verb_extensions('ros2trace_analysis.verb')
+        add_subparsers(
+            parser, cli_name, '_verb', verb_extensions, required=False)
+
+    def main(self, *, parser, args):
+        if not hasattr(args, '_verb'):
+            # in case no verb was passed
+            self._subparser.print_help()
+            return 0
+
+        extension = getattr(args, '_verb')
+
+        # call the verb's main method
+        return extension.main(args=args)
diff --git a/ros2trace_analysis/ros2trace_analysis/verb/__init__.py b/ros2trace_analysis/ros2trace_analysis/verb/__init__.py
new file mode 100644
index 0000000..35ffb6e
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/verb/__init__.py
@@ -0,0 +1,13 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
diff --git a/ros2trace_analysis/ros2trace_analysis/verb/convert.py b/ros2trace_analysis/ros2trace_analysis/verb/convert.py
new file mode 100644
index 0000000..62ea13f
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/verb/convert.py
@@ -0,0 +1,30 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ros2cli.verb import VerbExtension
+from tracetools_analysis.convert import add_args
+from tracetools_analysis.convert import convert
+
+
+class ConvertVerb(VerbExtension):
+    """Convert trace data to a file."""
+
+    def add_arguments(self, parser, cli_name):
+        add_args(parser)
+
+    def main(self, *, args):
+        return convert(
+            args.trace_directory,
+            args.output_file_name,
+        )
diff --git a/ros2trace_analysis/ros2trace_analysis/verb/process.py b/ros2trace_analysis/ros2trace_analysis/verb/process.py
new file mode 100644
index 0000000..9d650e8
--- /dev/null
+++ b/ros2trace_analysis/ros2trace_analysis/verb/process.py
@@ -0,0 +1,30 @@
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ros2cli.verb import VerbExtension
+from tracetools_analysis.process import add_args
+from tracetools_analysis.process import process
+
+
+class ProcessVerb(VerbExtension):
+    """Process a file converted from a trace directory and output model data."""
+
+    def add_arguments(self, parser, cli_name):
+        add_args(parser)
+
+    def main(self, *, args):
+        return process(
+            args.input_path,
+            args.force_conversion,
+        )
diff --git a/ros2trace_analysis/setup.py b/ros2trace_analysis/setup.py
new file mode 100644
index 0000000..744965f
--- /dev/null
+++ b/ros2trace_analysis/setup.py
@@ -0,0 +1,44 @@
+from setuptools import find_packages
+from setuptools import setup
+
+package_name = 'ros2trace_analysis'
+
+setup(
+    name=package_name,
+    version='0.1.1',
+    packages=find_packages(exclude=['test']),
+    data_files=[
+        ('share/' + package_name, ['package.xml']),
+    ],
+    install_requires=['ros2cli'],
+    zip_safe=True,
+    maintainer=(
+        'Christophe Bedard'
+    ),
+    maintainer_email=(
+        'christophe.bedard@apex.ai'
+    ),
+    author='Christophe Bedard',
+    author_email='christophe.bedard@apex.ai',
+    url='https://gitlab.com/micro-ROS/ros_tracing/tracetools_analysis',
+    keywords=[],
+    description='The trace analysis command for ROS 2 command line tools.',
+    long_description=(
+        'The package provides the trace analysis '
+        'command for the ROS 2 command line tools.'
+    ),
+    license='Apache 2.0',
+    tests_require=['pytest'],
+    entry_points={
+        'ros2cli.command': [
+            f'trace-analysis = {package_name}.command.trace_analysis:TraceAnalysisCommand',
+        ],
+        'ros2cli.extension_point': [
+            f'{package_name}.verb = {package_name}.verb:VerbExtension',
+        ],
+        f'{package_name}.verb': [
+            f'convert = {package_name}.verb.convert:ConvertVerb',
+            f'process = {package_name}.verb.process:ProcessVerb',
+        ],
+    }
+)
diff --git a/ros2trace_analysis/test/test_copyright.py b/ros2trace_analysis/test/test_copyright.py
new file mode 100644
index 0000000..cf0fae3
--- /dev/null
+++ b/ros2trace_analysis/test/test_copyright.py
@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_copyright.main import main
+import pytest
+
+
+@pytest.mark.copyright
+@pytest.mark.linter
+def test_copyright():
+    rc = main(argv=['.', 'test'])
+    assert rc == 0, 'Found errors'
diff --git a/ros2trace_analysis/test/test_flake8.py b/ros2trace_analysis/test/test_flake8.py
new file mode 100644
index 0000000..eff8299
--- /dev/null
+++ b/ros2trace_analysis/test/test_flake8.py
@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_flake8.main import main
+import pytest
+
+
+@pytest.mark.flake8
+@pytest.mark.linter
+def test_flake8():
+    rc = main(argv=[])
+    assert rc == 0, 'Found errors'
diff --git a/ros2trace_analysis/test/test_pep257.py b/ros2trace_analysis/test/test_pep257.py
new file mode 100644
index 0000000..0e38a6c
--- /dev/null
+++ b/ros2trace_analysis/test/test_pep257.py
@@ -0,0 +1,23 @@
+# Copyright 2017 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_pep257.main import main
+import pytest
+
+
+@pytest.mark.linter
+@pytest.mark.pep257
+def test_pep257():
+    rc = main(argv=[])
+    assert rc == 0, 'Found code style errors / warnings'
diff --git a/ros2trace_analysis/test/test_xmllint.py b/ros2trace_analysis/test/test_xmllint.py
new file mode 100644
index 0000000..f46285e
--- /dev/null
+++ b/ros2trace_analysis/test/test_xmllint.py
@@ -0,0 +1,23 @@
+# Copyright 2019 Open Source Robotics Foundation, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+from ament_xmllint.main import main
+import pytest
+
+
+@pytest.mark.linter
+@pytest.mark.xmllint
+def test_xmllint():
+    rc = main(argv=[])
+    assert rc == 0, 'Found errors'
diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index 7ae73a5..545bc73 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -18,6 +18,7 @@
 import argparse
 import os
 import time
+from typing import Optional
 
 from tracetools_analysis.conversion import ctf
 
@@ -25,24 +26,28 @@ from tracetools_analysis.conversion import ctf
 DEFAULT_CONVERT_FILE_NAME = 'converted'
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(
-        description='Convert CTF trace data to a file.')
+def add_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         'trace_directory',
-        help='the path to the main CTF trace directory')
+        help='the path to the main trace directory')
     parser.add_argument(
         '-o', '--output-file-name', dest='output_file_name',
         default=DEFAULT_CONVERT_FILE_NAME,
         help='the name of the output file to generate, '
         'under $trace_directory (default: %(default)s)')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(
+        description='Convert trace data to a file.')
+    add_args(parser)
     return parser.parse_args()
 
 
 def convert(
     trace_directory: str,
     output_file_name: str = DEFAULT_CONVERT_FILE_NAME,
-) -> None:
+) -> Optional[int]:
     """
     Convert trace directory to a file.
 
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 90fb615..77f7714 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -19,6 +19,7 @@ import argparse
 import os
 import sys
 import time
+from typing import Optional
 from typing import Tuple
 
 from tracetools_analysis.convert import convert
@@ -28,9 +29,7 @@ from tracetools_analysis.processor.ros2 import Ros2Handler
 from tracetools_read.trace import is_trace_directory
 
 
-def parse_args():
-    parser = argparse.ArgumentParser(description='Process a file converted from a trace '
-                                                 'directory and output model data.')
+def add_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
         'input_path',
         help='the path to a converted file to import and process, '
@@ -39,6 +38,12 @@ def parse_args():
         '-f', '--force-conversion', dest='force_conversion',
         action='store_true', default=False,
         help='re-convert trace directory even if converted file is found')
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description='Process a file converted from a trace '
+                                                 'directory and output model data.')
+    add_args(parser)
     return parser.parse_args()
 
 
@@ -106,7 +111,7 @@ def inspect_input_path(
 def process(
     input_path: str,
     force_conversion: bool = False,
-) -> None:
+) -> Optional[int]:
     """
     Process converted trace file.
 

From 7c9533655128aa5d9805c791fe0d16c37b5c7857 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 15:30:12 -0700
Subject: [PATCH 17/22] Add link to ros2_tracing design document

---
 README.md | 4 ++++
 1 file changed, 4 insertions(+)

diff --git a/README.md b/README.md
index 9635cf1..30a6670 100644
--- a/README.md
+++ b/README.md
@@ -18,3 +18,7 @@ $ jupyter notebook
 ```
 
 Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, and select one of the provided notebooks, or create your own!
+
+## Design
+
+See the [`ros2_tracing` design document](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md), especially the [*Goals and requirements*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#goals-and-requirements) and [*Analysis architecture*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#analysis-architecture) sections.

From 2bf8aedac4239e12f5cbdd305562b35fb1589e84 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 15:30:29 -0700
Subject: [PATCH 18/22] Improve readme with command examples and analysis steps

---
 README.md | 20 +++++++++++++++++---
 1 file changed, 17 insertions(+), 3 deletions(-)

diff --git a/README.md b/README.md
index 30a6670..20685c3 100644
--- a/README.md
+++ b/README.md
@@ -2,16 +2,30 @@
 
 Analysis tools for [ROS 2 tracing](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing).
 
-# Setup
+## Setup
 
 To display results, install:
 
 * [Jupyter](https://jupyter.org/install)
 * [Bokeh](https://bokeh.pydata.org/en/latest/docs/user_guide/quickstart.html#userguide-quickstart-install)
 
-# Use
+## Trace analysis
 
-Start Jupyter Notebook:
+After generating a trace (see [`ros2_tracing`](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing#tracing)), we can analyze it to extract useful execution data.
+
+Since CTF traces (the output format of the [LTTng](https://lttng.org/) tracer) are very slow to read, we first convert them into a single file which can be read much faster.
+
+```
+$ ros2 trace-analysis convert /path/to/trace/directory
+```
+
+Then we can process it to create a data model which could be queried for analysis.
+
+```
+$ ros2 trace-analysis process /path/to/trace/directory
+```
+
+This last command will process and output the raw data models, but to actually display results, process and analyze using a Jupyter Notebook.
 
 ```
 $ jupyter notebook

From 8c77b3bbb4658552fd904e43c2ad72df1060af2e Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 16:10:49 -0700
Subject: [PATCH 19/22] Use os.path.expanduser() in load_file

---
 tracetools_analysis/analysis/callback_duration.ipynb        | 2 --
 tracetools_analysis/tracetools_analysis/loading/__init__.py | 3 ++-
 2 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/tracetools_analysis/analysis/callback_duration.ipynb b/tracetools_analysis/analysis/callback_duration.ipynb
index faac368..9b10a7e 100644
--- a/tracetools_analysis/analysis/callback_duration.ipynb
+++ b/tracetools_analysis/analysis/callback_duration.ipynb
@@ -44,7 +44,6 @@
     "sys.path.insert(0, '../')\n",
     "sys.path.insert(0, '../../../micro-ROS/ros_tracing/ros2_tracing/tracetools_read/')\n",
     "import datetime as dt\n",
-    "import os\n",
     "\n",
     "from bokeh.plotting import figure\n",
     "from bokeh.plotting import output_notebook\n",
@@ -68,7 +67,6 @@
    "outputs": [],
    "source": [
     "# Process\n",
-    "converted_file_path = os.path.expanduser(converted_file_path)\n",
     "events = load_file(converted_file_path)\n",
     "handler = Ros2Handler.process(events)\n",
     "#handler.data.print_model()"
diff --git a/tracetools_analysis/tracetools_analysis/loading/__init__.py b/tracetools_analysis/tracetools_analysis/loading/__init__.py
index 0347e7e..96abc39 100644
--- a/tracetools_analysis/tracetools_analysis/loading/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/loading/__init__.py
@@ -14,6 +14,7 @@
 
 """Module for converted trace file loading."""
 
+import os
 import pickle
 from typing import Dict
 from typing import List
@@ -27,7 +28,7 @@ def load_file(file_path: str) -> List[Dict]:
     :return: the list of events read from the file
     """
     events = []
-    with open(file_path, 'rb') as f:
+    with open(os.path.expanduser(file_path), 'rb') as f:
         p = pickle.Unpickler(f)
         while True:
             try:

From 402dc9dd93f2396499b3d791e008adaa46155fe2 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 16:11:00 -0700
Subject: [PATCH 20/22] Add jupyter example

---
 README.md | 36 +++++++++++++++++++++++++++++++++++-
 1 file changed, 35 insertions(+), 1 deletion(-)

diff --git a/README.md b/README.md
index 20685c3..7586c88 100644
--- a/README.md
+++ b/README.md
@@ -13,6 +13,8 @@ To display results, install:
 
 After generating a trace (see [`ros2_tracing`](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing#tracing)), we can analyze it to extract useful execution data.
 
+### Commands
+
 Since CTF traces (the output format of the [LTTng](https://lttng.org/) tracer) are very slow to read, we first convert them into a single file which can be read much faster.
 
 ```
@@ -25,7 +27,9 @@ Then we can process it to create a data model which could be queried for analysi
 $ ros2 trace-analysis process /path/to/trace/directory
 ```
 
-This last command will process and output the raw data models, but to actually display results, process and analyze using a Jupyter Notebook.
+### Jupyter
+
+The last command will process and output the raw data models, but to actually display results, process and analyze using a Jupyter Notebook.
 
 ```
 $ jupyter notebook
@@ -33,6 +37,36 @@ $ jupyter notebook
 
 Then navigate to the [`analysis/`](./tracetools_analysis/analysis/) directory, and select one of the provided notebooks, or create your own!
 
+For example:
+
+```python
+from tracetools_analysis import utils
+from tracetools_analysis.loading import load_file
+from tracetools_analysis.processor import Processor
+from tracetools_analysis.processor.cpu_time import CpuTimeHandler
+from tracetools_analysis.processor.ros2 import Ros2Handler
+
+# Load converted trace file
+events = load_file('/path/to/converted/file')
+
+# Process
+ros2_handler = Ros2Handler()
+cpu_handler = CpuTimeHandler()
+
+Processor(ros2_handler, cpu_handler).process(events)
+
+# Use data model utils to extract information
+ros2_util = utils.RosDataModelUtil(ros2_handler.data)
+cpu_util = CpuTimeDataModelUtil(cpu_handler.data)
+
+callback_durations = ros2_util.get_callback_durations()
+time_per_thread = cpu_util.get_time_per_thread()
+# ...
+
+# Display, e.g. with bokeh or matplotlib
+# ...
+```
+
 ## Design
 
 See the [`ros2_tracing` design document](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md), especially the [*Goals and requirements*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#goals-and-requirements) and [*Analysis architecture*](https://gitlab.com/micro-ROS/ros_tracing/ros2_tracing/blob/master/doc/design_ros_2.md#analysis-architecture) sections.

From 0f4f035d51ab4d6c20a6d0fd0ad4cfe37d411749 Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 17:24:31 -0700
Subject: [PATCH 21/22] Add time difference formatting function

---
 .../tracetools_analysis/__init__.py           | 22 +++++++++++++++++--
 .../tracetools_analysis/convert.py            |  4 +++-
 .../tracetools_analysis/process.py            |  4 +++-
 3 files changed, 26 insertions(+), 4 deletions(-)

diff --git a/tracetools_analysis/tracetools_analysis/__init__.py b/tracetools_analysis/tracetools_analysis/__init__.py
index e4deba3..94788ac 100644
--- a/tracetools_analysis/tracetools_analysis/__init__.py
+++ b/tracetools_analysis/tracetools_analysis/__init__.py
@@ -12,5 +12,23 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-"""Reading and interpreting of LTTng trace data."""
-__author__ = 'Luetkebohle Ingo (CR/AEX3)'
+"""Tools for analysing trace data."""
+
+
+def time_diff_to_str(
+    time_diff: float,
+) -> str:
+    """
+    Format time difference as a string.
+
+    :param time_diff: the difference between two timepoints (e.g. `time.time()`)
+    """
+    if time_diff < 1.0:
+        # ms
+        return f'{time_diff * 1000:.0f} ms'
+    elif time_diff < 60.0:
+        # s
+        return f'{time_diff:.1f} s'
+    else:
+        # m s
+        return f'{time_diff // 60.0:.0f} m {time_diff % 60.0:.0f} s'
diff --git a/tracetools_analysis/tracetools_analysis/convert.py b/tracetools_analysis/tracetools_analysis/convert.py
index 545bc73..c2384a2 100644
--- a/tracetools_analysis/tracetools_analysis/convert.py
+++ b/tracetools_analysis/tracetools_analysis/convert.py
@@ -22,6 +22,8 @@ from typing import Optional
 
 from tracetools_analysis.conversion import ctf
 
+from . import time_diff_to_str
+
 
 DEFAULT_CONVERT_FILE_NAME = 'converted'
 
@@ -61,7 +63,7 @@ def convert(
     start_time = time.time()
     count = ctf.convert(trace_directory, output_file_path)
     time_diff = time.time() - start_time
-    print(f'converted {count} events in {time_diff * 1000:.2f} ms')
+    print(f'converted {count} events in {time_diff_to_str(time_diff)}')
     print(f'output written to: {output_file_path}')
 
 
diff --git a/tracetools_analysis/tracetools_analysis/process.py b/tracetools_analysis/tracetools_analysis/process.py
index 77f7714..d7ea399 100644
--- a/tracetools_analysis/tracetools_analysis/process.py
+++ b/tracetools_analysis/tracetools_analysis/process.py
@@ -28,6 +28,8 @@ from tracetools_analysis.loading import load_file
 from tracetools_analysis.processor.ros2 import Ros2Handler
 from tracetools_read.trace import is_trace_directory
 
+from . import time_diff_to_str
+
 
 def add_args(parser: argparse.ArgumentParser) -> None:
     parser.add_argument(
@@ -136,7 +138,7 @@ def process(
 
     time_diff = time.time() - start_time
     ros2_handler.data.print_model()
-    print(f'processed {len(events)} events in {time_diff * 1000:.2f} ms')
+    print(f'processed {len(events)} events in {time_diff_to_str(time_diff)}')
 
 
 def main():

From e0dc3b0a2186e2e6c76c3428fd123f33f6d79c5a Mon Sep 17 00:00:00 2001
From: Christophe Bedard <christophe.bedard@apex.ai>
Date: Sun, 13 Oct 2019 18:06:04 -0700
Subject: [PATCH 22/22] Add test for time_diff_to_str

---
 tracetools_analysis/test/test_utils.py | 33 ++++++++++++++++++++++++++
 1 file changed, 33 insertions(+)
 create mode 100644 tracetools_analysis/test/test_utils.py

diff --git a/tracetools_analysis/test/test_utils.py b/tracetools_analysis/test/test_utils.py
new file mode 100644
index 0000000..16d0886
--- /dev/null
+++ b/tracetools_analysis/test/test_utils.py
@@ -0,0 +1,33 @@
+#!/usr/bin/env python3
+# Copyright 2019 Apex.AI, Inc.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+import unittest
+
+from tracetools_analysis import time_diff_to_str
+
+
+class TestUtils(unittest.TestCase):
+
+    def __init__(self, *args) -> None:
+        super().__init__(
+            *args,
+        )
+
+    def test_time_diff_to_str(self) -> None:
+        self.assertEqual('11 ms', time_diff_to_str(0.0106))
+        self.assertEqual('6.9 s', time_diff_to_str(6.9069))
+        self.assertEqual('1 m 10 s', time_diff_to_str(69.6969))
+        self.assertEqual('6 m 10 s', time_diff_to_str(369.6969))
+        self.assertEqual('2 m 0 s', time_diff_to_str(120.499999999))