WIP: backport message flow tracing to foxy

.devcontainer/devcontainer.json

@@ -1,7 +1,5 @@
-// For format details, see https://aka.ms/devcontainer.json. For config options, see the
-// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
 {
-	"name": "Existing Dockerfile",
+	"name": "LTTng Traced ROS2 Scheduling Experiments",
 	"build": {
 		// Sets the run context to one level up instead of the .devcontainer folder.
 		"context": "..",
@@ -29,7 +27,7 @@
 	// "forwardPorts": [],
 
 	// Uncomment the next line to run commands after the container is created.
-	// "postCreateCommand": "cat /etc/os-release",
+	"postCreateCommand": "bash postCreate.sh",
 
 	// Uncomment the next line to run commands after the container is created and started.
 	"postStartCommand": "sudo modprobe lttng-tracer && sudo modprobe lttng-ring-buffer-client-discard && sudo modprobe lttng-probe-sched && sudo modprobe lttng-probe-irq && sudo modprobe lttng-probe-timer && (sudo lttng-sessiond --daemonize || true)",
@@ -38,6 +36,7 @@
 	"customizations": {
 		"vscode": {
 			"settings": {
+				"terminal.integrated.shell.linux": "/bin/bash",
 				"terminal.integrated.defaultProfile.linux": "bash",
 				"C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools"
 			},
@@ -59,8 +58,19 @@
 				"ms-python.vscode-python-envs"
 			]
 		}
-	}
+	},
+
+	"features": {
+		// optional but convenient
+		"ghcr.io/devcontainers/features/common-utils:2": {}
+	},
+
+	"workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind,consistency=cached",
+	"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
+	"containerEnv": {
+		"TRACETOOLS_TRACEPOINTS_ENABLED": "1"
+	},
 	
 	// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
-	// "remoteUser": "devcontainer"
+	"remoteUser": "dev"
 }

.gitmodules

@@ -1,11 +1,23 @@
 [submodule "src/ros_edf"]
 	path = src/ros_edf
 	url = git@git.niklashalle.net:niklas/ROS-Dynamic-Executor.git
-[submodule "src/tools/ros2_tracing"]
-	path = src/tools/ros2_tracing
-	url = git@git.niklashalle.net:niklas/ros2_tracing.git
-	branch = foxy
 [submodule "src/tools/tracetools_analysis"]
-	path = src/tools/tracetools_analysis
+	path = src/tracetools_analysis
 	url = git@git.niklashalle.net:niklas/tracetools_analysis.git
 	branch = foxy
+[submodule "src/rcl"]
+	path = src/rcl
+	url = git@git.niklashalle.net:niklas/rcl.git
+	branch = foxy
+[submodule "src/rclcpp"]
+	path = src/rclcpp
+	url = git@git.niklashalle.net:niklas/rclcpp.git
+	branch = foxy
+[submodule "src/ros2_tracing"]
+	path = src/ros2_tracing
+	url = git@git.niklashalle.net:niklas/ros2_tracing.git
+	branch = foxy
+[submodule "src/rmw_cyclonedds"]
+	path = src/rmw_cyclonedds
+	url = git@git.niklashalle.net:niklas/rmw_cyclonedds.git
+	branch = foxy

Dockerfile

@@ -37,11 +37,8 @@ RUN apt-get update -q && \
     apt-get install -yq --no-install-recommends \
       apt-utils wget curl git build-essential vim sudo \
       lsb-release locales bash-completion tzdata gosu \
-      gedit htop nano libserial-dev ninja-build clang-18
-
-# Update packages and install dependencies
-RUN apt-get update && \
-    apt-get install -y software-properties-common curl build-essential && \
+      gedit htop nano libserial-dev ninja-build clang-18 \
+      software-properties-common && \
     add-apt-repository ppa:deadsnakes/ppa && \
     apt-get update && \
     apt-get install -y \
@@ -57,7 +54,8 @@ RUN apt-get update && \
 # These packages help with building and managing ROS 2 workspaces
 RUN apt-get update -q && \
     apt-get install -y gnupg2 iputils-ping usbutils \
-      python3-argcomplete python3-colcon-common-extensions python3-networkx python3-pip python3-rosdep python3-vcstool
+      python3-argcomplete python3-colcon-common-extensions \
+      python3-networkx python3-pip python3-rosdep python3-vcstool
 
 # Set up the ROS 2 environment
 # This ensures that ROS 2 commands are available in the shell
@@ -72,18 +70,13 @@ RUN curl -sS https://bootstrap.pypa.io/pip/3.8/get-pip.py | python3.8
 # Set Python 3.8 as default Python3
 RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
 
-# Install modern nodejs (optional but good practice for JupyterLab widgets)
-RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
-    apt-get install -y nodejs && \
-    npm install -g npm
-
-# Install JupyterLab globally
-RUN python3 -m pip install --upgrade jupyterlab
-
 # Install dependencies for ros2_tracing
-RUN apt-get update -q && \
-    apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust-dev
-RUN apt-get install -y python3-babeltrace python3-lttng python3-pytest python3-pytest-cov python3-pandas
+RUN add-apt-repository ppa:lttng/stable-2.13 && \
+    apt-get update -q && \
+    apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust-dev && \
+    apt-get install -y python3-babeltrace python3-lttng python3-pytest \
+     python3-pytest-cov python3-pandas ros-foxy-rmw-cyclonedds-cpp && \
+    rm -rf /var/lib/apt/lists/*
 
 # Python bokeh is not available in the default apt repository
 # So we install it using pip

README.md

@@ -157,3 +157,13 @@ Contributions are welcome! Please follow these steps:
 ## License
 
 Apache License 2.0 - See LICENSE file for details
+
+---
+
+colcon build
+# later also just:
+colcon build --packages-select full_topology
+source install/setup.bash
+ros2 launch full_topology trace_full_topology.launch.py
+ros2 trace-analysis convert ./analysis/tracing/full_topology_tracing
+ros2 trace-analysis process ./analysis/tracing/full_topology_tracing

build.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+colcon build --packages-select tracetools
+source install/setup.bash
+colcon build --packages-select rmw_cyclonedds_cpp
+source install/setup.bash
+colcon build --packages-select rcl rcl_yaml_param_parser
+source install/setup.bash
+colcon build --packages-select rclcpp
+source install/setup.bash
+colcon build
+source install/setup.bash
+export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp

colcon_defaults.yaml

@@ -8,6 +8,8 @@
       # clang
       "-DCMAKE_C_COMPILER=clang-18",
       "-DCMAKE_CXX_COMPILER=clang++-18",
+      # no tests
+      "-DBUILD_TESTING=OFF",
     ],
     "event-handlers": ["console_cohesion+"],
     # "mixin": "clang",

debug.sh

@@ -0,0 +1,22 @@
+#!/usr/bin/env bash
+
+export TRACETOOLS_TRACEPOINTS_ENABLED=1
+ros2 run full_topology full_topology &
+pid=$!
+
+sleep 2
+sudo lttng create test --output=/tmp/test
+sudo lttng enable-event --userspace 'ros2:rclcpp_publish'
+sudo lttng enable-event --userspace 'ros2:rmw_take'
+sudo lttng start
+sleep 5          # let the node run a bit
+sudo lttng stop; sudo lttng destroy
+kill $pid
+
+sudo chown dev: -R /tmp/test
+
+python - <<'PY'
+from tracetools_analysis.loading import load_file
+ev = load_file('/tmp/test/ust')
+print({e['_name'] for e in ev})
+PY

postCreate.sh

@@ -0,0 +1,50 @@
+#!/usr/bin/env bash
+set -e
+
+echo "▶ Setting up tracing overlay workspaces"
+
+###############################################################################
+# 0.  fresh terminal                                                           
+###############################################################################
+source /opt/ros/foxy/setup.bash
+
+###############################################################################
+# 1.  build the tracing stack only   (workspace: trace_ws)                     
+###############################################################################
+mkdir -p ~/trace_ws/src && cd ~/trace_ws/src
+
+git clone -b iron git@github.com:ros2/ros2_tracing.git
+
+#  ament will generate tracetools_config.h with TRACEPOINTS enabled by default
+cd ~/trace_ws
+colcon build --symlink-install --cmake-args -DBUILD_TESTING=OFF
+
+# make the new tracetools *override* the distro one for everything we compile
+source install/setup.bash
+echo "overlay order ✔  (tracetools version = $(pkg-config --modversion tracetools))"
+
+###############################################################################
+# 2.  build rclcpp, rcl, rmw… against that new tracetools  (core_ws)          
+###############################################################################
+mkdir -p ~/core_ws/src && cd ~/core_ws/src
+
+git clone -b iron git@github.com:ros2/rosidl_dynamic_typesupport.git
+git clone -b iron git@github.com:ros2/rclcpp.git
+git clone -b iron git@github.com:ros2/rcl.git
+git clone -b iron git@github.com:ros2/rmw.git
+git clone -b iron git@github.com:ros2/rmw_fastrtps.git
+# add any other core packages you really need (rclcpp_action, rclcpp_components…)
+
+cd ~/core_ws
+colcon build --symlink-install --cmake-args -DBUILD_TESTING=OFF
+
+###############################################################################
+# 3.  use it                                                                   
+###############################################################################
+source install/setup.bash        # rclcpp, rcl… now come from the overlay
+
+echo "overlay order ✔  (rclcpp version = $(pkg-config --modversion rclcpp))"
+
+echo "Source the following in your terminal to use the new tracing stack:"
+echo "~/trace_ws/install/setup.bash"
+echo "~/core_ws/install/setup.bash"

src/casestudy/CMakeLists.txt

@@ -13,6 +13,7 @@ endif()
 
 # Find dependencies
 find_package(ament_cmake REQUIRED)
+find_package(tracetools REQUIRED)
 find_package(rclcpp REQUIRED)
 find_package(priority_executor REQUIRED)
 find_package(simple_timer REQUIRED)
@@ -39,6 +40,7 @@ function(new_experiment experiment_name)
   
   # Add dependencies with ament
   ament_target_dependencies(${experiment_name}
+    tracetools
     simple_timer
     casestudy_tools
     rclcpp

src/casestudy/package.xml

@@ -12,6 +12,7 @@
   <buildtool_depend>ament_cmake</buildtool_depend>
 
   <!-- Build Dependencies -->
+  <depend>tracetools</depend>
   <depend>rclcpp</depend>
   <depend>priority_executor</depend>
   <depend>simple_timer</depend>

src/casestudy_tools/CMakeLists.txt

@@ -13,6 +13,7 @@ endif()
 
 # Find dependencies
 find_package(ament_cmake REQUIRED)
+find_package(tracetools REQUIRED)
 find_package(rclcpp REQUIRED)
 find_package(priority_executor REQUIRED)
 find_package(simple_timer REQUIRED)
@@ -26,6 +27,7 @@ target_include_directories(primes_workload PUBLIC
   $<INSTALL_INTERFACE:include>
 )
 ament_target_dependencies(primes_workload
+  tracetools
   simple_timer
   rclcpp
   priority_executor
@@ -38,6 +40,7 @@ target_include_directories(test_nodes PUBLIC
 )
 target_link_libraries(test_nodes primes_workload)
 ament_target_dependencies(test_nodes
+  tracetools
   rclcpp
   simple_timer
 )
@@ -53,6 +56,7 @@ target_link_libraries(experiment_host
   test_nodes
 )
 ament_target_dependencies(experiment_host
+  tracetools
   rclcpp
   simple_timer
   priority_executor

src/casestudy_tools/package.xml

@@ -11,6 +11,7 @@
 
   <buildtool_depend>ament_cmake</buildtool_depend>
 
+  <depend>tracetools</depend>
   <depend>rclcpp</depend>
   <depend>priority_executor</depend>
   <depend>simple_timer</depend>

src/full_topology/CMakeLists.txt

@@ -16,7 +16,10 @@ find_package(ament_cmake REQUIRED)
 find_package(rclcpp REQUIRED)
 find_package(std_msgs REQUIRED)
 
+add_compile_definitions(RCLCPP_ENABLE_TRACEPOINTS)
+
 add_executable(${PROJECT_NAME} src/full_topology.cpp)
+#add_executable(${PROJECT_NAME} src/test.cpp)
 target_include_directories(${PROJECT_NAME} PUBLIC
   $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
   $<INSTALL_INTERFACE:include>
@@ -26,6 +29,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC
 ament_target_dependencies(${PROJECT_NAME}
   rclcpp
   std_msgs
+  tracetools
 )
 
 # Installation

src/full_topology/launch/trace_full_topology.launch.py

@@ -1,22 +1,60 @@
-from launch import LaunchDescription
+import os
+import launch
+from pathlib import Path
 from launch_ros.actions import Node
 from tracetools_launch.action import Trace
 
 def generate_launch_description():
-    return LaunchDescription([
+    length_arg = launch.actions.DeclareLaunchArgument(
+        'length',
+        default_value='60.0',
+        description='length of execution in seconds',
+    )
+
+    file_dir = Path(__file__).resolve().parent
+    target_path = file_dir.parents[2] / 'analysis' / 'tracing'
+
+    return launch.LaunchDescription([
+        length_arg,
         Trace(
             session_name='full_topology_tracing',
+            append_timestamp=True,
+            base_path=target_path,
             events_ust=[
-                'rclcpp:*', 'rcl:*', 'rmw:*', 'ros2:*', 'rclcpp:*callback*', 'rclcpp:*executor*'
+                'dds:*',
+                'ros2:*',
+                'rcl:*',
+                'rmw:*',
+                'rclcpp:*',
+                'rclcpp:*callback*',
+                'rclcpp:*executor*',
+                'rmw_cyclonedds:*',
+                # explicit list of message‑flow events
+                'ros2:rclcpp_publish',
+                'ros2:rclcpp_subscription_callback',
+                'ros2:rclcpp_timer_callback',
+                'ros2:rclcpp_executor_execute',
+                'ros2:rclcpp_intra_publish',
+                'ros2:rmw_take',
+                'ros2:callback_start',
+                'ros2:callback_end',
+                # ---- (optional) add whatever else you like ----
+                # scheduling / executor / memory events:
+                'ros2:rclcpp_executor_*',
+                'ros2:*callback_added',
             ],
             events_kernel=[
                 'sched_switch', 'sched_wakeup'
             ],
         ),
-        
         Node(
             package='full_topology',
             executable='full_topology',
             output='screen',
         ),
+        # Shut down after some time, otherwise the system would run indefinitely
+        launch.actions.TimerAction(
+            period=launch.substitutions.LaunchConfiguration(length_arg.name),
+            actions=[launch.actions.Shutdown(reason='stopping system')],
+        ),
     ])

src/full_topology/package.xml

@@ -17,6 +17,7 @@
   <!-- Build Dependencies -->
   <depend>rclcpp</depend>
   <depend>std_msgs</depend>
+  <depend>tracetools</depend>
 
   <test_depend>ament_copyright</test_depend>
   <test_depend>ament_flake8</test_depend>

src/full_topology/src/full_topology.cpp

@@ -5,12 +5,14 @@
 #include <string>
 #include <vector>
 #include <thread>
-#include "rclcpp/rclcpp.hpp"
+// the order here matters
+#include "tracetools/utils.hpp"
+
+#include <rclcpp/rclcpp.hpp>
 #include "std_msgs/msg/string.hpp"
 
 using namespace std::chrono_literals;
 
-constexpr int RUN_DURATION_SECONDS = 20;
 constexpr bool DEBUG = false;
 
 // A generic sensor node that publishes at a given rate
@@ -138,12 +140,13 @@ public:
           RCLCPP_INFO(get_logger(), "%s received: %s", name.c_str(), msg->data.c_str());
         }
 
+        /* TODO
         // random: 1 in 5 it will be 10s instead of delay_ms
         if (rand() % 5 == 0) {
           std::this_thread::sleep_for(std::chrono::milliseconds(10000));
         } else {
           std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
-        }
+        }*/
 
         // Custom transformation of data
         auto out = std_msgs::msg::String();
@@ -255,6 +258,7 @@ int main(int argc, char **argv)
   auto fusion = std::make_shared<FusionNode>(
       "sensor_fusion_node", std::vector<std::string>{"/gps/fix", "/imu/data", "/baro/alt"}, "/sensors/fused", 10);
   auto lidar = std::make_shared<LidarNode>("lidar_node", "/lidar/scan", 15);
+  // TODO: operator or autonomous flight plan
   auto cmd = std::make_shared<CommandNode>("operator_cmd_node", "/operator/commands", 1);
   auto mgmt = std::make_shared<FlightManagementNode>(
       "flight_mgmt_node", std::vector<std::string>{"/sensors/fused", "/lidar/scan", "/operator/commands"}, "/flight/plan", 20);
@@ -283,13 +287,7 @@ int main(int argc, char **argv)
   executor.add_node(telemetry);
   executor.add_node(radio);
 
-  // ToDo: find a better way to handle time limited executions
-  auto start_time = std::chrono::steady_clock::now();
-  while (rclcpp::ok() && (std::chrono::steady_clock::now() - start_time) < std::chrono::seconds(RUN_DURATION_SECONDS))
-  {
-    executor.spin_some();
-    std::this_thread::sleep_for(1ms);
-  }
+  executor.spin();
 
   rclcpp::shutdown();
   return 0;

src/full_topology/src/test.cpp

@@ -0,0 +1,9 @@
+#include "tracetools/utils.hpp"
+#include <rclcpp/rclcpp.hpp>
+
+int main(int argc, char** argv) {
+  rclcpp::init(argc, argv);
+  auto node = std::make_shared<rclcpp::Node>("test_node");
+  rclcpp::shutdown();
+  return 0;
+}

src/rcl

@@ -0,0 +1 @@
+Subproject commit 5da7e193371c25cf1ebd2489978768327c382c04

src/rclcpp

@@ -0,0 +1 @@
+Subproject commit e57a3a393e46f0c0e0e5cc64090ac34e5cef32bc

src/rmw_cyclonedds

@@ -0,0 +1 @@
+Subproject commit 79e07924d66b205838d28da4c9df6a941df6795c

src/ros2_tracing

@@ -0,0 +1 @@
+Subproject commit c0a5cf2d1dc8706a62bae59123e2c71a6707ac33

src/ros2trace

@@ -1 +0,0 @@
-tools/ros2_tracing/ros2trace

src/ros2trace_analysis

@@ -1 +0,0 @@
-tools/tracetools_analysis/ros2trace_analysis

src/ros_edf

@@ -1 +1 @@
-Subproject commit d7ecdf0108d8decdda33b6dd7fb1bbc5f1d3ac43
+Subproject commit c44b7b8c3c527a8d16739349b50340a7de9b6f51

src/simple_topology/CMakeLists.txt

@@ -13,6 +13,7 @@ endif()
 
 # find dependencies
 find_package(ament_cmake REQUIRED)
+find_package(tracetools REQUIRED)
 find_package(rclcpp REQUIRED)
 find_package(std_msgs REQUIRED)
 
@@ -24,6 +25,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC
 
 # Add dependencies with ament
 ament_target_dependencies(${PROJECT_NAME}
+  tracetools
   rclcpp
   std_msgs
 )

src/simple_topology/package.xml

@@ -11,6 +11,7 @@
 
   <depend>launch</depend>
   <depend>launch_ros</depend>
+  <depend>tracetools</depend>
   <depend>tracetools_trace</depend>
   <depend>tracetools_launch</depend>
   

src/tools/ros2_tracing

@@ -1 +0,0 @@
-Subproject commit 449a0123fd12032519ca6bea2209cc0505304771

src/tools/tracetools_analysis

@@ -1 +0,0 @@
-Subproject commit d99bb4041665d77d6f9502d614af498d31d7b6de

src/tracetools

@@ -1 +0,0 @@
-tools/ros2_tracing/tracetools

src/tracetools_analysis

@@ -1 +0,0 @@
-tools/tracetools_analysis/tracetools_analysis

src/tracetools_analysis

@@ -0,0 +1 @@
+Subproject commit 0feb705d5bf8e3578a28a6a5f9397695228c2f26

src/tracetools_launch

@@ -1 +0,0 @@
-tools/ros2_tracing/tracetools_launch

src/tracetools_read

@@ -1 +0,0 @@
-tools/ros2_tracing/tracetools_read

src/tracetools_test

@@ -1 +0,0 @@
-tools/ros2_tracing/tracetools_test

src/tracetools_trace

@@ -1 +0,0 @@
-tools/ros2_tracing/tracetools_trace