niklas/ROS-Dynamic-Executor-Experiments
1
0
Fork 0
Code Issues Pull requests Projects Releases Packages Wiki Activity Actions

WIP: backport message flow tracing to foxy

Browse source
This commit is contained in:
Niklas Halle 2025-05-17 14:29:03 +02:00
parent 1ed67d4a7d
commit bd2620ef8d
34 changed files with 221 additions and 53 deletions
Download patch file Download diff file Expand all files Collapse all files

22
.devcontainer/devcontainer.json
View file

@ -1,7 +1,5 @@
// For format details, see https://aka.ms/devcontainer.json. For config options, see the
// README at: https://github.com/devcontainers/templates/tree/main/src/docker-existing-dockerfile
{
"name": "Existing Dockerfile",
"name": "LTTng Traced ROS2 Scheduling Experiments",
"build": {
// Sets the run context to one level up instead of the .devcontainer folder.
"context": "..",
@ -29,7 +27,7 @@
// "forwardPorts": [],
// Uncomment the next line to run commands after the container is created.
// "postCreateCommand": "cat /etc/os-release",
"postCreateCommand": "bash postCreate.sh",
// Uncomment the next line to run commands after the container is created and started.
"postStartCommand": "sudo modprobe lttng-tracer && sudo modprobe lttng-ring-buffer-client-discard && sudo modprobe lttng-probe-sched && sudo modprobe lttng-probe-irq && sudo modprobe lttng-probe-timer && (sudo lttng-sessiond --daemonize || true)",
@ -38,6 +36,7 @@
"customizations": {
"vscode": {
"settings": {
"terminal.integrated.shell.linux": "/bin/bash",
"terminal.integrated.defaultProfile.linux": "bash",
"C_Cpp.default.configurationProvider": "ms-vscode.cmake-tools"
},
@ -59,8 +58,19 @@
"ms-python.vscode-python-envs"
]
}
}
},
"features": {
// optional but convenient
"ghcr.io/devcontainers/features/common-utils:2": {}
},
"workspaceMount": "source=${localWorkspaceFolder},target=/workspaces/${localWorkspaceFolderBasename},type=bind,consistency=cached",
"workspaceFolder": "/workspaces/${localWorkspaceFolderBasename}",
"containerEnv": {
"TRACETOOLS_TRACEPOINTS_ENABLED": "1"
},
// Uncomment to connect as an existing user other than the container default. More info: https://aka.ms/dev-containers-non-root.
// "remoteUser": "devcontainer"
"remoteUser": "dev"
}

22
.gitmodules vendored
View file

@ -1,11 +1,23 @@
[submodule "src/ros_edf"]
path = src/ros_edf
url = git@git.niklashalle.net:niklas/ROS-Dynamic-Executor.git
[submodule "src/tools/ros2_tracing"]
path = src/tools/ros2_tracing
url = git@git.niklashalle.net:niklas/ros2_tracing.git
branch = foxy
[submodule "src/tools/tracetools_analysis"]
path = src/tools/tracetools_analysis
path = src/tracetools_analysis
url = git@git.niklashalle.net:niklas/tracetools_analysis.git
branch = foxy
[submodule "src/rcl"]
path = src/rcl
url = git@git.niklashalle.net:niklas/rcl.git
branch = foxy
[submodule "src/rclcpp"]
path = src/rclcpp
url = git@git.niklashalle.net:niklas/rclcpp.git
branch = foxy
[submodule "src/ros2_tracing"]
path = src/ros2_tracing
url = git@git.niklashalle.net:niklas/ros2_tracing.git
branch = foxy
[submodule "src/rmw_cyclonedds"]
path = src/rmw_cyclonedds
url = git@git.niklashalle.net:niklas/rmw_cyclonedds.git
branch = foxy

27
Dockerfile
View file

@ -37,11 +37,8 @@ RUN apt-get update -q && \
apt-get install -yq --no-install-recommends \
apt-utils wget curl git build-essential vim sudo \
lsb-release locales bash-completion tzdata gosu \
gedit htop nano libserial-dev ninja-build clang-18
# Update packages and install dependencies
RUN apt-get update && \
apt-get install -y software-properties-common curl build-essential && \
gedit htop nano libserial-dev ninja-build clang-18 \
software-properties-common && \
add-apt-repository ppa:deadsnakes/ppa && \
apt-get update && \
apt-get install -y \
@ -57,7 +54,8 @@ RUN apt-get update && \
# These packages help with building and managing ROS 2 workspaces
RUN apt-get update -q && \
apt-get install -y gnupg2 iputils-ping usbutils \
python3-argcomplete python3-colcon-common-extensions python3-networkx python3-pip python3-rosdep python3-vcstool
python3-argcomplete python3-colcon-common-extensions \
python3-networkx python3-pip python3-rosdep python3-vcstool
# Set up the ROS 2 environment
# This ensures that ROS 2 commands are available in the shell
@ -72,18 +70,13 @@ RUN curl -sS https://bootstrap.pypa.io/pip/3.8/get-pip.py | python3.8
# Set Python 3.8 as default Python3
RUN update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 1
# Install modern nodejs (optional but good practice for JupyterLab widgets)
RUN curl -fsSL https://deb.nodesource.com/setup_20.x | bash - && \
apt-get install -y nodejs && \
npm install -g npm
# Install JupyterLab globally
RUN python3 -m pip install --upgrade jupyterlab
# Install dependencies for ros2_tracing
RUN apt-get update -q && \
apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust-dev
RUN apt-get install -y python3-babeltrace python3-lttng python3-pytest python3-pytest-cov python3-pandas
RUN add-apt-repository ppa:lttng/stable-2.13 && \
apt-get update -q && \
apt-get install -y lttng-tools lttng-modules-dkms liblttng-ust-dev && \
apt-get install -y python3-babeltrace python3-lttng python3-pytest \
python3-pytest-cov python3-pandas ros-foxy-rmw-cyclonedds-cpp && \
rm -rf /var/lib/apt/lists/*
# Python bokeh is not available in the default apt repository
# So we install it using pip

10
README.md
View file

@ -157,3 +157,13 @@ Contributions are welcome! Please follow these steps:
## License
Apache License 2.0 - See LICENSE file for details
---
colcon build
# later also just:
colcon build --packages-select full_topology
source install/setup.bash
ros2 launch full_topology trace_full_topology.launch.py
ros2 trace-analysis convert ./analysis/tracing/full_topology_tracing
ros2 trace-analysis process ./analysis/tracing/full_topology_tracing

13
build.sh Executable file
View file

@ -0,0 +1,13 @@
#!/usr/bin/env bash
colcon build --packages-select tracetools
source install/setup.bash
colcon build --packages-select rmw_cyclonedds_cpp
source install/setup.bash
colcon build --packages-select rcl rcl_yaml_param_parser
source install/setup.bash
colcon build --packages-select rclcpp
source install/setup.bash
colcon build
source install/setup.bash
export RMW_IMPLEMENTATION=rmw_cyclonedds_cpp

2
colcon_defaults.yaml
View file

@ -8,6 +8,8 @@
# clang
"-DCMAKE_C_COMPILER=clang-18",
"-DCMAKE_CXX_COMPILER=clang++-18",
# no tests
"-DBUILD_TESTING=OFF",
],
"event-handlers": ["console_cohesion+"],
# "mixin": "clang",

22
debug.sh Executable file
View file

@ -0,0 +1,22 @@
#!/usr/bin/env bash
export TRACETOOLS_TRACEPOINTS_ENABLED=1
ros2 run full_topology full_topology &
pid=$!
sleep 2
sudo lttng create test --output=/tmp/test
sudo lttng enable-event --userspace 'ros2:rclcpp_publish'
sudo lttng enable-event --userspace 'ros2:rmw_take'
sudo lttng start
sleep 5 # let the node run a bit
sudo lttng stop; sudo lttng destroy
kill $pid
sudo chown dev: -R /tmp/test
python - <<'PY'
from tracetools_analysis.loading import load_file
ev = load_file('/tmp/test/ust')
print({e['_name'] for e in ev})
PY

50
postCreate.sh Executable file
View file

@ -0,0 +1,50 @@
#!/usr/bin/env bash
set -e
echo "▶ Setting up tracing overlay workspaces"
###############################################################################
# 0. fresh terminal
###############################################################################
source /opt/ros/foxy/setup.bash
###############################################################################
# 1. build the tracing stack only (workspace: trace_ws)
###############################################################################
mkdir -p ~/trace_ws/src && cd ~/trace_ws/src
git clone -b iron git@github.com:ros2/ros2_tracing.git
# ament will generate tracetools_config.h with TRACEPOINTS enabled by default
cd ~/trace_ws
colcon build --symlink-install --cmake-args -DBUILD_TESTING=OFF
# make the new tracetools *override* the distro one for everything we compile
source install/setup.bash
echo "overlay order ✔ (tracetools version = $(pkg-config --modversion tracetools))"
###############################################################################
# 2. build rclcpp, rcl, rmw… against that new tracetools (core_ws)
###############################################################################
mkdir -p ~/core_ws/src && cd ~/core_ws/src
git clone -b iron git@github.com:ros2/rosidl_dynamic_typesupport.git
git clone -b iron git@github.com:ros2/rclcpp.git
git clone -b iron git@github.com:ros2/rcl.git
git clone -b iron git@github.com:ros2/rmw.git
git clone -b iron git@github.com:ros2/rmw_fastrtps.git
# add any other core packages you really need (rclcpp_action, rclcpp_components…)
cd ~/core_ws
colcon build --symlink-install --cmake-args -DBUILD_TESTING=OFF
###############################################################################
# 3. use it
###############################################################################
source install/setup.bash # rclcpp, rcl… now come from the overlay
echo "overlay order ✔ (rclcpp version = $(pkg-config --modversion rclcpp))"
echo "Source the following in your terminal to use the new tracing stack:"
echo "~/trace_ws/install/setup.bash"
echo "~/core_ws/install/setup.bash"

2
src/casestudy/CMakeLists.txt
View file

@ -13,6 +13,7 @@ endif()
# Find dependencies
find_package(ament_cmake REQUIRED)
find_package(tracetools REQUIRED)
find_package(rclcpp REQUIRED)
find_package(priority_executor REQUIRED)
find_package(simple_timer REQUIRED)
@ -39,6 +40,7 @@ function(new_experiment experiment_name)
# Add dependencies with ament
ament_target_dependencies(${experiment_name}
tracetools
simple_timer
casestudy_tools
rclcpp

1
src/casestudy/package.xml
View file

@ -12,6 +12,7 @@
<buildtool_depend>ament_cmake</buildtool_depend>
<!-- Build Dependencies -->
<depend>tracetools</depend>
<depend>rclcpp</depend>
<depend>priority_executor</depend>
<depend>simple_timer</depend>

4
src/casestudy_tools/CMakeLists.txt
View file

@ -13,6 +13,7 @@ endif()
# Find dependencies
find_package(ament_cmake REQUIRED)
find_package(tracetools REQUIRED)
find_package(rclcpp REQUIRED)
find_package(priority_executor REQUIRED)
find_package(simple_timer REQUIRED)
@ -26,6 +27,7 @@ target_include_directories(primes_workload PUBLIC
$<INSTALL_INTERFACE:include>
)
ament_target_dependencies(primes_workload
tracetools
simple_timer
rclcpp
priority_executor
@ -38,6 +40,7 @@ target_include_directories(test_nodes PUBLIC
)
target_link_libraries(test_nodes primes_workload)
ament_target_dependencies(test_nodes
tracetools
rclcpp
simple_timer
)
@ -53,6 +56,7 @@ target_link_libraries(experiment_host
test_nodes
)
ament_target_dependencies(experiment_host
tracetools
rclcpp
simple_timer
priority_executor

1
src/casestudy_tools/package.xml
View file

@ -11,6 +11,7 @@
<buildtool_depend>ament_cmake</buildtool_depend>
<depend>tracetools</depend>
<depend>rclcpp</depend>
<depend>priority_executor</depend>
<depend>simple_timer</depend>

4
src/full_topology/CMakeLists.txt
View file

@ -16,7 +16,10 @@ find_package(ament_cmake REQUIRED)
find_package(rclcpp REQUIRED)
find_package(std_msgs REQUIRED)
add_compile_definitions(RCLCPP_ENABLE_TRACEPOINTS)
add_executable(${PROJECT_NAME} src/full_topology.cpp)
#add_executable(${PROJECT_NAME} src/test.cpp)
target_include_directories(${PROJECT_NAME} PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/include>
$<INSTALL_INTERFACE:include>
@ -26,6 +29,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC
ament_target_dependencies(${PROJECT_NAME}
rclcpp
std_msgs
tracetools
)
# Installation

46
src/full_topology/launch/trace_full_topology.launch.py
View file

@ -1,22 +1,60 @@
from launch import LaunchDescription
import os
import launch
from pathlib import Path
from launch_ros.actions import Node
from tracetools_launch.action import Trace
def generate_launch_description():
return LaunchDescription([
length_arg = launch.actions.DeclareLaunchArgument(
'length',
default_value='60.0',
description='length of execution in seconds',
)
file_dir = Path(__file__).resolve().parent
target_path = file_dir.parents[2] / 'analysis' / 'tracing'
return launch.LaunchDescription([
length_arg,
Trace(
session_name='full_topology_tracing',
append_timestamp=True,
base_path=target_path,
events_ust=[
'rclcpp:*', 'rcl:*', 'rmw:*', 'ros2:*', 'rclcpp:*callback*', 'rclcpp:*executor*'
'dds:*',
'ros2:*',
'rcl:*',
'rmw:*',
'rclcpp:*',
'rclcpp:*callback*',
'rclcpp:*executor*',
'rmw_cyclonedds:*',
# explicit list of message‑flow events
'ros2:rclcpp_publish',
'ros2:rclcpp_subscription_callback',
'ros2:rclcpp_timer_callback',
'ros2:rclcpp_executor_execute',
'ros2:rclcpp_intra_publish',
'ros2:rmw_take',
'ros2:callback_start',
'ros2:callback_end',
# ---- (optional) add whatever else you like ----
# scheduling / executor / memory events:
'ros2:rclcpp_executor_*',
'ros2:*callback_added',
],
events_kernel=[
'sched_switch', 'sched_wakeup'
],
),
Node(
package='full_topology',
executable='full_topology',
output='screen',
),
# Shut down after some time, otherwise the system would run indefinitely
launch.actions.TimerAction(
period=launch.substitutions.LaunchConfiguration(length_arg.name),
actions=[launch.actions.Shutdown(reason='stopping system')],
),
])

1
src/full_topology/package.xml
View file

@ -17,6 +17,7 @@
<!-- Build Dependencies -->
<depend>rclcpp</depend>
<depend>std_msgs</depend>
<depend>tracetools</depend>
<test_depend>ament_copyright</test_depend>
<test_depend>ament_flake8</test_depend>

18
src/full_topology/src/full_topology.cpp
View file

@ -5,12 +5,14 @@
#include <string>
#include <vector>
#include <thread>
#include "rclcpp/rclcpp.hpp"
// the order here matters
#include "tracetools/utils.hpp"
#include <rclcpp/rclcpp.hpp>
#include "std_msgs/msg/string.hpp"
using namespace std::chrono_literals;
constexpr int RUN_DURATION_SECONDS = 20;
constexpr bool DEBUG = false;
// A generic sensor node that publishes at a given rate
@ -138,12 +140,13 @@ public:
RCLCPP_INFO(get_logger(), "%s received: %s", name.c_str(), msg->data.c_str());
}
/* TODO
// random: 1 in 5 it will be 10s instead of delay_ms
if (rand() % 5 == 0) {
std::this_thread::sleep_for(std::chrono::milliseconds(10000));
} else {
std::this_thread::sleep_for(std::chrono::milliseconds(delay_ms));
}
}*/
// Custom transformation of data
auto out = std_msgs::msg::String();
@ -255,6 +258,7 @@ int main(int argc, char **argv)
auto fusion = std::make_shared<FusionNode>(
"sensor_fusion_node", std::vector<std::string>{"/gps/fix", "/imu/data", "/baro/alt"}, "/sensors/fused", 10);
auto lidar = std::make_shared<LidarNode>("lidar_node", "/lidar/scan", 15);
// TODO: operator or autonomous flight plan
auto cmd = std::make_shared<CommandNode>("operator_cmd_node", "/operator/commands", 1);
auto mgmt = std::make_shared<FlightManagementNode>(
"flight_mgmt_node", std::vector<std::string>{"/sensors/fused", "/lidar/scan", "/operator/commands"}, "/flight/plan", 20);
@ -283,13 +287,7 @@ int main(int argc, char **argv)
executor.add_node(telemetry);
executor.add_node(radio);
// ToDo: find a better way to handle time limited executions
auto start_time = std::chrono::steady_clock::now();
while (rclcpp::ok() && (std::chrono::steady_clock::now() - start_time) < std::chrono::seconds(RUN_DURATION_SECONDS))
{
executor.spin_some();
std::this_thread::sleep_for(1ms);
}
executor.spin();
rclcpp::shutdown();
return 0;

9
src/full_topology/src/test.cpp Normal file
View file

@ -0,0 +1,9 @@
#include "tracetools/utils.hpp"
#include <rclcpp/rclcpp.hpp>
int main(int argc, char** argv) {
rclcpp::init(argc, argv);
auto node = std::make_shared<rclcpp::Node>("test_node");
rclcpp::shutdown();
return 0;
}

1
src/rcl Submodule

@ -0,0 +1 @@
Subproject commit 5da7e193371c25cf1ebd2489978768327c382c04

1
src/rclcpp Submodule

@ -0,0 +1 @@
Subproject commit e57a3a393e46f0c0e0e5cc64090ac34e5cef32bc

1
src/rmw_cyclonedds Submodule

@ -0,0 +1 @@
Subproject commit 79e07924d66b205838d28da4c9df6a941df6795c

1
src/ros2_tracing Submodule

@ -0,0 +1 @@
Subproject commit c0a5cf2d1dc8706a62bae59123e2c71a6707ac33

1
src/ros2trace
View file

@ -1 +0,0 @@
tools/ros2_tracing/ros2trace

1
src/ros2trace_analysis
View file

@ -1 +0,0 @@
tools/tracetools_analysis/ros2trace_analysis

2
src/ros_edf

@ -1 +1 @@
Subproject commit d7ecdf0108d8decdda33b6dd7fb1bbc5f1d3ac43
Subproject commit c44b7b8c3c527a8d16739349b50340a7de9b6f51

2
src/simple_topology/CMakeLists.txt
View file

@ -13,6 +13,7 @@ endif()
# find dependencies
find_package(ament_cmake REQUIRED)
find_package(tracetools REQUIRED)
find_package(rclcpp REQUIRED)
find_package(std_msgs REQUIRED)
@ -24,6 +25,7 @@ target_include_directories(${PROJECT_NAME} PUBLIC
# Add dependencies with ament
ament_target_dependencies(${PROJECT_NAME}
tracetools
rclcpp
std_msgs
)

1
src/simple_topology/package.xml
View file

@ -11,6 +11,7 @@
<depend>launch</depend>
<depend>launch_ros</depend>
<depend>tracetools</depend>
<depend>tracetools_trace</depend>
<depend>tracetools_launch</depend>

1
src/tools/ros2_tracing

@ -1 +0,0 @@
Subproject commit 449a0123fd12032519ca6bea2209cc0505304771

1
src/tools/tracetools_analysis

@ -1 +0,0 @@
Subproject commit d99bb4041665d77d6f9502d614af498d31d7b6de

1
src/tracetools
View file

@ -1 +0,0 @@
tools/ros2_tracing/tracetools

1
src/tracetools_analysis
View file

@ -1 +0,0 @@
tools/tracetools_analysis/tracetools_analysis

1
src/tracetools_analysis Submodule

@ -0,0 +1 @@
Subproject commit 0feb705d5bf8e3578a28a6a5f9397695228c2f26

1
src/tracetools_launch
View file

@ -1 +0,0 @@
tools/ros2_tracing/tracetools_launch

1
src/tracetools_read
View file

@ -1 +0,0 @@
tools/ros2_tracing/tracetools_read

1
src/tracetools_test
View file

@ -1 +0,0 @@
tools/ros2_tracing/tracetools_test

1
src/tracetools_trace
View file

@ -1 +0,0 @@
tools/ros2_tracing/tracetools_trace