251 lines
11 KiB
Python
251 lines
11 KiB
Python
|
import pandas as pd
|
||
|
|
import numpy as np
|
||
|
|
import matplotlib.pyplot as plt
|
||
|
|
import seaborn as sns
|
||
|
|
import os
|
||
|
|
import glob
|
||
|
|
import argparse
|
||
|
|
from pathlib import Path
|
||
|
|
|
||
|
|
def parse_arguments():
|
||
|
|
parser = argparse.ArgumentParser(description='Cross-experiment analysis of chain performance.')
|
||
|
|
parser.add_argument('--experiments-dir', '-e', required=True,
|
||
|
|
help='Path to directory containing experiment subdirectories')
|
||
|
|
parser.add_argument('--supplementary', '-s', required=True,
|
||
|
|
help='Path to supplementary.csv file with input delays')
|
||
|
|
parser.add_argument('--output', '-o', default='cross_experiment_analysis.png',
|
||
|
|
help='Output filename for the plot')
|
||
|
|
parser.add_argument('--experiment-duration', '-d', type=int, default=20,
|
||
|
|
help='Duration of each experiment in seconds (default: 20)')
|
||
|
|
return parser.parse_args()
|
||
|
|
|
||
|
|
def load_supplementary_data(supplementary_path):
|
||
|
|
"""Load the supplementary data with input delays for each chain."""
|
||
|
|
supp_df = pd.read_csv(supplementary_path)
|
||
|
|
# Create a dictionary for quick lookup
|
||
|
|
delay_dict = dict(zip(supp_df['chain'], supp_df['input_delay']))
|
||
|
|
return delay_dict
|
||
|
|
|
||
|
|
def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s):
|
||
|
|
"""Calculate the theoretical maximum number of runs for a chain."""
|
||
|
|
runs_per_second = 1000 / input_delay_ms # Convert ms to runs per second
|
||
|
|
max_runs = runs_per_second * experiment_duration_s
|
||
|
|
return int(max_runs)
|
||
|
|
|
||
|
|
def load_experiment_data(experiments_dir, delay_dict, experiment_duration):
|
||
|
|
"""Load all experiment data and calculate performance metrics."""
|
||
|
|
all_data = []
|
||
|
|
|
||
|
|
# Find all subdirectories containing results.csv
|
||
|
|
experiment_dirs = [d for d in Path(experiments_dir).iterdir()
|
||
|
|
if d.is_dir() and (d / 'results.csv').exists()]
|
||
|
|
|
||
|
|
print(f"Found {len(experiment_dirs)} experiment directories")
|
||
|
|
|
||
|
|
for exp_dir in experiment_dirs:
|
||
|
|
results_path = exp_dir / 'results.csv'
|
||
|
|
|
||
|
|
try:
|
||
|
|
df = pd.read_csv(results_path)
|
||
|
|
|
||
|
|
# Extract experiment name (remove timestamp if present)
|
||
|
|
if 'experiment_name' in df.columns:
|
||
|
|
exp_name = df['experiment_name'].iloc[0]
|
||
|
|
exp_name = exp_name.split('-')[0] if '-' in exp_name else exp_name
|
||
|
|
else:
|
||
|
|
exp_name = exp_dir.name
|
||
|
|
|
||
|
|
# Group by chain and calculate metrics
|
||
|
|
for chain, chain_data in df.groupby('chain'):
|
||
|
|
if chain in delay_dict:
|
||
|
|
# Calculate theoretical maximum runs
|
||
|
|
input_delay = delay_dict[chain]
|
||
|
|
theoretical_max = calculate_theoretical_max_runs(
|
||
|
|
chain, input_delay, experiment_duration
|
||
|
|
)
|
||
|
|
|
||
|
|
# Calculate actual performance metrics
|
||
|
|
actual_runs = chain_data['count'].mean()
|
||
|
|
mean_latency = chain_data['mean'].mean()
|
||
|
|
std_latency = chain_data['std'].mean()
|
||
|
|
|
||
|
|
# Calculate percentage of theoretical maximum
|
||
|
|
completion_percentage = (actual_runs / theoretical_max) * 100
|
||
|
|
|
||
|
|
if completion_percentage > 100:
|
||
|
|
print(f"Warning: Completion percentage for {chain} in {exp_name} exceeds 100%: {completion_percentage:.2f}%")
|
||
|
|
# Cap at 105% for visualization purposes
|
||
|
|
# This is to avoid visual clutter in the plot
|
||
|
|
# and to handle cases where the actual runs exceed theoretical max.
|
||
|
|
# This is a safeguard and should be adjusted based on actual data characteristics.
|
||
|
|
# In practice, this might indicate an issue with the data or the calculation.
|
||
|
|
completion_percentage = 105
|
||
|
|
|
||
|
|
all_data.append({
|
||
|
|
'experiment_type': exp_name,
|
||
|
|
'experiment_dir': exp_dir.name,
|
||
|
|
'chain': chain,
|
||
|
|
'mean_latency_ms': mean_latency,
|
||
|
|
'std_latency_ms': std_latency,
|
||
|
|
'actual_runs': actual_runs,
|
||
|
|
'theoretical_max_runs': theoretical_max,
|
||
|
|
'completion_percentage': completion_percentage,
|
||
|
|
'input_delay_ms': input_delay
|
||
|
|
})
|
||
|
|
else:
|
||
|
|
print(f"Warning: Chain '{chain}' not found in supplementary data")
|
||
|
|
|
||
|
|
except Exception as e:
|
||
|
|
print(f"Error processing {results_path}: {e}")
|
||
|
|
|
||
|
|
return pd.DataFrame(all_data)
|
||
|
|
|
||
|
|
def create_visualization(data_df, output_path):
|
||
|
|
"""Create the main visualization plot."""
|
||
|
|
plt.style.use('seaborn-v0_8-darkgrid')
|
||
|
|
|
||
|
|
# Set up the figure
|
||
|
|
fig, ax = plt.subplots(figsize=(20, 12))
|
||
|
|
|
||
|
|
# Get unique experiment types and chains for color/marker assignment
|
||
|
|
experiment_types = data_df['experiment_type'].unique()
|
||
|
|
chains = data_df['chain'].unique()
|
||
|
|
|
||
|
|
# Create color palette for experiment types
|
||
|
|
exp_colors = sns.color_palette("husl", len(experiment_types))
|
||
|
|
exp_color_map = dict(zip(experiment_types, exp_colors))
|
||
|
|
|
||
|
|
# Create marker styles for chains (cycle through available markers)
|
||
|
|
markers = ['o', 's', '^', 'D', 'v', '<', '>', 'p', '*', 'h', 'H', '+', 'x']
|
||
|
|
chain_markers = dict(zip(chains, markers[:len(chains)]))
|
||
|
|
|
||
|
|
# Plot data points
|
||
|
|
for _, row in data_df.iterrows():
|
||
|
|
ax.scatter(
|
||
|
|
row['completion_percentage'],
|
||
|
|
row['mean_latency_ms'],
|
||
|
|
color=exp_color_map[row['experiment_type']],
|
||
|
|
marker=chain_markers[row['chain']],
|
||
|
|
s=100,
|
||
|
|
alpha=0.7,
|
||
|
|
edgecolors='black',
|
||
|
|
linewidth=0.5
|
||
|
|
)
|
||
|
|
|
||
|
|
# Set labels and title
|
||
|
|
ax.set_xlabel('Completion Rate (% of Theoretical Maximum)', fontsize=14, fontweight='bold')
|
||
|
|
ax.set_ylabel('Mean End-to-End Latency (ms)', fontsize=14, fontweight='bold')
|
||
|
|
ax.set_title('Cross-Experiment Performance Analysis\nMean Latency vs Chain Completion Rate',
|
||
|
|
fontsize=16, fontweight='bold', pad=20)
|
||
|
|
|
||
|
|
# Add grid for better readability
|
||
|
|
ax.grid(True, alpha=0.3)
|
||
|
|
|
||
|
|
# Create legends with better positioning
|
||
|
|
# Legend for experiment types (colors)
|
||
|
|
exp_legend_elements = [plt.Line2D([0], [0], marker='o', color='w',
|
||
|
|
markerfacecolor=color, markersize=10,
|
||
|
|
label=exp_type, markeredgecolor='black', markeredgewidth=1)
|
||
|
|
for exp_type, color in exp_color_map.items()]
|
||
|
|
|
||
|
|
# Legend for chains (markers) - limit to avoid overcrowding
|
||
|
|
max_chains_in_legend = min(len(chains), 11) # Show max 11 chains
|
||
|
|
chain_legend_elements = [plt.Line2D([0], [0], marker=marker, color='w',
|
||
|
|
markerfacecolor='gray', markersize=10,
|
||
|
|
label=chain.split(' --> ')[-1], markeredgecolor='black', markeredgewidth=1)
|
||
|
|
for chain, marker in list(chain_markers.items())[:max_chains_in_legend]]
|
||
|
|
|
||
|
|
# Position legends inside the plot area
|
||
|
|
legend1 = ax.legend(handles=exp_legend_elements, title='Experiment Type',
|
||
|
|
loc='upper right', fontsize=10, title_fontsize=12,
|
||
|
|
framealpha=0.9, fancybox=True, shadow=True)
|
||
|
|
|
||
|
|
# Temporarily remove first legend to add second one
|
||
|
|
legend1.remove()
|
||
|
|
|
||
|
|
legend2 = ax.legend(handles=chain_legend_elements, title='Chain Output',
|
||
|
|
loc='lower right', fontsize=9, title_fontsize=11,
|
||
|
|
framealpha=0.9, fancybox=True, shadow=True)
|
||
|
|
|
||
|
|
# Add both legends back
|
||
|
|
ax.add_artist(legend1)
|
||
|
|
ax.add_artist(legend2)
|
||
|
|
|
||
|
|
# Save the plot
|
||
|
|
plt.savefig(output_path, dpi=300, bbox_inches='tight')
|
||
|
|
plt.show()
|
||
|
|
|
||
|
|
return fig
|
||
|
|
|
||
|
|
def print_summary_statistics(data_df):
|
||
|
|
"""Print summary statistics for the analysis."""
|
||
|
|
print("\n" + "="*80)
|
||
|
|
print("CROSS-EXPERIMENT ANALYSIS SUMMARY")
|
||
|
|
print("="*80)
|
||
|
|
|
||
|
|
print(f"\nTotal experiments analyzed: {data_df['experiment_type'].nunique()}")
|
||
|
|
print(f"Total chains analyzed: {data_df['chain'].nunique()}")
|
||
|
|
print(f"Total data points: {len(data_df)}")
|
||
|
|
|
||
|
|
print("\nPer Experiment Type Summary:")
|
||
|
|
exp_summary = data_df.groupby('experiment_type').agg({
|
||
|
|
'completion_percentage': ['mean', 'std', 'min', 'max'],
|
||
|
|
'mean_latency_ms': ['mean', 'std', 'min', 'max'],
|
||
|
|
'chain': 'count'
|
||
|
|
}).round(2)
|
||
|
|
print(exp_summary)
|
||
|
|
|
||
|
|
print("\nPer Chain Summary:")
|
||
|
|
chain_summary = data_df.groupby('chain').agg({
|
||
|
|
'completion_percentage': ['mean', 'std'],
|
||
|
|
'mean_latency_ms': ['mean', 'std'],
|
||
|
|
'experiment_type': 'count'
|
||
|
|
}).round(2)
|
||
|
|
print(chain_summary)
|
||
|
|
|
||
|
|
# Find best and worst performing combinations
|
||
|
|
print("\nBest Performance (highest completion rate):")
|
||
|
|
best_completion = data_df.loc[data_df['completion_percentage'].idxmax()]
|
||
|
|
print(f" {best_completion['experiment_type']} - {best_completion['chain']}")
|
||
|
|
print(f" Completion: {best_completion['completion_percentage']:.1f}%, Latency: {best_completion['mean_latency_ms']:.1f}ms")
|
||
|
|
|
||
|
|
print("\nWorst Performance (lowest completion rate):")
|
||
|
|
worst_completion = data_df.loc[data_df['completion_percentage'].idxmin()]
|
||
|
|
print(f" {worst_completion['experiment_type']} - {worst_completion['chain']}")
|
||
|
|
print(f" Completion: {worst_completion['completion_percentage']:.1f}%, Latency: {worst_completion['mean_latency_ms']:.1f}ms")
|
||
|
|
|
||
|
|
def main():
|
||
|
|
args = parse_arguments()
|
||
|
|
|
||
|
|
print("Starting cross-experiment analysis...")
|
||
|
|
|
||
|
|
# Load supplementary data
|
||
|
|
print(f"Loading supplementary data from: {args.supplementary}")
|
||
|
|
delay_dict = load_supplementary_data(args.supplementary)
|
||
|
|
print(f"Found delay information for {len(delay_dict)} chains")
|
||
|
|
|
||
|
|
# Load all experiment data
|
||
|
|
print(f"Loading experiment data from: {args.experiments_dir}")
|
||
|
|
data_df = load_experiment_data(args.experiments_dir, delay_dict, args.experiment_duration)
|
||
|
|
|
||
|
|
if data_df.empty:
|
||
|
|
print("No data found! Please check your paths and file formats.")
|
||
|
|
return
|
||
|
|
|
||
|
|
print(f"Loaded data for {len(data_df)} experiment-chain combinations")
|
||
|
|
|
||
|
|
# Create visualization
|
||
|
|
print(f"Creating visualization...")
|
||
|
|
create_visualization(data_df, args.output)
|
||
|
|
|
||
|
|
# Print summary statistics
|
||
|
|
print_summary_statistics(data_df)
|
||
|
|
|
||
|
|
# Save detailed data to CSV for further analysis
|
||
|
|
csv_output = args.output.replace('.png', '_detailed_data.csv')
|
||
|
|
data_df.to_csv(csv_output, index=False)
|
||
|
|
print(f"\nDetailed data saved to: {csv_output}")
|
||
|
|
print(f"Visualization saved to: {args.output}")
|
||
|
|
|
||
|
|
if __name__ == "__main__":
|
||
|
|
main()
|