import pandas as pd import numpy as np import matplotlib.pyplot as plt import seaborn as sns import os import glob import argparse from pathlib import Path def parse_arguments(): parser = argparse.ArgumentParser(description='Cross-experiment analysis of chain performance.') parser.add_argument('--experiments-dir', '-e', required=True, help='Path to directory containing experiment subdirectories') parser.add_argument('--supplementary', '-s', required=True, help='Path to supplementary.csv file with input delays') parser.add_argument('--output', '-o', default='cross_experiment_analysis.png', help='Output filename for the plot') parser.add_argument('--experiment-duration', '-d', type=int, default=20, help='Duration of each experiment in seconds (default: 20)') return parser.parse_args() def load_supplementary_data(supplementary_path): """Load the supplementary data with input delays for each chain.""" supp_df = pd.read_csv(supplementary_path) # Create a dictionary for quick lookup delay_dict = dict(zip(supp_df['chain'], supp_df['input_delay'])) return delay_dict def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s): """Calculate the theoretical maximum number of runs for a chain.""" runs_per_second = 1000 / input_delay_ms # Convert ms to runs per second max_runs = runs_per_second * experiment_duration_s return int(max_runs) def load_experiment_data(experiments_dir, delay_dict, experiment_duration): """Load all experiment data and calculate performance metrics.""" all_data = [] # Find all subdirectories containing results.csv experiment_dirs = [d for d in Path(experiments_dir).iterdir() if d.is_dir() and (d / 'results.csv').exists()] print(f"Found {len(experiment_dirs)} experiment directories") for exp_dir in experiment_dirs: results_path = exp_dir / 'results.csv' try: df = pd.read_csv(results_path) # Extract experiment name (remove timestamp if present) if 'experiment_name' in df.columns: exp_name = df['experiment_name'].iloc[0] exp_name = exp_name.split('-')[0] if '-' in exp_name else exp_name else: exp_name = exp_dir.name # Group by chain and calculate metrics for chain, chain_data in df.groupby('chain'): if chain in delay_dict: # Calculate theoretical maximum runs input_delay = delay_dict[chain] theoretical_max = calculate_theoretical_max_runs( chain, input_delay, experiment_duration ) # Calculate actual performance metrics actual_runs = chain_data['count'].mean() mean_latency = chain_data['mean'].mean() std_latency = chain_data['std'].mean() # Calculate percentage of theoretical maximum completion_percentage = (actual_runs / theoretical_max) * 100 if completion_percentage > 100: print(f"Warning: Completion percentage for {chain} in {exp_name} exceeds 100%: {completion_percentage:.2f}%") # Cap at 105% for visualization purposes # This is to avoid visual clutter in the plot # and to handle cases where the actual runs exceed theoretical max. # This is a safeguard and should be adjusted based on actual data characteristics. # In practice, this might indicate an issue with the data or the calculation. completion_percentage = 105 all_data.append({ 'experiment_type': exp_name, 'experiment_dir': exp_dir.name, 'chain': chain, 'mean_latency_ms': mean_latency, 'std_latency_ms': std_latency, 'actual_runs': actual_runs, 'theoretical_max_runs': theoretical_max, 'completion_percentage': completion_percentage, 'input_delay_ms': input_delay }) else: print(f"Warning: Chain '{chain}' not found in supplementary data") except Exception as e: print(f"Error processing {results_path}: {e}") return pd.DataFrame(all_data) def create_visualization(data_df, output_path): """Create the main visualization plot.""" plt.style.use('seaborn-v0_8-darkgrid') # Set up the figure fig, ax = plt.subplots(figsize=(20, 12)) # Get unique experiment types and chains for color/marker assignment experiment_types = data_df['experiment_type'].unique() chains = data_df['chain'].unique() # Create color palette for experiment types exp_colors = sns.color_palette("husl", len(experiment_types)) exp_color_map = dict(zip(experiment_types, exp_colors)) # Create marker styles for chains (cycle through available markers) markers = ['o', 's', '^', 'D', 'v', '<', '>', 'p', '*', 'h', 'H', '+', 'x'] chain_markers = dict(zip(chains, markers[:len(chains)])) # Plot data points for _, row in data_df.iterrows(): ax.scatter( row['completion_percentage'], row['mean_latency_ms'], color=exp_color_map[row['experiment_type']], marker=chain_markers[row['chain']], s=100, alpha=0.7, edgecolors='black', linewidth=0.5 ) # Set labels and title ax.set_xlabel('Completion Rate (% of Theoretical Maximum)', fontsize=14, fontweight='bold') ax.set_ylabel('Mean End-to-End Latency (ms)', fontsize=14, fontweight='bold') ax.set_title('Cross-Experiment Performance Analysis\nMean Latency vs Chain Completion Rate', fontsize=16, fontweight='bold', pad=20) # Add grid for better readability ax.grid(True, alpha=0.3) # Create legends with better positioning # Legend for experiment types (colors) exp_legend_elements = [plt.Line2D([0], [0], marker='o', color='w', markerfacecolor=color, markersize=10, label=exp_type, markeredgecolor='black', markeredgewidth=1) for exp_type, color in exp_color_map.items()] # Legend for chains (markers) - limit to avoid overcrowding max_chains_in_legend = min(len(chains), 11) # Show max 11 chains chain_legend_elements = [plt.Line2D([0], [0], marker=marker, color='w', markerfacecolor='gray', markersize=10, label=chain.split(' --> ')[-1], markeredgecolor='black', markeredgewidth=1) for chain, marker in list(chain_markers.items())[:max_chains_in_legend]] # Position legends inside the plot area legend1 = ax.legend(handles=exp_legend_elements, title='Experiment Type', loc='upper right', fontsize=10, title_fontsize=12, framealpha=0.9, fancybox=True, shadow=True) # Temporarily remove first legend to add second one legend1.remove() legend2 = ax.legend(handles=chain_legend_elements, title='Chain Output', loc='lower right', fontsize=9, title_fontsize=11, framealpha=0.9, fancybox=True, shadow=True) # Add both legends back ax.add_artist(legend1) ax.add_artist(legend2) # Save the plot plt.savefig(output_path, dpi=300, bbox_inches='tight') plt.show() return fig def print_summary_statistics(data_df): """Print summary statistics for the analysis.""" print("\n" + "="*80) print("CROSS-EXPERIMENT ANALYSIS SUMMARY") print("="*80) print(f"\nTotal experiments analyzed: {data_df['experiment_type'].nunique()}") print(f"Total chains analyzed: {data_df['chain'].nunique()}") print(f"Total data points: {len(data_df)}") print("\nPer Experiment Type Summary:") exp_summary = data_df.groupby('experiment_type').agg({ 'completion_percentage': ['mean', 'std', 'min', 'max'], 'mean_latency_ms': ['mean', 'std', 'min', 'max'], 'chain': 'count' }).round(2) print(exp_summary) print("\nPer Chain Summary:") chain_summary = data_df.groupby('chain').agg({ 'completion_percentage': ['mean', 'std'], 'mean_latency_ms': ['mean', 'std'], 'experiment_type': 'count' }).round(2) print(chain_summary) # Find best and worst performing combinations print("\nBest Performance (highest completion rate):") best_completion = data_df.loc[data_df['completion_percentage'].idxmax()] print(f" {best_completion['experiment_type']} - {best_completion['chain']}") print(f" Completion: {best_completion['completion_percentage']:.1f}%, Latency: {best_completion['mean_latency_ms']:.1f}ms") print("\nWorst Performance (lowest completion rate):") worst_completion = data_df.loc[data_df['completion_percentage'].idxmin()] print(f" {worst_completion['experiment_type']} - {worst_completion['chain']}") print(f" Completion: {worst_completion['completion_percentage']:.1f}%, Latency: {worst_completion['mean_latency_ms']:.1f}ms") def main(): args = parse_arguments() print("Starting cross-experiment analysis...") # Load supplementary data print(f"Loading supplementary data from: {args.supplementary}") delay_dict = load_supplementary_data(args.supplementary) print(f"Found delay information for {len(delay_dict)} chains") # Load all experiment data print(f"Loading experiment data from: {args.experiments_dir}") data_df = load_experiment_data(args.experiments_dir, delay_dict, args.experiment_duration) if data_df.empty: print("No data found! Please check your paths and file formats.") return print(f"Loaded data for {len(data_df)} experiment-chain combinations") # Create visualization print(f"Creating visualization...") create_visualization(data_df, args.output) # Print summary statistics print_summary_statistics(data_df) # Save detailed data to CSV for further analysis csv_output = args.output.replace('.png', '_detailed_data.csv') data_df.to_csv(csv_output, index=False) print(f"\nDetailed data saved to: {csv_output}") print(f"Visualization saved to: {args.output}") if __name__ == "__main__": main()