diff --git a/eye_catcher_plot.py b/eye_catcher_plot.py index 519f2ac..21883bb 100644 --- a/eye_catcher_plot.py +++ b/eye_catcher_plot.py @@ -13,18 +13,26 @@ def parse_arguments(): help='Path to directory containing experiment subdirectories') parser.add_argument('--supplementary', '-s', required=True, help='Path to supplementary.csv file with input delays') - parser.add_argument('--output', '-o', default='cross_experiment_analysis.png', - help='Output filename for the plot') + parser.add_argument('--output', '-o', default='cross_experiment_analysis', + help='Output filename prefix for the plots (will add experiment type and .png)') parser.add_argument('--experiment-duration', '-d', type=int, default=20, help='Duration of each experiment in seconds (default: 20)') return parser.parse_args() def load_supplementary_data(supplementary_path): - """Load the supplementary data with input delays for each chain.""" + """Load the supplementary data with input delays and theoretical perfect times for each chain.""" supp_df = pd.read_csv(supplementary_path) - # Create a dictionary for quick lookup + # Create dictionaries for quick lookup delay_dict = dict(zip(supp_df['chain'], supp_df['input_delay'])) - return delay_dict + + # Load theoretical perfect e2e time (assuming the third column exists) + if len(supp_df.columns) >= 3: + perfect_time_dict = dict(zip(supp_df['chain'], supp_df.iloc[:, 2])) # Third column + return delay_dict, perfect_time_dict + else: + print("Warning: No third column found for theoretical perfect times. Using input_delay as fallback.") + perfect_time_dict = delay_dict.copy() # Fallback to input_delay + return delay_dict, perfect_time_dict def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s): """Calculate the theoretical maximum number of runs for a chain.""" @@ -32,7 +40,7 @@ def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s) max_runs = runs_per_second * experiment_duration_s return int(max_runs) -def load_experiment_data(experiments_dir, delay_dict, experiment_duration): +def load_experiment_data(experiments_dir, delay_dict, perfect_time_dict, experiment_duration): """Load all experiment data and calculate performance metrics.""" all_data = [] @@ -57,9 +65,10 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration): # Group by chain and calculate metrics for chain, chain_data in df.groupby('chain'): - if chain in delay_dict: + if chain in delay_dict and chain in perfect_time_dict: # Calculate theoretical maximum runs input_delay = delay_dict[chain] + perfect_time = perfect_time_dict[chain] theoretical_max = calculate_theoretical_max_runs( chain, input_delay, experiment_duration ) @@ -69,6 +78,9 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration): mean_latency = chain_data['mean'].mean() std_latency = chain_data['std'].mean() + # Normalize latency by theoretical perfect time + normalized_latency = mean_latency / perfect_time + # Calculate percentage of theoretical maximum completion_percentage = (actual_runs / theoretical_max) * 100 @@ -86,96 +98,179 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration): 'experiment_dir': exp_dir.name, 'chain': chain, 'mean_latency_ms': mean_latency, + 'normalized_latency': normalized_latency, 'std_latency_ms': std_latency, 'actual_runs': actual_runs, 'theoretical_max_runs': theoretical_max, 'completion_percentage': completion_percentage, - 'input_delay_ms': input_delay + 'input_delay_ms': input_delay, + 'perfect_time_ms': perfect_time }) else: - print(f"Warning: Chain '{chain}' not found in supplementary data") + missing_info = [] + if chain not in delay_dict: + missing_info.append("input delay") + if chain not in perfect_time_dict: + missing_info.append("perfect time") + print(f"Warning: Chain '{chain}' missing {', '.join(missing_info)} in supplementary data") except Exception as e: print(f"Error processing {results_path}: {e}") return pd.DataFrame(all_data) -def create_visualization(data_df, output_path): - """Create the main visualization plot.""" +def create_visualizations(data_df, output_prefix): + """Create separate visualization plots for each experiment type.""" plt.style.use('seaborn-v0_8-darkgrid') - # Set up the figure - fig, ax = plt.subplots(figsize=(20, 12)) + # Get unique experiment types + experiment_types = sorted(data_df['experiment_type'].unique()) - # Get unique experiment types and chains for color/marker assignment - experiment_types = data_df['experiment_type'].unique() - chains = data_df['chain'].unique() + print(f"Creating {len(experiment_types)} separate plots for experiment types: {experiment_types}") - # Create color palette for experiment types - exp_colors = sns.color_palette("husl", len(experiment_types)) - exp_color_map = dict(zip(experiment_types, exp_colors)) + created_files = [] - # Create marker styles for chains (cycle through available markers) - markers = ['o', 's', '^', 'D', 'v', '<', '>', 'p', '*', 'h', 'H', '+', 'x'] - chain_markers = dict(zip(chains, markers[:len(chains)])) + for exp_type in experiment_types: + # Filter data for this experiment type + exp_data = data_df[data_df['experiment_type'] == exp_type] + + # Get unique chains for this experiment + chains = sorted(exp_data['chain'].unique()) + + # Create color palette for chains + chain_colors = sns.color_palette("husl", len(chains)) + chain_color_map = dict(zip(chains, chain_colors)) + + # Set up the figure + fig, ax = plt.subplots(figsize=(14, 10)) + + # Plot data points for each chain + for chain in chains: + chain_data = exp_data[exp_data['chain'] == chain] + + ax.scatter( + chain_data['completion_percentage'], + chain_data['normalized_latency'], + color=chain_color_map[chain], + label=chain, + s=120, + alpha=0.8, + edgecolors='black', + linewidth=0.8 + ) + + # Set labels and title + ax.set_xlabel('Completion Rate (% of Theoretical Maximum)', fontsize=14, fontweight='bold') + ax.set_ylabel('Normalized Latency (Actual / Theoretical Perfect)', fontsize=14, fontweight='bold') + ax.set_title(f'Performance Analysis: {exp_type}\nNormalized Latency vs Chain Completion Rate', + fontsize=16, fontweight='bold', pad=20) + + # Add grid for better readability + ax.grid(True, alpha=0.3) + + # Set axis limits + ax.set_xlim(0, 107) + ax.set_ylim(bottom=1) + + # Create legend for chains + legend = ax.legend(title='Chain Output', + loc='best', + fontsize=10, + title_fontsize=12, + framealpha=0.9, + fancybox=True, + shadow=True, + bbox_to_anchor=(1.05, 1)) + + # Adjust layout to accommodate legend + plt.tight_layout() + + # Save the plot + safe_exp_name = exp_type.replace('/', '_').replace(' ', '_') + output_path = f"{output_prefix}_{safe_exp_name}.png" + plt.savefig(output_path, dpi=300, bbox_inches='tight') + created_files.append(output_path) + + # Show the plot + plt.show() + + # Close the figure to free memory + plt.close() - # Plot data points - for _, row in data_df.iterrows(): - ax.scatter( - row['completion_percentage'], - row['mean_latency_ms'], - color=exp_color_map[row['experiment_type']], - marker=chain_markers[row['chain']], - s=100, - alpha=0.7, - edgecolors='black', - linewidth=0.5 - ) + return created_files + +def create_combined_summary_plot(data_df, output_prefix): + """Create a combined summary plot showing all experiment types in subplots.""" + experiment_types = sorted(data_df['experiment_type'].unique()) + n_experiments = len(experiment_types) - # Set labels and title - ax.set_xlabel('Completion Rate (% of Theoretical Maximum)', fontsize=14, fontweight='bold') - ax.set_ylabel('Mean End-to-End Latency (ms)', fontsize=14, fontweight='bold') - ax.set_title('Cross-Experiment Performance Analysis\nMean Latency vs Chain Completion Rate', - fontsize=16, fontweight='bold', pad=20) + # Calculate subplot grid dimensions + n_cols = min(3, n_experiments) # Max 3 columns + n_rows = (n_experiments + n_cols - 1) // n_cols # Ceiling division - # Add grid for better readability - ax.grid(True, alpha=0.3) + fig, axes = plt.subplots(n_rows, n_cols, figsize=(6*n_cols, 5*n_rows)) - # Create legends with better positioning - # Legend for experiment types (colors) - exp_legend_elements = [plt.Line2D([0], [0], marker='o', color='w', - markerfacecolor=color, markersize=10, - label=exp_type, markeredgecolor='black', markeredgewidth=1) - for exp_type, color in exp_color_map.items()] + # Ensure axes is always a 2D array + if n_rows == 1 and n_cols == 1: + axes = np.array([[axes]]) + elif n_rows == 1: + axes = axes.reshape(1, -1) + elif n_cols == 1: + axes = axes.reshape(-1, 1) - # Legend for chains (markers) - limit to avoid overcrowding - max_chains_in_legend = min(len(chains), 11) # Show max 11 chains - chain_legend_elements = [plt.Line2D([0], [0], marker=marker, color='w', - markerfacecolor='gray', markersize=10, - label=chain.split(' --> ')[-1], markeredgecolor='black', markeredgewidth=1) - for chain, marker in list(chain_markers.items())[:max_chains_in_legend]] + plt.style.use('seaborn-v0_8-darkgrid') - # Position legends inside the plot area - legend1 = ax.legend(handles=exp_legend_elements, title='Experiment Type', - loc='upper right', fontsize=10, title_fontsize=12, - framealpha=0.9, fancybox=True, shadow=True) + for i, exp_type in enumerate(experiment_types): + row = i // n_cols + col = i % n_cols + ax = axes[row, col] + + # Filter data for this experiment type + exp_data = data_df[data_df['experiment_type'] == exp_type] + chains = sorted(exp_data['chain'].unique()) + + # Create color palette for chains + chain_colors = sns.color_palette("husl", len(chains)) + chain_color_map = dict(zip(chains, chain_colors)) + + # Plot data points + for chain in chains: + chain_data = exp_data[exp_data['chain'] == chain] + ax.scatter( + chain_data['completion_percentage'], + chain_data['normalized_latency'], + color=chain_color_map[chain], + s=60, + alpha=0.7, + edgecolors='black', + linewidth=0.5 + ) + + ax.set_title(exp_type, fontsize=12, fontweight='bold') + ax.set_xlabel('Completion Rate (%)', fontsize=10) + ax.set_ylabel('Normalized Latency', fontsize=10) + ax.grid(True, alpha=0.3) + + # Set axis limits for consistency + ax.set_xlim(0, 107) + ax.set_ylim(bottom=1) - # Temporarily remove first legend to add second one - legend1.remove() + # Hide unused subplots + for i in range(n_experiments, n_rows * n_cols): + row = i // n_cols + col = i % n_cols + axes[row, col].set_visible(False) - legend2 = ax.legend(handles=chain_legend_elements, title='Chain Output', - loc='lower right', fontsize=9, title_fontsize=11, - framealpha=0.9, fancybox=True, shadow=True) + plt.suptitle('Performance Analysis Summary - All Experiment Types\n(Normalized Latency vs Completion Rate)', + fontsize=16, fontweight='bold', y=0.98) + plt.tight_layout() - # Add both legends back - ax.add_artist(legend1) - ax.add_artist(legend2) - - # Save the plot - plt.savefig(output_path, dpi=300, bbox_inches='tight') + summary_output = f"{output_prefix}_summary.png" + plt.savefig(summary_output, dpi=300, bbox_inches='tight') plt.show() + plt.close() - return fig + return summary_output def print_summary_statistics(data_df): """Print summary statistics for the analysis.""" @@ -190,6 +285,7 @@ def print_summary_statistics(data_df): print("\nPer Experiment Type Summary:") exp_summary = data_df.groupby('experiment_type').agg({ 'completion_percentage': ['mean', 'std', 'min', 'max'], + 'normalized_latency': ['mean', 'std', 'min', 'max'], 'mean_latency_ms': ['mean', 'std', 'min', 'max'], 'chain': 'count' }).round(2) @@ -198,6 +294,7 @@ def print_summary_statistics(data_df): print("\nPer Chain Summary:") chain_summary = data_df.groupby('chain').agg({ 'completion_percentage': ['mean', 'std'], + 'normalized_latency': ['mean', 'std'], 'mean_latency_ms': ['mean', 'std'], 'experiment_type': 'count' }).round(2) @@ -207,12 +304,22 @@ def print_summary_statistics(data_df): print("\nBest Performance (highest completion rate):") best_completion = data_df.loc[data_df['completion_percentage'].idxmax()] print(f" {best_completion['experiment_type']} - {best_completion['chain']}") - print(f" Completion: {best_completion['completion_percentage']:.1f}%, Latency: {best_completion['mean_latency_ms']:.1f}ms") + print(f" Completion: {best_completion['completion_percentage']:.1f}%, Normalized Latency: {best_completion['normalized_latency']:.2f}x, Raw Latency: {best_completion['mean_latency_ms']:.1f}ms") print("\nWorst Performance (lowest completion rate):") worst_completion = data_df.loc[data_df['completion_percentage'].idxmin()] print(f" {worst_completion['experiment_type']} - {worst_completion['chain']}") - print(f" Completion: {worst_completion['completion_percentage']:.1f}%, Latency: {worst_completion['mean_latency_ms']:.1f}ms") + print(f" Completion: {worst_completion['completion_percentage']:.1f}%, Normalized Latency: {worst_completion['normalized_latency']:.2f}x, Raw Latency: {worst_completion['mean_latency_ms']:.1f}ms") + + print("\nBest Normalized Latency (closest to theoretical perfect):") + best_latency = data_df.loc[data_df['normalized_latency'].idxmin()] + print(f" {best_latency['experiment_type']} - {best_latency['chain']}") + print(f" Normalized Latency: {best_latency['normalized_latency']:.2f}x, Completion: {best_latency['completion_percentage']:.1f}%, Raw Latency: {best_latency['mean_latency_ms']:.1f}ms") + + print("\nWorst Normalized Latency (furthest from theoretical perfect):") + worst_latency = data_df.loc[data_df['normalized_latency'].idxmax()] + print(f" {worst_latency['experiment_type']} - {worst_latency['chain']}") + print(f" Normalized Latency: {worst_latency['normalized_latency']:.2f}x, Completion: {worst_latency['completion_percentage']:.1f}%, Raw Latency: {worst_latency['mean_latency_ms']:.1f}ms") def main(): args = parse_arguments() @@ -221,12 +328,13 @@ def main(): # Load supplementary data print(f"Loading supplementary data from: {args.supplementary}") - delay_dict = load_supplementary_data(args.supplementary) + delay_dict, perfect_time_dict = load_supplementary_data(args.supplementary) print(f"Found delay information for {len(delay_dict)} chains") + print(f"Found perfect time information for {len(perfect_time_dict)} chains") # Load all experiment data print(f"Loading experiment data from: {args.experiments_dir}") - data_df = load_experiment_data(args.experiments_dir, delay_dict, args.experiment_duration) + data_df = load_experiment_data(args.experiments_dir, delay_dict, perfect_time_dict, args.experiment_duration) if data_df.empty: print("No data found! Please check your paths and file formats.") @@ -234,18 +342,26 @@ def main(): print(f"Loaded data for {len(data_df)} experiment-chain combinations") - # Create visualization - print(f"Creating visualization...") - create_visualization(data_df, args.output) + # Create individual visualizations for each experiment type + print(f"Creating individual visualizations...") + created_files = create_visualizations(data_df, args.output) + + # Create combined summary plot + print(f"Creating combined summary plot...") + summary_file = create_combined_summary_plot(data_df, args.output) + created_files.append(summary_file) # Print summary statistics print_summary_statistics(data_df) # Save detailed data to CSV for further analysis - csv_output = args.output.replace('.png', '_detailed_data.csv') + csv_output = f"{args.output}_detailed_data.csv" data_df.to_csv(csv_output, index=False) print(f"\nDetailed data saved to: {csv_output}") - print(f"Visualization saved to: {args.output}") + + print(f"\nCreated visualization files:") + for file in created_files: + print(f" - {file}") if __name__ == "__main__": main() \ No newline at end of file