wip: split plots and normalize y axis as well

eye_catcher_plot.py

@@ -13,18 +13,26 @@ def parse_arguments():
                        help='Path to directory containing experiment subdirectories')
     parser.add_argument('--supplementary', '-s', required=True,
                        help='Path to supplementary.csv file with input delays')
-    parser.add_argument('--output', '-o', default='cross_experiment_analysis.png',
+    parser.add_argument('--output', '-o', default='cross_experiment_analysis',
-                       help='Output filename for the plot')
+                       help='Output filename prefix for the plots (will add experiment type and .png)')
     parser.add_argument('--experiment-duration', '-d', type=int, default=20,
                        help='Duration of each experiment in seconds (default: 20)')
     return parser.parse_args()
 
 def load_supplementary_data(supplementary_path):
-    """Load the supplementary data with input delays for each chain."""
+    """Load the supplementary data with input delays and theoretical perfect times for each chain."""
     supp_df = pd.read_csv(supplementary_path)
-    # Create a dictionary for quick lookup
+    # Create dictionaries for quick lookup
     delay_dict = dict(zip(supp_df['chain'], supp_df['input_delay']))
-    return delay_dict
+    
+    # Load theoretical perfect e2e time (assuming the third column exists)
+    if len(supp_df.columns) >= 3:
+        perfect_time_dict = dict(zip(supp_df['chain'], supp_df.iloc[:, 2]))  # Third column
+        return delay_dict, perfect_time_dict
+    else:
+        print("Warning: No third column found for theoretical perfect times. Using input_delay as fallback.")
+        perfect_time_dict = delay_dict.copy()  # Fallback to input_delay
+        return delay_dict, perfect_time_dict
 
 def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s):
     """Calculate the theoretical maximum number of runs for a chain."""
@@ -32,7 +40,7 @@ def calculate_theoretical_max_runs(chain, input_delay_ms, experiment_duration_s)
     max_runs = runs_per_second * experiment_duration_s
     return int(max_runs)
 
-def load_experiment_data(experiments_dir, delay_dict, experiment_duration):
+def load_experiment_data(experiments_dir, delay_dict, perfect_time_dict, experiment_duration):
     """Load all experiment data and calculate performance metrics."""
     all_data = []
     
@@ -57,9 +65,10 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration):
             
             # Group by chain and calculate metrics
             for chain, chain_data in df.groupby('chain'):
-                if chain in delay_dict:
+                if chain in delay_dict and chain in perfect_time_dict:
                     # Calculate theoretical maximum runs
                     input_delay = delay_dict[chain]
+                    perfect_time = perfect_time_dict[chain]
                     theoretical_max = calculate_theoretical_max_runs(
                         chain, input_delay, experiment_duration
                     )
@@ -69,6 +78,9 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration):
                     mean_latency = chain_data['mean'].mean()
                     std_latency = chain_data['std'].mean()
                     
+                    # Normalize latency by theoretical perfect time
+                    normalized_latency = mean_latency / perfect_time
+                    
                     # Calculate percentage of theoretical maximum
                     completion_percentage = (actual_runs / theoretical_max) * 100
 
@@ -86,96 +98,179 @@ def load_experiment_data(experiments_dir, delay_dict, experiment_duration):
                         'experiment_dir': exp_dir.name,
                         'chain': chain,
                         'mean_latency_ms': mean_latency,
+                        'normalized_latency': normalized_latency,
                         'std_latency_ms': std_latency,
                         'actual_runs': actual_runs,
                         'theoretical_max_runs': theoretical_max,
                         'completion_percentage': completion_percentage,
-                        'input_delay_ms': input_delay
+                        'input_delay_ms': input_delay,
+                        'perfect_time_ms': perfect_time
                     })
                 else:
-                    print(f"Warning: Chain '{chain}' not found in supplementary data")
+                    missing_info = []
+                    if chain not in delay_dict:
+                        missing_info.append("input delay")
+                    if chain not in perfect_time_dict:
+                        missing_info.append("perfect time")
+                    print(f"Warning: Chain '{chain}' missing {', '.join(missing_info)} in supplementary data")
                     
         except Exception as e:
             print(f"Error processing {results_path}: {e}")
     
     return pd.DataFrame(all_data)
 
-def create_visualization(data_df, output_path):
+def create_visualizations(data_df, output_prefix):
-    """Create the main visualization plot."""
+    """Create separate visualization plots for each experiment type."""
     plt.style.use('seaborn-v0_8-darkgrid')
     
+    # Get unique experiment types
+    experiment_types = sorted(data_df['experiment_type'].unique())
+    
+    print(f"Creating {len(experiment_types)} separate plots for experiment types: {experiment_types}")
+    
+    created_files = []
+    
+    for exp_type in experiment_types:
+        # Filter data for this experiment type
+        exp_data = data_df[data_df['experiment_type'] == exp_type]
+        
+        # Get unique chains for this experiment
+        chains = sorted(exp_data['chain'].unique())
+        
+        # Create color palette for chains
+        chain_colors = sns.color_palette("husl", len(chains))
+        chain_color_map = dict(zip(chains, chain_colors))
+        
         # Set up the figure
-    fig, ax = plt.subplots(figsize=(20, 12))
+        fig, ax = plt.subplots(figsize=(14, 10))
         
-    # Get unique experiment types and chains for color/marker assignment
+        # Plot data points for each chain
-    experiment_types = data_df['experiment_type'].unique()
+        for chain in chains:
-    chains = data_df['chain'].unique()
+            chain_data = exp_data[exp_data['chain'] == chain]
             
-    # Create color palette for experiment types
-    exp_colors = sns.color_palette("husl", len(experiment_types))
-    exp_color_map = dict(zip(experiment_types, exp_colors))
-    
-    # Create marker styles for chains (cycle through available markers)
-    markers = ['o', 's', '^', 'D', 'v', '<', '>', 'p', '*', 'h', 'H', '+', 'x']
-    chain_markers = dict(zip(chains, markers[:len(chains)]))
-    
-    # Plot data points
-    for _, row in data_df.iterrows():
             ax.scatter(
-            row['completion_percentage'], 
+                chain_data['completion_percentage'], 
-            row['mean_latency_ms'],
+                chain_data['normalized_latency'],
-            color=exp_color_map[row['experiment_type']],
+                color=chain_color_map[chain],
-            marker=chain_markers[row['chain']],
+                label=chain,
-            s=100,
+                s=120,
-            alpha=0.7,
+                alpha=0.8,
                 edgecolors='black',
-            linewidth=0.5
+                linewidth=0.8
             )
         
         # Set labels and title
         ax.set_xlabel('Completion Rate (% of Theoretical Maximum)', fontsize=14, fontweight='bold')
-    ax.set_ylabel('Mean End-to-End Latency (ms)', fontsize=14, fontweight='bold')
+        ax.set_ylabel('Normalized Latency (Actual / Theoretical Perfect)', fontsize=14, fontweight='bold')
-    ax.set_title('Cross-Experiment Performance Analysis\nMean Latency vs Chain Completion Rate', 
+        ax.set_title(f'Performance Analysis: {exp_type}\nNormalized Latency vs Chain Completion Rate', 
                     fontsize=16, fontweight='bold', pad=20)
         
         # Add grid for better readability
         ax.grid(True, alpha=0.3)
         
-    # Create legends with better positioning
+        # Set axis limits
-    # Legend for experiment types (colors)
+        ax.set_xlim(0, 107)
-    exp_legend_elements = [plt.Line2D([0], [0], marker='o', color='w', 
+        ax.set_ylim(bottom=1)
-                                     markerfacecolor=color, markersize=10, 
-                                     label=exp_type, markeredgecolor='black', markeredgewidth=1)
-                          for exp_type, color in exp_color_map.items()]
         
-    # Legend for chains (markers) - limit to avoid overcrowding
+        # Create legend for chains
-    max_chains_in_legend = min(len(chains), 11)  # Show max 11 chains
+        legend = ax.legend(title='Chain Output', 
-    chain_legend_elements = [plt.Line2D([0], [0], marker=marker, color='w', 
+                          loc='best', 
-                                       markerfacecolor='gray', markersize=10, 
+                          fontsize=10, 
-                                       label=chain.split(' --> ')[-1], markeredgecolor='black', markeredgewidth=1)
+                          title_fontsize=12,
-                            for chain, marker in list(chain_markers.items())[:max_chains_in_legend]]
+                          framealpha=0.9, 
+                          fancybox=True, 
+                          shadow=True,
+                          bbox_to_anchor=(1.05, 1))
         
-    # Position legends inside the plot area
+        # Adjust layout to accommodate legend
-    legend1 = ax.legend(handles=exp_legend_elements, title='Experiment Type', 
+        plt.tight_layout()
-                       loc='upper right', fontsize=10, title_fontsize=12,
-                       framealpha=0.9, fancybox=True, shadow=True)
-    
-    # Temporarily remove first legend to add second one
-    legend1.remove()
-    
-    legend2 = ax.legend(handles=chain_legend_elements, title='Chain Output', 
-                       loc='lower right', fontsize=9, title_fontsize=11,
-                       framealpha=0.9, fancybox=True, shadow=True)
-    
-    # Add both legends back
-    ax.add_artist(legend1)
-    ax.add_artist(legend2)
         
         # Save the plot
+        safe_exp_name = exp_type.replace('/', '_').replace(' ', '_')
+        output_path = f"{output_prefix}_{safe_exp_name}.png"
         plt.savefig(output_path, dpi=300, bbox_inches='tight')
+        created_files.append(output_path)
+        
+        # Show the plot
         plt.show()
         
-    return fig
+        # Close the figure to free memory
+        plt.close()
+    
+    return created_files
+
+def create_combined_summary_plot(data_df, output_prefix):
+    """Create a combined summary plot showing all experiment types in subplots."""
+    experiment_types = sorted(data_df['experiment_type'].unique())
+    n_experiments = len(experiment_types)
+    
+    # Calculate subplot grid dimensions
+    n_cols = min(3, n_experiments)  # Max 3 columns
+    n_rows = (n_experiments + n_cols - 1) // n_cols  # Ceiling division
+    
+    fig, axes = plt.subplots(n_rows, n_cols, figsize=(6*n_cols, 5*n_rows))
+    
+    # Ensure axes is always a 2D array
+    if n_rows == 1 and n_cols == 1:
+        axes = np.array([[axes]])
+    elif n_rows == 1:
+        axes = axes.reshape(1, -1)
+    elif n_cols == 1:
+        axes = axes.reshape(-1, 1)
+    
+    plt.style.use('seaborn-v0_8-darkgrid')
+    
+    for i, exp_type in enumerate(experiment_types):
+        row = i // n_cols
+        col = i % n_cols
+        ax = axes[row, col]
+        
+        # Filter data for this experiment type
+        exp_data = data_df[data_df['experiment_type'] == exp_type]
+        chains = sorted(exp_data['chain'].unique())
+        
+        # Create color palette for chains
+        chain_colors = sns.color_palette("husl", len(chains))
+        chain_color_map = dict(zip(chains, chain_colors))
+        
+        # Plot data points
+        for chain in chains:
+            chain_data = exp_data[exp_data['chain'] == chain]
+            ax.scatter(
+                chain_data['completion_percentage'], 
+                chain_data['normalized_latency'],
+                color=chain_color_map[chain],
+                s=60,
+                alpha=0.7,
+                edgecolors='black',
+                linewidth=0.5
+            )
+        
+        ax.set_title(exp_type, fontsize=12, fontweight='bold')
+        ax.set_xlabel('Completion Rate (%)', fontsize=10)
+        ax.set_ylabel('Normalized Latency', fontsize=10)
+        ax.grid(True, alpha=0.3)
+        
+        # Set axis limits for consistency
+        ax.set_xlim(0, 107)
+        ax.set_ylim(bottom=1)
+    
+    # Hide unused subplots
+    for i in range(n_experiments, n_rows * n_cols):
+        row = i // n_cols
+        col = i % n_cols
+        axes[row, col].set_visible(False)
+    
+    plt.suptitle('Performance Analysis Summary - All Experiment Types\n(Normalized Latency vs Completion Rate)', 
+                 fontsize=16, fontweight='bold', y=0.98)
+    plt.tight_layout()
+    
+    summary_output = f"{output_prefix}_summary.png"
+    plt.savefig(summary_output, dpi=300, bbox_inches='tight')
+    plt.show()
+    plt.close()
+    
+    return summary_output
 
 def print_summary_statistics(data_df):
     """Print summary statistics for the analysis."""
@@ -190,6 +285,7 @@ def print_summary_statistics(data_df):
     print("\nPer Experiment Type Summary:")
     exp_summary = data_df.groupby('experiment_type').agg({
         'completion_percentage': ['mean', 'std', 'min', 'max'],
+        'normalized_latency': ['mean', 'std', 'min', 'max'],
         'mean_latency_ms': ['mean', 'std', 'min', 'max'],
         'chain': 'count'
     }).round(2)
@@ -198,6 +294,7 @@ def print_summary_statistics(data_df):
     print("\nPer Chain Summary:")
     chain_summary = data_df.groupby('chain').agg({
         'completion_percentage': ['mean', 'std'],
+        'normalized_latency': ['mean', 'std'],
         'mean_latency_ms': ['mean', 'std'],
         'experiment_type': 'count'
     }).round(2)
@@ -207,12 +304,22 @@ def print_summary_statistics(data_df):
     print("\nBest Performance (highest completion rate):")
     best_completion = data_df.loc[data_df['completion_percentage'].idxmax()]
     print(f"  {best_completion['experiment_type']} - {best_completion['chain']}")
-    print(f"  Completion: {best_completion['completion_percentage']:.1f}%, Latency: {best_completion['mean_latency_ms']:.1f}ms")
+    print(f"  Completion: {best_completion['completion_percentage']:.1f}%, Normalized Latency: {best_completion['normalized_latency']:.2f}x, Raw Latency: {best_completion['mean_latency_ms']:.1f}ms")
     
     print("\nWorst Performance (lowest completion rate):")
     worst_completion = data_df.loc[data_df['completion_percentage'].idxmin()]
     print(f"  {worst_completion['experiment_type']} - {worst_completion['chain']}")
-    print(f"  Completion: {worst_completion['completion_percentage']:.1f}%, Latency: {worst_completion['mean_latency_ms']:.1f}ms")
+    print(f"  Completion: {worst_completion['completion_percentage']:.1f}%, Normalized Latency: {worst_completion['normalized_latency']:.2f}x, Raw Latency: {worst_completion['mean_latency_ms']:.1f}ms")
+    
+    print("\nBest Normalized Latency (closest to theoretical perfect):")
+    best_latency = data_df.loc[data_df['normalized_latency'].idxmin()]
+    print(f"  {best_latency['experiment_type']} - {best_latency['chain']}")
+    print(f"  Normalized Latency: {best_latency['normalized_latency']:.2f}x, Completion: {best_latency['completion_percentage']:.1f}%, Raw Latency: {best_latency['mean_latency_ms']:.1f}ms")
+    
+    print("\nWorst Normalized Latency (furthest from theoretical perfect):")
+    worst_latency = data_df.loc[data_df['normalized_latency'].idxmax()]
+    print(f"  {worst_latency['experiment_type']} - {worst_latency['chain']}")
+    print(f"  Normalized Latency: {worst_latency['normalized_latency']:.2f}x, Completion: {worst_latency['completion_percentage']:.1f}%, Raw Latency: {worst_latency['mean_latency_ms']:.1f}ms")
 
 def main():
     args = parse_arguments()
@@ -221,12 +328,13 @@ def main():
     
     # Load supplementary data
     print(f"Loading supplementary data from: {args.supplementary}")
-    delay_dict = load_supplementary_data(args.supplementary)
+    delay_dict, perfect_time_dict = load_supplementary_data(args.supplementary)
     print(f"Found delay information for {len(delay_dict)} chains")
+    print(f"Found perfect time information for {len(perfect_time_dict)} chains")
     
     # Load all experiment data
     print(f"Loading experiment data from: {args.experiments_dir}")
-    data_df = load_experiment_data(args.experiments_dir, delay_dict, args.experiment_duration)
+    data_df = load_experiment_data(args.experiments_dir, delay_dict, perfect_time_dict, args.experiment_duration)
     
     if data_df.empty:
         print("No data found! Please check your paths and file formats.")
@@ -234,18 +342,26 @@ def main():
     
     print(f"Loaded data for {len(data_df)} experiment-chain combinations")
     
-    # Create visualization
+    # Create individual visualizations for each experiment type
-    print(f"Creating visualization...")
+    print(f"Creating individual visualizations...")
-    create_visualization(data_df, args.output)
+    created_files = create_visualizations(data_df, args.output)
+    
+    # Create combined summary plot
+    print(f"Creating combined summary plot...")
+    summary_file = create_combined_summary_plot(data_df, args.output)
+    created_files.append(summary_file)
     
     # Print summary statistics
     print_summary_statistics(data_df)
     
     # Save detailed data to CSV for further analysis
-    csv_output = args.output.replace('.png', '_detailed_data.csv')
+    csv_output = f"{args.output}_detailed_data.csv"
     data_df.to_csv(csv_output, index=False)
     print(f"\nDetailed data saved to: {csv_output}")
-    print(f"Visualization saved to: {args.output}")
+    
+    print(f"\nCreated visualization files:")
+    for file in created_files:
+        print(f"  - {file}")
 
 if __name__ == "__main__":
     main()