⚡️ Speed up function _get_available_includes_from_csv by 41%

[codeflash-ai]

📄 41% (0.41x) speedup for _get_available_includes_from_csv in pdd/auto_include.py

⏱️ Runtime : 41.0 milliseconds → 29.1 milliseconds (best of 170 runs)

📝 Explanation and details

The optimized code achieves a 40% speedup by replacing pandas' slow row-wise apply() function with vectorized string operations.

Key optimization:

• Eliminated dataframe.apply(lambda ...) bottleneck: The original code used pandas apply() with a lambda function that processed each row individually, which is inherently slow (59.8% of original runtime).
• Replaced with vectorized string concatenation: Direct pandas Series string operations ("File: " + file_col + "\nSummary: " + summary_col) that process all rows at once.
• Added column validation: Early return when required columns are missing, preventing exceptions during string operations.

Why it's faster:

• Vectorized operations in pandas use optimized C code under the hood, while apply() with lambda functions involves Python function call overhead for each row
• String concatenation on pandas Series is much more efficient than row-by-row processing
• The explicit astype(str) ensures consistent data types, avoiding potential type conversion overhead during concatenation

Performance characteristics:

• Excellent for large datasets: 285% faster on 1000-row CSV, 269% faster on another 1000-row test
• Handles edge cases better: 65-70% faster when columns are missing due to early validation
• Slight overhead on small datasets: 10-12% slower on single/few rows due to additional column checks and type conversions, but this is negligible compared to gains on realistic workloads

✅ Correctness verification report:

Test	Status
⚙️ Existing Unit Tests	🔘 None Found
🌀 Generated Regression Tests	✅ 42 Passed
⏪ Replay Tests	🔘 None Found
🔎 Concolic Coverage Tests	🔘 None Found
📊 Tests Coverage	100.0%

🌀 Generated Regression Tests and Runtime

from io import StringIO

# function to test
import pandas as pd
# imports
import pytest  # used for our unit tests
from pdd.auto_include import _get_available_includes_from_csv
from rich.console import Console

console = Console()
from pdd.auto_include import _get_available_includes_from_csv

# unit tests

# ------------------ BASIC TEST CASES ------------------

def test_empty_string_returns_empty_list():
    # Test: empty string input should return empty list
    codeflash_output = _get_available_includes_from_csv("") # 399ns -> 396ns (0.758% faster)

def test_single_row_csv():
    # Test: single row, normal input
    csv = "full_path,file_summary\nfoo/bar.py,This is a summary."
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 647μs -> 732μs (11.7% slower)

def test_multiple_rows_csv():
    # Test: multiple rows, normal input
    csv = "full_path,file_summary\nfoo.py,first\nbar.py,second"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 607μs -> 692μs (12.3% slower)

def test_trailing_newline():
    # Test: CSV with trailing newline
    csv = "full_path,file_summary\nfoo.py,first\nbar.py,second\n"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 603μs -> 681μs (11.4% slower)

def test_whitespace_in_fields():
    # Test: CSV with leading/trailing whitespace in fields
    csv = "full_path,file_summary\n foo.py , first summary \nbar.py,second"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 602μs -> 688μs (12.5% slower)

# ------------------ EDGE TEST CASES ------------------

def test_missing_columns():
    # Test: CSV missing one required column
    csv = "full_path\nfoo.py"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 750μs -> 453μs (65.5% faster)

def test_extra_columns():
    # Test: CSV with extra columns
    csv = "full_path,file_summary,extra\nfoo.py,summary,extra1\nbar.py,summary2,extra2"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 647μs -> 711μs (8.93% slower)

def test_column_order_swapped():
    # Test: columns in different order
    csv = "file_summary,full_path\nsummary1,foo.py\nsummary2,bar.py"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 607μs -> 692μs (12.2% slower)

def test_empty_fields():
    # Test: empty fields in CSV
    csv = "full_path,file_summary\n,summary1\nfoo.py,"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 623μs -> 699μs (11.0% slower)

def test_non_utf8_characters():
    # Test: non-ASCII characters in fields
    csv = "full_path,file_summary\nfoo.py,Résumé\nbar.py,naïve"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 612μs -> 692μs (11.5% slower)

def test_incorrect_csv_format():
    # Test: not a CSV at all
    csv = "not,a,csv"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 921μs -> 565μs (62.8% faster)

def test_headers_only():
    # Test: CSV with only headers, no data rows
    csv = "full_path,file_summary"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 627μs -> 734μs (14.5% slower)

def test_headers_with_blank_row():
    # Test: CSV with only headers and one blank row
    csv = "full_path,file_summary\n,"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 607μs -> 693μs (12.4% slower)

def test_csv_with_commas_in_fields():
    # Test: CSV with commas inside quoted fields
    csv = 'full_path,file_summary\n"foo,bar.py","summary, with comma"\nbar.py,"another,summary"'
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 609μs -> 692μs (12.1% slower)

def test_csv_with_newlines_in_fields():
    # Test: CSV with newlines inside quoted fields
    csv = 'full_path,file_summary\nfoo.py,"summary with\nnewline"\nbar.py,"another\nsummary"'
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 603μs -> 687μs (12.2% slower)

def test_csv_with_unicode_emojis():
    # Test: CSV with emojis
    csv = "full_path,file_summary\nfoo.py,Summary 😊\nbar.py,Another 🚀"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 607μs -> 692μs (12.3% slower)

# ------------------ LARGE SCALE TEST CASES ------------------

def test_large_csv_1000_rows():
    # Test: CSV with 1000 rows
    rows = ["full_path,file_summary"]
    for i in range(1000):
        rows.append(f"file{i}.py,summary{i}")
    csv = "\n".join(rows)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 4.71ms -> 1.22ms (285% faster)

def test_large_csv_with_long_fields():
    # Test: CSV with very long fields
    long_path = "a" * 500
    long_summary = "b" * 400
    csv = f"full_path,file_summary\n{long_path},{long_summary}\nshort.py,short summary"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 634μs -> 706μs (10.2% slower)

def test_large_csv_with_special_characters():
    # Test: CSV with many rows with special characters
    rows = ["full_path,file_summary"]
    for i in range(100):
        path = f"file{i}.py"
        summary = f"summary{i} 🚀✨"
        rows.append(f"{path},{summary}")
    csv = "\n".join(rows)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 1.04ms -> 769μs (35.7% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.
#------------------------------------------------
from io import StringIO

# function to test
import pandas as pd
# imports
import pytest  # used for our unit tests
from pdd.auto_include import _get_available_includes_from_csv
from rich.console import Console

console = Console()
from pdd.auto_include import _get_available_includes_from_csv

# unit tests

# 1. Basic Test Cases

def test_basic_single_row():
    # Test with a simple, valid CSV with one row
    csv = "full_path,file_summary\n/foo/bar.py,This is a test file."
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 604μs -> 690μs (12.5% slower)

def test_basic_multiple_rows():
    # Test with multiple valid rows
    csv = "full_path,file_summary\n/a.py,First file.\n/b.py,Second file."
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 610μs -> 684μs (10.8% slower)

def test_basic_trailing_newline():
    # Test with trailing newline at end
    csv = "full_path,file_summary\n/a.py,First file.\n/b.py,Second file.\n"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 613μs -> 681μs (9.97% slower)

def test_basic_whitespace_in_fields():
    # Test with extra whitespace in fields
    csv = "full_path,file_summary\n /a.py , First file. "
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 597μs -> 675μs (11.6% slower)

def test_basic_empty_file_summary():
    # Test with empty file summary
    csv = "full_path,file_summary\n/a.py,"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 662μs -> 710μs (6.85% slower)

def test_basic_empty_full_path():
    # Test with empty full_path
    csv = "full_path,file_summary\n,/summary"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 657μs -> 720μs (8.75% slower)

# 2. Edge Test Cases

def test_edge_empty_string():
    # Test with empty input string
    codeflash_output = _get_available_includes_from_csv(""); result = codeflash_output # 373ns -> 360ns (3.61% faster)

def test_edge_only_headers():
    # Test with only header row, no data
    csv = "full_path,file_summary"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 625μs -> 760μs (17.7% slower)

def test_edge_missing_file_summary_column():
    # Test with missing required column 'file_summary'
    csv = "full_path,other_col\n/a.py,something"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 789μs -> 476μs (65.6% faster)

def test_edge_missing_full_path_column():
    # Test with missing required column 'full_path'
    csv = "other_col,file_summary\nfoo,bar"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 790μs -> 463μs (70.5% faster)

def test_edge_malformed_csv():
    # Test with malformed CSV (unclosed quote)
    csv = 'full_path,file_summary\n"a.py,Missing quote'
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 381μs -> 400μs (4.68% slower)

def test_edge_non_utf8_characters():
    # Test with non-UTF8 characters (simulate with unicode)
    csv = "full_path,file_summary\n/foo/β.py,Contains β character."
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 642μs -> 720μs (10.9% slower)

def test_edge_commas_in_fields():
    # Test with commas inside quoted fields
    csv = 'full_path,file_summary\n"/foo/bar,baz.py","Summary, with comma."'
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 605μs -> 685μs (11.6% slower)

def test_edge_newlines_in_fields():
    # Test with newlines inside quoted fields
    csv = 'full_path,file_summary\n"/foo/bar.py","Summary with\nnewline."'
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 600μs -> 688μs (12.9% slower)

def test_edge_extra_columns():
    # Test with extra columns in CSV
    csv = "full_path,file_summary,extra\n/a.py,First file.,extra1\n/b.py,Second file.,extra2"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 626μs -> 708μs (11.4% slower)

def test_edge_duplicate_rows():
    # Test with duplicate rows
    csv = "full_path,file_summary\n/a.py,First file.\n/a.py,First file."
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 604μs -> 688μs (12.1% slower)

def test_edge_leading_trailing_whitespace_header():
    # Test with whitespace in header names
    csv = " full_path , file_summary \n/foo.py,summary"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 759μs -> 471μs (61.1% faster)

def test_edge_headers_in_different_order():
    # Test with headers in different order
    csv = "file_summary,full_path\nsummary,/foo.py"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 629μs -> 688μs (8.51% slower)

# 3. Large Scale Test Cases

def test_large_scale_1000_rows():
    # Test with 1000 rows
    csv_lines = ["full_path,file_summary"]
    for i in range(1000):
        csv_lines.append(f"/file_{i}.py,Summary {i}")
    csv = "\n".join(csv_lines)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 4.71ms -> 1.28ms (269% faster)

def test_large_scale_long_fields():
    # Test with very long file paths and summaries
    long_path = "/foo/" + "bar" * 200
    long_summary = "summary " + "baz" * 200
    csv = f"full_path,file_summary\n{long_path},{long_summary}"
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 626μs -> 712μs (12.0% slower)

def test_large_scale_all_empty_fields():
    # Test with 1000 rows, all fields empty
    csv_lines = ["full_path,file_summary"]
    for _ in range(1000):
        csv_lines.append(",")
    csv = "\n".join(csv_lines)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 4.75ms -> 1.37ms (248% faster)

def test_large_scale_headers_with_spaces_and_many_rows():
    # Test with headers with spaces and many rows (should fail gracefully)
    csv_lines = [" full_path , file_summary "]
    for i in range(1000):
        csv_lines.append(f"/file_{i}.py,Summary {i}")
    csv = "\n".join(csv_lines)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 1.24ms -> 914μs (35.1% faster)

def test_large_scale_commas_and_newlines_in_fields():
    # Test with 500 rows, each field contains commas and newlines
    csv_lines = ["full_path,file_summary"]
    for i in range(500):
        path = f'"/file_{i},foo.py"'
        summary = f'"Summary {i}, with newline\nand comma."'
        csv_lines.append(f"{path},{summary}")
    csv = "\n".join(csv_lines)
    codeflash_output = _get_available_includes_from_csv(csv); result = codeflash_output # 2.78ms -> 1.09ms (154% faster)
# codeflash_output is used to check that the output of the original code is the same as that of the optimized code.
#------------------------------------------------
Error parsing CSV: bad argument type for built-in operation
Error parsing CSV: bad argument type for built-in operation
Error parsing CSV: bad argument type for built-in operation
from pdd.auto_include import _get_available_includes_from_csv

def test__get_available_includes_from_csv():
    _get_available_includes_from_csv('\x00')

def test__get_available_includes_from_csv_2():
    _get_available_includes_from_csv('')

To edit these changes git checkout codeflash/optimize-_get_available_includes_from_csv-mgmyoasa and push.