utils.py

"""Utility functions for Rightmove scraper."""

import re
from typing import Optional
from urllib.parse import urlencode, quote


def build_search_url(
    base_url: str,
    listing_type: str = "property-for-sale",
    location_id: str = "REGION%5E87490",
    index: int = 0,
    sort_type: str = "6",
    property_types: str = "",
    min_bedrooms: Optional[int] = None,
    max_bedrooms: Optional[int] = None,
    min_price: Optional[int] = None,
    max_price: Optional[int] = None,
) -> str:
    """Build search URL with filters.

    Args:
        base_url: Base URL for rightmove.co.uk
        listing_type: 'property-for-sale' or 'property-to-rent'
        location_id: Location identifier (e.g., REGION%5E87490 for London)
        index: Pagination index (0, 24, 48, ...)
        sort_type: Sort option (6=newest, 1=price_low, etc.)
        property_types: Property type filter
        min_bedrooms: Minimum bedrooms filter
        max_bedrooms: Maximum bedrooms filter
        min_price: Minimum price filter
        max_price: Maximum price filter

    Returns:
        Complete search URL
    """
    # Build base path
    path = f"/{listing_type}/find.html"

    # Build query parameters
    params = {
        "locationIdentifier": location_id,
        "sortType": sort_type,
        "includeSSTC": "false",
    }

    if index > 0:
        params["index"] = index

    if property_types:
        params["propertyTypes"] = property_types

    if min_bedrooms:
        params["minBedrooms"] = min_bedrooms

    if max_bedrooms:
        params["maxBedrooms"] = max_bedrooms

    if min_price:
        params["minPrice"] = min_price

    if max_price:
        params["maxPrice"] = max_price

    # Build URL
    query = "&".join(f"{k}={v}" for k, v in params.items())
    return f"{base_url}{path}?{query}"


def clean_price(price_text: str) -> str:
    """Clean and format price text.

    Args:
        price_text: Raw price text (e.g., "FEATURED PROPERTY£399,000")

    Returns:
        Cleaned price string (e.g., "£399,000")
    """
    if not price_text:
        return ""

    # Extract price with pound sign
    match = re.search(r"£[\d,]+(?:\s*(?:pcm|pw|pa))?", price_text, re.IGNORECASE)
    if match:
        return match.group(0)

    # Try to find just numbers with pound sign
    match = re.search(r"£[\d,]+", price_text)
    if match:
        return match.group(0)

    return price_text.strip()


def clean_address(address_text: str) -> str:
    """Clean and format address text.

    Args:
        address_text: Raw address text

    Returns:
        Cleaned address string
    """
    if not address_text:
        return ""

    # Remove extra whitespace and newlines
    address = " ".join(address_text.split())
    return address.strip()


def extract_property_id(url: str) -> Optional[str]:
    """Extract property ID from URL.

    Args:
        url: Property URL

    Returns:
        Property ID or None
    """
    if not url:
        return None

    # URL format: /properties/123456789#/...
    match = re.search(r"/properties/(\d+)", url)
    return match.group(1) if match else None


def extract_bedrooms(text: str) -> str:
    """Extract bedroom count from text.

    Args:
        text: Text containing bedroom info

    Returns:
        Bedroom count string
    """
    if not text:
        return ""

    # Look for number before "bed"
    match = re.search(r"(\d+)\s*bed", text, re.IGNORECASE)
    if match:
        return match.group(1)

    # Check for just a number in bedroom element
    match = re.search(r"(\d+)", text)
    if match:
        return match.group(1)

    return text.strip()


def extract_bathrooms(text: str) -> str:
    """Extract bathroom count from text.

    Args:
        text: Text containing bathroom info

    Returns:
        Bathroom count string
    """
    if not text:
        return ""

    # Look for number before "bath"
    match = re.search(r"(\d+)\s*bath", text, re.IGNORECASE)
    if match:
        return match.group(1)

    # Check for just a number
    match = re.search(r"(\d+)", text)
    if match:
        return match.group(1)

    return text.strip()


def extract_agent_name(text: str) -> str:
    """Extract agent name from listing text.

    Args:
        text: Full listing text

    Returns:
        Agent name
    """
    if not text:
        return ""

    # Pattern: "Added ... by Agent Name"
    match = re.search(r"by\s+([^,\n]+(?:,\s*[^,\n]+)?)", text, re.IGNORECASE)
    if match:
        agent = match.group(1).strip()
        # Clean up phone numbers
        agent = re.sub(r"\d{3,}", "", agent).strip()
        return agent

    return ""


def extract_added_date(text: str) -> str:
    """Extract added date from listing text.

    Args:
        text: Full listing text

    Returns:
        Added date string
    """
    if not text:
        return ""

    # Pattern: "Added today", "Added yesterday", "Added on DD/MM/YYYY"
    match = re.search(r"Added\s+(today|yesterday|on\s+\d{2}/\d{2}/\d{4})", text, re.IGNORECASE)
    if match:
        return match.group(1).strip()

    return ""


def extract_description(text: str, max_length: int = 500) -> str:
    """Extract description from listing text.

    Args:
        text: Full listing text
        max_length: Maximum description length

    Returns:
        Description string
    """
    if not text:
        return ""

    # Find description after property details and before "Added"
    # Usually follows bathrooms count and ends with "..." or "Added"
    lines = text.split("\n")
    description_lines = []

    for line in lines:
        line = line.strip()
        # Skip short lines, numbers, and metadata
        if len(line) < 20:
            continue
        if line.startswith("Added"):
            break
        if re.match(r"^[\d\s/|]+$", line):
            continue
        if any(x in line.lower() for x in ["local call", "save", "contact"]):
            continue
        description_lines.append(line)

    description = " ".join(description_lines)

    # Truncate if needed
    if len(description) > max_length:
        description = description[:max_length - 3] + "..."

    return description


def calculate_page_index(page_number: int, items_per_page: int = 24) -> int:
    """Calculate the index parameter for pagination.

    Args:
        page_number: Page number (1-based)
        items_per_page: Number of items per page

    Returns:
        Index value for URL parameter
    """
    return (page_number - 1) * items_per_page