zoom_interface.py

#!/usr/bin/env python
"""
Zoom Interface for Robot Communication

This module provides a bidirectional interface between Zoom meetings and ROS topics.
It handles:
- Starting and managing a Zoom web client via Selenium
- Reading user messages/commands from Zoom chat
- Listening to voice commands through microphone with speech recognition
- Publishing user instructions to ROS topics
- Writing robot responses back to Zoom chat

The interface acts as a bridge between remote human operators (via Zoom) and 
robotic systems (via ROS).

Author: Robot Communication Team
License: MIT
"""

import rospy
from std_msgs.msg import String
import time
import re
import threading
import os
from typing import List, Optional
from dotenv import load_dotenv
import speech_recognition as sr
from selenium import webdriver
from selenium.webdriver.firefox.service import Service
from selenium.webdriver.common.by import By
from selenium.webdriver.support.wait import WebDriverWait
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.common.keys import Keys

# Load environment variables from .env file
load_dotenv()


class ZoomInterface:
    """
    Main interface class for bridging Zoom meetings with ROS.
    
    This class manages a Zoom web client session and provides bidirectional
    communication between Zoom (chat/voice) and ROS topics.
    
    Attributes:
        MAIN_URL: URL of the Zoom meeting to join
        driver: Selenium WebDriver for controlling Zoom web client
        wake_up_words: List of words that activate the robot
        recognizer: Speech recognition engine
        user_instruction_pub: ROS publisher for user instructions
        robot_feedback_sub: ROS subscriber for robot feedback
    """
    
    def __init__(self, zoom_url: Optional[str] = None, 
                 geckodriver_path: str = './geckodriver',
                 microphone_device: Optional[int] = None):
        """
        Initialize the Zoom interface.
        
        Args:
            zoom_url: URL of the Zoom meeting (if None, uses default)
            geckodriver_path: Path to geckodriver executable
            microphone_device: Index of microphone device (None for default)
        """
        # Zoom configuration - read from .env file, parameter, or use default
        self.MAIN_URL = (
            zoom_url or 
            os.getenv('ZOOM_MEETING_URL') or
            rospy.get_param('~zoom_url', None)
        )
        rospy.loginfo(f"Using Zoom URL: {self.MAIN_URL}")
        
        # Selenium WebDriver setup
        self.options = webdriver.FirefoxOptions()
        self.service = Service(geckodriver_path)
        self.driver = webdriver.Firefox(service=self.service, options=self.options)
        
        # Chat tracking
        self.chattag = 'chat-message-content-'
        self.chatmsgsxpath = f"//div[starts-with(@id, '{self.chattag}')]"
        self.lastchatidx = -1
        self.last_chat_message = ""
        self.last_robot_response = ""
        self.is_writing = False
        
        # Wake-up words for activation
        self.wake_up_words = [
            'robi', 'ruby', 'robbie', 'robby', 'roby', 'robie', 
            'no be', 'lobby', 'whoopi', 'rugby', 'bobby', 'toby', 
            'ravi', 'will be', 'robin', 'robee', 'obi', 'll be', 'yubi'
        ]
        
        # Speech recognition setup
        self.recognizer = sr.Recognizer()
        self.recognizer.energy_threshold = rospy.get_param('~energy_threshold', 35)
        self.recognizer.dynamic_energy_threshold = False
        self.mic_device = microphone_device
        self.mic_thread: Optional[threading.Thread] = None
        self.mic_running = False
        
        # ROS Publishers and Subscribers
        self.user_instruction_pub = rospy.Publisher(
            '/user_instruction', 
            String, 
            queue_size=10
        )
        
        self.robot_feedback_sub = rospy.Subscriber(
            '/robot_feedback', 
            String, 
            self.robot_feedback_callback
        )
        
        # Timer for reading chat (10 Hz)
        self.timer_chat = rospy.Timer(
            rospy.Duration(0.1), 
            self.reading_from_chat
        )
        
        rospy.loginfo("Zoom Interface initialized")

    def start_zoom(self, wait_time: float = 35.0, iframe_wait: float = 10.0):
        """
        Start the Zoom web client and join the meeting.
        
        Args:
            wait_time: Time to wait for Zoom page to load (seconds)
            iframe_wait: Time to wait after switching to iframe (seconds)
        """
        rospy.loginfo(f"Starting Zoom interface, connecting to: {self.MAIN_URL}")
        
        self.driver.get(self.MAIN_URL)
        self.driver.set_page_load_timeout(20)
        
        rospy.loginfo(f"Waiting {wait_time}s for Zoom to load...")
        time.sleep(wait_time)
        
        try:
            # Switch to the webclient iframe
            self.driver.switch_to.frame("webclient")
            rospy.loginfo("Successfully switched to webclient iframe")
            time.sleep(iframe_wait)
            
            # Debug: log available page elements
            if rospy.get_param('~debug', False):
                self.debug_page_elements()
                
        except Exception as e:
            rospy.logerr(f"Failed to switch to iframe: {e}")
            
            # List all available iframes for debugging
            frames = self.driver.find_elements(By.TAG_NAME, "iframe")
            rospy.loginfo(f"Found {len(frames)} iframe(s) on page:")
            
            for i, frame in enumerate(frames):
                frame_id = frame.get_attribute("id")
                frame_src = frame.get_attribute("src")
                rospy.loginfo(f"  Frame {i}: id='{frame_id}', src='{frame_src}'")
        
        # Start microphone listening in a separate thread
        self.start_microphone_listening()
        
        rospy.loginfo("Zoom interface fully started and ready!")

    def debug_page_elements(self):
        """
        Debug method to log available page elements.
        Helps identify chat-related elements on the Zoom page.
        """
        try:
            rospy.loginfo("=== DEBUG: Page Elements ===")
            rospy.loginfo(f"Current URL: {self.driver.current_url}")
            rospy.loginfo(f"Page title: {self.driver.title}")
            
            # Find all elements with class attributes containing 'chat' or 'message'
            all_elements = self.driver.find_elements(By.XPATH, "//*[@class]")
            chat_related = []
            
            for element in all_elements:
                class_name = element.get_attribute("class") or ""
                if any(keyword in class_name.lower() for keyword in ["chat", "message"]):
                    chat_related.append((element.tag_name, class_name))
            
            rospy.loginfo(f"Total elements with class: {len(all_elements)}")
            rospy.loginfo(f"Chat/message related elements: {len(chat_related)}")
            
            # Show first 10 chat-related elements
            for tag, class_name in chat_related[:10]:
                rospy.loginfo(f"  {tag}: {class_name}")
                
            rospy.loginfo("=== END DEBUG ===")
            
        except Exception as e:
            rospy.logerr(f"Debug failed: {e}")

    def start_microphone_listening(self):
        """
        Start microphone listening in a separate thread.
        Runs continuously until stop_microphone_listening() is called.
        """
        self.mic_running = True
        self.mic_thread = threading.Thread(
            target=self.listen_microphone, 
            daemon=True,
            name="MicrophoneListener"
        )
        self.mic_thread.start()
        rospy.loginfo("Microphone listening started")

    def listen_microphone(self):
        """
        Continuously listen to microphone for voice commands.
        Processes audio using Whisper speech recognition.
        Publishes commands to /user_instruction when wake-up words detected.
        """
        try:
            with sr.Microphone(device_index=self.mic_device) as source:
                rospy.loginfo("Microphone initialized, listening for commands...")
                
                # Adjust for ambient noise once at start
                rospy.loginfo("Adjusting for ambient noise...")
                self.recognizer.adjust_for_ambient_noise(source, duration=1.0)
                
                while self.mic_running and not rospy.is_shutdown():
                    try:
                        # Listen for audio with timeout
                        audio = self.recognizer.listen(source, phrase_time_limit=15)
                        rospy.logdebug("Audio captured, processing...")
                        
                        # Recognize speech using Whisper
                        command = self.recognizer.recognize_whisper(
                            audio, 
                            model=rospy.get_param('~whisper_model', 'medium.en'),
                            language="english"
                        ).lower()

                        rospy.loginfo(f"Detected speech: {command}")
                        
                        # Check for wake-up words
                        detected_wake_up_word = next(
                            (word for word in self.wake_up_words if word in command), 
                            None
                        )
                        
                        if detected_wake_up_word:
                            rospy.loginfo(f"Wake-up word '{detected_wake_up_word}' detected!")
                            
                            # Clean up the command
                            cleaned_command = self._clean_command(command, detected_wake_up_word)
                            
                            rospy.loginfo(f"Publishing voice instruction: {cleaned_command}")
                            
                            # Publish to /user_instruction topic
                            instruction_msg = String()
                            instruction_msg.data = cleaned_command
                            self.user_instruction_pub.publish(instruction_msg)

                    except sr.UnknownValueError:
                        # Speech was unintelligible - this is normal, just continue
                        pass
                    except sr.RequestError as e:
                        rospy.logerr(f"Speech recognition service error: {e}")
                        time.sleep(1.0)  # Brief pause before retrying
                    except Exception as e:
                        rospy.logerr(f"Error in microphone listening: {e}")
                        time.sleep(1.0)
                        
        except Exception as e:
            rospy.logerr(f"Failed to initialize microphone: {e}")

    def stop_microphone_listening(self):
        """Stop the microphone listening thread."""
        rospy.loginfo("Stopping microphone listening...")
        self.mic_running = False
        
        if self.mic_thread and self.mic_thread.is_alive():
            self.mic_thread.join(timeout=2.0)
        
        rospy.loginfo("Microphone listening stopped")
    
    def _clean_command(self, command: str, wake_word: str) -> str:
        """
        Clean up voice command by replacing wake-up word variants with 'Robi'.
        
        Args:
            command: Raw voice command text
            wake_word: Detected wake-up word
            
        Returns:
            Cleaned command text
        """
        # Replace wake-up word and its variants with 'Robi'
        pattern = r"\b\w*{}\w*\b".format(re.escape(wake_word))
        cleaned = re.sub(pattern, "Robi", command, flags=re.IGNORECASE)
        
        # Common speech recognition corrections
        cleaned = cleaned.replace("coach", "couch")
        
        return cleaned

    def robot_feedback_callback(self, msg: String):
        """
        ROS callback for robot feedback messages.
        Writes robot responses to Zoom chat.
        
        Args:
            msg: String message containing robot feedback
        """
        feedback = msg.data
        if feedback and feedback != self.last_robot_response:
            self.write_to_zoom(feedback)

    def reading_from_chat(self, event):
        """
        ROS timer callback to periodically read messages from Zoom chat.
        Publishes user instructions to /user_instruction topic when wake-up words detected.
        
        Args:
            event: ROS timer event (unused)
        """
        # Don't read while writing to avoid conflicts
        if self.is_writing:
            return
            
        try:
            wait = WebDriverWait(self.driver, 0.1)
            element_chat = None

            # Try multiple selectors for different Zoom versions/layouts
            chat_selectors = [
                "//div[@class='ReactVirtualized__Grid__innerScrollContainer']",
                "//div[contains(@class, 'chat-message-list')]",
                "//div[contains(@class, 'message-list')]",
                "//div[contains(@class, 'chat-container')]",
                "//div[contains(@class, 'chat-messages')]",
                "//div[@role='log']",
                "//div[contains(@class, 'chat')]//div[contains(@class, 'message')]",
                "//div[@data-testid='chat-messages']",
                "//*[contains(@class, 'chat') and contains(@class, 'list')]"
            ]

            # Try each selector until one works
            for selector in chat_selectors:
                try:
                    element_chat = wait.until(
                        EC.presence_of_element_located((By.XPATH, selector))
                    )
                    break
                except Exception:
                    continue

            # Fallback: search for any chat-related elements
            if element_chat is None:
                try:
                    all_divs = self.driver.find_elements(By.TAG_NAME, "div")
                    for div in all_divs:
                        class_name = div.get_attribute("class") or ""
                        if any(keyword in class_name.lower() 
                               for keyword in ["chat", "message"]):
                            element_chat = div
                            break
                except Exception as e:
                    rospy.logdebug(f"Alternative chat search failed: {e}")

            if element_chat is None:
                return
            
            # Find message elements within chat container
            message_selectors = [
                self.chatmsgsxpath,  # Original selector
                ".//div[contains(@class, 'message')]",
                ".//div[contains(@class, 'chat-message')]",
                ".//div[contains(@id, 'message')]",
                ".//div[contains(@class, 'text-content')]",
                ".//span[contains(@class, 'message')]",
                ".//*[contains(@class, 'message-text')]"
            ]
            
            children = []
            for msg_selector in message_selectors:
                try:
                    children = element_chat.find_elements(By.XPATH, msg_selector)
                    if children:
                        break
                except Exception:
                    continue
            
            if not children:
                return

            # Get the last message
            lastchatblock = children[-1]
            
            # Get message ID or use index
            chatdivid = lastchatblock.get_attribute('id')
            if not chatdivid:
                chatnum = str(len(children) - 1)
            else:
                chatnum = (chatdivid[len(self.chattag):] 
                          if chatdivid.startswith(self.chattag) 
                          else chatdivid)

            # Check if this is a new message
            if chatnum != self.lastchatidx:
                # Try multiple ways to extract message text
                message_text = self._extract_message_text(lastchatblock)

                if self.last_chat_message != message_text and message_text:
                    self.last_chat_message = message_text
                    rospy.loginfo(f"New chat message: {self.last_chat_message}")

                    # Check for wake-up words
                    detected_wake_up_word = next(
                        (word for word in self.wake_up_words 
                         if word in self.last_chat_message.lower()), 
                        None
                    )
                    
                    if detected_wake_up_word:
                        # Clean up the message
                        cleaned_message = self._clean_command(
                            self.last_chat_message, 
                            detected_wake_up_word
                        )
                        
                        rospy.loginfo(f"Publishing user instruction: {cleaned_message}")
                        
                        # Publish to /user_instruction topic
                        instruction_msg = String()
                        instruction_msg.data = cleaned_message
                        self.user_instruction_pub.publish(instruction_msg)

                self.lastchatidx = chatnum

        except Exception as e:
            rospy.logdebug(f"Error in reading_from_chat: {e}")
    
    def _extract_message_text(self, message_element) -> str:
        """
        Extract text from a message element using multiple strategies.
        
        Args:
            message_element: Selenium WebElement containing the message
            
        Returns:
            Extracted message text, or empty string if extraction fails
        """
        message_text = ""
        
        # Try different class names for message text
        text_selectors = [
            "new-chat-message__text-content",
            "message-text",
            "chat-message-text",
            "text-content",
            "message-content"
        ]
        
        for text_selector in text_selectors:
            try:
                chatbox = message_element.find_element(By.CLASS_NAME, text_selector)
                message_text = chatbox.text
                if message_text:
                    break
            except Exception:
                continue
        
        # Fallback: get text directly from the element
        if not message_text:
            message_text = message_element.text
        
        return message_text

    def write_to_zoom(self, response: str):
        """
        Write robot response to Zoom chat.
        
        Args:
            response: Text message to send to Zoom chat
        """
        if response != "" and self.last_robot_response != response:
            self.is_writing = True
            time.sleep(1)
            
            try:
                # Wait for chat box to be available
                chat_box = WebDriverWait(self.driver, 10).until(
                    EC.presence_of_element_located((By.CLASS_NAME, "ProseMirror"))
                )
                chat_box.click()
                time.sleep(0.3)
                
                # Replace Robi with Robot for chat display
                response = response.replace("Robi", "Robot")
                
                # Type and send the message
                chat_box.send_keys(response)
                chat_box.send_keys(Keys.ENTER)
                
                time.sleep(1)
                self.last_robot_response = response
                rospy.loginfo(f"Sent to Zoom chat: {response}")
                
            except Exception as e:
                rospy.logerr(f"Error writing to Zoom: {e}")
            finally:
                self.is_writing = False

    def shutdown(self):
        """Clean shutdown of the Zoom interface."""
        rospy.loginfo("Shutting down Zoom Interface...")
        
        # Stop microphone listening
        self.stop_microphone_listening()
        
        # Stop chat reading timer
        if hasattr(self, 'timer_chat'):
            self.timer_chat.shutdown()
        
        # Quit the browser
        try:
            self.driver.quit()
            rospy.loginfo("Browser closed successfully")
        except Exception as e:
            rospy.logerr(f"Error during browser shutdown: {e}")


def main():
    """Main function to run the Zoom interface node."""
    rospy.init_node('zoom_interface')
    rospy.loginfo("Zoom Interface node starting...")
    
    # Get parameters
    zoom_url = rospy.get_param('~zoom_url', None)
    geckodriver_path = rospy.get_param('~geckodriver_path', './geckodriver')
    mic_device = rospy.get_param('~microphone_device', None)
    
    # Create interface
    zoom_interface = ZoomInterface(
        zoom_url=zoom_url,
        geckodriver_path=geckodriver_path,
        microphone_device=mic_device
    )
    
    try:
        # Start Zoom interface (this will also start microphone listening)
        zoom_interface.start_zoom()
        
        rospy.loginfo("="*60)
        rospy.loginfo("Zoom Interface fully initialized and ready!")
        rospy.loginfo("="*60)
        rospy.loginfo("Listening for:")
        rospy.loginfo("  - Zoom chat messages with wake-up words")
        rospy.loginfo("  - Voice commands through microphone")
        rospy.loginfo("Publishing to:")
        rospy.loginfo("  - /user_instruction (user commands)")
        rospy.loginfo("Subscribing to:")
        rospy.loginfo("  - /robot_feedback (robot responses)")
        rospy.loginfo("="*60)
        
        # Keep the main thread alive
        rospy.spin()
        
    except KeyboardInterrupt:
        rospy.loginfo("Received interrupt signal...")
    except Exception as e:
        rospy.logerr(f"Unexpected error: {e}")
    finally:
        zoom_interface.shutdown()


if __name__ == "__main__":
    main()