added HTML util methods

Author: DavertMik

CLAUDE.md

@@ -37,6 +37,8 @@ After big changes run linter: `npm run lint:fix`
 **Never use NodeJS**
 This application is only Bun
 
+
+
 ## Core Architecture
 
 Explorbot uses layered architecture with AI-driven automation:
@@ -86,6 +88,12 @@ Application is built via React Ink with interactive TUI
 ]
 ```
 
+DO NEVER REMOVE FROM COMPONENTS:
+
+```
+import React from 'react';
+```
+
 ### User Input in TUI
 
 There are application commands available in TUI

explorbot.config.example.ts

@@ -61,12 +61,35 @@ interface AIConfig {
   retryDelay: number;
 }
 
+interface HtmlConfig {
+  minimal?: {
+    include?: string[];
+    exclude?: string[];
+  };
+  combined?: {
+    include?: string[];
+    exclude?: string[];
+  };
+  text?: {
+    include?: string[];
+    exclude?: string[];
+  };
+}
+
+interface DirsConfig {
+  knowledge: string;
+  experience: string;
+  output: string;
+}
+
 interface ExplorbotConfig {
   playwright: PlaywrightConfig;
   app: AppConfig;
   output: OutputConfig;
   test: TestConfig;
   ai: AIConfig;
+  html?: HtmlConfig;
+  dirs?: DirsConfig;
 }
 
 const config: ExplorbotConfig = {
@@ -155,6 +178,72 @@ const config: ExplorbotConfig = {
     retryAttempts: 3,
     retryDelay: 1000,
   },
+
+  // Optional HTML parsing configuration
+  // Use CSS selectors to customize which elements are included in snapshots
+  html: {
+    // Minimal UI snapshot - keeps only interactive elements
+    minimal: {
+      include: [
+        // Include elements with test IDs (not included by default)
+        '[data-testid]',
+        '[data-cy]',
+        // Include custom interactive components
+        '[role="toolbar"]',
+        // Include elements with specific data attributes
+        'div[data-id]',
+      ],
+      exclude: [
+        // Exclude cookie banners
+        '#onetrust-consent-sdk',
+        // Exclude notification toasts
+        '.toast',
+        '.notification',
+        // Exclude loading spinners
+        '.spinner',
+        '.loading',
+      ],
+    },
+    // Combined snapshot - keeps interactive elements + meaningful text
+    combined: {
+      include: [
+        // Include content areas with specific classes
+        '.content',
+        '.main-content',
+        '[data-content]',
+        // Include article content
+        'article',
+        '.article',
+      ],
+      exclude: [
+        // Exclude navigation menus
+        '.nav-menu',
+        '.navigation',
+        'nav',
+        // Exclude metadata
+        '.metadata',
+        '.meta',
+        'time',
+      ],
+    },
+    // Text snapshot - converts to markdown text
+    text: {
+      include: [
+        // Include specific text containers
+        '.prose',
+        '.markdown',
+        '[data-markdown]',
+      ],
+      exclude: [
+        // Exclude code blocks (if not needed)
+        'code',
+        'pre',
+        // Exclude small text
+        'small',
+        '.fine-print',
+      ],
+    },
+  },
 };
 
 export default config;
@@ -165,4 +254,6 @@ export type {
   OutputConfig,
   TestConfig,
   AIConfig,
+  HtmlConfig,
+  DirsConfig,
 };

src/ai/researcher.ts

@@ -14,7 +14,7 @@ import { z } from 'zod';
 const debugLog = createDebug('explorbot:researcher');
 
 export class Research {
-  expandDOMCalled: boolean = false;
+  expandDOMCalled = false;
 }
 
 export class Researcher {

src/components/ActivityPane.tsx.old

@@ -0,0 +1,73 @@
+import React from 'react';
+import { useState, useEffect } from 'react';
+import { Box, Text } from 'ink';
+import {
+  addActivityListener,
+  removeActivityListener,
+  type ActivityEntry,
+} from '../activity.ts';
+
+const ActivityPane: React.FC = () => {
+  const [activity, setActivity] = useState<ActivityEntry | null>(null);
+  const [animationState, setAnimationState] = useState(0);
+
+  useEffect(() => {
+    const listener = (newActivity: ActivityEntry | null) => {
+      setActivity(newActivity);
+    };
+
+    addActivityListener(listener);
+
+    return () => {
+      removeActivityListener(listener);
+    };
+  }, []);
+
+  useEffect(() => {
+    if (!activity) return;
+
+    const interval = setInterval(() => {
+      setAnimationState((prev) => (prev + 1) % 4);
+    }, 500);
+
+    return () => clearInterval(interval);
+  }, [activity]);
+
+  if (!activity) {
+    return (
+      <Box height={1} paddingX={1}>
+        <Text dimColor>Done. Press [ESC] to enable input</Text>
+      </Box>
+    );
+  }
+
+  const getActivityColor = (type: ActivityEntry['type']) => {
+    switch (type) {
+      case 'ai':
+        return 'cyan';
+      case 'action':
+        return 'green';
+      case 'navigation':
+        return 'blue';
+      default:
+        return 'yellow';
+    }
+  };
+
+  const getDots = () => {
+    return '.'.repeat(animationState);
+  };
+
+  const isDimmed = animationState % 2 === 0;
+
+  return (
+    <Box height={1} marginY={1} paddingX={1}>
+      <Text color={getActivityColor(activity.type)} dimColor={isDimmed}>
+        {activity.message}
+        {getDots()}
+      </Text>
+    </Box>
+  );
+};
+
+export default ActivityPane;

src/components/LogPane.tsx.old

@@ -0,0 +1,149 @@
+import React, { useState, useCallback, useEffect } from 'react';
+import dedent from 'dedent';
+import { marked } from 'marked';
+import { markedTerminal } from 'marked-terminal';
+
+marked.use(markedTerminal());
+
+import { Box, Text } from 'ink';
+import type { TaggedLogEntry, LogType } from '../utils/logger.js';
+import {
+  registerLogPane,
+  setVerboseMode,
+  unregisterLogPane,
+} from '../utils/logger.js';
+
+// marked.use(new markedTerminal());
+
+type LogEntry = TaggedLogEntry;
+
+interface LogPaneProps {
+  verboseMode: boolean;
+}
+
+const LogPane: React.FC<LogPaneProps> = ({ verboseMode }) => {
+  const [logs, setLogs] = useState<TaggedLogEntry[]>([]);
+
+  const addLog = useCallback((logEntry: TaggedLogEntry) => {
+    setLogs((prevLogs: TaggedLogEntry[]) => {
+      // Skip duplicate consecutive logs
+      if (prevLogs.length === 0) return [logEntry];
+
+      const lastLog = prevLogs[prevLogs.length - 1];
+      if (
+        lastLog.type === logEntry.type &&
+        lastLog.content === logEntry.content &&
+        // Check if it's within 1 second to avoid legitimate duplicates
+        Math.abs(
+          (lastLog.timestamp?.getTime() || 0) -
+            (logEntry.timestamp?.getTime() || 0)
+        ) < 1000
+      ) {
+        return prevLogs;
+      }
+
+      return [...prevLogs, logEntry];
+    });
+  }, []);
+
+  useEffect(() => {
+    registerLogPane(addLog);
+
+    return () => {
+      unregisterLogPane(addLog);
+    };
+  }, []); // Empty dependency array to ensure this only runs once
+  const getLogStyles = (type: LogType) => {
+    switch (type) {
+      case 'success':
+        return { color: 'green' as const };
+      case 'error':
+        return { color: 'red' as const };
+      case 'warning':
+        return { color: 'yellow' as const };
+      case 'debug':
+        return { color: 'gray' as const, dimColor: true };
+      case 'substep':
+        return { color: 'gray' as const, dimColor: true };
+      case 'step':
+        return { color: 'cyan' as const, dimColor: true };
+      case 'multiline':
+        return { color: 'gray' as const, dimColor: true };
+      default:
+        return {};
+    }
+  };
+
+  const processLogContent = (content: string): string[] => {
+    return content.split('\n').filter((line) => line.length > 0);
+  };
+
+  const renderLogEntry = (log: TaggedLogEntry, index: number) => {
+    // Skip debug logs when not in verbose mode AND DEBUG env var is not set
+    const shouldShowDebug =
+      verboseMode || Boolean(process.env.DEBUG?.includes('explorbot:'));
+    if (log.type === 'debug' && !shouldShowDebug) {
+      return null;
+    }
+    const styles = getLogStyles(log.type);
+
+    if (log.type === 'multiline') {
+      return (
+        <Box
+          key={index}
+          borderStyle="classic"
+          marginY={1}
+          borderColor="dim"
+          height={15}
+          overflow="hidden"
+        >
+          <Text>{dedent(marked.parse(String(log.content)).toString())}</Text>
+        </Box>
+      );
+    }
+
+    const lines = processLogContent(String(log.content));
+
+    if (log.type === 'substep') {
+      return (
+        <Box key={index} flexDirection="column">
+          {lines.map((line, lineIndex) => (
+            <Text key={`${index}-${lineIndex}`} {...styles}>
+              {lineIndex === 0 ? `> ${line}` : `   ${line}`}
+            </Text>
+          ))}
+        </Box>
+      );
+    }
+
+    if (log.type === 'step') {
+      return (
+        <Box key={index} flexDirection="column" paddingLeft={2}>
+          {lines.map((line, lineIndex) => (
+            <Text key={`${index}-${lineIndex}`} {...styles}>
+              {line}
+            </Text>
+          ))}
+        </Box>
+      );
+    }
+
+    return (
+      <Box key={index} flexDirection="column">
+        {lines.map((line, lineIndex) => (
+          <Text key={`${index}-${lineIndex}`} {...styles}>
+            {line}
+          </Text>
+        ))}
+      </Box>
+    );
+  };
+
+  return (
+    <Box flexDirection="column">
+      {logs.map((log, index) => renderLogEntry(log, index)).filter(Boolean)}
+    </Box>
+  );
+};
+
+export default LogPane;