Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
2 changes: 1 addition & 1 deletion .config/jp/tools/src/cargo/test.rs
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,7 @@ fn cargo_test_impl<R: ProcessRunner>(
let kind = l.get("type").and_then(Value::as_str).unwrap_or_default();
let event = l.get("event").and_then(Value::as_str).unwrap_or_default();

if kind != "test" {
if kind != "test" || event == "started" {
continue;
}
total_tests += 1;
Expand Down
3 changes: 2 additions & 1 deletion .config/jp/tools/src/fs/create_file.rs
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,14 @@ pub(crate) async fn fs_create_file(
lang => lang,
};

let mut response = format!("Create file '{}'", path.as_str().bold().blue());
let mut response = format!("Created file '{}'", path.as_str().bold().blue());
if let Some(content) = content {
let code_block = format!("`````{lang}\n{content}\n`````");
let highlighted = Formatter::new()
.format_terminal(&code_block)
.unwrap_or(code_block);
response.push_str(&format!(" with content:\n\n{highlighted}\n"));
response.push_str(&format!("\n{response}\n"));
}

return Ok(response.into());
Expand Down
30 changes: 28 additions & 2 deletions .config/jp/tools/src/fs/list_files.rs
Original file line number Diff line number Diff line change
Expand Up @@ -39,10 +39,28 @@ pub(crate) async fn fs_list_files(

let mut entries = vec![];
for prefix in &prefixes {
let prefixed = root.join(prefix.trim_start_matches('/'));
let normalized = prefix.trim_start_matches('/');
let prefixed = root.join(normalized);

// When the prefix points to an existing directory, walk it directly.
// Otherwise, walk the parent directory and filter to entries whose
// root-relative path starts with the prefix. This supports partial
// filename prefixes like "docs/rfd/D".
let (walk_dir, path_filter): (Utf8PathBuf, Option<String>) =
if prefixed.is_dir() || normalized.is_empty() {
(prefixed, None)
} else {
let parent = prefixed
.parent()
.map_or_else(|| root.to_owned(), Utf8PathBuf::from);
(
parent,
Some(normalized.replace('/', std::path::MAIN_SEPARATOR_STR)),
)
};

let (tx, matches) = crossbeam_channel::unbounded();
WalkBuilder::new(&prefixed)
WalkBuilder::new(&walk_dir)
// Include hidden and otherwise ignored files.
.standard_filters(false)
.follow_links(false)
Expand All @@ -53,6 +71,7 @@ pub(crate) async fn fs_list_files(
.run(|| {
let tx = tx.clone();
let extensions = extensions.clone();
let path_filter = path_filter.clone();
Box::new(move |entry| {
// Ignore invalid entries.
let Ok(entry) = entry else {
Expand Down Expand Up @@ -82,6 +101,13 @@ pub(crate) async fn fs_list_files(
return WalkState::Continue;
};

// Filter by partial prefix if the original prefix wasn't a directory.
if let Some(filter) = &path_filter
&& !path.as_str().starts_with(filter.as_str())
{
return WalkState::Continue;
}

let _result = tx.send(path.to_string());

WalkState::Continue
Expand Down
18 changes: 18 additions & 0 deletions .config/jp/tools/src/fs/list_files_tests.rs
Original file line number Diff line number Diff line change
Expand Up @@ -52,6 +52,24 @@ async fn test_list_files() {
given: vec!["test/b.txt", "test/c.md", "test/d/e.txt"],
expected: vec!["test/b.txt", "test/c.md", "test/d/e.txt"],
}),
("partial-prefix", TestCase {
prefixes: vec!["rfd/D"],
extensions: vec![],
given: vec!["rfd/D01-foo.md", "rfd/D02-bar.md", "rfd/001-baz.md"],
expected: vec!["rfd/D01-foo.md", "rfd/D02-bar.md"],
}),
("partial-prefix-with-extension", TestCase {
prefixes: vec!["rfd/D"],
extensions: vec!["md"],
given: vec!["rfd/D01-foo.md", "rfd/D02-bar.txt", "rfd/001-baz.md"],
expected: vec!["rfd/D01-foo.md"],
}),
("partial-prefix-nested", TestCase {
prefixes: vec!["src/foo"],
extensions: vec![],
given: vec!["src/foo.rs", "src/foo_tests.rs", "src/bar.rs"],
expected: vec!["src/foo.rs", "src/foo_tests.rs"],
}),
]);

for (
Expand Down
57 changes: 23 additions & 34 deletions .config/jp/tools/src/web/fetch.rs
Original file line number Diff line number Diff line change
@@ -1,6 +1,5 @@
use fancy_regex::Regex;
use htmd::HtmlToMarkdown;
use reqwest::header::CONTENT_TYPE;
use reqwest::header::{CONTENT_TYPE, HeaderMap, HeaderValue, USER_AGENT};
use serde::Deserialize;
use url::Url;

Expand All @@ -16,7 +15,7 @@ const HAIKU_MODEL: &str = "claude-haiku-4-5";
const ANTHROPIC_API_URL: &str = "https://api.anthropic.com/v1/messages";

pub(crate) async fn web_fetch(url: Url) -> ToolResult {
let response = reqwest::get(url.clone()).await?;
let response = http_client().get(url.clone()).send().await?;

let content_type = response
.headers()
Expand All @@ -37,24 +36,18 @@ pub(crate) async fn web_fetch(url: Url) -> ToolResult {
return Ok(truncate(&body, SUMMARIZE_THRESHOLD).into());
}

let title = extract_title(&body);
let md = html_to_markdown(&body)?;

let content = match title {
Some(ref t) if !t.is_empty() => format!("# {t}\n\n{md}"),
_ => md,
};

if content.len() <= SUMMARIZE_THRESHOLD {
return Ok(content.into());
if md.len() <= SUMMARIZE_THRESHOLD {
return Ok(md.into());
}

// Try Haiku summarization for large pages
if let Some(summary) = try_summarize(&url, &content).await {
if let Some(summary) = try_summarize(&url, &md).await {
return Ok(summary.into());
}

Ok(truncate(&content, SUMMARIZE_THRESHOLD).into())
Ok(truncate(&md, SUMMARIZE_THRESHOLD).into())
}

fn html_to_markdown(html: &str) -> Result<String, Error> {
Expand All @@ -66,26 +59,6 @@ fn html_to_markdown(html: &str) -> Result<String, Error> {
Ok(collapse_blank_lines(&md))
}

fn extract_title(html: &str) -> Option<String> {
let re = Regex::new(r"(?is)<title[^>]*>(.*?)</title>").ok()?;
let caps = re.captures(html).ok()??;
let raw = caps.get(1)?.as_str().trim();
if raw.is_empty() {
return None;
}

Some(decode_html_entities(raw))
}

fn decode_html_entities(s: &str) -> String {
s.replace("&amp;", "&")
.replace("&lt;", "<")
.replace("&gt;", ">")
.replace("&quot;", "\"")
.replace("&#39;", "'")
.replace("&apos;", "'")
}

fn is_binary(content_type: &str) -> bool {
let ct = content_type.to_ascii_lowercase();
ct.starts_with("image/")
Expand Down Expand Up @@ -157,7 +130,7 @@ async fn summarize(api_key: &str, url: &Url, content: &str) -> Result<String, Er
"messages": [{"role": "user", "content": prompt}]
});

let resp = reqwest::Client::new()
let resp = http_client()
.post(ANTHROPIC_API_URL)
.header("x-api-key", api_key)
.header("anthropic-version", "2023-06-01")
Expand Down Expand Up @@ -192,6 +165,22 @@ struct HaikuResponse {
content: Vec<ContentBlock>,
}

fn http_client() -> reqwest::Client {
let mut headers = HeaderMap::new();
headers.insert(
USER_AGENT,
HeaderValue::from_static(
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like \
Gecko) Chrome/137.0.0.0 Safari/537.36",
),
);

reqwest::Client::builder()
.default_headers(headers)
.build()
.expect("failed to build HTTP client")
}

#[derive(Deserialize)]
struct ContentBlock {
#[serde(rename = "type")]
Expand Down
63 changes: 0 additions & 63 deletions .config/jp/tools/src/web/fetch_tests.rs
Original file line number Diff line number Diff line change
@@ -1,51 +1,5 @@
use super::*;

mod extract_title {
use super::*;

#[test]
fn basic() {
let html = "<html><head><title>Hello World</title></head><body></body></html>";
assert_eq!(extract_title(html), Some("Hello World".into()));
}

#[test]
fn with_html_entities() {
let html = "<title>Foo &amp; Bar &lt;Baz&gt;</title>";
assert_eq!(extract_title(html), Some("Foo & Bar <Baz>".into()));
}

#[test]
fn with_whitespace() {
let html = "<title> \n Some Title \n </title>";
assert_eq!(extract_title(html), Some("Some Title".into()));
}

#[test]
fn case_insensitive() {
let html = "<TITLE>Upper Case</TITLE>";
assert_eq!(extract_title(html), Some("Upper Case".into()));
}

#[test]
fn with_attributes() {
let html = r#"<title lang="en">Attributed</title>"#;
assert_eq!(extract_title(html), Some("Attributed".into()));
}

#[test]
fn empty_title() {
let html = "<title> </title>";
assert_eq!(extract_title(html), None);
}

#[test]
fn no_title() {
let html = "<html><head></head><body>Hello</body></html>";
assert_eq!(extract_title(html), None);
}
}

mod is_binary {
use super::*;

Expand Down Expand Up @@ -211,20 +165,3 @@ mod html_to_markdown {
assert!(md.trim().is_empty(), "expected empty but got: {md:?}");
}
}

mod decode_html_entities {
use super::*;

#[test]
fn all_entities() {
assert_eq!(
decode_html_entities("&amp; &lt; &gt; &quot; &#39; &apos;"),
"& < > \" ' '"
);
}

#[test]
fn no_entities() {
assert_eq!(decode_html_entities("plain text"), "plain text");
}
}
Loading