{"id":48913,"library":"pi-web-fetch","title":"pi-web-fetch","description":"pi-web-fetch is a pi coding agent extension that gives AI agents a proper web browser via headless Chrome, clean content extraction with trafilatura, and optional LLM processing. Version 1.1.0 brings concurrent batch fetching (up to 10 URLs with 6 parallel tabs), a 15-minute cache, smart large-page summarization, cross-host redirect detection, and an extensible hook system for site-specific handling. Unlike static fetchers like Claude Code's WebFetch, it handles JavaScript-rendered SPAs, strips boilerplate using ML-based extraction, and supports optional LLM distillation via a pi sub-agent. Requires Node.js 18+, a Python tool runner (uvx recommended), and puppeteer's bundled Chromium.","status":"active","version":"1.1.0","language":"javascript","source_language":"en","source_url":"https://github.com/georgebashi/pi-web-fetch","tags":["javascript","pi-package"],"install":[{"cmd":"npm install pi-web-fetch","lang":"bash","label":"npm"},{"cmd":"yarn add pi-web-fetch","lang":"bash","label":"yarn"},{"cmd":"pnpm add pi-web-fetch","lang":"bash","label":"pnpm"}],"dependencies":[{"reason":"Provides the pi coding agent runtime and tool registration system","package":"@mariozechner/pi-coding-agent","optional":false},{"reason":"Provides terminal UI components for progress display","package":"@mariozechner/pi-tui","optional":false},{"reason":"Used for tool schema validation","package":"@sinclair/typebox","optional":false},{"reason":"Downloads and manages headless Chrome for JavaScript rendering","package":"puppeteer","optional":false}],"imports":[{"note":"Package is ESM-only; CJS require() will not work.","wrong":"const piWebFetch = require('pi-web-fetch')","symbol":"default","correct":"import piWebFetch from 'pi-web-fetch'"},{"note":"Named export, not default. Use named import syntax.","wrong":"import WebFetchExtension from 'pi-web-fetch'","symbol":"WebFetchExtension","correct":"import { WebFetchExtension } from 'pi-web-fetch'"},{"note":"TypeScript type; import with 'import type' to avoid runtime errors.","wrong":"import { WebFetchToolOptions } from 'pi-web-fetch'","symbol":"type WebFetchToolOptions","correct":"import type { WebFetchToolOptions } from 'pi-web-fetch'"}],"quickstart":{"code":"import { WebFetchExtension } from 'pi-web-fetch';\nimport { PiAgent } from '@mariozechner/pi-coding-agent';\n\nconst agent = new PiAgent({\n  extensions: [new WebFetchExtension()]\n});\n\n// The agent can now call web_fetch tool\n// Example: web_fetch({ urls: ['https://example.com'], prompt: 'What is the main topic?' });\n\n// Single URL fetch without LLM:\n// web_fetch({ urls: ['https://example.com'] });\n\n// Batch fetch (up to 10 URLs):\n// web_fetch({ urls: ['https://example.com', 'https://example.org'] });","lang":"typescript","description":"Demonstrates importing and initializing the pi-web-fetch extension with a pi agent, including example tool calls for single and batch URL fetching."},"warnings":[{"fix":"Set PUPPETEER_EXECUTABLE_PATH to existing Chrome binary to skip Chromium download.","message":"First run triggers puppeteer Chromium download (~300MB) and trafilatura via uvx (~10MB).","severity":"gotcha","affected_versions":">=1.0.0"},{"fix":"Ensure uv is installed and uvx alias is available, or rely on pipx fallback.","message":"uvx alias may not exist on all systems; fallback to uv run may be slower.","severity":"deprecated","affected_versions":">=1.0.0"},{"fix":"Update Node.js to version 18 or later.","message":"Node.js 18+ is required; older versions may fail due to puppeteer compatibility.","severity":"gotcha","affected_versions":">=1.0.0"},{"fix":"Use import syntax or convert project to ESM.","message":"ESM-only package: CJS require() will throw an error.","severity":"breaking","affected_versions":">=1.0.0"},{"fix":"Split fetches into batches of 10 or fewer URLs.","message":"Maximum 10 URLs per batch fetch; exceeding limit will cause tool failure.","severity":"gotcha","affected_versions":">=1.0.0"}],"env_vars":null,"search_vec":"'1.1.0':36 '10':43 '15':50 '18':107 '6':46 'agent':13,18,104 'ai':17 'base':92 'batch':39 'boilerpl':88 'bring':37 'browser':22 'bundl':117 'cach':52 'chrome':25 'chromium':118 'claud':77 'clean':26 'code':12,78 'concurr':38 'content':27 'cross':59 'cross-host':58 'detect':62 'distil':98 'extens':14,65 'extract':28,93 'fetch':4,8,40 'fetcher':75 'give':16 'handl':72,82 'headless':24 'hook':66 'host':60 'javascript':84,119 'javascript-rend':83 'larg':55 'large-pag':54 'like':76 'llm':33,97 'minut':51 'ml':91 'ml-base':90 'node.js':106 'option':32,96 'packag':122 'page':56 'parallel':47 'pi':2,6,11,101,121 'pi-packag':120 'pi-web-fetch':1,5 'process':34 'proper':20 'puppet':115 'python':109 'recommend':113 'redirect':61 'render':85 'requir':105 'runner':111 'site':70 'site-specif':69 'smart':53 'spas':86 'specif':71 'static':74 'strip':87 'sub':103 'sub-ag':102 'summar':57 'support':95 'system':67 'tab':48 'tool':110 'trafilatura':30 'unlik':73 'url':44 'use':89 'uvx':112 'version':35 'via':23,99 'web':3,7,21 'webfetch':80","created_at":"2026-06-07T16:59:06.929752+00:00","updated_at":"2026-06-07T16:59:06.929752+00:00","problems":[{"fix":"Run 'npm install pi-web-fetch' in your project directory.","cause":"Package not installed or not in node_modules.","error":"Error: Cannot find module 'pi-web-fetch'"},{"fix":"Correct: 'new WebFetchExtension()'","cause":"Attempted to call WebFetchExtension as a function instead of using 'new'.","error":"TypeError: Class constructor WebFetchExtension cannot be invoked without 'new'"},{"fix":"Ensure the extension object is passed to PiAgent constructor in 'extensions' array.","cause":"The extension was not loaded correctly or agent does not have the extension registered.","error":"Error: The 'web_fetch' tool is not registered. Available tools: ..."},{"fix":"Check that Chrome is installed and puppeteer can launch it. Set PUPPETEER_EXECUTABLE_PATH if needed.","cause":"Chrome remote debugging port is not accessible. Puppeteer cannot start Chrome.","error":"Error: connect ECONNREFUSED ::1:9222"}],"ecosystem":"npm","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/georgebashi/pi-web-fetch#readme","github":"https://github.com/georgebashi/pi-web-fetch","docs":null,"changelog":null,"pypi":null,"npm":"pi-web-fetch","openapi_spec":null,"status_page":null,"smithery":null,"categories":["devops"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-07","next_check":"2026-09-05","install_tag":null}}