{"id":47334,"library":"crawlex","title":"CrawleX","description":"A stealth web crawler for Node.js (≥20) that achieves Chrome-perfect TLS and HTTP/2 fingerprinting, making bot detection nearly impossible. v1.0.4 (April 2025) is actively maintained with weekly releases focused on stability and feature parity. It includes a browser render pool for JavaScript-heavy pages, persistent queue management, lifecycle hooks, and built-in sitemap support. Unlike Playwright or Puppeteer, CrawleX provides a higher-level orchestration layer optimized for large-scale scraping with anti-bot evasion as a core feature. Ships TypeScript types and is ESM-only. Key differentiators: automatic fingerprint rotation, adaptive throttling, and a plugin system for custom middleware.","status":"active","version":"1.0.4","language":"javascript","source_language":"en","source_url":"https://github.com/forattini-dev/crawlex","tags":["javascript","crawler","scraper","stealth","tls","impersonate","headless","chrome","sitemap","typescript"],"install":[{"cmd":"npm install crawlex","lang":"bash","label":"npm"},{"cmd":"yarn add crawlex","lang":"bash","label":"yarn"},{"cmd":"pnpm add crawlex","lang":"bash","label":"pnpm"}],"dependencies":[{"reason":"Underlying browser automation engine for headless rendering and rendering pool","package":"playwright-core","optional":false},{"reason":"HTTP client used for simple requests without rendering (fallback mode)","package":"got","optional":false}],"imports":[{"note":"CrawleX is ESM-only and does not support CommonJS require. Default export is the main crawler function.","wrong":"const crawlex = require('crawlex')","symbol":"crawlex","correct":"import crawlex from 'crawlex'"},{"note":"Config type is only available as a type import. Using a value import will cause a runtime error in TypeScript.","wrong":"import { CrawleXConfig } from 'crawlex'","symbol":"CrawleXConfig","correct":"import type { CrawleXConfig } from 'crawlex'","typeOnly":true},{"note":"Named export for hook enum used in lifecycle hooks. Works with ESM import.","symbol":"HookType","correct":"import { HookType } from 'crawlex'"},{"note":"The persistent queue class is exported as CrawlQueue, not Queue.","wrong":"import { Queue } from 'crawlex'","symbol":"CrawlQueue","correct":"import { CrawlQueue } from 'crawlex'"}],"quickstart":{"code":"import crawlex from 'crawlex';\n\nconst results = await crawlex({\n  urls: ['https://example.com', 'https://httpbin.org/html'],\n  followLinks: true,\n  maxDepth: 2,\n  concurrency: 5,\n  renderJavaScript: true, // use headless browser for JS\n  headless: true,         // run in headless mode (default true)\n  stealth: true,          // enable TLS/H2 fingerprint spoofing\n  hooks: {\n    onRequest: (url) => console.log(`Crawling: ${url}`),\n    onResponse: (response) => console.log(`Status: ${response.status}`),\n    onError: (url, error) => console.error(`Error: ${url} - ${error.message}`)\n  },\n  onProgress: (progress) => console.log(`Progress: ${progress.completed}/${progress.total}`)\n});\n\nconsole.log(`Collected ${results.length} pages`);","lang":"typescript","description":"Demonstrates basic usage of CrawleX with a rendering pool, stealth mode, hooks, and progress reporting."},"warnings":[{"fix":"Upgrade Node.js to version 20 or later.","message":"Node.js >=20 is required; lower versions will throw engine errors.","severity":"breaking","affected_versions":"<1.0.0"},{"fix":"Remove 'headless' option from config. Use renderJavaScript to toggle headless browser usage.","message":"The option 'headless' is deprecated in favor of 'headless: true' (which is the default). Future versions may remove it.","severity":"deprecated","affected_versions":">=1.0.0"},{"fix":"Reduce concurrency or implement custom throttling via hooks.","message":"Using stealth mode with concurrency >10 may trigger rate limiting on some sites due to rapid fingerprint rotation.","severity":"gotcha","affected_versions":">=1.0.0"},{"fix":"Use 'import type { CrawleXResult } from 'crawlex' for type annotations.","message":"The default export 'crawlex' returns an array; the return type is imported as CrawleXResult[].","severity":"gotcha","affected_versions":">=1.0.0"},{"fix":"Update imports to use 'CrawlQueue' instead of 'Queue'.","message":"In v1.0.0, the persistent queue was renamed from 'Queue' to 'CrawlQueue'. Imports using 'Queue' will break.","severity":"breaking","affected_versions":">=1.0.0"}],"env_vars":null,"search_vec":"'20':8 '2025':25 'achiev':10 'activ':27 'adapt':100 'anti':80 'anti-bot':79 'april':24 'automat':97 'bot':19,81 'browser':41 'built':56 'built-in':55 'chrome':12,116 'chrome-perfect':11 'core':85 'crawler':5,110 'crawlex':1,64 'custom':107 'detect':20 'differenti':96 'esm':93 'esm-on':92 'evas':82 'featur':36,86 'fingerprint':17,98 'focus':32 'headless':115 'heavi':47 'higher':68 'higher-level':67 'hook':53 'http/2':16 'imperson':114 'imposs':22 'includ':39 'javascript':46,109 'javascript-heavi':45 'key':95 'larg':75 'large-scal':74 'layer':71 'level':69 'lifecycl':52 'maintain':28 'make':18 'manag':51 'middlewar':108 'near':21 'node.js':7 'optim':72 'orchestr':70 'page':48 'pariti':37 'perfect':13 'persist':49 'playwright':61 'plugin':104 'pool':43 'provid':65 'puppet':63 'queue':50 'releas':31 'render':42 'rotat':99 'scale':76 'scrape':77 'scraper':111 'ship':87 'sitemap':58,117 'stabil':34 'stealth':3,112 'support':59 'system':105 'throttl':101 'tls':14,113 'type':89 'typescript':88,118 'unlik':60 'v1.0.4':23 'web':4 'week':30","created_at":"2026-06-07T16:51:00.250016+00:00","updated_at":"2026-06-07T16:51:00.250016+00:00","problems":[{"fix":"Change to ESM imports: 'import crawlex from 'crawlex'' or use dynamic import() in CommonJS.","cause":"Using CommonJS require() to import CrawleX, which is ESM-only.","error":"Error: Cannot find module 'crawlex' require() of ES Module"},{"fix":"Use default import: 'import crawlex from 'crawlex''.","cause":"Importing named export instead of default export, or using wrong import syntax.","error":"TypeError: crawlex is not a function"},{"fix":"Set 'type': 'module' in package.json or use import syntax in .mjs files.","cause":"Trying to require() the package from a CommonJS context.","error":"Error [ERR_REQUIRE_ESM]: require() of ES Module .../crawlex/index.js from ... not supported"}],"ecosystem":"npm","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/forattini-dev/crawlex","github":"https://github.com/forattini-dev/crawlex","docs":null,"changelog":null,"pypi":null,"npm":"crawlex","openapi_spec":null,"status_page":null,"smithery":null,"categories":["devops"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-07","next_check":"2026-09-05","install_tag":null}}