{"id":45008,"library":"gatsby-source-filesystem-with-queue","title":"gatsby-source-filesystem-with-queue","description":"A Gatsby source plugin for sourcing local files into a Gatsby application, identical in usage to gatsby-source-filesystem but with an asynchronous queue that limits concurrent file reads to prevent EMFILE errors on large sites. Version 5.3.1 requires Node >=18 and Gatsby ^5.0.0-next. Key differentiator: configurable concurrent file limit (default 800) via GATSBY_CONCURRENT_FILES environment variable, and concurrent download limit (default 200) via GATSBY_CONCURRENT_DOWNLOAD. Ships TypeScript types. Maintains full compatibility with transformer plugins like gatsby-transformer-remark and gatsby-transformer-json. Only difference from upstream is the queuing mechanism; all options, helper functions (createFilePath, createRemoteFileNode, createFileNodeFromBuffer), and query patterns are identical.","status":"active","version":"5.3.1","language":"javascript","source_language":"en","source_url":"https://github.com/FraserThompson/gatsby-source-filesystem-with-queue","tags":["javascript","gatsby","gatsby-plugin","typescript"],"install":[{"cmd":"npm install gatsby-source-filesystem-with-queue","lang":"bash","label":"npm"},{"cmd":"yarn add gatsby-source-filesystem-with-queue","lang":"bash","label":"yarn"},{"cmd":"pnpm add gatsby-source-filesystem-with-queue","lang":"bash","label":"pnpm"}],"dependencies":[{"reason":"peer dependency - requires Gatsby ^5.0.0-next to run","package":"gatsby","optional":false}],"imports":[{"note":"ESM-only. Named export, not default. Don't use destructuring from default import.","wrong":"const createFilePath = require('gatsby-source-filesystem-with-queue').createFilePath","symbol":"createFilePath","correct":"import { createFilePath } from 'gatsby-source-filesystem-with-queue'"},{"note":"Named export. Mistaking it for default export is common.","wrong":"import createRemoteFileNode from 'gatsby-source-filesystem-with-queue'","symbol":"createRemoteFileNode","correct":"import { createRemoteFileNode } from 'gatsby-source-filesystem-with-queue'"},{"note":"Relative path import is unnecessary and breaks. Import from package name only.","wrong":"import { createFileNodeFromBuffer } from '../node_modules/gatsby-source-filesystem-with-queue'","symbol":"createFileNodeFromBuffer","correct":"import { createFileNodeFromBuffer } from 'gatsby-source-filesystem-with-queue'"}],"quickstart":{"code":"// gatsby-config.js\nmodule.exports = {\n  plugins: [\n    {\n      resolve: `gatsby-source-filesystem-with-queue`,\n      options: {\n        name: `pages`,\n        path: `${__dirname}/src/pages/`,\n      },\n    },\n  ],\n}\n\n// Query in Gatsby page/component\nimport { graphql } from 'gatsby'\n\nexport const query = graphql`\n  {\n    allFile(filter: { sourceInstanceName: { eq: \"pages\" } }) {\n      edges {\n        node {\n          extension\n          dir\n          modifiedTime\n        }\n      }\n    }\n  }\n`","lang":"javascript","description":"Configure plugin in gatsby-config.js with a source directory and query the created File nodes in a page component."},"warnings":[{"fix":"Update Node to >=18.0.0, or install an older version of the package (e.g., 4.x).","message":"Minimum Node version is 18.0.0 — using older versions will fail with cryptic errors.","severity":"breaking","affected_versions":">=5.3.1"},{"fix":"Ensure your project uses Gatsby 5.x. If on Gatsby 4, use gatsby-source-filesystem instead.","message":"Peer dependency on Gatsby ^5.0.0-next — incompatible with Gatsby 4.x or earlier.","severity":"breaking","affected_versions":">=5.0.0"},{"fix":"Switch to gatsby-source-filesystem unless you specifically need queue concurrency control.","message":"The package is a fork of gatsby-source-filesystem with identical API — if you don't need the queue, consider using the original to avoid maintenance risk.","severity":"deprecated","affected_versions":">=5.3.1"},{"fix":"Set GATSBY_CONCURRENT_FILES to a lower value (e.g., 200) if you experience 'EMFILE: too many open files' errors.","message":"Environment variable GATSBY_CONCURRENT_FILES default is 800, not 8000 as older docs might suggest — lowering it may help EMFILE errors but slow down builds.","severity":"gotcha","affected_versions":">=5.3.1"},{"fix":"Keep GATSBY_CONCURRENT_DOWNLOAD reasonable (e.g., <=500) or adjust based on system limits.","message":"Environment variable GATSBY_CONCURRENT_DOWNLOAD controls remote file node downloads (default 200) — setting too high may exhaust system resources.","severity":"gotcha","affected_versions":">=5.3.1"},{"fix":"If you need to remove a default ignore, you'll need to fork the plugin. Alternatively, use gatsby-source-filesystem which may allow customization.","message":"ignore option globs are merged with defaults — you cannot override defaults, only add to them.","severity":"gotcha","affected_versions":">=5.3.1"},{"fix":"Always pass node (File node or transformer-created node with parent) and getNode function from onCreateNode API.","message":"createFilePath's node and getNode parameters are required — omitting them causes the query to silently fail.","severity":"deprecated","affected_versions":">=5.3.1"}],"env_vars":null,"search_vec":"'18':48 '200':72 '5.0.0':51 '5.3.1':45 '800':60 'applic':18 'asynchron':30 'compat':82 'concurr':34,56,63,68,75 'configur':55 'createfilenodefrombuff':110 'createfilepath':108 'createremotefilenod':109 'default':59,71 'differ':97 'differenti':54 'download':69,76 'emfil':39 'environ':65 'error':40 'file':14,35,57,64 'filesystem':4,26 'full':81 'function':107 'gatsbi':2,8,17,24,50,62,74,88,93,117,119 'gatsby-plugin':118 'gatsby-source-filesystem':23 'gatsby-source-filesystem-with-queu':1 'gatsby-transformer-json':92 'gatsby-transformer-remark':87 'helper':106 'ident':19,115 'javascript':116 'json':95 'key':53 'larg':42 'like':86 'limit':33,58,70 'local':13 'maintain':80 'mechan':103 'next':52 'node':47 'option':105 'pattern':113 'plugin':10,85,120 'prevent':38 'queri':112 'queu':102 'queue':6,31 'read':36 'remark':90 'requir':46 'ship':77 'site':43 'sourc':3,9,12,25 'transform':84,89,94 'type':79 'typescript':78,121 'upstream':99 'usag':21 'variabl':66 'version':44 'via':61,73","created_at":"2026-06-07T12:52:55.585816+00:00","updated_at":"2026-06-07T12:52:55.585816+00:00","problems":[{"fix":"Set environment variable GATSBY_CONCURRENT_FILES to a lower value, e.g., 100, or increase the OS file limit using `ulimit -n 2048`.","cause":"Simultaneous file reads exceed OS limit (often around 256 on macOS/Linux).","error":"EMFILE: too many open files"},{"fix":"Run `npm install gatsby-source-filesystem-with-queue` or `yarn add gatsby-source-filesystem-with-queue`.","cause":"Package not installed or not in node_modules.","error":"Cannot find module 'gatsby-source-filesystem-with-queue'"},{"fix":"Ensure the 'path' option is a string, e.g., path: `${__dirname}/src/pages/`.","cause":"Missing or undefined 'path' option in plugin configuration.","error":"The \"path\" argument must be of type string. Received undefined"},{"fix":"Import as named export: `import { createFilePath } from 'gatsby-source-filesystem-with-queue'`.","cause":"Attempting to call createFilePath without importing it, or importing as default instead of named.","error":"TypeError: Cannot read properties of undefined (reading 'createFilePath')"}],"ecosystem":"npm","meta_description":null,"install_score":null,"quickstart_score":null,"quickstart_tag":null,"pypi_latest":null,"cli_name":null,"cli_version":null,"type":"library","homepage":"https://github.com/FraserThompson/gatsby-source-filesystem-with-queue","github":"https://github.com/FraserThompson/gatsby-source-filesystem-with-queue","docs":null,"changelog":null,"pypi":null,"npm":"gatsby-source-filesystem-with-queue","openapi_spec":null,"status_page":null,"smithery":null,"categories":["devops","storage"],"base_url":null,"auth_type":null,"provenance":{"verified_status":null,"verified_at":null,"last_verified":"2026-06-07","next_check":"2026-09-05","install_tag":null}}