Add concurrentFsLimit option

README.md

@@ -31,6 +31,7 @@ await repo.dispose();
     - `'include'` or `true` (default) - process all packs
     - `'exclude'` or `false` - exclude cruft packs from processing
     - `'only'` - process cruft packs only
+  - `concurrentFsLimit` – number of concurrent file system operations (default: 50)
 
 ### Refs
 

src/index.ts

@@ -17,8 +17,8 @@ export async function createGitReader(gitdir: string, options?: GitReaderOptions
     const normalizedOptions = normalizeOptions(options);
     const resolvedGitDir = await resolveGitDir(gitdir);
     const [refIndex, looseObjectIndex, packedObjectIndex] = await Promise.all([
-        createRefIndex(resolvedGitDir),
-        createLooseObjectIndex(resolvedGitDir),
+        createRefIndex(resolvedGitDir, normalizedOptions),
+        createLooseObjectIndex(resolvedGitDir, normalizedOptions),
         createPackedObjectIndex(resolvedGitDir, normalizedOptions)
     ]);
     const { readObjectHeaderByHash, readObjectByHash, readObjectHeaderByOid, readObjectByOid } =
@@ -38,20 +38,23 @@ export async function createGitReader(gitdir: string, options?: GitReaderOptions
         async dispose() {
             await Promise.all([looseObjectIndex.dispose(), packedObjectIndex.dispose()]);
         },
-        stat: createStatMethod({
-            gitdir: resolvedGitDir,
-            refIndex,
-            looseObjectIndex,
-            packedObjectIndex
-        }),
+        stat: createStatMethod(
+            {
+                gitdir: resolvedGitDir,
+                refIndex,
+                looseObjectIndex,
+                packedObjectIndex
+            },
+            normalizedOptions
+        ),
 
         initTime: Date.now() - startInitTime
     };
 }
 
 function normalizeOptions(options?: GitReaderOptions): NormalizedGitReaderOptions {
     if (!options || options.cruftPacks === undefined) {
-        return { cruftPacks: 'include' };
+        return { cruftPacks: 'include', concurrentFsLimit: 50 };
     }
 
     return {
@@ -60,7 +63,8 @@ function normalizeOptions(options?: GitReaderOptions): NormalizedGitReaderOption
                 ? validateCruftPackMode(options.cruftPacks)
                 : options.cruftPacks // expands true/false aliases
                 ? 'include'
-                : 'exclude'
+                : 'exclude',
+        concurrentFsLimit: options.concurrentFsLimit ?? 50
     };
 }
 

src/loose-object-index.ts

@@ -5,6 +5,7 @@ import {
     GitObject,
     InternalGitObjectContent,
     InternalGitObjectHeader,
+    NormalizedGitReaderOptions,
     ObjectsTypeStat,
     PackedObjectType
 } from './types.js';
@@ -15,13 +16,16 @@ import { promiseAllThreaded } from './utils/threads.js';
 type LooseObjectMap = Map<string, string>;
 type LooseObjectMapEntry = [oid: string, relpath: string];
 
-async function createLooseObjectMap(gitdir: string): Promise<LooseObjectMap> {
+async function createLooseObjectMap(
+    gitdir: string,
+    { concurrentFsLimit }: NormalizedGitReaderOptions
+): Promise<LooseObjectMap> {
     const objectsPath = pathJoin(gitdir, 'objects');
     const looseDirs = (await fsPromises.readdir(objectsPath)).filter((p) =>
         /^[0-9a-f]{2}$/.test(p)
     );
 
-    const objectDirs = await promiseAllThreaded(20, looseDirs, (dir) =>
+    const objectDirs = await promiseAllThreaded(concurrentFsLimit, looseDirs, (dir) =>
         fsPromises
             .readdir(pathJoin(objectsPath, dir))
             .then((files) =>
@@ -76,8 +80,8 @@ function parseLooseObject(buffer: Buffer): InternalGitObjectContent {
     };
 }
 
-export async function createLooseObjectIndex(gitdir: string) {
-    const looseObjectMap = await createLooseObjectMap(gitdir);
+export async function createLooseObjectIndex(gitdir: string, options: NormalizedGitReaderOptions) {
+    const looseObjectMap = await createLooseObjectMap(gitdir, options);
     const { fanoutTable, binaryNames, names } = indexObjectNames([...looseObjectMap.keys()]);
 
     const getOidFromHash = (hash: Buffer) => {

src/packed-object-index.ts

@@ -20,7 +20,7 @@ const PACKDIR = 'objects/pack';
  */
 export async function createPackedObjectIndex(
     gitdir: string,
-    { cruftPacks }: NormalizedGitReaderOptions
+    { cruftPacks, concurrentFsLimit }: NormalizedGitReaderOptions
 ) {
     function readObjectHeaderByHash(
         hash: Buffer,
@@ -76,7 +76,7 @@ export async function createPackedObjectIndex(
             : !cruftPackFilenames.includes(filename);
     });
 
-    const packFiles = await promiseAllThreaded(20, packFilenames, async (filename) =>
+    const packFiles = await promiseAllThreaded(concurrentFsLimit, packFilenames, async (filename) =>
         readPackFile(gitdir, `${PACKDIR}/${filename}`, readObjectHeaderByHash, readObjectByHash)
     );
 

src/resolve-ref.ts

@@ -2,6 +2,7 @@ import { promises as fsPromises, existsSync } from 'fs';
 import { join as pathJoin, basename, sep as pathSep } from 'path';
 import { promiseAllThreaded } from './utils/threads.js';
 import { scanFs } from '@discoveryjs/scan-fs';
+import { NormalizedGitReaderOptions } from './types.js';
 
 type Ref = {
     name: string;
@@ -50,7 +51,10 @@ function isOid(value: unknown) {
     return typeof value === 'string' && value.length === 40 && /^[0-9a-f]{40}$/.test(value);
 }
 
-export async function createRefIndex(gitdir: string) {
+export async function createRefIndex(
+    gitdir: string,
+    { concurrentFsLimit }: NormalizedGitReaderOptions
+) {
     const refResolver = await createRefResolver(gitdir);
 
     // expand a ref into a full form
@@ -137,7 +141,7 @@ export async function createRefIndex(gitdir: string) {
         let cachedRefsWithOid = listRefsWithOidCache.get(prefix);
 
         if (cachedRefsWithOid === undefined) {
-            const oids = await promiseAllThreaded(20, cachedRefs, (name) =>
+            const oids = await promiseAllThreaded(50, cachedRefs, (name) =>
                 refResolver.resolveOid(prefix + name)
             );
 
@@ -211,8 +215,10 @@ export async function createRefIndex(gitdir: string) {
 
         async stat() {
             const remotes = listRemotes();
-            const branchesByRemote = await promiseAllThreaded(20, remotes, (remote) =>
-                listRemoteBranches(remote)
+            const branchesByRemote = await promiseAllThreaded(
+                concurrentFsLimit,
+                remotes,
+                (remote) => listRemoteBranches(remote)
             );
 
             return {

src/stat.ts

@@ -6,18 +6,22 @@ import { promiseAllThreaded } from './utils/threads.js';
 import { createRefIndex } from './resolve-ref.js';
 import { createLooseObjectIndex } from './loose-object-index.js';
 import { createPackedObjectIndex } from './packed-object-index.js';
+import { NormalizedGitReaderOptions } from './types.js';
 
-export function createStatMethod({
-    gitdir,
-    refIndex,
-    looseObjectIndex,
-    packedObjectIndex
-}: {
-    gitdir: string;
-    refIndex: Awaited<ReturnType<typeof createRefIndex>>;
-    looseObjectIndex: Awaited<ReturnType<typeof createLooseObjectIndex>>;
-    packedObjectIndex: Awaited<ReturnType<typeof createPackedObjectIndex>>;
-}) {
+export function createStatMethod(
+    {
+        gitdir,
+        refIndex,
+        looseObjectIndex,
+        packedObjectIndex
+    }: {
+        gitdir: string;
+        refIndex: Awaited<ReturnType<typeof createRefIndex>>;
+        looseObjectIndex: Awaited<ReturnType<typeof createLooseObjectIndex>>;
+        packedObjectIndex: Awaited<ReturnType<typeof createPackedObjectIndex>>;
+    },
+    { concurrentFsLimit }: NormalizedGitReaderOptions
+) {
     return async function () {
         const [refs, looseObjects, packedObjects, { files }] = await Promise.all([
             refIndex.stat(),
@@ -26,7 +30,7 @@ export function createStatMethod({
             scanFs(gitdir)
         ]);
 
-        const fileStats = await promiseAllThreaded(20, files, (file) =>
+        const fileStats = await promiseAllThreaded(concurrentFsLimit, files, (file) =>
             fsPromises.stat(path.join(gitdir, file.path))
         );
 

src/types.ts

@@ -132,8 +132,14 @@ export interface GitReaderOptions {
      * @default 'include'
      */
     cruftPacks?: CruftPackMode | boolean;
+    /**
+     * Maximum number of concurrent file system operations.
+     * @default 50
+     */
+    concurrentFsLimit?: number;
 }
 
 export interface NormalizedGitReaderOptions {
     cruftPacks: CruftPackMode;
+    concurrentFsLimit: number;
 }