diff --git a/README.md b/README.md index e0af469..1e17909 100644 --- a/README.md +++ b/README.md @@ -31,6 +31,7 @@ await repo.dispose(); - `'include'` or `true` (default) - process all packs - `'exclude'` or `false` - exclude cruft packs from processing - `'only'` - process cruft packs only + - `concurrentFsLimit` – number of concurrent file system operations (default: 50) ### Refs diff --git a/src/index.ts b/src/index.ts index 0738984..c1415fd 100644 --- a/src/index.ts +++ b/src/index.ts @@ -17,8 +17,8 @@ export async function createGitReader(gitdir: string, options?: GitReaderOptions const normalizedOptions = normalizeOptions(options); const resolvedGitDir = await resolveGitDir(gitdir); const [refIndex, looseObjectIndex, packedObjectIndex] = await Promise.all([ - createRefIndex(resolvedGitDir), - createLooseObjectIndex(resolvedGitDir), + createRefIndex(resolvedGitDir, normalizedOptions), + createLooseObjectIndex(resolvedGitDir, normalizedOptions), createPackedObjectIndex(resolvedGitDir, normalizedOptions) ]); const { readObjectHeaderByHash, readObjectByHash, readObjectHeaderByOid, readObjectByOid } = @@ -38,12 +38,15 @@ export async function createGitReader(gitdir: string, options?: GitReaderOptions async dispose() { await Promise.all([looseObjectIndex.dispose(), packedObjectIndex.dispose()]); }, - stat: createStatMethod({ - gitdir: resolvedGitDir, - refIndex, - looseObjectIndex, - packedObjectIndex - }), + stat: createStatMethod( + { + gitdir: resolvedGitDir, + refIndex, + looseObjectIndex, + packedObjectIndex + }, + normalizedOptions + ), initTime: Date.now() - startInitTime }; @@ -51,7 +54,7 @@ export async function createGitReader(gitdir: string, options?: GitReaderOptions function normalizeOptions(options?: GitReaderOptions): NormalizedGitReaderOptions { if (!options || options.cruftPacks === undefined) { - return { cruftPacks: 'include' }; + return { cruftPacks: 'include', concurrentFsLimit: 50 }; } return { @@ -60,7 +63,8 @@ function normalizeOptions(options?: GitReaderOptions): NormalizedGitReaderOption ? validateCruftPackMode(options.cruftPacks) : options.cruftPacks // expands true/false aliases ? 'include' - : 'exclude' + : 'exclude', + concurrentFsLimit: options.concurrentFsLimit ?? 50 }; } diff --git a/src/loose-object-index.ts b/src/loose-object-index.ts index b388aba..2842f41 100644 --- a/src/loose-object-index.ts +++ b/src/loose-object-index.ts @@ -5,6 +5,7 @@ import { GitObject, InternalGitObjectContent, InternalGitObjectHeader, + NormalizedGitReaderOptions, ObjectsTypeStat, PackedObjectType } from './types.js'; @@ -15,13 +16,16 @@ import { promiseAllThreaded } from './utils/threads.js'; type LooseObjectMap = Map; type LooseObjectMapEntry = [oid: string, relpath: string]; -async function createLooseObjectMap(gitdir: string): Promise { +async function createLooseObjectMap( + gitdir: string, + { concurrentFsLimit }: NormalizedGitReaderOptions +): Promise { const objectsPath = pathJoin(gitdir, 'objects'); const looseDirs = (await fsPromises.readdir(objectsPath)).filter((p) => /^[0-9a-f]{2}$/.test(p) ); - const objectDirs = await promiseAllThreaded(20, looseDirs, (dir) => + const objectDirs = await promiseAllThreaded(concurrentFsLimit, looseDirs, (dir) => fsPromises .readdir(pathJoin(objectsPath, dir)) .then((files) => @@ -76,8 +80,8 @@ function parseLooseObject(buffer: Buffer): InternalGitObjectContent { }; } -export async function createLooseObjectIndex(gitdir: string) { - const looseObjectMap = await createLooseObjectMap(gitdir); +export async function createLooseObjectIndex(gitdir: string, options: NormalizedGitReaderOptions) { + const looseObjectMap = await createLooseObjectMap(gitdir, options); const { fanoutTable, binaryNames, names } = indexObjectNames([...looseObjectMap.keys()]); const getOidFromHash = (hash: Buffer) => { diff --git a/src/packed-object-index.ts b/src/packed-object-index.ts index ba5f40b..b859ab8 100644 --- a/src/packed-object-index.ts +++ b/src/packed-object-index.ts @@ -20,7 +20,7 @@ const PACKDIR = 'objects/pack'; */ export async function createPackedObjectIndex( gitdir: string, - { cruftPacks }: NormalizedGitReaderOptions + { cruftPacks, concurrentFsLimit }: NormalizedGitReaderOptions ) { function readObjectHeaderByHash( hash: Buffer, @@ -76,7 +76,7 @@ export async function createPackedObjectIndex( : !cruftPackFilenames.includes(filename); }); - const packFiles = await promiseAllThreaded(20, packFilenames, async (filename) => + const packFiles = await promiseAllThreaded(concurrentFsLimit, packFilenames, async (filename) => readPackFile(gitdir, `${PACKDIR}/${filename}`, readObjectHeaderByHash, readObjectByHash) ); diff --git a/src/resolve-ref.ts b/src/resolve-ref.ts index 42b3219..7dfe7a6 100644 --- a/src/resolve-ref.ts +++ b/src/resolve-ref.ts @@ -2,6 +2,7 @@ import { promises as fsPromises, existsSync } from 'fs'; import { join as pathJoin, basename, sep as pathSep } from 'path'; import { promiseAllThreaded } from './utils/threads.js'; import { scanFs } from '@discoveryjs/scan-fs'; +import { NormalizedGitReaderOptions } from './types.js'; type Ref = { name: string; @@ -50,7 +51,10 @@ function isOid(value: unknown) { return typeof value === 'string' && value.length === 40 && /^[0-9a-f]{40}$/.test(value); } -export async function createRefIndex(gitdir: string) { +export async function createRefIndex( + gitdir: string, + { concurrentFsLimit }: NormalizedGitReaderOptions +) { const refResolver = await createRefResolver(gitdir); // expand a ref into a full form @@ -137,7 +141,7 @@ export async function createRefIndex(gitdir: string) { let cachedRefsWithOid = listRefsWithOidCache.get(prefix); if (cachedRefsWithOid === undefined) { - const oids = await promiseAllThreaded(20, cachedRefs, (name) => + const oids = await promiseAllThreaded(50, cachedRefs, (name) => refResolver.resolveOid(prefix + name) ); @@ -211,8 +215,10 @@ export async function createRefIndex(gitdir: string) { async stat() { const remotes = listRemotes(); - const branchesByRemote = await promiseAllThreaded(20, remotes, (remote) => - listRemoteBranches(remote) + const branchesByRemote = await promiseAllThreaded( + concurrentFsLimit, + remotes, + (remote) => listRemoteBranches(remote) ); return { diff --git a/src/stat.ts b/src/stat.ts index 2062667..da97e3e 100644 --- a/src/stat.ts +++ b/src/stat.ts @@ -6,18 +6,22 @@ import { promiseAllThreaded } from './utils/threads.js'; import { createRefIndex } from './resolve-ref.js'; import { createLooseObjectIndex } from './loose-object-index.js'; import { createPackedObjectIndex } from './packed-object-index.js'; +import { NormalizedGitReaderOptions } from './types.js'; -export function createStatMethod({ - gitdir, - refIndex, - looseObjectIndex, - packedObjectIndex -}: { - gitdir: string; - refIndex: Awaited>; - looseObjectIndex: Awaited>; - packedObjectIndex: Awaited>; -}) { +export function createStatMethod( + { + gitdir, + refIndex, + looseObjectIndex, + packedObjectIndex + }: { + gitdir: string; + refIndex: Awaited>; + looseObjectIndex: Awaited>; + packedObjectIndex: Awaited>; + }, + { concurrentFsLimit }: NormalizedGitReaderOptions +) { return async function () { const [refs, looseObjects, packedObjects, { files }] = await Promise.all([ refIndex.stat(), @@ -26,7 +30,7 @@ export function createStatMethod({ scanFs(gitdir) ]); - const fileStats = await promiseAllThreaded(20, files, (file) => + const fileStats = await promiseAllThreaded(concurrentFsLimit, files, (file) => fsPromises.stat(path.join(gitdir, file.path)) ); diff --git a/src/types.ts b/src/types.ts index 73337d6..912749c 100644 --- a/src/types.ts +++ b/src/types.ts @@ -132,8 +132,14 @@ export interface GitReaderOptions { * @default 'include' */ cruftPacks?: CruftPackMode | boolean; + /** + * Maximum number of concurrent file system operations. + * @default 50 + */ + concurrentFsLimit?: number; } export interface NormalizedGitReaderOptions { cruftPacks: CruftPackMode; + concurrentFsLimit: number; }