forked from ducktors/turborepo-remote-cache
-
Notifications
You must be signed in to change notification settings - Fork 0
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
feat: add git-repository storage provider
- Loading branch information
Showing
9 changed files
with
453 additions
and
1 deletion.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,80 @@ | ||
| import cp from 'child_process' | ||
|
|
||
| class ProcessError extends Error { | ||
| code: number | ||
|
|
||
| constructor(code: number, message: string) { | ||
| super(message) | ||
| Error.captureStackTrace(this) | ||
| this.code = code | ||
| } | ||
| } | ||
|
|
||
| /** | ||
| * Util function for handling spawned processes as promises. | ||
| * @param {string} exe Executable. | ||
| * @param {Array<string>} args Arguments. | ||
| * @param {string} cwd Working directory. | ||
| * @return {Promise} A promise. | ||
| */ | ||
| function spawn(exe: string, args: string[], cwd: string): Promise<string> { | ||
| return new Promise((resolve, reject) => { | ||
| const child = cp.spawn(exe, args, { cwd: cwd || process.cwd() }) | ||
| const buffer: string[] = [] | ||
| child.stderr.on('data', chunk => { | ||
| buffer.push(chunk.toString()) | ||
| }) | ||
| child.stdout.on('data', chunk => { | ||
| buffer.push(chunk.toString()) | ||
| }) | ||
| child.on('close', code => { | ||
| const output = buffer.join('') | ||
| if (code) { | ||
| const msg = output || `Process failed: ${code}. (command: ${exe} ${args.join(' ')})` | ||
| reject(new ProcessError(code, msg)) | ||
| } else { | ||
| resolve(output) | ||
| } | ||
| }) | ||
| }) | ||
| } | ||
|
|
||
| /** | ||
| * Create an object for executing git commands. | ||
| * @param {string} cwd Repository directory. | ||
| * @param {string} cmd Git executable (full path if not already on path). | ||
| * @function Object() { [native code] } | ||
| */ | ||
| export class Git { | ||
| cwd: string | ||
| cmd: string | ||
| output = '' | ||
|
|
||
| constructor(cwd: string, cmd?: string) { | ||
| this.cwd = cwd | ||
| this.cmd = cmd || 'git' | ||
| } | ||
|
|
||
| async exec(...args: string[]): Promise<Git> { | ||
| const output = await spawn(this.cmd, [...args], this.cwd) | ||
| this.output = output | ||
| return this | ||
| } | ||
|
|
||
| async add(_files: string | string[]) { | ||
| const files = Array.isArray(_files) ? _files : [_files] | ||
| return await this.exec('add', ...files) | ||
| } | ||
|
|
||
| async commit(message: string) { | ||
| try { | ||
| await this.exec('diff-index', '--quiet', 'HEAD') | ||
| } catch (e) { | ||
| await this.exec('commit', '-m', message) | ||
| } | ||
| } | ||
|
|
||
| async push(remote: string, branch: string) { | ||
| return await this.exec('push', '--tags', remote, branch) | ||
| } | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,142 @@ | ||
| import { Git } from '../../../git' | ||
| import fsBlob from 'fs-blob-store' | ||
| import fsExtra from 'fs-extra' | ||
| import fs from 'fs' | ||
| import path from 'path' | ||
| import { tmpdir } from 'os' | ||
| import { StorageProvider } from './index' | ||
|
|
||
| export type GitRepositoryOptions = { | ||
| repo: string | ||
| branch: string | ||
| remote: string | ||
| path?: string | ||
| userPassword: string | ||
| userName: string | ||
| userEmail: string | ||
| host: string | ||
| } | ||
|
|
||
| export async function createGitRepository(options: GitRepositoryOptions): Promise<StorageProvider> { | ||
| const tempDir = tmpdir() | ||
| const repoUrl = options.repo | ||
| const cacheDir = path.join(tempDir, 'git-repository', options.path ?? '') | ||
| const credentialPath = path.join(tempDir, 'git-repository', '.git-credentials') | ||
| const git: Git = new Git(cacheDir) | ||
| let skipClone = false | ||
|
|
||
| const ensureCacheDir = async () => { | ||
| if (fsExtra.pathExistsSync(cacheDir)) { | ||
| console.debug('Cache directory %s already exists', cacheDir) | ||
| if (fsExtra.pathExistsSync(path.join(cacheDir, '.git'))) { | ||
| console.debug('.git directory already exists') | ||
| await git.exec('config', '--get', 'remote.' + options.remote + '.url') | ||
| const url = git.output?.trim() ?? '' | ||
| if (url !== repoUrl) { | ||
| console.debug('Url mismatch. Got "%s" but expected "%s"', url, repoUrl) | ||
| console.debug('Empty cache directory %s', cacheDir) | ||
| fsExtra.emptyDirSync(cacheDir) | ||
| } else { | ||
| skipClone = true | ||
| } | ||
| } else { | ||
| console.debug('Empty cache directory %s', cacheDir) | ||
| fsExtra.emptyDirSync(cacheDir) | ||
| } | ||
| } else { | ||
| console.debug('Make cache directory %s', cacheDir) | ||
| fsExtra.mkdirpSync(cacheDir) | ||
| } | ||
| } | ||
| const clone = async () => { | ||
| if (skipClone) { | ||
| console.debug('Skipping clone') | ||
| return | ||
| } | ||
| console.debug('Cloning %s into %s', repoUrl, cacheDir) | ||
| await git.exec( | ||
| 'clone', | ||
| repoUrl, | ||
| cacheDir, | ||
| '--branch', | ||
| options.branch, | ||
| '--single-branch', | ||
| '--origin', | ||
| options.remote, | ||
| ) | ||
| } | ||
| const configGit = async () => { | ||
| console.debug('Configuring git user %s <%s>', options.userName, options.userEmail) | ||
| await git.exec('config', 'user.email', options.userEmail) | ||
| await git.exec('config', 'user.name', options.userName) | ||
| await git.exec('config', 'user.password', options.userPassword) | ||
| await git.exec('config', 'commit.gpgsign', 'false') | ||
| await git.exec('config', 'credential.helper', `store --file=${credentialPath}`) | ||
| if (!fs.existsSync(credentialPath)) { | ||
| fs.writeFileSync( | ||
| credentialPath, | ||
| `https://${options.userName}:${options.userPassword}@${options.host}`, | ||
| ) | ||
| } | ||
| } | ||
|
|
||
| const pull = async () => { | ||
| console.debug('Pulling fast-forward only from %s/%s', options.remote, options.branch) | ||
| await git.exec('merge', '--ff-only', `${options.remote}/${options.branch}`) | ||
| } | ||
|
|
||
| const checkout = async () => { | ||
| console.debug('Checking out %s/%s ', options.remote, options.branch) | ||
| await git.exec('ls-remote', '--exit-code', '.', `${options.remote}/${options.branch}`) | ||
| await git.exec('checkout', options.branch) | ||
| await git.exec('reset', '--hard', `${options.remote}/${options.branch}`) | ||
| } | ||
|
|
||
| const init = async () => { | ||
| await ensureCacheDir() | ||
| await clone() | ||
| await checkout() | ||
| await configGit() | ||
| console.debug('Git storage is ready') | ||
| } | ||
|
|
||
| const commitAndPush = async (artifactPath: string) => { | ||
| console.debug('Adding %s', artifactPath) | ||
| await git.add(artifactPath) | ||
|
|
||
| console.debug('Committing') | ||
| await git.commit(`chore: update cache ${artifactPath}`) | ||
|
|
||
| console.debug('Pushing to %s/%s', options.remote, options.branch) | ||
| await git.push(options.remote, options.branch) | ||
| } | ||
|
|
||
| const location: StorageProvider = { | ||
| exists: (artifactPath, cb) => { | ||
| pull().then(() => { | ||
| const exists = fs.existsSync(path.join(cacheDir, artifactPath)) | ||
| console.debug('exists: ', exists, path.join(cacheDir, artifactPath)) | ||
| cb(null, exists) | ||
| }) | ||
| }, | ||
| createReadStream: artifactPath => { | ||
| return fsBlob(cacheDir).createReadStream(artifactPath) | ||
| }, | ||
| createWriteStream: artifactPath => { | ||
| return fsBlob(cacheDir).createWriteStream(artifactPath) | ||
| }, | ||
| afterCreateWriteStream: async artifactPath => { | ||
| try { | ||
| await pull() | ||
| await commitAndPush(artifactPath) | ||
| } catch (e) { | ||
| console.error(e) | ||
| throw e | ||
| } | ||
| }, | ||
| } | ||
|
|
||
| await init() | ||
|
|
||
| return location | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,12 @@ | ||
| NODE_ENV=test | ||
| PORT=3000 | ||
| TURBO_TOKEN=changeme | ||
| STORAGE_PROVIDER=git-repository | ||
| STORAGE_PATH=turborepo-remote-cache-test | ||
| GIT_REPOSITORY=https://github.com/your_id/your_repo.git | ||
| GIT_BRANCH=main | ||
| GIT_REMOTE=origin | ||
| GIT_USER_NAME="Your Name" | ||
| [email protected] | ||
| GIT_USER_PASSWORD=your_github_token | ||
| GIT_HOST=github.com |
Oops, something went wrong.