Implement spawn-able github scan api

apps/github-scan-api/.dockerignore

@@ -0,0 +1,7 @@
+node_modules/
+dist/
+.git
+.gitignore
+Dockerfile
+.dockerignore
+.env

apps/github-scan-api/.env.example

@@ -0,0 +1,3 @@
+CLONING_DIR=/home/node/keyshade-github-scan
+GITHUB_SCAN_API_USERNAME=
+GITHUB_SCAN_API_PASSWORD=

apps/github-scan-api/.eslintrc.js

@@ -0,0 +1,6 @@
+/** @type {import("eslint").Linter.Config} */
+module.exports = {
+  root: true,
+  plugins: ['@typescript-eslint'],
+  extends: ['eslint:recommended', 'plugin:@typescript-eslint/recommended']
+}

apps/github-scan-api/Dockerfile

@@ -0,0 +1,35 @@
+FROM node:20.0.0-alpine AS build
+
+WORKDIR /app
+
+RUN corepack enable
+
+COPY --chown=root:root --chmod=755 package.json turbo.json pnpm-*.yaml ./
+COPY --chown=root:root --chmod=755 apps/github-scan-api/package.json apps/github-scan-api/tsconfig.json apps/github-scan-api/
+COPY --chown=root:root --chmod=755 packages packages
+
+RUN --mount=type=cache,id=pnpm,target=/pnpm/store \
+	pnpm install --ignore-scripts --frozen-lockfile && \
+	rm -rf /root/.npm /root/.node-gyp /tmp/npm-*
+
+COPY --chown=root:root --chmod=755 apps/github-scan-api/src apps/github-scan-api/src
+
+RUN pnpm build:github-scan-api
+
+USER node
+
+FROM node:20-alpine AS prod
+
+RUN apk add --no-cache git
+# Don't run production as root
+USER node
+
+WORKDIR /app
+
+COPY --chown=root:root --chmod=755 --from=build /app /app
+
+EXPOSE 8080
+
+RUN mkdir /home/node/keyshade-github-scan
+
+CMD ["node", "apps/github-scan-api/dist/src/index.js"]

apps/github-scan-api/package.json

@@ -0,0 +1,28 @@
+{
+  "name": "github-scan-api",
+  "version": "0.0.0",
+  "private": true,
+  "scripts": {
+    "build": "tsc",
+    "clean": "rm -rf dist",
+    "dev": "nodemon --exec \"pnpm build && pnpm start\" -e ts --ignore dist/",
+    "lint": "eslint \"src/**/*.ts\" --fix",
+    "start": "dotenv -e .env -- node ./dist/src/index.js"
+  },
+  "dependencies": {
+    "@keyshade/secret-scan": "workspace:*",
+    "express": "^4.18.3",
+    "glob": "^11.0.0",
+    "simple-git": "^3.27.0"
+  },
+  "devDependencies": {
+    "@types/cors": "^2.8.17",
+    "@types/express": "^4.17.17",
+    "@types/node": "^20.11.24",
+    "@typescript-eslint/eslint-plugin": "^6.0.0",
+    "@typescript-eslint/parser": "^6.0.0",
+    "eslint": "^8.57.1",
+    "nodemon": "^3.1.0",
+    "typescript": "5.5.4"
+  }
+}

apps/github-scan-api/src/app.ts

@@ -0,0 +1,33 @@
+import express, { type Express } from 'express'
+import { scanRepo } from './util'
+
+export const createServer = (): Express => {
+  const app = express()
+  app.use(express.json())
+
+  app.get('/', (_, res) => {
+    res.send(`GitHub Scan API from Keyshade.xyz`)
+  })
+  app.post('/scan', async (req, res) => {
+    const { username, password, githubUrl } = req.body
+    if (
+      username !== process.env.GITHUB_SCAN_API_USERNAME ||
+      password !== process.env.GITHUB_SCAN_API_PASSWORD
+    ) {
+      return res.status(401).json({ message: 'Unauthorized' })
+    }
+    if (!githubUrl || !githubUrl.startsWith('https://github.com/')) {
+      return res
+        .status(400)
+        .json({ message: 'Invalid or missing githubUrl parameter' })
+    }
+    try {
+      res.status(200).json({ files: await scanRepo(githubUrl) })
+    } catch (error) {
+      console.error(`Error scanning repo, githubUrl:${githubUrl} `, error)
+      res.status(500).json({ message: 'Internal Server Error' })
+    }
+  })
+
+  return app
+}

apps/github-scan-api/src/index.ts

@@ -0,0 +1,5 @@
+import { createServer } from './app'
+
+createServer().listen(8080, () => {
+  console.log('Server is running on http://localhost:8080')
+})

apps/github-scan-api/src/util.ts

@@ -0,0 +1,170 @@
+import simpleGit from 'simple-git'
+import path from 'path'
+import { readFileSync, statSync, unlinkSync, rmSync } from 'node:fs'
+import { globSync } from 'glob'
+import secretDetector from '@keyshade/secret-scan'
+
+interface ScanResult {
+  file: string
+  line: number
+  content: string
+}
+
+const git = simpleGit()
+const CLONING_DIR = process.env.CLONING_DIR
+
+const ignoredExtensions = [
+  'png',
+  'jpg',
+  'jpeg',
+  'gif',
+  'svg',
+  'ico',
+  'woff',
+  'woff2',
+  'ttf',
+  'eot',
+  'pdf',
+  'mp4',
+  'mp3',
+  'wav',
+  'avi',
+  'mov',
+  'webm',
+  'zip',
+  'tar',
+  'gz',
+  '7z',
+  'rar',
+  'iso',
+  'bin',
+  'exe',
+  'dll',
+  'so',
+  'a',
+  'o',
+  'dylib',
+  'lib',
+  'obj',
+  'jar',
+  'war',
+  'ear'
+]
+
+function getAllFiles(localPath: string): string[] {
+  const currentWorkDir = localPath
+  let gitIgnorePatterns: string[] = []
+  try {
+    const gitIgnorePath = path.resolve(currentWorkDir, '.gitignore')
+
+    const gitIgnoreContent = readFileSync(gitIgnorePath, 'utf8')
+
+    gitIgnorePatterns = gitIgnoreContent
+      .split('\n')
+      .filter((line) => line.trim() !== '' && !line.startsWith('#'))
+  } catch {
+    // Repository doesn't have .gitignore file
+  }
+
+  return globSync(currentWorkDir + '/**/**', {
+    dot: true,
+    ignore: {
+      ignored: (p) => {
+        return gitIgnorePatterns.some((pattern) => {
+          return p.isNamed(pattern)
+        })
+      },
+      childrenIgnored: (p) => {
+        return gitIgnorePatterns.some((pattern) => {
+          return p.isNamed(pattern)
+        })
+      }
+    }
+  })
+}
+
+function scanSecrets(localPath: string): ScanResult[] {
+  const foundSecrets = []
+  let skipNextLine = false
+  const allFiles = getAllFiles(localPath)
+  for (const file of allFiles) {
+    const stats = statSync(file)
+    if (stats.isFile()) {
+      // Skip the file if it has an ignored extension like images, videos, etc.
+      if (ignoredExtensions.includes(file.split('.').pop())) {
+        // Delete the file
+        try {
+          unlinkSync(file)
+        } catch (err) {
+          console.error(`Failed to delete file ${file}:`, err)
+        }
+        continue
+      }
+
+      const content = readFileSync(file, 'utf8').split(/\r?\n/)
+
+      // Delete the file after reading
+      try {
+        unlinkSync(file)
+      } catch (err) {
+        console.error(`Failed to delete file ${file}:`, err)
+      }
+
+      // Skip the file if ignore comment is found in the first line
+      if (content[0].includes('keyshade-ignore-all')) {
+        continue
+      }
+
+      content.forEach((line, index) => {
+        // Skip the next line if ignore comment is found in the previous line
+        if (skipNextLine) {
+          skipNextLine = false
+          return
+        }
+
+        if (line.includes('keyshade-ignore')) {
+          skipNextLine = true
+          return
+        }
+        const { found, regex } = secretDetector.detect(line) as {
+          found: boolean
+          regex: RegExp
+        }
+        if (found) {
+          const matched = line.match(regex)
+          const highlightedLine = line.replace(regex, matched[0]).trim()
+          foundSecrets.push({
+            file: file.split(localPath)[1],
+            line: index + 1,
+            content: highlightedLine
+          })
+        }
+      })
+    }
+  }
+
+  // Delete the directory after scanning
+  try {
+    rmSync(localPath, { recursive: true })
+  } catch (err) {
+    console.error(`Failed to delete directory ${localPath}:`, err)
+  }
+
+  return foundSecrets
+}
+
+export async function scanRepo(githubUrl: string) {
+  let repoName = githubUrl.split('https://github.com/')[1]
+  if (repoName.endsWith('.git')) {
+    repoName = repoName.slice(0, -4)
+  }
+  const dirName = repoName.split('/').join('_')
+  const localPath = path.resolve(CLONING_DIR, dirName)
+
+  try {
+    await git.clone(githubUrl, localPath)
+    return scanSecrets(localPath)
+  } catch (error) {
+    console.error('Failed to clone repository:', error)
+  }
+}

apps/github-scan-api/tsconfig.json

@@ -0,0 +1,13 @@
+{
+  "extends": "tsconfig/base.json",
+  "compilerOptions": {
+    "lib": ["ES2015"],
+    "outDir": "./dist",
+    "rootDir": ".",
+    "target": "ES2022",
+    "module": "NodeNext",
+    "moduleResolution": "nodenext"
+  },
+  "exclude": ["node_modules"],
+  "include": ["src"]
+}

apps/github-scan/.env.example

@@ -1 +1,2 @@
-GITHUB_TOKEN=
+GITHUB_SCAN_API_USERNAME=
+GITHUB_SCAN_API_PASSWORD=

apps/github-scan/package.json

@@ -3,9 +3,9 @@
   "version": "0.1.0",
   "private": true,
   "scripts": {
-    "dev": "next dev",
+    "dev": "next dev -p 5173",
     "build": "next build",
-    "start": "next start",
+    "start": "next start -p 5173",
     "lint": "next lint"
   },
   "dependencies": {

apps/github-scan/src/app/api/scan/route.ts

@@ -1,5 +1,9 @@
 import { NextRequest, NextResponse } from 'next/server'
-import gitHubScanner from '@/util/github-scanner'
+import { ScanResult } from '@/util/types'
+
+const GITHUB_SCAN_API = process.env.GITHUB_SCAN_API
+const username = process.env.GITHUB_SCAN_API_USERNAME
+const password = process.env.GITHUB_SCAN_API_PASSWORD
 
 export async function GET(req: NextRequest) {
   try {
@@ -12,37 +16,18 @@ export async function GET(req: NextRequest) {
         { status: 400 }
       )
     }
-    try {
-      const result = await gitHubScanner.scanRepo(githubUrl)
-      if (result.length > 0)
-        return NextResponse.json(
-          { isVulnerable: true, files: result },
-          { status: 200 }
-        )
-      else
-        return NextResponse.json(
-          {
-            isVulnerable: false
-          },
-          { status: 200 }
-        )
-    } catch (error) {
-      if (error instanceof Error) {
-        if (error.message === 'Rate limit hit') {
-          return NextResponse.json({ error: 'Rate limit hit' }, { status: 429 })
-        } else if (error.message === 'Invalid GitHub URL') {
-          return NextResponse.json(
-            { error: 'Invalid GitHub URL' },
-            { status: 400 }
-          )
-        } else {
-          return NextResponse.json(
-            { error: 'Internal Server Error' },
-            { status: 500 }
-          )
-        }
-      }
+    const respone = await fetch(GITHUB_SCAN_API + '/scan', {
+      method: 'POST',
+      headers: {
+        'Content-Type': 'application/json'
+      },
+      body: JSON.stringify({ username, password, githubUrl: githubUrl })
+    })
+    const data: { files: ScanResult[] } = await respone.json()
+    if (data.files.length === 0) {
+      return NextResponse.json({ isVulnerable: false })
     }
+    return NextResponse.json({ isVulnerable: true, files: data.files })
   } catch (error) {
     return NextResponse.json(
       { error: 'Internal Server Error' },