Init storage shipper

.eslintrc.js

@@ -0,0 +1,24 @@
+module.exports = {
+    "env": {
+        "commonjs": true,
+        "es2021": true,
+        "node": true
+    },
+    "extends": [
+        "plugin:jest/all",
+        "standard-with-typescript"
+    ],
+    "parserOptions": {
+        "ecmaVersion": "latest",
+        "sourceType": "module",
+        "project": [
+            "./tsconfig.json"
+        ]
+    },
+    "plugins": [
+        "jest"
+    ],
+    "rules": {
+        "jest/prefer-expect-assertions": 0
+    }
+}

.github/workflows/ci.yml

@@ -24,10 +24,10 @@ jobs:
           node-version: 18.17.0
 
       - name: Install dependencies
-        run: npm ci
+        run: npm i
 
       - name: Run ESLint
-        run: npm run eslint
+        run: npx lerna run eslint
 
       - name: Run tests
-        run: npm run test
+        run: npx lerna run test

.gitignore

@@ -0,0 +1,5 @@
+.idea/
+dist/
+node_modules/
+lerna-debug.log
+package-lock.json

lerna.json

@@ -0,0 +1,4 @@
+{
+  "$schema": "node_modules/lerna/schemas/lerna-schema.json",
+  "version": "0.0.0"
+}

package.json

@@ -0,0 +1,23 @@
+{
+  "name": "root",
+  "private": false,
+  "workspaces": [
+    "packages/*"
+  ],
+  "devDependencies": {
+    "@azure/storage-blob": "12.15.0",
+    "@types/jest": "29.5.3",
+    "@typescript-eslint/eslint-plugin": "5.52.0",
+    "eslint": "8.46.0",
+    "eslint-config-standard-with-typescript": "37.0.0",
+    "eslint-plugin-import": "2.28.0",
+    "eslint-plugin-jest": "27.2.3",
+    "eslint-plugin-n": "16.0.1",
+    "eslint-plugin-promise": "6.1.1",
+    "jest": "29.6.2",
+    "lerna": "7.1.4",
+    "ts-jest": "29.1.1",
+    "ts-node": "10.9.1",
+    "typescript": "5.1.6"
+  }
+}

packages/core/README.md

@@ -0,0 +1,11 @@
+# `core`
+
+> TODO: description
+
+## Usage
+
+```
+const core = require('core');
+
+// TODO: DEMONSTRATE API
+```

packages/core/__tests__/manifest/ManifestValidator.test.ts

@@ -0,0 +1,85 @@
+import { ManifestValidator } from '../../src/manifest/ManifestValidator'
+
+describe('manifest validator test', () => {
+  it('should not throw exception when json is valid', () => {
+    // given
+    const manifest = {
+      version: '1.0',
+      deploy: [
+        {
+          name: 'resources',
+          include: 'v1-rc6/resources/*',
+          headers: {
+            'content-cache': 'public, max-age=2592000'
+          }
+        },
+        {
+          name: 'main',
+          include: 'v1-rc6/*.js',
+          headers: {
+            'content-encoding': 'gzip',
+            'content-type': 'text/javascript',
+            'content-cache': 'public, max-age=2592000'
+          }
+        },
+        {
+          name: 'chat-loader',
+          include: 'chat-loader.js',
+          headers: {
+            'content-encoding': 'gzip',
+            'content-typ': 'text/javascript',
+            'content-cache': 'public, max-age=300'
+          }
+        }
+      ]
+    }
+
+    const manifestValidator = new ManifestValidator()
+
+    // when/then
+    expect(() => {
+      manifestValidator.validate(manifest)
+    }).not.toThrow()
+  })
+
+  it('should throw exception when json is not valid', () => {
+    // given
+    const manifest = {
+      version: 'invalid-version',
+      deploy: [
+        {
+          name: 'resources',
+          include: 'v1-rc6/resources/*',
+          headers: {
+            'content-cache': 'public, max-age=2592000'
+          }
+        },
+        {
+          name: 'main',
+          include: 'v1-rc6/*.js',
+          headers: {
+            'content-encoding': 'gzip',
+            'content-type': 'text/javascript',
+            'content-cache': 'public, max-age=2592000'
+          }
+        },
+        {
+          name: 'chat-loader',
+          include: 'chat-loader.js',
+          headers: {
+            'content-encoding': 'gzip',
+            'content-typ': 'text/javascript',
+            'content-cache': 'public, max-age=300'
+          }
+        }
+      ]
+    }
+
+    const manifestValidator = new ManifestValidator()
+
+    // when/then
+    expect(() => {
+      manifestValidator.validate(manifest)
+    }).toThrow('Storage shipper manifest is not valid. Error: [path: \'/version\'] [message: \'must match pattern "^([1-9]\\d*)\\.(?:0|[1-9]\\d*)$"\']')
+  })
+})

packages/core/jest.config.ts

@@ -0,0 +1,7 @@
+import { type JestConfigWithTsJest } from 'ts-jest'
+
+const config: JestConfigWithTsJest = {
+  preset: 'ts-jest'
+}
+
+export default config

packages/core/package.json

@@ -0,0 +1,33 @@
+{
+  "name": "@thulium/storage-shipper",
+  "version": "0.0.0",
+  "description": "Core module for ship files to storage",
+  "homepage": "https://github.com/thulium/storage-shipper#readme",
+  "license": "MIT",
+  "main": "src/core.ts",
+  "directories": {
+    "lib": "src",
+    "test": "__tests__"
+  },
+  "files": [
+    "src"
+  ],
+  "repository": {
+    "type": "git",
+    "url": "git+https://github.com/thulium/storage-shipper.git"
+  },
+  "scripts": {
+    "build": "tsc",
+    "test": "jest",
+    "eslint": "eslint \"src/**\""
+  },
+  "bugs": {
+    "url": "https://github.com/thulium/storage-shipper/issues"
+  },
+  "dependencies": {
+    "ajv": "8.12.0",
+    "jszip": "3.10.1",
+    "minimatch": "9.0.3",
+    "remeda": "1.24.0"
+  }
+}

packages/core/src/Destination.ts

@@ -0,0 +1,3 @@
+export interface Destination {
+  container: string
+}

packages/core/src/StorageShipper.ts

@@ -0,0 +1,24 @@
+import { type StorageUploader } from './StorageUploader'
+import { type Destination } from './Destination'
+import { type ArtifactRepositoryFactory } from './artifact/ArtifactRepositoryFactory'
+import { type Artifact } from './artifact/Artifact'
+
+export class StorageShipper {
+  constructor (
+    private readonly artifactRepositoryFactory: ArtifactRepositoryFactory,
+    private readonly storageUploader: StorageUploader
+  ) {
+  }
+
+  public async shipIt (artifact: Artifact, destination: Destination): Promise<void> {
+    const artifactRepository = await this.artifactRepositoryFactory.create(artifact)
+
+    const manifest = await artifactRepository.getManifest()
+
+    const deploys = manifest.deploy
+    for (const deploy of deploys) {
+      const artifactFiles = await artifactRepository.getMatchingArtifactFiles(deploy.include)
+      this.storageUploader.upload(artifactFiles, destination, artifact)
+    }
+  }
+}

packages/core/src/StorageUploader.ts

@@ -0,0 +1,6 @@
+import { type Destination } from './Destination'
+import { type Artifact, type ArtifactFile } from './artifact/Artifact'
+
+export interface StorageUploader {
+  upload: (artifactFiles: ArtifactFile[], destination: Destination, artifact: Artifact) => void
+}

packages/core/src/artifact/Artifact.ts

@@ -0,0 +1,17 @@
+export interface Artifact {
+  parentDir?: string
+  path: string
+}
+
+export interface ArtifactFile {
+  name: string
+  data: ArrayBuffer | null
+}
+
+export function sanitizedParentDir (artifact: Artifact): string | null | undefined {
+  const parentDir = artifact.parentDir
+  if (parentDir == null) {
+    return parentDir
+  }
+  return parentDir.endsWith('/') ? '' : '/'
+}

packages/core/src/artifact/ArtifactRepository.ts

@@ -0,0 +1,8 @@
+import { type Manifest } from '../manifest/Manifest'
+import { type ArtifactFile } from './Artifact'
+
+export interface ArtifactRepository {
+  getManifest: () => Promise<Manifest>
+
+  getMatchingArtifactFiles: (pattern: string) => Promise<ArtifactFile[]>
+}

packages/core/src/artifact/ArtifactRepositoryFactory.ts

@@ -0,0 +1,6 @@
+import { type Artifact } from './Artifact'
+import { type ArtifactRepository } from './ArtifactRepository'
+
+export interface ArtifactRepositoryFactory {
+  create: (artifact: Artifact) => Promise<ArtifactRepository>
+}

packages/core/src/artifact/zip/ZipArtifactRepository.ts

@@ -0,0 +1,63 @@
+import type JSZip from 'jszip'
+import { minimatch } from 'minimatch/dist/mjs'
+import { type Artifact, type ArtifactFile, sanitizedParentDir } from '../Artifact'
+import { ManifestValidator } from '../../manifest/ManifestValidator'
+import { type Manifest } from '../../manifest/Manifest'
+import { type ArtifactRepository } from '../ArtifactRepository'
+import * as R from 'remeda'
+
+export class ZipArtifactRepository implements ArtifactRepository {
+  private readonly manifestValidator: ManifestValidator = new ManifestValidator()
+
+  constructor (
+    private readonly artifact: Artifact,
+    private readonly jsZip: JSZip
+  ) {
+  }
+
+  public async getManifest (): Promise<Manifest> {
+    const manifests = this.jsZip.filter(relativePath => {
+      return relativePath.includes('storage-shipper-manifest.json')
+    })
+
+    const manifestString = await this.getManifestString(manifests)
+    const manifest: Manifest = JSON.parse(manifestString)
+    this.manifestValidator.validate(manifest)
+
+    return manifest
+  }
+
+  public async getMatchingArtifactFiles (pattern: string): Promise<ArtifactFile[]> {
+    const jsZipObjects = this.jsZip.filter(relativePath => {
+      const parentDir = sanitizedParentDir(this.artifact)
+      if (parentDir != null) {
+        relativePath = relativePath.replace(parentDir, '')
+      }
+      return minimatch(relativePath, pattern)
+    })
+
+    const artifactFilesPromise = R.map(jsZipObjects, async (jsZipObject): Promise<ArtifactFile> => {
+      const parentDir = sanitizedParentDir(this.artifact) ?? ''
+      const data = await jsZipObject.async('arraybuffer') ?? null
+      return {
+        name: jsZipObject.name.replace(parentDir, ''),
+        data
+      }
+    })
+    return await Promise.all(artifactFilesPromise)
+  }
+
+  private async getManifestString (manifests: JSZip.JSZipObject[]): Promise<string> {
+    if (manifests.length !== 1) {
+      throw new Error('Cannot determine correct manifest file')
+    }
+
+    const manifestFile = manifests.at(0)
+    const manifestString = await manifestFile?.async('text') ?? null
+    if (manifestString === null) {
+      throw new Error('Cannot read manifest file')
+    }
+
+    return manifestString
+  }
+}

packages/core/src/artifact/zip/ZipArtifactRepositoryFactory.ts

@@ -0,0 +1,22 @@
+import JSZip from 'jszip'
+import { type ArtifactRepository } from '../ArtifactRepository'
+import * as fs from 'fs'
+import { type ArtifactRepositoryFactory } from '../ArtifactRepositoryFactory'
+import { type Artifact } from '../Artifact'
+import { ZipArtifactRepository } from './ZipArtifactRepository'
+
+export class ZipArtifactRepositoryFactory implements ArtifactRepositoryFactory {
+  private readonly jsZip = new JSZip()
+
+  public async create (artifact: Artifact): Promise<ArtifactRepository> {
+    const path = artifact.path
+
+    if (!fs.existsSync(path)) {
+      throw new Error(`Artifact '${path}' does not exists`)
+    }
+
+    const archive = fs.readFileSync(path)
+    const jsZip = await this.jsZip.loadAsync(archive)
+    return new ZipArtifactRepository(artifact, jsZip)
+  }
+}

packages/core/src/core.ts

@@ -0,0 +1,12 @@
+export * from './Destination'
+export * from './StorageShipper'
+export * from './StorageUploader'
+
+export * from './artifact/Artifact'
+export * from './artifact/ArtifactRepositoryFactory'
+
+export * from './artifact/zip/ZipArtifactRepository'
+export * from './artifact/zip/ZipArtifactRepositoryFactory'
+
+export * from './manifest/Manifest'
+export * from './manifest/ManifestValidator'

packages/core/src/manifest/Manifest.ts

@@ -0,0 +1,10 @@
+export interface Manifest {
+  version: string
+  deploy: Deploy[]
+}
+
+export interface Deploy {
+  name: string
+  include: string
+  headers?: object
+}