[scripts] Add automatic helm-chart bump on teraslice bump (#3968)

This PR makes the following changes:

- When using `ts-scripts` to bump teraslice, it will also update and
bump the chart automatically
  - Updates the `version` and `appVersion` fields in `Chart.yaml` 
  - Updates the `image.nodeVersion` field in `values.yaml`
- **Functionality**: When bumping teraslice a major, the chart is bumped
a major. When teraslice is bumped a minor/patch, the chart is bumped a
minor.

Ref to issue #3965

e2e/package.json

@@ -45,7 +45,7 @@
         "ms": "~2.1.3"
     },
     "devDependencies": {
-        "@terascope/scripts": "~1.10.5",
+        "@terascope/scripts": "~1.11.0",
         "@terascope/types": "~1.4.1",
         "@terascope/utils": "~1.7.6",
         "bunyan": "~1.8.15",

package.json

@@ -53,7 +53,7 @@
         "@eslint/js": "~9.21.0",
         "@swc/core": "1.10.18",
         "@swc/jest": "~0.2.37",
-        "@terascope/scripts": "~1.10.5",
+        "@terascope/scripts": "~1.11.0",
         "@types/bluebird": "~3.5.42",
         "@types/convict": "~6.1.6",
         "@types/elasticsearch": "~5.0.43",

packages/scripts/package.json

@@ -1,7 +1,7 @@
 {
     "name": "@terascope/scripts",
     "displayName": "Scripts",
-    "version": "1.10.5",
+    "version": "1.11.0",
     "description": "A collection of terascope monorepo scripts",
     "homepage": "https://github.com/terascope/teraslice/tree/master/packages/scripts#readme",
     "bugs": {

packages/scripts/src/helpers/bump/index.ts

@@ -4,7 +4,7 @@ import {
     AssetJsonInfo, BumpAssetOnlyOptions, BumpPkgInfo,
     BumpPackageOptions
 } from './interfaces.js';
-import { listPackages, isMainPackage, updatePkgJSON } from '../packages.js';
+import { listPackages, isMainPackage, updatePkgJSON, bumpHelmChart } from '../packages.js';
 import { PackageInfo } from '../interfaces.js';
 import { getRootInfo, writeIfChanged } from '../misc.js';
 import {
@@ -46,6 +46,13 @@ export async function bumpPackages(options: BumpPackageOptions, isAsset: boolean
 
     await setup();
 
+    if (bumpedMain) {
+        // If main package is bumped we need to bump the chart
+        // We want to do this AFTER all packages have been updated.
+        signale.info(`Bumping teraslice chart...`);
+        await bumpHelmChart(options.release);
+    }
+
     signale.success(`
 
 Please commit these changes:

packages/scripts/src/helpers/interfaces.ts

@@ -160,3 +160,27 @@ export interface TsVolumeSet {
     volumes: V1Volume[];
     volumeMounts: V1VolumeMount[];
 }
+
+export interface OCIImageManifest {
+    mediaType: string;
+    digest: string;
+    size: number;
+    platform: {
+        architecture: string;
+    };
+}
+
+export interface OCIindexManifest {
+    schemaVersion: number;
+    mediaType: string;
+    manifests: OCIImageManifest[];
+    config: {
+        digest: string;
+    };
+}
+
+export interface OCIimageConfig {
+    config: {
+        Labels: Record<string, string>;
+    };
+}

packages/scripts/src/helpers/packages.ts

@@ -4,7 +4,7 @@ import fse from 'fs-extra';
 import semver from 'semver';
 import { isDynamicPattern, globbySync } from 'globby';
 import {
-    uniq, fastCloneDeep, get, trim
+    uniq, fastCloneDeep, get, trim,
 } from '@terascope/utils';
 import toposort from 'toposort';
 import { MultiMap } from 'mnemonist';
@@ -16,6 +16,11 @@ import {
     writeIfChanged
 } from './misc.js';
 import * as i from './interfaces.js';
+import { ReleaseType } from 'semver';
+import signale from './signale.js';
+import {
+    updateHelmChart, getCurrentHelmChartVersion
+} from '../helpers/scripts.js';
 
 let _packages: i.PackageInfo[] = [];
 let _e2eDir: string | undefined;
@@ -352,3 +357,33 @@ export function getPublishTag(version: string): 'prerelease' | 'latest' {
     if (parsed.prerelease.length) return 'prerelease';
     return 'latest';
 }
+
+/**
+ * Updates the Teraslice Helm chart version based on the specified release type
+ *
+ * @param {'major' | 'minor' | 'patch'} releaseType - The type of version bump for Teraslice.
+ *    - `major`: Bumps the Helm chart by a major version.
+ *    - `minor` or `patch`: Bumps the Helm chart by a minor version.
+ *    - Other values will result in no update.
+ * @returns {Promise<void>} Resolves when the Helm chart version is updated.
+ */
+export async function bumpHelmChart(releaseType: ReleaseType): Promise<void> {
+    const currentChartVersion = await getCurrentHelmChartVersion();
+
+    if (!['major', 'minor', 'patch'].includes(releaseType)) {
+        signale.warn('Teraslice Helm chart won\'t be updated');
+        return;
+    }
+
+    const bumpType = releaseType === 'major' ? 'major' : 'minor';
+    const newVersion = semver.inc(currentChartVersion, bumpType);
+
+    if (!newVersion) {
+        signale.error('Failed to determine new chart version');
+        return;
+    }
+
+    signale.info(`Bumping teraslice-chart from ${currentChartVersion} to ${newVersion}`);
+    await updateHelmChart(newVersion);
+    signale.success(`Successfully bumped teraslice-chart to v${newVersion}`);
+}

packages/scripts/src/helpers/scripts.ts

@@ -5,12 +5,17 @@ import path from 'node:path';
 import { execa, execaCommand, type Options } from 'execa';
 import fse from 'fs-extra';
 import yaml from 'js-yaml';
+import got from 'got';
+import { parseDocument } from 'yaml';
 import {
     debugLogger, isString, get,
     pWhile, pDelay, TSError
 } from '@terascope/utils';
-import { TSCommands, PackageInfo } from './interfaces.js';
-import { getRootDir } from './misc.js';
+import {
+    TSCommands, PackageInfo,
+    OCIImageManifest, OCIimageConfig, OCIindexManifest
+} from './interfaces.js';
+import { getRootDir, getRootInfo } from './misc.js';
 import signale from './signale.js';
 import * as config from './config.js';
 import { getE2EDir, getE2eK8sDir } from '../helpers/packages.js';
@@ -868,3 +873,212 @@ function createValuesStringFromServicesArray() {
     logger.debug('helmfile command values: ', values);
     return values;
 }
+
+/**
+ * Gets the current version of the Teraslice Helm chart from `Chart.yaml`.
+ *
+ * @throws {Error} If the `Chart.yaml` file cannot be read
+ * @returns {Promise<string>} Resolves with the Helm chart version as a string
+ */
+export async function getCurrentHelmChartVersion(): Promise<string> {
+    const chartYamlPath = path.join(getRootDir(), '/helm/teraslice/Chart.yaml');
+    const chartYAML = await yaml.load(fs.readFileSync(chartYamlPath, 'utf8')) as any;
+    return chartYAML.version as string;
+}
+
+function getTerasliceVersion() {
+    const rootPackageInfo = getRootInfo();
+    return rootPackageInfo.version;
+}
+
+/**
+ * Extracts the base Docker image information from the top-level Dockerfile
+ *
+ * This function parses the Dockerfile to determine the base image name,
+ * node version, registry, and repository.
+ *
+ * @throws {TSError} If the Dockerfile cannot be read or the base image format is unexpected.
+ * @returns {{
+*   name: string;
+*   tag: string;
+*   registry: string;
+*   repo: string;
+* }} An object containing:
+*    - `name: Full base image name
+*    - `tag`: Node version used in the image
+*    - `registry`: Docker registry (defaults to `docker.io` if not specified)
+*    - `repo`: Repository name including organization
+*/
+export function getDockerBaseImageInfo() {
+    try {
+        const dockerFilePath = path.join(getRootDir(), 'Dockerfile');
+        const dockerfileContent = fs.readFileSync(dockerFilePath, 'utf8');
+        // Grab the "ARG NODE_VERSION" line in the Dockerfile
+        const nodeVersionDefault = dockerfileContent.match(/^ARG NODE_VERSION=(\d+)/m);
+        // Search "FROM" line that includes "NODE_VERSION" in it
+        const dockerImageName = dockerfileContent.match(/^FROM (.+):\$\{NODE_VERSION\}/m);
+
+        if (nodeVersionDefault && dockerImageName) {
+            const nodeVersion = nodeVersionDefault[1];
+            const baseImage = dockerImageName[1];
+            // Regex to extract registry (if present) and keep the rest as `repo`
+            const imagePattern = /^(?:(.+?)\/)?([^/]+\/[^/]+)$/;
+            const match = baseImage.match(imagePattern);
+
+            if (!match) {
+                throw new TSError(`Unexpected image format: ${baseImage}`);
+            }
+            // Default to Docker Hub if no registry
+            const registry = match[1] || 'docker.io';
+            // Keep org and repo together
+            const repo = match[2];
+            signale.debug(`Base Image: ${baseImage}:${nodeVersion}`);
+            return {
+                name: baseImage,
+                tag: nodeVersion,
+                registry,
+                repo
+            };
+        } else {
+            throw new TSError('Failed to parse Dockerfile for base image.');
+        }
+    } catch (err) {
+        throw new TSError('Failed to read top-level Dockerfile to get base image.', err);
+    }
+}
+
+/**
+ * Retrieves the Node.js version from the base Docker image specified in the Teraslice `Dockerfile`.
+ *
+ * This function:
+ * - Extracts the base image details from the `Dockerfile`
+ * - Authenticates with the container registry to retrieve a token
+ * - Fetches the image manifest and configuration
+ * - Extracts the `node_version` label from the image config
+ *
+ * @throws {TSError} If any request to the registry fails or expected data is missing.
+ * @returns {Promise<string>} Resolves with the Node.js version string.
+ */
+export async function grabCurrentTSNodeVersion(): Promise<string> {
+    // Extract base image details from the Dockerfile
+    const baseImage = getDockerBaseImageInfo();
+    let token: string;
+
+    // Request authentication token for accessing image manifests
+    try {
+        const authUrl = `https://${baseImage.registry}/token?scope=repository:${baseImage.repo}:pull`;
+        const authResponse = await got(authUrl);
+        token = JSON.parse(authResponse.body).token;
+    } catch (err) {
+        throw new TSError(`Unable to retrieve token from ${baseImage.registry} for repo ${baseImage.repo}: `, err);
+    }
+
+    // Grab the manifest list to find the right architecture digest
+    let manifestDigest: string;
+    try {
+        const manifestUrl = `https://${baseImage.registry}/v2/${baseImage.repo}/manifests/${baseImage.tag}`;
+        const response = await got(manifestUrl, {
+            headers: {
+                Authorization: `Bearer ${token}`,
+                Accept: 'application/vnd.oci.image.index.v1+json, application/vnd.docker.distribution.manifest.v2+json'
+            },
+            responseType: 'json'
+        });
+
+        const manifestList = response.body as OCIindexManifest;
+        const amd64Manifest = manifestList.manifests.find(
+            (manifest: OCIImageManifest) => manifest.platform.architecture === 'amd64'
+        );
+
+        if (!amd64Manifest) {
+            throw new TSError(`No amd64 manifest found for ${baseImage.repo}:${baseImage.tag}`);
+        }
+
+        manifestDigest = amd64Manifest.digest;
+    } catch (err) {
+        throw new TSError(`Unable to retrieve image manifest list from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}: `, err);
+    }
+
+    // Get the specific manifest using the digest
+    let configBlobSha: string;
+    try {
+        const manifestDetailUrl = `https://${baseImage.registry}/v2/${baseImage.repo}/manifests/${manifestDigest}`;
+        const response = await got(manifestDetailUrl, {
+            headers: {
+                Authorization: `Bearer ${token}`,
+                Accept: 'application/vnd.oci.image.manifest.v1+json'
+            },
+            responseType: 'json'
+        });
+
+        const amd64Manifest = response.body as OCIindexManifest;
+        if (!amd64Manifest.config?.digest) {
+            throw new TSError(`Manifest does not contain a config digest for ${baseImage.repo}:${baseImage.tag}`);
+        }
+
+        configBlobSha = amd64Manifest.config.digest;
+    } catch (err) {
+        throw new TSError(`Unable to get manifest details from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}: `, err);
+    }
+
+    // Retrieve the image configuration and extract the Node.js version label
+    try {
+        const configUrl = `https://${baseImage.registry}/v2/${baseImage.repo}/blobs/${configBlobSha}`;
+        const response = await got(configUrl, {
+            headers: {
+                Authorization: `Bearer ${token}`,
+                Accept: 'application/vnd.oci.image.config.v1+json'
+            },
+            responseType: 'json'
+        });
+
+        const imageConfig = response.body as OCIimageConfig;
+        const nodeVersion = imageConfig.config?.Labels['io.terascope.image.node_version'];
+
+        if (!nodeVersion) {
+            throw new TSError(`Node version label missing in config for ${baseImage.repo}:${baseImage.tag}`);
+        }
+
+        return nodeVersion;
+    } catch (err) {
+        throw new TSError(`Unable to grab image config from ${baseImage.registry} for ${baseImage.repo}:${baseImage.tag}: `, err);
+    }
+}
+
+/**
+ * Updates the Teraslice Helm chart YAML files (`Chart.yaml` and `values.yaml`)
+ * with the new chart version
+ *
+ * @param {string | null} newChartVersion - The new version to set in `Chart.yaml`.
+ *    - If `null`, the function does not update the chart version.
+ * @throws {TSError} If the function fails to read or write YAML files.
+ * @returns {Promise<void>} Resolves when the Helm chart files have been successfully updated
+ */
+export async function updateHelmChart(newChartVersion: string | null): Promise<void> {
+    const currentNodeVersion = await grabCurrentTSNodeVersion();
+    const rootDir = getRootDir();
+    const chartYamlPath = path.join(rootDir, 'helm/teraslice/Chart.yaml');
+    const valuesYamlPath = path.join(rootDir, 'helm/teraslice/values.yaml');
+
+    try {
+        // Read YAML files and parse them into objects
+        const chartFileContent = fs.readFileSync(chartYamlPath, 'utf8');
+        const valuesFileContent = fs.readFileSync(valuesYamlPath, 'utf8');
+
+        const chartDoc = parseDocument(chartFileContent);
+        const valuesDoc = parseDocument(valuesFileContent);
+
+        // Update specific values for the chart
+        if (newChartVersion) {
+            chartDoc.set('version', newChartVersion);
+        }
+        chartDoc.set('appVersion', `v${getTerasliceVersion()}`);
+        valuesDoc.setIn(['image', 'nodeVersion'], `v${currentNodeVersion}`);
+
+        // Write the updated YAML back to the files
+        fs.writeFileSync(chartYamlPath, chartDoc.toString(), 'utf8');
+        fs.writeFileSync(valuesYamlPath, valuesDoc.toString(), 'utf8');
+    } catch (err) {
+        throw new TSError('Unable to read or write Helm chart YAML files', err);
+    }
+}