Add content hashes to release (#441)

* Add content hashes to release

* Add prefix to release files in multi variants

* Only match needed files

* Upload the avatar first

* Create upload avatar

* Upload content-hash too

* Use isMultivariants from cli args

src/commands/publish/handler.ts

@@ -63,6 +63,8 @@ export async function publishHandler({
 
   const variantsDirPath = path.join(dir, variantsDirName);
 
+  const isMultiVariant = !!allVariants || !!variants;
+
   const publishTasks = new Listr(
     publish({
       releaseType,
@@ -84,7 +86,8 @@ export async function publishHandler({
         rootDir: dir,
         variantsDirPath,
         composeFileName
-      })
+      }),
+      isMultiVariant
     }),
     verbosityOptions
   );

src/files/manifest/compactManifestIfCore.ts

@@ -16,26 +16,26 @@ import { writeManifest } from "./writeManifest.js";
  * @param buildDir `build_0.1.0`
  */
 export function compactManifestIfCore(buildDir: string): void {
-  const { manifest, format } = readManifest([{ dir: buildDir }]);
+    const { manifest, format } = readManifest([{ dir: buildDir }]);
 
-  if (manifest.type !== "dncore") return;
+    if (manifest.type !== "dncore") return;
 
-  const setupWizard = readSetupWizardIfExists(buildDir);
-  if (setupWizard) {
-    manifest.setupWizard = setupWizard;
-  }
+    const setupWizard = readSetupWizardIfExists(buildDir);
+    if (setupWizard) {
+        manifest.setupWizard = setupWizard;
+    }
 
-  writeManifest(manifest, format, { dir: buildDir });
+    writeManifest(manifest, format, { dir: buildDir });
 }
 
 // Utils
 
 function readSetupWizardIfExists(buildDir: string): SetupWizard | null {
-  const files = fs.readdirSync(buildDir);
-  const setupWizardFile = files.find(file =>
-    releaseFiles.setupWizard.regex.test(file)
-  );
-  if (!setupWizardFile) return null;
-  const setupWizardPath = path.join(buildDir, setupWizardFile);
-  return yaml.load(fs.readFileSync(setupWizardPath, "utf8"));
-}
+    const files = fs.readdirSync(buildDir);
+    const setupWizardFile = files.find(file =>
+        releaseFiles.setupWizard.regex.test(file)
+    );
+    if (!setupWizardFile) return null;
+    const setupWizardPath = path.join(buildDir, setupWizardFile);
+    return yaml.load(fs.readFileSync(setupWizardPath, "utf8"));
+}

src/providers/github/Github.ts

@@ -1,4 +1,7 @@
 import fs from "fs";
+import path from "path";
+import retry from "async-retry";
+import mime from "mime-types";
 import { Octokit } from "@octokit/rest";
 import { RequestError } from "@octokit/request-error";
 import { getRepoSlugFromManifest } from "../../files/index.js";
@@ -185,7 +188,7 @@ export class Github {
   }
 
   /**
-   * Create a Github release
+   * Create a Github release and return the release id
    * @param tag "v0.2.0"
    * @param options
    */
@@ -195,9 +198,9 @@ export class Github {
       body?: string;
       prerelease?: boolean;
     }
-  ): Promise<void> {
+  ): Promise<number> {
     const { body, prerelease } = options || {};
-    await this.octokit.rest.repos
+    const release = await this.octokit.rest.repos
       .createRelease({
         owner: this.owner,
         repo: this.repo,
@@ -214,6 +217,50 @@ export class Github {
         e.message = `Error creating release: ${e.message}`;
         throw e;
       });
+
+    return release.data.id;
+  }
+
+  async uploadReleaseAssets({
+    releaseId,
+    assetsDir,
+    matchPattern,
+    fileNamePrefix
+  }: {
+    releaseId: number;
+    assetsDir: string;
+    matchPattern?: RegExp;
+    fileNamePrefix?: string;
+  }) {
+    for (const file of fs.readdirSync(assetsDir)) {
+      // Used to ignore duplicated legacy .tar.xz image
+      if (matchPattern && !matchPattern.test(file)) continue;
+
+      const filepath = path.resolve(assetsDir, file);
+      const contentType = mime.lookup(filepath) || "application/octet-stream";
+      try {
+        // The uploadReleaseAssetApi fails sometimes, retry 3 times
+        await retry(
+          async () => {
+            await this.octokit.repos.uploadReleaseAsset({
+              owner: this.owner,
+              repo: this.repo,
+              release_id: releaseId,
+              data: fs.createReadStream(filepath) as any,
+              headers: {
+                "content-type": contentType,
+                "content-length": fs.statSync(filepath).size
+              },
+              name: `${fileNamePrefix || ""}${path.basename(filepath)}`
+            });
+          },
+          { retries: 3 }
+        );
+      } catch (e) {
+        e.message = `Error uploading release asset: ${e.message}`;
+        throw e;
+      }
+    }
   }
 
   /**

src/tasks/createGithubRelease/index.ts

@@ -12,9 +12,11 @@ import { getCreateReleaseTask } from "./subtasks/getCreateReleaseTask.js";
 export function createGithubRelease({
   dir: rootDir = defaultDir,
   compose_file_name: composeFileName,
-  verbosityOptions
+  verbosityOptions,
+  isMultiVariant
 }: {
   verbosityOptions: VerbosityOptions;
+  isMultiVariant: boolean;
 } & CliGlobalOptions): Listr<ListrContextPublish> {
   // OAuth2 token from Github
   if (!process.env.GITHUB_TOKEN)
@@ -27,7 +29,8 @@ export function createGithubRelease({
       getHandleTagsTask({ github }),
       getCreateReleaseTask({
         github,
-        composeFileName
+        composeFileName,
+        isMultiVariant
       })
     ],
     verbosityOptions

src/tasks/createGithubRelease/subtasks/getCreateReleaseTask.ts

@@ -3,10 +3,6 @@ import fs from "fs";
 import { Github } from "../../../providers/github/Github.js";
 import { ListrContextPublish, TxData } from "../../../types.js";
 import { ListrTask } from "listr";
-import {
-  compactManifestIfCore,
-  composeDeleteBuildProperties
-} from "../../../files/index.js";
 import {
   getInstallDnpLink,
   getPublishTxLink
@@ -15,6 +11,7 @@ import { getNextGitTag } from "../getNextGitTag.js";
 import { contentHashFileName } from "../../../params.js";
 import { ReleaseDetailsMap } from "../types.js";
 import { buildReleaseDetailsMap } from "../buildReleaseDetailsMap.js";
+import { compactManifestIfCore, composeDeleteBuildProperties } from "../../../files/index.js";
 
 /**
  * Create release
@@ -23,10 +20,12 @@ import { buildReleaseDetailsMap } from "../buildReleaseDetailsMap.js";
  */
 export function getCreateReleaseTask({
   github,
-  composeFileName
+  composeFileName,
+  isMultiVariant
 }: {
   github: Github;
   composeFileName?: string;
+  isMultiVariant: boolean;
 }): ListrTask<ListrContextPublish> {
   return {
     title: `Create release`,
@@ -38,74 +37,102 @@ export function getCreateReleaseTask({
       task.output = "Deleting existing release...";
       await github.deleteReleaseAndAssets(tag);
 
-      const contentHashPaths = await handleReleaseVariantFiles({
-        releaseDetailsMap,
-        composeFileName
-      });
-
       task.output = `Creating release for tag ${tag}...`;
-      await github.createRelease(tag, {
+      const releaseId = await github.createRelease(tag, {
         body: await getReleaseBody({ releaseDetailsMap }),
         prerelease: true, // Until it is actually published to mainnet
       });
 
-      // Clean content hash file so the directory uploaded to IPFS is the same
-      // as the local build_* dir. User can then `ipfs add -r` and get the same hash
-      contentHashPaths.map(contentHashPath => fs.unlinkSync(contentHashPath));
+      task.output = "Preparing release directories for Github release...";
+      prepareGithubReleaseFiles({ releaseDetailsMap, composeFileName });
+
+      task.output = "Uploading assets...";
+      await uploadAssets({ releaseDetailsMap, github, releaseId, isMultiVariant });
+
     }
   };
 }
 
-async function handleReleaseVariantFiles({
+function prepareGithubReleaseFiles({
   releaseDetailsMap,
   composeFileName
 }: {
   releaseDetailsMap: ReleaseDetailsMap;
   composeFileName?: string;
-}): Promise<string[]> {
-  const contentHashPaths: string[] = [];
-
-  for (const [, { variant, releaseDir, releaseMultiHash }] of Object.entries(
-    releaseDetailsMap
-  )) {
-    if (!releaseMultiHash) {
-      throw new Error(
-        `Release hash not found for variant ${variant} of ${name}`
-      );
-    }
+}) {
+  for (const [, { releaseMultiHash, releaseDir }] of Object.entries(releaseDetailsMap)) {
 
-    const contentHashPath = writeContentHashToFile({
-      releaseDir,
-      releaseMultiHash
-    });
+    const contentHashPath = path.join(releaseDir, `${contentHashFileName}`);
+
+    try {
+
+      /**
+       * Plain text file which should contain the IPFS hash of the release
+       * Necessary for the installer script to fetch the latest content hash
+       * of the eth clients. The resulting hashes are used by the DAPPMANAGER
+       * to install an eth client when the user does not want to use a remote node
+       */
+      fs.writeFileSync(contentHashPath, releaseMultiHash);
 
-    contentHashPaths.push(contentHashPath);
+      compactManifestIfCore(releaseDir);
+      composeDeleteBuildProperties({ dir: releaseDir, composeFileName });
 
-    compactManifestIfCore(releaseDir);
-    composeDeleteBuildProperties({ dir: releaseDir, composeFileName });
+    } catch (e) {
+      console.error(`Error found while preparing files in ${releaseDir} for Github release`, e);
+    }
   }
+}
 
-  return contentHashPaths;
+async function uploadAssets({
+  releaseDetailsMap,
+  github,
+  releaseId,
+  isMultiVariant
+}: {
+  releaseDetailsMap: ReleaseDetailsMap;
+  github: Github;
+  releaseId: number;
+  isMultiVariant: boolean;
+}) {
+  const releaseEntries = Object.entries(releaseDetailsMap);
+  const [, { releaseDir: firstReleaseDir }] = releaseEntries[0];
+
+  await uploadAvatar({ github, releaseId, avatarDir: firstReleaseDir });
+
+  for (const [dnpName, { releaseDir }] of releaseEntries) {
+    const shortDnpName = dnpName.split(".")[0];
+
+    await github.uploadReleaseAssets({
+      releaseId,
+      assetsDir: releaseDir,
+      // Only upload yml, txz and dappnode_package.json files
+      matchPattern: /(.*\.ya?ml$)|(.*\.txz$)|(dappnode_package\.json)|(content-hash)/,
+      fileNamePrefix: isMultiVariant ? `${shortDnpName}_` : ""
+    }).catch((e) => {
+      console.error(`Error uploading assets from ${releaseDir}`, e);
+    });
+  }
 }
 
-/**
- * Plain text file which should contain the IPFS hash of the release
- * Necessary for the installer script to fetch the latest content hash
- * of the eth clients. The resulting hashes are used by the DAPPMANAGER
- * to install an eth client when the user does not want to use a remote node
- */
-function writeContentHashToFile({
-  releaseDir,
-  releaseMultiHash
+async function uploadAvatar({
+  github,
+  releaseId,
+  avatarDir
 }: {
-  releaseDir: string;
-  releaseMultiHash: string;
-}): string {
-  const contentHashPath = path.join(releaseDir, contentHashFileName);
-  fs.writeFileSync(contentHashPath, releaseMultiHash);
-  return contentHashPath;
+  github: Github;
+  releaseId: number;
+  avatarDir: string;
+}): Promise<void> {
+  await github.uploadReleaseAssets({
+    releaseId,
+    assetsDir: avatarDir,
+    matchPattern: /.*\.png/,
+  }).catch((e) => {
+    console.error(`Error uploading avatar from ${avatarDir}`, e);
+  });
 }
 
+
 /**
  * Write the release body
  *

src/tasks/publish/index.ts

@@ -23,6 +23,7 @@ export function publish({
   verbosityOptions,
   variantsDirPath,
   packagesToBuildProps,
+  isMultiVariant
 }: PublishOptions): ListrTask<ListrContextPublish>[] {
   return [
     getVerifyEthConnectionTask({ ethProvider }),
@@ -63,7 +64,8 @@ export function publish({
       dir,
       githubRelease: Boolean(githubRelease),
       verbosityOptions,
-      composeFileName
+      composeFileName,
+      isMultiVariant
     })
   ];
 }

src/tasks/publish/subtasks/getCreateGithubReleaseTask.ts

@@ -7,12 +7,14 @@ export function getCreateGithubReleaseTask({
   githubRelease,
   dir,
   composeFileName,
-  verbosityOptions
+  verbosityOptions,
+  isMultiVariant
 }: {
   githubRelease: boolean;
   dir: string;
   composeFileName: string;
   verbosityOptions: VerbosityOptions;
+  isMultiVariant: boolean;
 }): ListrTask<ListrContextPublish> {
   return {
     title: "Release on github",
@@ -21,7 +23,8 @@ export function getCreateGithubReleaseTask({
       createGithubRelease({
         dir,
         compose_file_name: composeFileName,
-        verbosityOptions
+        verbosityOptions,
+        isMultiVariant
       })
   };
 }

src/tasks/publish/types.ts

@@ -17,4 +17,5 @@ export interface PublishOptions {
   verbosityOptions: VerbosityOptions;
   variantsDirPath: string;
   packagesToBuildProps: PackageToBuildProps[];
+  isMultiVariant: boolean;
 }