Merge branch 'main' into feat/helia-ipns

* main: (27 commits)
  feat(e2e): add /api/v0/repo/gc test
  chore: disable METRICS on CI e2e test runs
  test: add e2e test for /api/v0/version endpoint
  feat(server): ⚡️ Subdomain Gateway Using Fastify (#31)
  feat: use production level docker settings (#26)
  chore: remove unused playwright init code
  fix: use active LTS in package.json engines
  fix: playwright CI node-version=20
  test: get clinic flame & doctor output from e2e tests
  test: e2e updates
  fix: use HOST constant in healthcheck
  fix: use HOST constant
  test: cleanup playwright test code
  test: add playwright tests
  feat: move HOST,PORT to src/constants.ts
  fix: ✏️  Fixing urls. (#23)
  fix: ✏️ helia-docker -> helia-http-gateway (#22)
  build(deps): Bump @babel/traverse and depcheck (#13)
  feat: add health-check (#21)
  fix(server): 🩹 Using sessionID as a fallback to requests where referer is missing. (#20)
  ...

Signed-off-by: Nishant Arora <[email protected]>

.dockerignore

@@ -129,3 +129,20 @@ build
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+
+# other dotfiles
+.git
+.gitignore
+.dockerignore
+.github
+.tool-versions
+.envrc
+
+# playwright stuff
+e2e-tests
+playwright.config.ts
+playwright-report
+screenshots
+
+# Other misc files that don't make sense in the docker container
+README.md

.github/workflows/docker.yml

@@ -26,6 +26,8 @@ jobs:
 
       - name: Set up Docker Buildx
         uses: docker/setup-buildx-action@v3
+        with:
+          platforms: linux/amd64,linux/arm64
 
       - name: Docker meta
         id: meta
@@ -38,6 +40,7 @@ jobs:
         with:
           context: .
           file: ./Dockerfile
+          provenance: false
           push: ${{ github.event_name == 'push' || github.event.inputs.push == 'true' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}

.github/workflows/playwright.yml

@@ -0,0 +1,31 @@
+name: Playwright Tests
+on:
+  push:
+    branches: [ main ]
+  pull_request:
+    branches: [ main ]
+  workflow_dispatch:
+
+jobs:
+  test:
+    timeout-minutes: 60
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/checkout@v3
+    - uses: actions/setup-node@v3
+      with:
+        node-version: 20
+    - name: Install dependencies
+      run: npm ci
+    - name: Install Playwright Browsers
+      run: npx playwright install --with-deps
+    - name: Run Playwright tests
+      run: npm run test:e2e
+      env:
+        METRICS: false
+    - uses: actions/upload-artifact@v3
+      if: always()
+      with:
+        name: playwright-report
+        path: playwright-report/
+        retention-days: 30

.gitignore

@@ -129,3 +129,8 @@ build
 .yarn/build-state.yml
 .yarn/install-state.gz
 .pnp.*
+/test-results/
+/playwright-report/
+/playwright/.cache/
+screenshots
+.envrc

Dockerfile

@@ -1,11 +1,40 @@
-FROM node:20-slim
+FROM node:20-slim as builder
 
 RUN apt-get update
-RUN apt-get install -y build-essential cmake git libssl-dev
+RUN apt-get install -y build-essential cmake git libssl-dev tini
 
 WORKDIR /app
 
-COPY . .
+COPY package*.json ./
+
 RUN npm ci --quiet
+
+COPY . .
+
 RUN npm run build
-CMD [ "npm", "start" ]
+
+RUN npm prune --omit=dev
+
+FROM node:20-slim as app
+ENV NODE_ENV production
+WORKDIR /app
+# built src without dev dependencies
+COPY --from=builder /app ./
+# tini is used to handle signals properly, see https://github.com/krallin/tini#using-tini
+COPY --from=builder /usr/bin/tini /usr/bin/tini
+
+# copy shared libraries (without having artifacts from apt-get install that is needed to build our application)
+COPY --from=builder /usr/lib/**/libcrypto* /usr/lib/
+COPY --from=builder /usr/lib/**/libssl* /usr/lib/
+
+HEALTHCHECK --interval=12s --timeout=12s --start-period=10s CMD npm run healthcheck
+
+# Use tini to handle signals properly, see https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md#handling-kernel-signals
+ENTRYPOINT ["/usr/bin/tini", "-p", "SIGKILL", "--"]
+
+CMD [ "node", "dist/src/index.js" ]
+
+# for best practices, see:
+# * https://snyk.io/blog/10-best-practices-to-containerize-nodejs-web-applications-with-docker/
+# * https://github.com/nodejs/docker-node/blob/main/docs/BestPractices.md
+# * https://nodejs.org/en/docs/guides/nodejs-docker-webapp

README.md

@@ -1,4 +1,4 @@
-# helia-docker
+# helia-http-gateway
 
 Docker images for Helia.
 
@@ -28,7 +28,7 @@ $ docker build . --tag helia --platform linux/arm64
 ### Running
 
 ```sh
-$ docker run -it -p 8080:8080 -e DEBUG="helia-server" helia
+$ docker run -it -p 8080:8080 -e DEBUG="helia-http-gateway" helia
 ```
 
 ## Supported Environment Variables
@@ -38,6 +38,80 @@ $ docker run -it -p 8080:8080 -e DEBUG="helia-server" helia
 | `DEBUG` | Debug level | `''`|
 | `PORT` | Port to listen on | `8080` |
 | `HOST` | Host to listen on | `0.0.0.0` |
+| `METRICS` | Whether to enable prometheus metrics. Any value other than 'true' will disable metrics. | `true` |
+| `USE_BITSWAP` | Use bitswap to fetch content from IPFS | `true` |
+| `USE_TRUSTLESS_GATEWAYS` | Whether to fetch content from trustless-gateways or not | `true` |
+| `TRUSTLESS_GATEWAYS` | Comma separated list of trusted gateways to fetch content from | [Defined in Helia](https://github.com/ipfs/helia/blob/main/packages/helia/src/block-brokers/trustless-gateway/index.ts) |
+| `USE_LIBP2P` | Whether to use libp2p networking | `true` |
+
+<!--
+TODO: currently broken when used in docker, but they work when running locally (you can cache datastore and blockstore locally to speed things up if you want)
+| `FILE_DATASTORE_PATH` | Path to use with a datastore-level passed to Helia as the datastore | `null`; memory datastore is used by default. |
+| `FILE_BLOCKSTORE_PATH` | Path to use with a blockstore-level passed to Helia as the blockstore | `null`; memory blockstore is used by default. |
+-->
+
+See the source of truth for all `process.env.<name>` environment variables at [src/constants.ts](src/constants.ts).
+
+### Running with custom configurations
+
+Note that any of the following calls to docker can be replaced with something like `MY_ENV_VAR="MY_VALUE" npm run start`
+
+#### Disable libp2p
+```sh
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_LIBP2P="false" helia
+```
+
+#### Disable bitswap
+```sh
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_BITSWAP="false" helia
+```
+
+#### Disable trustless gateways
+```sh
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e USE_TRUSTLESS_GATEWAYS="false" helia
+```
+
+#### Customize trustless gateways
+```sh
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e TRUSTLESS_GATEWAYS="https://ipfs.io,https://dweb.link" helia
+```
+
+<!--
+#### With file datastore and blockstore
+
+**NOTE:** Not currently supported due to docker volume? issues.
+
+```sh
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e FILE_DATASTORE_PATH="./datastore" -e FILE_BLOCKSTORE_PATH="./blockstore" helia
+# and if you want to re-use a volume from your host:
+$ docker run -it -p $PORT:8080 -e DEBUG="helia-http-gateway*" -e FILE_DATASTORE_PATH="./datastore" -e FILE_BLOCKSTORE_PATH="./blockstore" -v ./datastore:/datastore -v ./blockstore:/blockstore helia
+```
+-->
+
+## E2E Testing
+
+We have some tests enabled that simulate running inside of [ProbeLab's Tiros](https://github.com/plprobelab/tiros), via playwright. These tests request the same paths from ipfs.io and ensure that the resulting text matches. This is not a direct replacement for [gateway conformance testing](https://github.com/ipfs/gateway-conformance), but helps us ensure the helia-http-gateway is working as expected.
+
+By default, these tests:
+
+1. Run in serial
+2. Allow for up to 5 failures before failing the whole suite run.
+3. Have an individual test timeout of two minutes.
+
+### Run e2e tests locally
+
+```sh
+$ npm run test:e2e # run all tests
+$ npm run test:e2e -- ${PLAYWRIGHT_OPTIONS} # run tests with custom playwright options.
+
+```
+
+### Get clinicjs flamecharts and doctor reports from e2e tests
+
+```sh
+$ npm run test:e2e-doctor # Run the dev server with clinicjs doctor, execute e2e tests, and generate a report.
+$ npm run test:e2e-flame # Run the dev server with clinicjs flame, execute e2e tests, and generate a report.
+```
 
 ## Author
 

docker-compose.yml

@@ -1,10 +1,10 @@
 version: "3.8"
 
 services:
-  server:
+  helia-http-gateway:
     build: .
     restart: always
     ports:
-      - "8080:8080"
+      - "${PORT:-8080}:8080"
     environment:
-      - DEBUG="helia-server*"
+      - DEBUG="${DEBUG:-helia-http-gateway*}"

e2e-tests/compare-to-ipfs-io.spec.ts

@@ -0,0 +1,48 @@
+import { test, expect } from '@playwright/test'
+import { PORT } from '../src/constants.js'
+
+// test all the same pages listed at https://probelab.io/websites/
+const pages = [
+  // '/ipns/blog.ipfs.tech', // currently timing out for Helia.
+  '/ipns/blog.libp2p.io',
+  '/ipns/consensuslab.world',
+  '/ipns/docs.ipfs.tech',
+  '/ipns/docs.libp2p.io',
+  '/ipns/drand.love',
+  // '/ipns/fil.org', // currently timing out for Helia.
+  // '/ipns/filecoin.io', // currently timing out for Helia.
+  '/ipns/green.filecoin.io',
+  // '/ipns/ipfs.tech', // currently timing out for Helia.
+  '/ipns/ipld.io',
+  '/ipns/libp2p.io',
+  // '/ipns/n0.computer', // currently timing out for Helia.
+  '/ipns/probelab.io',
+  '/ipns/protocol.ai', // very slow, but can pass.
+  '/ipns/research.protocol.ai', // slow-ish, but can pass.
+  '/ipns/singularity.storage',
+  '/ipns/specs.ipfs.tech',
+  '/ipns/strn.network'
+  // '/ipns/web3.storage' // currently timing out for Helia
+]
+
+// increase default test timeout to 2 minutes
+test.setTimeout(120000)
+
+// now for each page, make sure we can request the website, the content is not empty, and status code is 200
+pages.forEach((pagePath) => {
+  // afterEach, we should request /api/v0/repo/gc to clear the cache
+  test(`helia-http-gateway matches ipfs.io for path '${pagePath}'`, async ({ page }) => {
+    const ipfsIoResponse = await page.goto(`http://ipfs.io${pagePath}`)
+    expect(ipfsIoResponse?.status()).toBe(200)
+    const ipfsIoContent = await ipfsIoResponse?.text()
+    // TODO: enable screenshot testing? maybe not needed if we're confirming text content matches.
+    // await page.screenshot({ path: `screenshots${pagePath}.png`, fullPage: true });
+    const heliaGatewayResponse = await page.goto(`http://localhost:${PORT}${pagePath}`)
+    expect(heliaGatewayResponse?.status()).toBe(200)
+    // expect(page).toHaveScreenshot(`screenshots${pagePath}.png`, { fullPage: true, maxDiffPixelRatio: 0 });
+
+    // expect the response text content to be the same
+    const heliaGatewayContent = await heliaGatewayResponse?.text()
+    expect(heliaGatewayContent).toEqual(ipfsIoContent)
+  })
+})

e2e-tests/gc.spec.ts

@@ -0,0 +1,10 @@
+import { test, expect } from '@playwright/test'
+import { PORT } from '../src/constants.js'
+
+test('POST /api/v0/repo/gc', async ({ page }) => {
+  const result = await page.request.post(`http://localhost:${PORT}/api/v0/repo/gc`)
+  expect(result?.status()).toBe(200)
+
+  const maybeContent = await result?.text()
+  expect(maybeContent).toEqual('OK')
+})

e2e-tests/version-response.spec.ts

@@ -0,0 +1,28 @@
+import { test, expect } from '@playwright/test'
+import { PORT } from '../src/constants.js'
+
+function validateResponse (content: string): void {
+  expect(() => JSON.parse(content)).not.toThrow()
+  const versionObj = JSON.parse(content)
+
+  expect(versionObj).toHaveProperty('Version')
+  expect(versionObj).toHaveProperty('Commit')
+}
+
+test('GET /api/v0/version', async ({ page }) => {
+  const result = await page.goto(`http://localhost:${PORT}/api/v0/version`)
+  expect(result?.status()).toBe(200)
+
+  const maybeContent = await result?.text()
+  expect(maybeContent).not.toBe(undefined)
+  validateResponse(maybeContent as string)
+})
+
+test('POST /api/v0/version', async ({ page }) => {
+  const result = await page.request.post(`http://localhost:${PORT}/api/v0/version`)
+  expect(result?.status()).toBe(200)
+
+  const maybeContent = await result?.text()
+  expect(maybeContent).not.toBe(undefined)
+  validateResponse(maybeContent)
+})