reorganize workspaces into packages and apps (#128)

* reorganize workspaces into packages and apps

* remove phone-number-privacy namespsace from docker files directory

* use proper yarn berry caching

* save dist folder aka output of build ts for apps

---------

Co-authored-by: Aaron <[email protected]>

.changeset/tidy-cups-kneel.md

@@ -0,0 +1,2 @@
+---
+---

.github/actions/sync-workspace/action.yml

@@ -17,9 +17,12 @@ runs:
         REBUILD_PACKAGE: ${{ inputs.rebuild-package }}
       with:
         path: |
+          ./.yarn/cache
           node_modules
+          app/**/node_modules
           packages/**/node_modules
-        key: node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-${{ hashFiles('**/yarn.lock') }}-${{ inputs.package-json-checksum }}
+          examples/docs/node_modules
+        key: node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-${{ hashFiles('yarn.lock') }}-${{ inputs.package-json-checksum }}
         enableCrossOsArchive: true
         fail-on-cache-miss: "!${{ fromJSON(env.REBUILD_PACKAGE) }}"
     - name: Restore build artifacts cache
@@ -33,7 +36,7 @@ runs:
         path: |
           packages/**/lib
           packages/**/dist
-          packages/**/build
+          apps/**/dist
         key: code-${{ github.sha }}
         restore-keys: |
           code-${{ github.sha }}

.github/workflows/odis-combiner-container.yml

@@ -4,14 +4,14 @@ name: Build ODIS combiner image
 on:
   push:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-combiner'
-      - 'packages/phone-number-privacy/combiner/**'
+      - 'dockerfiles/Dockerfile-combiner'
+      - 'apps/combiner/**'
     branches:
       - main
   pull_request:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-combiner'
-      - 'packages/phone-number-privacy/combiner/**'
+      - 'dockerfiles/Dockerfile-combiner'
+      - 'apps/combiner/**'
   workflow_dispatch:
 
 jobs:
@@ -26,7 +26,7 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/odis-combiner
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-combiner
+      file: dockerfiles/Dockerfile-combiner
       trivy: true
 
   odis-combiner-build:
@@ -40,5 +40,5 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/social-connect/odis-combiner
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-combiner
+      file: dockerfiles/Dockerfile-combiner
       trivy: true

.github/workflows/odis-loadtest-container.yml

@@ -4,14 +4,14 @@ name: Build ODIS loadtest image
 on:
   push:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-loadtest'
-      - 'packages/phone-number-privacy/monitor/**'
+      - 'dockerfiles/Dockerfile-loadtest'
+      - 'apps/monitor/**'
     branches:
       - main
   pull_request:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-loadtest'
-      - 'packages/phone-number-privacy/monitor/**'
+      - 'dockerfiles/Dockerfile-loadtest'
+      - 'apps/monitor/**'
   workflow_dispatch:
 
 jobs:
@@ -26,7 +26,7 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/odis-loadtest
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-loadtest
+      file: dockerfiles/Dockerfile-loadtest
       trivy: true
 
   odis-loadtest-build:
@@ -40,5 +40,5 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/social-connect/odis-loadtest
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-loadtest
+      file: dockerfiles/Dockerfile-loadtest
       trivy: true

.github/workflows/odis-signer-container.yml

@@ -4,14 +4,14 @@ name: Build ODIS signer image
 on:
   push:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-signer'
-      - 'packages/phone-number-privacy/signer/**'
+      - 'dockerfiles/Dockerfile-signer'
+      - 'apps/signer/**'
     branches:
       - main
   pull_request:
     paths:
-      - 'dockerfiles/phone-number-privacy/Dockerfile-signer'
-      - 'packages/phone-number-privacy/signer/**'
+      - 'dockerfiles/Dockerfile-signer'
+      - 'apps/signer/**'
   workflow_dispatch:
 
 jobs:
@@ -26,7 +26,7 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/dev-images/odis-signer
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-signer
+      file: dockerfiles/Dockerfile-signer
       trivy: true
 
   odis-signer-build:
@@ -40,5 +40,5 @@ jobs:
       artifact-registry: us-west1-docker.pkg.dev/devopsre/social-connect/odis-signer
       tag: ${{ github.sha }}
       context: .
-      file: dockerfiles/phone-number-privacy/Dockerfile-signer
+      file: dockerfiles/Dockerfile-signer
       trivy: true

.github/workflows/social-connect.yml

@@ -85,9 +85,12 @@ jobs:
           # We need to cache all the artifacts generated by yarn install+build
           # Update this list also in .github/actions/sync-workspace/action.yml with exactly the same list
           path: |
+            ./.yarn/cache
             node_modules
+            app/**/node_modules
             packages/**/node_modules
-          key: node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-${{ hashFiles('**/yarn.lock') }}-${{ steps.node-checksums.outputs.PACKAGE_JSON_CHECKSUM }}
+            examples/docs/node_modules
+          key: node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-${{ hashFiles('yarn.lock') }}-${{ steps.node-checksums.outputs.PACKAGE_JSON_CHECKSUM }}
           restore-keys: |
             node-${{ runner.os }}-${{ runner.arch }}-${{ env.NODE_MODULE_CACHE_VERSION }}-
       # We use cache to share the build artifacts between jobs (gh artifacts are too slow...)
@@ -101,15 +104,14 @@ jobs:
           path: |
             packages/**/lib
             packages/**/dist
-            packages/**/build
+            apps/**/dist
           key: code-${{ github.sha }}
           restore-keys: |
             code-${{ github.sha }}
       - name: "enable corepack"
         run : sudo corepack enable yarn
       - name: Install yarn dependencies
         run: git config --global url."https://".insteadOf ssh:// && yarn install
-        if: steps.cache_node.outputs.cache-hit == 'true'
       - name: Fail if any file changed on git
         run: |
           # This fails if there is any change
@@ -201,10 +203,10 @@ jobs:
           yarn --cwd=packages/odis-identifiers test
       - name: Run Encrypted Backup tests
         run: |
-          yarn --cwd=packages/sdk/encrypted-backup test
+          yarn --cwd=packages/encrypted-backup test
       - name: Run Identity Tests
         run: |
-          yarn --cwd=packages/sdk/identity test
+          yarn --cwd=packages/identity test
 
       - name: Upload Jest Test Results
         uses: actions/upload-artifact@v3
@@ -220,7 +222,7 @@ jobs:
     needs: install-dependencies
     if: |
       github.base_ref == 'main' || contains(github.base_ref, 'staging') || contains(github.base_ref, 'production') ||
-      contains(needs.install-dependencies.outputs.all_modified_files, 'packages/phone-number-privacy/combiner') ||
+      contains(needs.install-dependencies.outputs.all_modified_files, 'apps/combiner') ||
       contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') ||
       contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') ||
       false
@@ -237,7 +239,7 @@ jobs:
           package-json-checksum: ${{ needs.install-dependencies.outputs.package-json-checksum }}
       - name: Run Tests for combiner
         run: |
-          yarn --cwd=packages/phone-number-privacy/combiner test:coverage
+          yarn --cwd=apps/combiner test:coverage
   odis-test:
     name: ODIS (signer + common) test
     needs: install-dependencies
@@ -247,7 +249,8 @@ jobs:
     timeout-minutes: 30
     if: |
       github.base_ref == 'main' || contains(github.base_ref, 'staging') || contains(github.base_ref, 'production') ||
-      contains(needs.install-dependencies.outputs.all_modified_files, 'packages/phone-number-privacy') ||
+      contains(needs.install-dependencies.outputs.all_modified_files, 'packages') ||
+      contains(needs.install-dependencies.outputs.all_modified_files, 'apps/signer') ||
       contains(needs.install-dependencies.outputs.all_modified_files, ',package.json') ||
       contains(needs.install-dependencies.outputs.all_modified_files, ',yarn.lock') ||
       false
@@ -264,7 +267,7 @@ jobs:
           package-json-checksum: ${{ needs.install-dependencies.outputs.package-json-checksum }}
       - name: Run Tests for common package
         run: |
-          yarn --cwd=packages/phone-number-privacy/common test:coverage
+          yarn --cwd=packages/common test:coverage
       - name: Run Tests for signer
         run: |
-          yarn --cwd=packages/phone-number-privacy/signer test:coverage
+          yarn --cwd=apps/signer test:coverage

.prettierignore

@@ -2,9 +2,9 @@ package.json
 **/.next
 **/coverage
 **/node_modules
-**/privacy/*
 .git/
 
+apps/**/dist
 packages/**/dist
 packages/**/lib
 docs/examples/**/lib

.vscode/launch.json

@@ -9,9 +9,9 @@
         "--inspect-brk",
         "${workspaceRoot}/node_modules/.bin/jest",
         "--rootDir",
-        "${workspaceFolder}/packages/phone-number-privacy",
+        "${workspaceFolder}/packages",
         "--runInBand",
-        "${workspaceFolder}/packages/phone-number-privacy/combiner/test/**",
+        "${workspaceFolder}/apps/combiner/test/**",
       ],
       "console": "integratedTerminal",
       "internalConsoleOptions": "neverOpen",
@@ -25,9 +25,9 @@
         "--inspect-brk",
         "${workspaceRoot}/node_modules/.bin/jest",
         "--rootDir",
-        "${workspaceFolder}/packages/phone-number-privacy/signer",
+        "${workspaceFolder}/apps/signer",
         "--runInBand",
-        "${workspaceFolder}/packages/phone-number-privacy/signer/test/**",
+        "${workspaceFolder}/apps/signer/test/**",
       ],
       "console": "integratedTerminal",
       "internalConsoleOptions": "neverOpen",

packages/phone-number-privacy/combiner/package.json → apps/combiner/package.json

@@ -54,6 +54,7 @@
     "uuid": "^7.0.3"
   },
   "devDependencies": {
+    "@celo/phone-number-privacy-signer": "workspace:^",
     "@celo/phone-utils": "^5.0.4",
     "@celo/utils": "^5.0.4",
     "@types/express": "^4.17.6",

packages/phone-number-privacy/combiner/test/end-to-end/pnp.test.ts → apps/combiner/test/end-to-end/pnp.test.ts

@@ -14,7 +14,7 @@ import { normalizeAddressWith0x } from '@celo/utils/lib/address'
 import threshold_bls from 'blind-threshold-bls'
 import { randomBytes } from 'crypto'
 import fetch from 'node-fetch'
-import { config as signerConfig } from '../../../signer/src/config'
+import { config as signerConfig } from '@celo/phone-number-privacy-signer/src/config'
 import { getCombinerVersion } from '../../src'
 import {
   ACCOUNT_ADDRESS,

packages/phone-number-privacy/combiner/test/integration/domain.test.ts → apps/combiner/test/integration/domain.test.ts

@@ -31,6 +31,7 @@ import {
   DefaultKeyName,
   KeyProvider,
 } from '@celo/phone-number-privacy-signer/dist/common/key-management/key-provider-base'
+import { MockKeyProvider } from '@celo/phone-number-privacy-signer/dist/common/key-management/mock-key-provider'
 import {
   SignerConfig,
   SupportedDatabase,
@@ -44,7 +45,6 @@ import { Server } from 'http'
 import { Server as HttpsServer } from 'https'
 import { Knex } from 'knex'
 import request from 'supertest'
-import { MockKeyProvider } from '../../../signer/dist/common/key-management/mock-key-provider'
 import config from '../../src/config'
 import { startCombiner } from '../../src/server'
 import { serverClose } from '../utils'

packages/phone-number-privacy/signer/README.md → apps/signer/README.md

@@ -174,4 +174,4 @@ You can test your mainnet service is set up correctly by running specific tests
 
 ### Logs
 
-Error logs will be prefixed with `CELO_ODIS_ERROR_XX`. You can see a full list of them in [errors.ts](https://github.com/celo-org/celo-monorepo/blob/master/packages/phone-number-privacy/common/src/interfaces/errors.ts) in the common package.
+Error logs will be prefixed with `CELO_ODIS_ERROR_XX`. You can see a full list of them in [errors.ts](https://github.com/celo-org/celo-monorepo/blob/master/packages/common/src/interfaces/errors.ts) in the common package.

packages/phone-number-privacy/signer/docs/deploy-on-aws.md → apps/signer/docs/deploy-on-aws.md

@@ -2,8 +2,8 @@
 
 ## Prerequisites
 
-- awscli (https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html)
-- AWS credentials configured (https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html)
+- awscli (<https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-install.html>)
+- AWS credentials configured (<https://docs.aws.amazon.com/cli/latest/userguide/cli-chap-configure.html>)
 
 ## Steps
 
@@ -35,7 +35,7 @@ For the DB Instance size, `Free tier` should be enough.
 ### Running the Signer on Fargate
 
 ECS Fargate is a container execution service provided by AWS. It runs containers without requiring explicit management of hosts or virtual machines.
-Alternatively the signer service can be run using any other service that allows to run containers, such as EC2 or EKS. In the case of EC2, you will need to install docker, configure the instance profile and follow the documentation from [the signer readme](https://github.com/celo-org/celo-monorepo/tree/master/packages/phone-number-privacy/signer).
+Alternatively the signer service can be run using any other service that allows to run containers, such as EC2 or EKS. In the case of EC2, you will need to install docker, configure the instance profile and follow the documentation from [the signer readme](https://github.com/celo-org/celo-monorepo/tree/master/apps/signer).
 
 1. Create the service-linked role. If it is the first time you run ECS on your account you will need to run this command.
 
@@ -44,7 +44,7 @@ Alternatively the signer service can be run using any other service that allows
     ```
 
 1. Create a Task Role for the signer ([documentation](https://docs.amazonaws.cn/en_us/AmazonECS/latest/userguide/ecs-cli-tutorial-fargate.html)).
-First we will create the `assume-role` policy that allows ECS tasks to be assigned to this task role. 
+First we will create the `assume-role` policy that allows ECS tasks to be assigned to this task role.
 
     ```bash
     cat <<'EOF' > /tmp/task-execution-assume-role.json
@@ -70,7 +70,7 @@ First we will create the `assume-role` policy that allows ECS tasks to be assign
     aws iam --region us-east-2 create-role --role-name signerTaskExecutionRole --assume-role-policy-document file:///tmp/task-execution-assume-role.json
     ```
 
-    Finally we create the policy assigned to this task-role that allows retrieval of secrets from AWS Secret Manager. Then we attach that policy to the task role. 
+    Finally we create the policy assigned to this task-role that allows retrieval of secrets from AWS Secret Manager. Then we attach that policy to the task role.
 
     ```bash
     cat <<'EOF' > /tmp/secret-manager-signer-policy.json

codecov.yml

@@ -5,10 +5,10 @@ coverage:
         target: auto
         threshold: null
         removed_code_behavior: adjust_base
-      odis:
+      packages:
         paths:
-          - packages/phone-number-privacy
-      sdk:
+          - packages/
+      apps:
         paths:
-          - packages/sdk
+          - apps/
     patch: off

dockerfiles/phone-number-privacy/Dockerfile-combiner → dockerfiles/Dockerfile-combiner

@@ -5,10 +5,11 @@ FROM scratch AS packages
 # Assemble all dependencies into the packages folder so the second stage can select whether to
 # include all packages, or just the phone-number-privacy packages.
 WORKDIR /celo-phone-number-privacy/
-COPY packages/phone-number-privacy/combiner packages/phone-number-privacy/combiner
-COPY packages/phone-number-privacy/common packages/phone-number-privacy/common
-COPY packages/sdk/encrypted-backup packages/sdk/encrypted-backup
-COPY packages/sdk/identity packages/sdk/identity
+COPY apps/combiner apps/combiner
+COPY apps/signer apps/signer
+COPY packages/common packages/common
+COPY packages/encrypted-backup packages/encrypted-backup
+COPY packages/identity packages/identity
 COPY packages/odis-identifiers packages/odis-identifiers
 
 ##### Main stage
@@ -28,17 +29,19 @@ COPY .git .git
 # phone-number-privacy package being copied into the image, and therefore it will only build using
 # published dependencies. Setting ONLY_PUBLISHED_DEPENDENCIES to "" will copy in all dependecies.
 ARG ONLY_PUBLISHED_DEPENDENCIES=""
-ARG PACKAGE_SELECTOR=${ONLY_PUBLISHED_DEPENDENCIES:+phone-number-privacy/combiner}
-COPY --from=packages celo-phone-number-privacy/packages/${PACKAGE_SELECTOR} packages/${PACKAGE_SELECTOR}
+ARG PACKAGE_SELECTOR=${ONLY_PUBLISHED_DEPENDENCIES:+combiner}
+COPY --from=packages celo-phone-number-privacy/packages/ packages/
+COPY --from=packages celo-phone-number-privacy/apps/${PACKAGE_SELECTOR} apps/${PACKAGE_SELECTOR}
 
 # Install dependencies and build.
 RUN yarn install --network-timeout 100000  --inline-builds && yarn cache clean
+RUN yarn workspaces list
 RUN yarn workspaces foreach -ipv --topological-dev  -R --from @celo/phone-number-privacy-combiner run build
-RUN yarn workspaces focus @celo/phone-number-privacy-combiner --production
+
 RUN rm -r .git
 
 # Setup and run the combiner application.
 ENV NODE_ENV production
-WORKDIR /celo-phone-number-privacy/packages/phone-number-privacy/combiner
+WORKDIR /celo-phone-number-privacy/apps/combiner
 EXPOSE 8080
 ENTRYPOINT ["yarn", "start:docker"]