Skip to content

Commit

Permalink
Build system rework (#2169)
Browse files Browse the repository at this point in the history
* refactor(crypto): remove circular dependency

* refactor(crypto): expose compress/decompress as part of the DidKeyPlugin interface

* fix(crypto): remove import from private file

* refactor: isolate tsconfig

* fix: remove unused bench file

* chore(repo): remove unused deps

* fix(ozone): properly list dependencies

* fix(services): do lint js files

* fix(services/pds): remove unused deps

* chore(pds): remove bench

* chore(dev-env): remove unused deps

* chore(api): remove bench

* remove unused babel.config.js files

* fix: remove .ts extension from import

* fix(pds): remove imports of src files

* fix(tsconfig): properly list all projects

* fix(dev-env): remove imports of src files

* fix(bsky): remove direct import to crypto src

* fix(api): remove imports to api internals

* chore(build): prevent bundling of built output

* chore(dev): add "dev" script to build in watch mode

* chore(deps): move ts-node dependency where it is actually used

* fix(deps): add dev-env as project dependency

* fix(xrpc-server): properly type kexicon

* fix(bsky): improve typings

* fix(pds): fully type formatRecordEmbedInternal return value

* fix(repo): remove imports from @ipld/car/api

* feat(dev-env): re-export BskyIngester

* fix: properly lint & type jest config & test files

* fix(ci): test after build

* fix(types): use NodeJS.Timeout instead of NodeJS.Timer

* fix(bsky): make types exportable

* fix(ozone): make types exportable

* fix(xrpc-server): make types exportable

* fix(xprc-server): make code compliant with "node" types

* fix(xrpc-server): avoid accessing properties of unknown

* chore(deps): update @types/node

* feat(tsconfig): narrow down available types depending on the package's target environment

* fix(pds): remove unused prop

* fix(bsync): Database's migrator not always initialized

* fix(dev-env): remove unreachable code

* fix(xrpc-server): remove unused import

* fix(xrpc-server): mark header property as abstract

* fix(pds): initialize LeakyTxPlugin's txOver property

* fix(bsky): initialize LeakyTxPlugin's txOver property

* fix(bsky): remove unused migrator from DatabaseCoordinator

* fix(bsky): Properly initialize LabelService's cache property

* fix(ozone): Database's migrator not initialized

* fix(ozone): initialize LeakyTxPlugin's txOver property

* fix(crypto): ignore unused variable error

* feat(tsconfig): use stricter rules

* feat(tsconfig): enable useDefineForClassFields

* feat(xrpc-server): add support for brotli incoming payload

* fix(xrpc-server): properly parse & process content-encoding

* fix(common:stream): always call cb in _transform

* tidy/fix tests and service entrypoints

* Revert "fix(xrpc-server): properly parse & process content-encoding"

This reverts commit 2b1c66e.

* Revert "feat(xrpc-server): add support for brotli incoming payload"

This reverts commit e710c21.

* remove special node env for tests (defaults to jest val of "test")

* kill mute sync handler on disconnect

* work around connect-es bug w/ request aborts

* style(crypto): rename imports from uint8arrays

* fix update package-lock

* fix lint

* force hbs files to be bundled as cjs

* fix: use concurrently instead of npm-run-all

npm-run-all seems not to be maintained anymore. Additionally, concurrently better forwards signals to child processes.

* remove concurrently alltogether

* ignore sqlite files in services/pds

* fix verify

* fix verify

* tidy, fix verify

* fix blob diversion test

* build rework changeset

---------

Co-authored-by: Devin Ivy <[email protected]>
  • Loading branch information
matthieusieben and devinivy authored Mar 18, 2024
1 parent 0a2464c commit f689bd5
Show file tree
Hide file tree
Showing 232 changed files with 1,571 additions and 1,557 deletions.
21 changes: 21 additions & 0 deletions .changeset/long-planes-bathe.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,21 @@
---
'@atproto/xrpc-server': minor
'@atproto/common-web': minor
'@atproto/identity': minor
'@atproto/dev-env': minor
'@atproto/lex-cli': minor
'@atproto/lexicon': minor
'@atproto/common': minor
'@atproto/crypto': minor
'@atproto/syntax': minor
'@atproto/repo': minor
'@atproto/xrpc': minor
'@atproto/api': minor
'@atproto/aws': minor
'@atproto/bsync': patch
'@atproto/ozone': patch
'@atproto/bsky': patch
'@atproto/pds': patch
---

Build system rework, stop bundling dependencies.
29 changes: 18 additions & 11 deletions .eslintrc
Original file line number Diff line number Diff line change
Expand Up @@ -13,16 +13,7 @@
"plugin:prettier/recommended",
"prettier"
],
"ignorePatterns": [
"dist",
"node_modules",
"jest.config.base.js",
"jest.bench.config.js",
"jest.config.js",
"babel.config.js",
"build.js",
"update-pkg.js"
],
"ignorePatterns": ["dist", "node_modules"],
"rules": {
"no-var": "error",
"prefer-const": "warn",
Expand All @@ -36,5 +27,21 @@
"@typescript-eslint/explicit-module-boundary-types": "off",
"@typescript-eslint/no-empty-function": "off",
"@typescript-eslint/no-explicit-any": "off"
}
},
"overrides": [
{
"files": ["jest.config.js"],
"env": { "commonjs": true }
},
{
"files": ["jest.setup.js"],
"env": { "jest": true }
},
{
"files": "*.js",
"rules": {
"@typescript-eslint/no-var-requires": "off"
}
}
]
}
1 change: 1 addition & 0 deletions .github/workflows/publish.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- run: pnpm build
- run: pnpm verify
- name: Publish
id: changesets
Expand Down
15 changes: 15 additions & 0 deletions .github/workflows/repo.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,8 +24,14 @@ jobs:
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- run: pnpm build
- uses: actions/upload-artifact@v4
with:
name: dist
path: packages/*/dist
retention-days: 1
test:
name: Test
needs: build
strategy:
matrix:
shard: [1/8, 2/8, 3/8, 4/8, 5/8, 6/8, 7/8, 8/8]
Expand All @@ -40,9 +46,14 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm i --frozen-lockfile
- uses: actions/download-artifact@v4
with:
name: dist
path: packages
- run: pnpm test:withFlags --maxWorkers=1 --shard=${{ matrix.shard }} --passWithNoTests
verify:
name: Verify
needs: build
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@v3
Expand All @@ -54,4 +65,8 @@ jobs:
node-version: 18
cache: 'pnpm'
- run: pnpm install --frozen-lockfile
- uses: actions/download-artifact@v4
with:
name: dist
path: packages
- run: pnpm verify
3 changes: 0 additions & 3 deletions babel.config.js

This file was deleted.

20 changes: 0 additions & 20 deletions jest.config.base.js

This file was deleted.

5 changes: 1 addition & 4 deletions jest.config.js
Original file line number Diff line number Diff line change
@@ -1,7 +1,4 @@
// jest.config.js
const base = require('./jest.config.base.js')

/** @type {import('jest').Config} */
module.exports = {
...base,
projects: ['<rootDir>/packages/*/jest.config.js'],
}
File renamed without changes.
30 changes: 16 additions & 14 deletions package.json
Original file line number Diff line number Diff line change
Expand Up @@ -10,44 +10,46 @@
},
"scripts": {
"lint:fix": "pnpm lint --fix",
"lint": "eslint . --ext .ts,.tsx",
"verify": "prettier --check . && pnpm lint",
"format": "prettier --write .",
"build": "pnpm -r --stream build",
"update-main-to-dist": "pnpm -r --stream update-main-to-dist",
"test": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
"test:withFlags": "LOG_ENABLED=false NODE_ENV=development ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
"lint": "eslint . --ext .ts,.js",
"style:fix": "prettier --write .",
"style": "prettier --check .",
"verify": "pnpm --stream '/^verify:.+$/'",
"verify:style": "pnpm run style",
"verify:lint": "pnpm lint",
"verify:types": "tsc --build tsconfig.json",
"format": "pnpm lint:fix && pnpm style:fix",
"build": "pnpm --recursive --stream build",
"dev": "pnpm --stream '/^dev:.+$/'",
"dev:tsc": "tsc --build tsconfig.json --watch",
"dev:pkg": "pnpm --recursive --parallel --stream dev",
"test": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test",
"test:withFlags": "LOG_ENABLED=false ./packages/dev-infra/with-test-redis-and-db.sh pnpm --stream -r test --",
"changeset": "changeset",
"release": "pnpm build && changeset publish",
"version-packages": "changeset version && git add ."
},
"devDependencies": {
"@atproto/dev-env": "workspace:^",
"@babel/core": "^7.18.6",
"@babel/preset-env": "^7.18.6",
"@changesets/changelog-github": "^0.4.8",
"@changesets/cli": "^2.26.2",
"@npmcli/package-json": "^3.0.0",
"@swc/core": "^1.3.42",
"@swc/jest": "^0.2.24",
"@types/jest": "^28.1.4",
"@types/node": "^18.0.0",
"@types/node": "^18.19.24",
"@typescript-eslint/eslint-plugin": "^6.14.0",
"@typescript-eslint/parser": "^6.14.0",
"babel-eslint": "^10.1.0",
"dotenv": "^16.0.3",
"esbuild": "^0.14.48",
"esbuild-node-externals": "^1.5.0",
"esbuild-plugin-handlebars": "^1.0.2",
"eslint": "^8.24.0",
"eslint-config-prettier": "^8.5.0",
"eslint-plugin-prettier": "^4.2.1",
"handlebars-jest": "^1.0.0",
"jest": "^28.1.2",
"node-gyp": "^9.3.1",
"pino-pretty": "^9.1.0",
"prettier": "^2.7.1",
"prettier-config-standard": "^5.0.0",
"ts-node": "^10.8.2",
"typescript": "^5.3.3"
},
"workspaces": {
Expand Down
3 changes: 0 additions & 3 deletions packages/api/babel.config.js

This file was deleted.

15 changes: 0 additions & 15 deletions packages/api/build.js

This file was deleted.

8 changes: 0 additions & 8 deletions packages/api/jest.bench.config.js

This file was deleted.

9 changes: 6 additions & 3 deletions packages/api/jest.config.js
Original file line number Diff line number Diff line change
@@ -1,6 +1,9 @@
const base = require('../../jest.config.base.js')

/** @type {import('jest').Config} */
module.exports = {
...base,
displayName: 'API',
transform: { '^.+\\.(t|j)s$': '@swc/jest' },
transformIgnorePatterns: [`<rootDir>/node_modules/(?!get-port)`],
testTimeout: 60000,
setupFiles: ['<rootDir>/../../jest.setup.ts'],
setupFilesAfterEnv: ['<rootDir>/jest.setup.ts'],
}
20 changes: 20 additions & 0 deletions packages/api/jest.d.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,20 @@
declare namespace jest {
// eslint-disable-next-line
interface Matchers<R, T = {}> {
toBeModerationResult(
expected: ModerationTestSuiteResultFlag[] | undefined,
context?: string,
stringifiedResult?: string,
ignoreCause?: boolean,
): R
}

interface Expect {
toBeModerationResult(
expected: ModerationTestSuiteResultFlag[] | undefined,
context?: string,
stringifiedResult?: string,
ignoreCause?: boolean,
): void
}
}
97 changes: 97 additions & 0 deletions packages/api/jest.setup.ts
Original file line number Diff line number Diff line change
@@ -0,0 +1,97 @@
import { ModerationUI } from './src'
import { ModerationTestSuiteResultFlag } from './tests/util/moderation-behavior'

expect.extend({
toBeModerationResult(
actual: ModerationUI,
expected: ModerationTestSuiteResultFlag[] | undefined,
context: string = '',
stringifiedResult: string | undefined = undefined,
_ignoreCause = false,
) {
const fail = (msg: string) => ({
pass: false,
message: () =>
`${msg}.${
stringifiedResult ? ` Full result: ${stringifiedResult}` : ''
}`,
})
// let cause = actual.causes?.type as string
// if (actual.cause?.type === 'label') {
// cause = `label:${actual.cause.labelDef.id}`
// } else if (actual.cause?.type === 'muted') {
// if (actual.cause.source.type === 'list') {
// cause = 'muted-by-list'
// }
// } else if (actual.cause?.type === 'blocking') {
// if (actual.cause.source.type === 'list') {
// cause = 'blocking-by-list'
// }
// }
if (!expected) {
// if (!ignoreCause && actual.cause) {
// return fail(`${context} expected to be a no-op, got ${cause}`)
// }
if (actual.inform) {
return fail(`${context} expected to be a no-op, got inform=true`)
}
if (actual.alert) {
return fail(`${context} expected to be a no-op, got alert=true`)
}
if (actual.blur) {
return fail(`${context} expected to be a no-op, got blur=true`)
}
if (actual.filter) {
return fail(`${context} expected to be a no-op, got filter=true`)
}
if (actual.noOverride) {
return fail(`${context} expected to be a no-op, got noOverride=true`)
}
} else {
// if (!ignoreCause && cause !== expected.cause) {
// return fail(`${context} expected to be ${expected.cause}, got ${cause}`)
// }
const expectedInform = expected.includes('inform')
if (!!actual.inform !== expectedInform) {
return fail(
`${context} expected to be inform=${expectedInform}, got ${
actual.inform || false
}`,
)
}
const expectedAlert = expected.includes('alert')
if (!!actual.alert !== expectedAlert) {
return fail(
`${context} expected to be alert=${expectedAlert}, got ${
actual.alert || false
}`,
)
}
const expectedBlur = expected.includes('blur')
if (!!actual.blur !== expectedBlur) {
return fail(
`${context} expected to be blur=${expectedBlur}, got ${
actual.blur || false
}`,
)
}
const expectedFilter = expected.includes('filter')
if (!!actual.filter !== expectedFilter) {
return fail(
`${context} expected to be filter=${expectedFilter}, got ${
actual.filter || false
}`,
)
}
const expectedNoOverride = expected.includes('noOverride')
if (!!actual.noOverride !== expectedNoOverride) {
return fail(
`${context} expected to be noOverride=${expectedNoOverride}, got ${
actual.noOverride || false
}`,
)
}
}
return { pass: true, message: () => '' }
},
})
19 changes: 6 additions & 13 deletions packages/api/package.json
Original file line number Diff line number Diff line change
Expand Up @@ -14,19 +14,12 @@
"url": "https://github.com/bluesky-social/atproto",
"directory": "packages/api"
},
"main": "src/index.ts",
"publishConfig": {
"main": "dist/index.js",
"types": "dist/index.d.ts"
},
"main": "dist/index.js",
"types": "dist/index.d.ts",
"scripts": {
"codegen": "node ./scripts/generate-code.mjs && lex gen-api ./src/client ../../lexicons/com/atproto/*/* ../../lexicons/app/bsky/*/* ../../lexicons/tools/ozone/*/*",
"build": "node ./build.js",
"postbuild": "tsc --build tsconfig.build.json",
"update-main-to-dist": "node ../../update-main-to-dist.js packages/api",
"test": "jest",
"bench": "jest --config jest.bench.config.js",
"bench:profile": "node --inspect-brk ../../node_modules/.bin/jest --config jest.bench.config.js"
"build": "tsc --build tsconfig.build.json",
"test": "jest"
},
"dependencies": {
"@atproto/common-web": "workspace:^",
Expand All @@ -38,7 +31,7 @@
},
"devDependencies": {
"@atproto/lex-cli": "workspace:^",
"@atproto/dev-env": "workspace:^",
"get-port": "^6.1.2"
"get-port": "^6.1.2",
"jest": "^28.1.2"
}
}
Loading

0 comments on commit f689bd5

Please sign in to comment.