From 2e08b691bb11e14abdaeaf33ff1ad13dec7cf9d8 Mon Sep 17 00:00:00 2001 From: Jake Gold <52801504+Jacob2161@users.noreply.github.com> Date: Thu, 29 Feb 2024 14:16:50 -0800 Subject: [PATCH 1/5] Add runit to the services/bsky Dockerfile (#2254) add runit to the services/bsky Dockerfile --- services/bsky/Dockerfile | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/services/bsky/Dockerfile b/services/bsky/Dockerfile index 84422945ae0..a15e6aa2ee7 100644 --- a/services/bsky/Dockerfile +++ b/services/bsky/Dockerfile @@ -35,7 +35,10 @@ WORKDIR services/bsky # Uses assets from build stage to reduce build size FROM node:20.11-alpine -RUN apk add --update dumb-init +# dumb-init is used to handle signals properly. +# runit is installed so it can be (optionally) used for logging via svlogd. +RUN apk add --update dumb-init runit + # Avoid zombie processes, handle signal forwarding ENTRYPOINT ["dumb-init", "--"] From 6ec8859929a16f9725319cc398b716acf913b01f Mon Sep 17 00:00:00 2001 From: Eric Bailey Date: Fri, 1 Mar 2024 12:44:52 -0600 Subject: [PATCH 2/5] Improve tag detection (#2260) * Allow tags to lead with and contain only numbers * Break tags on other whitespace characters * Export regexes from rich text detection * Add test * Add test * Disallow number-only tags * Avoid combining enclosing screen chars * Allow full-width number sign * Clarify tests * Fix punctuation edge case * Reorder * Simplify, add another test * Another test, comment --- .changeset/chatty-cows-kick.md | 5 +++ .changeset/lovely-pandas-pretend.md | 5 +++ .changeset/quick-ducks-joke.md | 5 +++ packages/api/src/index.ts | 1 + packages/api/src/rich-text/detection.ts | 18 ++++++++--- packages/api/src/rich-text/util.ts | 11 +++++++ .../api/tests/rich-text-detection.test.ts | 31 +++++++++++++++++-- 7 files changed, 68 insertions(+), 8 deletions(-) create mode 100644 .changeset/chatty-cows-kick.md create mode 100644 .changeset/lovely-pandas-pretend.md create mode 100644 .changeset/quick-ducks-joke.md create mode 100644 packages/api/src/rich-text/util.ts diff --git a/.changeset/chatty-cows-kick.md b/.changeset/chatty-cows-kick.md new file mode 100644 index 00000000000..76bd82f015d --- /dev/null +++ b/.changeset/chatty-cows-kick.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +Export regex from rich text detection diff --git a/.changeset/lovely-pandas-pretend.md b/.changeset/lovely-pandas-pretend.md new file mode 100644 index 00000000000..3a75be2877b --- /dev/null +++ b/.changeset/lovely-pandas-pretend.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +Disallow rare unicode whitespace characters from tags diff --git a/.changeset/quick-ducks-joke.md b/.changeset/quick-ducks-joke.md new file mode 100644 index 00000000000..923b2fe2fe0 --- /dev/null +++ b/.changeset/quick-ducks-joke.md @@ -0,0 +1,5 @@ +--- +'@atproto/api': patch +--- + +Allow tags to lead with numbers diff --git a/packages/api/src/index.ts b/packages/api/src/index.ts index 9e407142aba..87cf1ccf01a 100644 --- a/packages/api/src/index.ts +++ b/packages/api/src/index.ts @@ -14,6 +14,7 @@ export * from './agent' export * from './rich-text/rich-text' export * from './rich-text/sanitization' export * from './rich-text/unicode' +export * from './rich-text/util' export * from './moderation' export * from './moderation/types' export { LABELS } from './moderation/const/labels' diff --git a/packages/api/src/rich-text/detection.ts b/packages/api/src/rich-text/detection.ts index 7b5444a68a5..22c5db1b087 100644 --- a/packages/api/src/rich-text/detection.ts +++ b/packages/api/src/rich-text/detection.ts @@ -1,6 +1,12 @@ import TLDs from 'tlds' import { AppBskyRichtextFacet } from '../client' import { UnicodeString } from './unicode' +import { + URL_REGEX, + MENTION_REGEX, + TAG_REGEX, + TRAILING_PUNCTUATION_REGEX, +} from './util' export type Facet = AppBskyRichtextFacet.Main @@ -9,7 +15,7 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined { const facets: Facet[] = [] { // mentions - const re = /(^|\s|\()(@)([a-zA-Z0-9.-]+)(\b)/g + const re = MENTION_REGEX while ((match = re.exec(text.utf16))) { if (!isValidDomain(match[3]) && !match[3].endsWith('.test')) { continue // probably not a handle @@ -33,8 +39,7 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined { } { // links - const re = - /(^|\s|\()((https?:\/\/[\S]+)|((?[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*))/gim + const re = URL_REGEX while ((match = re.exec(text.utf16))) { let uri = match[2] if (!uri.startsWith('http')) { @@ -70,11 +75,14 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined { } } { - const re = /(^|\s)#((?!\ufe0f)[^\d\s]\S*)(?=\s)?/g + const re = TAG_REGEX while ((match = re.exec(text.utf16))) { let [, leading, tag] = match - tag = tag.trim().replace(/\p{P}+$/gu, '') // strip ending punctuation + if (!tag) continue + + // strip ending punctuation and any spaces + tag = tag.trim().replace(TRAILING_PUNCTUATION_REGEX, '') if (tag.length === 0 || tag.length > 64) continue diff --git a/packages/api/src/rich-text/util.ts b/packages/api/src/rich-text/util.ts new file mode 100644 index 00000000000..ab50c66212d --- /dev/null +++ b/packages/api/src/rich-text/util.ts @@ -0,0 +1,11 @@ +export const MENTION_REGEX = /(^|\s|\()(@)([a-zA-Z0-9.-]+)(\b)/g +export const URL_REGEX = + /(^|\s|\()((https?:\/\/[\S]+)|((?[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*))/gim +export const TRAILING_PUNCTUATION_REGEX = /\p{P}+$/gu + +/** + * `\ufe0f` emoji modifier + * `\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2` zero-width spaces (likely incomplete) + */ +export const TAG_REGEX = + /(^|\s)[#＃]((?!\ufe0f)[^\s\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]*[^\d\s\p{P}\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]+[^\s\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]*)?/gu diff --git a/packages/api/tests/rich-text-detection.test.ts b/packages/api/tests/rich-text-detection.test.ts index 0eafb65a3b1..084b5440a48 100644 --- a/packages/api/tests/rich-text-detection.test.ts +++ b/packages/api/tests/rich-text-detection.test.ts @@ -218,7 +218,7 @@ describe('detectFacets', () => { } }) - it('correctly detects tags inline', async () => { + describe('correctly detects tags inline', () => { const inputs: [ string, string[], @@ -234,11 +234,13 @@ describe('detectFacets', () => { ], ], ['#1', [], []], + ['#1a', ['1a'], [{ byteStart: 0, byteEnd: 3 }]], ['#tag', ['tag'], [{ byteStart: 0, byteEnd: 4 }]], ['body #tag', ['tag'], [{ byteStart: 5, byteEnd: 9 }]], ['#tag body', ['tag'], [{ byteStart: 0, byteEnd: 4 }]], ['body #tag body', ['tag'], [{ byteStart: 5, byteEnd: 9 }]], ['body #1', [], []], + ['body #1a', ['1a'], [{ byteStart: 5, byteEnd: 8 }]], ['body #a1', ['a1'], [{ byteStart: 5, byteEnd: 8 }]], ['#', [], []], ['#?', [], []], @@ -254,12 +256,18 @@ describe('detectFacets', () => { [], [], ], + [ + 'body #thisisa64characterstring_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa!', + ['thisisa64characterstring_aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa'], + [{ byteStart: 5, byteEnd: 70 }], + ], [ 'its a #double#rainbow', ['double#rainbow'], [{ byteStart: 6, byteEnd: 21 }], ], ['##hashash', ['#hashash'], [{ byteStart: 0, byteEnd: 9 }]], + ['##', [], []], ['some #n0n3s@n5e!', ['n0n3s@n5e'], [{ byteStart: 5, byteEnd: 15 }]], [ 'works #with,punctuation', @@ -319,9 +327,26 @@ describe('detectFacets', () => { }, ], ], + ['no match (\\u200B): #​', [], []], + ['no match (\\u200Ba): #​a', [], []], + ['match (a\\u200Bb): #a​b', ['a'], [{ byteStart: 18, byteEnd: 20 }]], + ['match (ab\\u200B): #ab​', ['ab'], [{ byteStart: 18, byteEnd: 21 }]], + ['no match (\\u20e2tag): #⃢tag', [], []], + ['no match (a\\u20e2b): #a⃢b', ['a'], [{ byteStart: 21, byteEnd: 23 }]], + [ + 'match full width number sign (tag): ＃tag', + ['tag'], + [{ byteStart: 36, byteEnd: 42 }], + ], + [ + 'match full width number sign (tag): ＃#️⃣tag', + ['#️⃣tag'], + [{ byteStart: 36, byteEnd: 49 }], + ], + ['no match 1?: #1?', [], []], ] - for (const [input, tags, indices] of inputs) { + it.each(inputs)('%s', async (input, tags, indices) => { const rt = new RichText({ text: input }) await rt.detectFacets(agent) @@ -340,7 +365,7 @@ describe('detectFacets', () => { expect(detectedTags).toEqual(tags) expect(detectedIndices).toEqual(indices) - } + }) }) }) From 27ff43b223e3d55133e2d475921c9bbdb7f45727 Mon Sep 17 00:00:00 2001 From: "github-actions[bot]" <41898282+github-actions[bot]@users.noreply.github.com> Date: Fri, 1 Mar 2024 13:03:04 -0600 Subject: [PATCH 3/5] Version packages (#2261) Co-authored-by: github-actions[bot] --- .changeset/chatty-cows-kick.md | 5 ----- .changeset/lovely-pandas-pretend.md | 5 ----- .changeset/quick-ducks-joke.md | 5 ----- packages/api/CHANGELOG.md | 10 ++++++++++ packages/api/package.json | 2 +- packages/bsky/CHANGELOG.md | 7 +++++++ packages/bsky/package.json | 2 +- packages/dev-env/CHANGELOG.md | 10 ++++++++++ packages/dev-env/package.json | 2 +- packages/ozone/CHANGELOG.md | 7 +++++++ packages/ozone/package.json | 2 +- packages/pds/CHANGELOG.md | 7 +++++++ packages/pds/package.json | 2 +- 13 files changed, 46 insertions(+), 20 deletions(-) delete mode 100644 .changeset/chatty-cows-kick.md delete mode 100644 .changeset/lovely-pandas-pretend.md delete mode 100644 .changeset/quick-ducks-joke.md diff --git a/.changeset/chatty-cows-kick.md b/.changeset/chatty-cows-kick.md deleted file mode 100644 index 76bd82f015d..00000000000 --- a/.changeset/chatty-cows-kick.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -Export regex from rich text detection diff --git a/.changeset/lovely-pandas-pretend.md b/.changeset/lovely-pandas-pretend.md deleted file mode 100644 index 3a75be2877b..00000000000 --- a/.changeset/lovely-pandas-pretend.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -Disallow rare unicode whitespace characters from tags diff --git a/.changeset/quick-ducks-joke.md b/.changeset/quick-ducks-joke.md deleted file mode 100644 index 923b2fe2fe0..00000000000 --- a/.changeset/quick-ducks-joke.md +++ /dev/null @@ -1,5 +0,0 @@ ---- -'@atproto/api': patch ---- - -Allow tags to lead with numbers diff --git a/packages/api/CHANGELOG.md b/packages/api/CHANGELOG.md index 48e4edc018b..a5bac9a4eaa 100644 --- a/packages/api/CHANGELOG.md +++ b/packages/api/CHANGELOG.md @@ -1,5 +1,15 @@ # @atproto/api +## 0.10.4 + +### Patch Changes + +- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Export regex from rich text detection + +- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Disallow rare unicode whitespace characters from tags + +- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Allow tags to lead with numbers + ## 0.10.3 ### Patch Changes diff --git a/packages/api/package.json b/packages/api/package.json index 4f07d5a01a1..dbb18da1786 100644 --- a/packages/api/package.json +++ b/packages/api/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/api", - "version": "0.10.3", + "version": "0.10.4", "license": "MIT", "description": "Client library for atproto and Bluesky", "keywords": [ diff --git a/packages/bsky/CHANGELOG.md b/packages/bsky/CHANGELOG.md index 27155388ef9..102c40050eb 100644 --- a/packages/bsky/CHANGELOG.md +++ b/packages/bsky/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/bsky +## 0.0.36 + +### Patch Changes + +- Updated dependencies [[`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f)]: + - @atproto/api@0.10.4 + ## 0.0.35 ### Patch Changes diff --git a/packages/bsky/package.json b/packages/bsky/package.json index 8ffa5e0911f..dd081d3c209 100644 --- a/packages/bsky/package.json +++ b/packages/bsky/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/bsky", - "version": "0.0.35", + "version": "0.0.36", "license": "MIT", "description": "Reference implementation of app.bsky App View (Bluesky API)", "keywords": [ diff --git a/packages/dev-env/CHANGELOG.md b/packages/dev-env/CHANGELOG.md index 814f89dd619..15a6ad3171f 100644 --- a/packages/dev-env/CHANGELOG.md +++ b/packages/dev-env/CHANGELOG.md @@ -1,5 +1,15 @@ # @atproto/dev-env +## 0.2.36 + +### Patch Changes + +- Updated dependencies [[`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f)]: + - @atproto/api@0.10.4 + - @atproto/bsky@0.0.36 + - @atproto/ozone@0.0.15 + - @atproto/pds@0.4.4 + ## 0.2.35 ### Patch Changes diff --git a/packages/dev-env/package.json b/packages/dev-env/package.json index 5924c18bfd0..46899cdb763 100644 --- a/packages/dev-env/package.json +++ b/packages/dev-env/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/dev-env", - "version": "0.2.35", + "version": "0.2.36", "license": "MIT", "description": "Local development environment helper for atproto development", "keywords": [ diff --git a/packages/ozone/CHANGELOG.md b/packages/ozone/CHANGELOG.md index 7f27390ce91..0953945b4b2 100644 --- a/packages/ozone/CHANGELOG.md +++ b/packages/ozone/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/ozone +## 0.0.15 + +### Patch Changes + +- Updated dependencies [[`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f)]: + - @atproto/api@0.10.4 + ## 0.0.14 ### Patch Changes diff --git a/packages/ozone/package.json b/packages/ozone/package.json index 3a3d3a690ff..3840f4b49ed 100644 --- a/packages/ozone/package.json +++ b/packages/ozone/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/ozone", - "version": "0.0.14", + "version": "0.0.15", "license": "MIT", "description": "Backend service for moderating the Bluesky network.", "keywords": [ diff --git a/packages/pds/CHANGELOG.md b/packages/pds/CHANGELOG.md index 3a323988102..1a3e87286f3 100644 --- a/packages/pds/CHANGELOG.md +++ b/packages/pds/CHANGELOG.md @@ -1,5 +1,12 @@ # @atproto/pds +## 0.4.4 + +### Patch Changes + +- Updated dependencies [[`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f), [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f)]: + - @atproto/api@0.10.4 + ## 0.4.3 ### Patch Changes diff --git a/packages/pds/package.json b/packages/pds/package.json index f345bbcbf63..062578aa531 100644 --- a/packages/pds/package.json +++ b/packages/pds/package.json @@ -1,6 +1,6 @@ { "name": "@atproto/pds", - "version": "0.4.3", + "version": "0.4.4", "license": "MIT", "description": "Reference implementation of atproto Personal Data Server (PDS)", "keywords": [ From 1102784cc89888c9887d13544c73f6002aa728dd Mon Sep 17 00:00:00 2001 From: Foysal Ahamed Date: Mon, 4 Mar 2024 17:31:50 +0100 Subject: [PATCH 4/5] :bug: Increment attempt count after each attempt to push ozone event (#2239) --- packages/ozone/src/daemon/event-pusher.ts | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/packages/ozone/src/daemon/event-pusher.ts b/packages/ozone/src/daemon/event-pusher.ts index faaee4529ed..01570595c0d 100644 --- a/packages/ozone/src/daemon/event-pusher.ts +++ b/packages/ozone/src/daemon/event-pusher.ts @@ -205,7 +205,7 @@ export class EventPusher { ? { confirmedAt: new Date() } : { lastAttempted: new Date(), - attempts: evt.attempts ?? 0 + 1, + attempts: (evt.attempts ?? 0) + 1, }, ) .where('subjectDid', '=', evt.subjectDid) @@ -244,7 +244,7 @@ export class EventPusher { ? { confirmedAt: new Date() } : { lastAttempted: new Date(), - attempts: evt.attempts ?? 0 + 1, + attempts: (evt.attempts ?? 0) + 1, }, ) .where('subjectUri', '=', evt.subjectUri) @@ -284,7 +284,7 @@ export class EventPusher { ? { confirmedAt: new Date() } : { lastAttempted: new Date(), - attempts: evt.attempts ?? 0 + 1, + attempts: (evt.attempts ?? 0) + 1, }, ) .where('subjectDid', '=', evt.subjectDid) From c7e6ef090798ae0e10a94ded5278646687130865 Mon Sep 17 00:00:00 2001 From: devin ivy Date: Tue, 5 Mar 2024 14:27:25 -0500 Subject: [PATCH 5/5] Ozone delegates email sending to actor's pds (#2272) * ozone delegates email sending to user's pds * lexicon: add content field to mod email event * test email sending via mod event --- lexicons/com/atproto/admin/defs.json | 4 ++ packages/api/src/client/lexicons.ts | 4 ++ .../client/types/com/atproto/admin/defs.ts | 2 + packages/bsky/src/lexicon/lexicons.ts | 4 ++ .../lexicon/types/com/atproto/admin/defs.ts | 2 + packages/dev-env/src/ozone.ts | 1 + .../src/api/admin/emitModerationEvent.ts | 19 +++++ packages/ozone/src/config/config.ts | 2 + packages/ozone/src/config/env.ts | 4 +- packages/ozone/src/context.ts | 14 ++-- packages/ozone/src/daemon/context.ts | 14 ++-- packages/ozone/src/lexicon/lexicons.ts | 4 ++ .../lexicon/types/com/atproto/admin/defs.ts | 2 + packages/ozone/src/mod-service/index.ts | 71 +++++++++++++++++-- packages/ozone/src/mod-service/views.ts | 16 ++--- .../ozone/tests/moderation-events.test.ts | 49 +++++++++++++ .../src/api/com/atproto/admin/sendEmail.ts | 44 ++++-------- packages/pds/src/lexicon/lexicons.ts | 4 ++ .../lexicon/types/com/atproto/admin/defs.ts | 2 + 19 files changed, 205 insertions(+), 57 deletions(-) diff --git a/lexicons/com/atproto/admin/defs.json b/lexicons/com/atproto/admin/defs.json index e1315eb7473..d2056f77cf8 100644 --- a/lexicons/com/atproto/admin/defs.json +++ b/lexicons/com/atproto/admin/defs.json @@ -585,6 +585,10 @@ "type": "string", "description": "The subject line of the email sent to the user." }, + "content": { + "type": "string", + "description": "The content of the email sent to the user." + }, "comment": { "type": "string", "description": "Additional comment about the outgoing comm." diff --git a/packages/api/src/client/lexicons.ts b/packages/api/src/client/lexicons.ts index 14e4c1cb81e..72ac23230f4 100644 --- a/packages/api/src/client/lexicons.ts +++ b/packages/api/src/client/lexicons.ts @@ -897,6 +897,10 @@ export const schemaDict = { type: 'string', description: 'The subject line of the email sent to the user.', }, + content: { + type: 'string', + description: 'The content of the email sent to the user.', + }, comment: { type: 'string', description: 'Additional comment about the outgoing comm.', diff --git a/packages/api/src/client/types/com/atproto/admin/defs.ts b/packages/api/src/client/types/com/atproto/admin/defs.ts index 4e3d35a869f..2de71524005 100644 --- a/packages/api/src/client/types/com/atproto/admin/defs.ts +++ b/packages/api/src/client/types/com/atproto/admin/defs.ts @@ -704,6 +704,8 @@ export function validateModEventUnmute(v: unknown): ValidationResult { export interface ModEventEmail { /** The subject line of the email sent to the user. */ subjectLine: string + /** The content of the email sent to the user. */ + content?: string /** Additional comment about the outgoing comm. */ comment?: string [k: string]: unknown diff --git a/packages/bsky/src/lexicon/lexicons.ts b/packages/bsky/src/lexicon/lexicons.ts index 14e4c1cb81e..72ac23230f4 100644 --- a/packages/bsky/src/lexicon/lexicons.ts +++ b/packages/bsky/src/lexicon/lexicons.ts @@ -897,6 +897,10 @@ export const schemaDict = { type: 'string', description: 'The subject line of the email sent to the user.', }, + content: { + type: 'string', + description: 'The content of the email sent to the user.', + }, comment: { type: 'string', description: 'Additional comment about the outgoing comm.', diff --git a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..acfda37abbc 100644 --- a/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/bsky/src/lexicon/types/com/atproto/admin/defs.ts @@ -704,6 +704,8 @@ export function validateModEventUnmute(v: unknown): ValidationResult { export interface ModEventEmail { /** The subject line of the email sent to the user. */ subjectLine: string + /** The content of the email sent to the user. */ + content?: string /** Additional comment about the outgoing comm. */ comment?: string [k: string]: unknown diff --git a/packages/dev-env/src/ozone.ts b/packages/dev-env/src/ozone.ts index 5fded7947c5..39cf26f5c81 100644 --- a/packages/dev-env/src/ozone.ts +++ b/packages/dev-env/src/ozone.ts @@ -34,6 +34,7 @@ export class TestOzone { const port = config.port || (await getPort()) const url = `http://localhost:${port}` const env: ozone.OzoneEnvironment = { + devMode: true, version: '0.0.0', port, didPlcUrl: config.plcUrl, diff --git a/packages/ozone/src/api/admin/emitModerationEvent.ts b/packages/ozone/src/api/admin/emitModerationEvent.ts index ef4c5fd2822..e8f65ab8dc7 100644 --- a/packages/ozone/src/api/admin/emitModerationEvent.ts +++ b/packages/ozone/src/api/admin/emitModerationEvent.ts @@ -2,12 +2,14 @@ import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../lexicon' import AppContext from '../../context' import { + isModEventEmail, isModEventLabel, isModEventReverseTakedown, isModEventTakedown, } from '../../lexicon/types/com/atproto/admin/defs' import { subjectFromInput } from '../../mod-service/subject' import { ModerationLangService } from '../../mod-service/lang' +import { retryHttp } from '../../util' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.emitModerationEvent({ @@ -75,6 +77,23 @@ export default function (server: Server, ctx: AppContext) { } } + if (isModEventEmail(event) && event.content) { + // sending email prior to logging the event to avoid a long transaction below + if (!subject.isRepo()) { + throw new InvalidRequestError( + 'Email can only be sent to a repo subject', + ) + } + const { content, subjectLine } = event + await retryHttp(() => + ctx.modService(db).sendEmail({ + subject: subjectLine, + content, + recipientDid: subject.did, + }), + ) + } + const moderationEvent = await db.transaction(async (dbTxn) => { const moderationTxn = ctx.modService(dbTxn) diff --git a/packages/ozone/src/config/config.ts b/packages/ozone/src/config/config.ts index a219dba8d5a..8c25f8ace0b 100644 --- a/packages/ozone/src/config/config.ts +++ b/packages/ozone/src/config/config.ts @@ -13,6 +13,7 @@ export const envToCfg = (env: OzoneEnvironment): OzoneConfig => { publicUrl: env.publicUrl, did: env.serverDid, version: env.version, + devMode: env.devMode, } assert(env.dbPostgresUrl) @@ -71,6 +72,7 @@ export type ServiceConfig = { publicUrl: string did: string version?: string + devMode?: boolean } export type DatabaseConfig = { diff --git a/packages/ozone/src/config/env.ts b/packages/ozone/src/config/env.ts index 1a93ad2b855..7175c846839 100644 --- a/packages/ozone/src/config/env.ts +++ b/packages/ozone/src/config/env.ts @@ -1,8 +1,9 @@ -import { envInt, envList, envStr } from '@atproto/common' +import { envBool, envInt, envList, envStr } from '@atproto/common' export const readEnv = (): OzoneEnvironment => { return { nodeEnv: envStr('NODE_ENV'), + devMode: envBool('OZONE_DEV_MODE'), version: envStr('OZONE_VERSION'), port: envInt('OZONE_PORT'), publicUrl: envStr('OZONE_PUBLIC_URL'), @@ -27,6 +28,7 @@ export const readEnv = (): OzoneEnvironment => { export type OzoneEnvironment = { nodeEnv?: string + devMode?: boolean version?: string port?: number publicUrl?: string diff --git a/packages/ozone/src/context.ts b/packages/ozone/src/context.ts index e65a6af2185..9701cc53d12 100644 --- a/packages/ozone/src/context.ts +++ b/packages/ozone/src/context.ts @@ -58,8 +58,6 @@ export class AppContext { aud, keypair: signingKey, }) - const appviewAuth = async () => - cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined const backgroundQueue = new BackgroundQueue(db) const eventPusher = new EventPusher(db, createAuthHeaders, { @@ -67,11 +65,17 @@ export class AppContext { pds: cfg.pds ?? undefined, }) + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const modService = ModerationService.creator( + cfg, backgroundQueue, + idResolver, eventPusher, appviewAgent, - appviewAuth, + createAuthHeaders, cfg.service.did, overrides?.imgInvalidator, cfg.cdn.paths, @@ -79,10 +83,6 @@ export class AppContext { const communicationTemplateService = CommunicationTemplateService.creator() - const idResolver = new IdResolver({ - plcUrl: cfg.identity.plcUrl, - }) - const sequencer = new Sequencer(db) return new AppContext( diff --git a/packages/ozone/src/daemon/context.ts b/packages/ozone/src/daemon/context.ts index 5af19d89bc4..3ed0596c2ed 100644 --- a/packages/ozone/src/daemon/context.ts +++ b/packages/ozone/src/daemon/context.ts @@ -7,6 +7,7 @@ import { EventPusher } from './event-pusher' import { EventReverser } from './event-reverser' import { ModerationService, ModerationServiceCreator } from '../mod-service' import { BackgroundQueue } from '../background' +import { IdResolver } from '@atproto/identity' export type DaemonContextOptions = { db: Database @@ -39,21 +40,26 @@ export class DaemonContext { keypair: signingKey, }) - const appviewAuth = async () => - cfg.appview.did ? createAuthHeaders(cfg.appview.did) : undefined - const eventPusher = new EventPusher(db, createAuthHeaders, { appview: cfg.appview, pds: cfg.pds ?? undefined, }) + const backgroundQueue = new BackgroundQueue(db) + const idResolver = new IdResolver({ + plcUrl: cfg.identity.plcUrl, + }) + const modService = ModerationService.creator( + cfg, backgroundQueue, + idResolver, eventPusher, appviewAgent, - appviewAuth, + createAuthHeaders, cfg.service.did, ) + const eventReverser = new EventReverser(db, modService) return new DaemonContext({ diff --git a/packages/ozone/src/lexicon/lexicons.ts b/packages/ozone/src/lexicon/lexicons.ts index 14e4c1cb81e..72ac23230f4 100644 --- a/packages/ozone/src/lexicon/lexicons.ts +++ b/packages/ozone/src/lexicon/lexicons.ts @@ -897,6 +897,10 @@ export const schemaDict = { type: 'string', description: 'The subject line of the email sent to the user.', }, + content: { + type: 'string', + description: 'The content of the email sent to the user.', + }, comment: { type: 'string', description: 'Additional comment about the outgoing comm.', diff --git a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..acfda37abbc 100644 --- a/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/ozone/src/lexicon/types/com/atproto/admin/defs.ts @@ -704,6 +704,8 @@ export function validateModEventUnmute(v: unknown): ValidationResult { export interface ModEventEmail { /** The subject line of the email sent to the user. */ subjectLine: string + /** The content of the email sent to the user. */ + content?: string /** Additional comment about the outgoing comm. */ comment?: string [k: string]: unknown diff --git a/packages/ozone/src/mod-service/index.ts b/packages/ozone/src/mod-service/index.ts index d0690fce57a..1ca793d5601 100644 --- a/packages/ozone/src/mod-service/index.ts +++ b/packages/ozone/src/mod-service/index.ts @@ -1,9 +1,13 @@ +import net from 'node:net' +import { Insertable, sql } from 'kysely' import { CID } from 'multiformats/cid' import { AtUri, INVALID_HANDLE } from '@atproto/syntax' import { InvalidRequestError } from '@atproto/xrpc-server' import { addHoursToDate } from '@atproto/common' +import { IdResolver } from '@atproto/identity' +import AtpAgent from '@atproto/api' import { Database } from '../db' -import { AppviewAuth, ModerationViews } from './views' +import { AuthHeaders, ModerationViews } from './views' import { Main as StrongRef } from '../lexicon/types/com/atproto/repo/strongRef' import { isModEventComment, @@ -30,9 +34,7 @@ import { } from './types' import { ModerationEvent } from '../db/schema/moderation_event' import { StatusKeyset, TimeIdKeyset, paginate } from '../db/pagination' -import AtpAgent from '@atproto/api' import { Label } from '../lexicon/types/com/atproto/label/defs' -import { Insertable, sql } from 'kysely' import { ModSubject, RecordSubject, @@ -46,26 +48,31 @@ import { BackgroundQueue } from '../background' import { EventPusher } from '../daemon' import { ImageInvalidator } from '../image-invalidator' import { httpLogger as log } from '../logger' +import { OzoneConfig } from '../config' export type ModerationServiceCreator = (db: Database) => ModerationService export class ModerationService { constructor( public db: Database, + public cfg: OzoneConfig, public backgroundQueue: BackgroundQueue, + public idResolver: IdResolver, public eventPusher: EventPusher, public appviewAgent: AtpAgent, - private appviewAuth: AppviewAuth, + private createAuthHeaders: (aud: string) => Promise, public serverDid: string, public imgInvalidator?: ImageInvalidator, public cdnPaths?: string[], ) {} static creator( + cfg: OzoneConfig, backgroundQueue: BackgroundQueue, + idResolver: IdResolver, eventPusher: EventPusher, appviewAgent: AtpAgent, - appviewAuth: AppviewAuth, + createAuthHeaders: (aud: string) => Promise, serverDid: string, imgInvalidator?: ImageInvalidator, cdnPaths?: string[], @@ -73,17 +80,21 @@ export class ModerationService { return (db: Database) => new ModerationService( db, + cfg, backgroundQueue, + idResolver, eventPusher, appviewAgent, - appviewAuth, + createAuthHeaders, serverDid, imgInvalidator, cdnPaths, ) } - views = new ModerationViews(this.db, this.appviewAgent, this.appviewAuth) + views = new ModerationViews(this.db, this.appviewAgent, () => + this.createAuthHeaders(this.cfg.appview.did), + ) async getEvent(id: number): Promise { return await this.db.db @@ -291,6 +302,9 @@ export class ModerationService { if (isModEventEmail(event)) { meta.subjectLine = event.subjectLine + if (event.content) { + meta.content = event.content + } } const subjectInfo = subject.info() @@ -903,6 +917,49 @@ export class ModerationService { ) .execute() } + + async sendEmail(opts: { + content: string + recipientDid: string + subject: string + }) { + const { subject, content, recipientDid } = opts + const { pds } = await this.idResolver.did.resolveAtprotoData(recipientDid) + const url = new URL(pds) + if (!this.cfg.service.devMode && !isSafeUrl(url)) { + throw new InvalidRequestError('Invalid pds service in DID doc') + } + const agent = new AtpAgent({ service: url }) + const { data: serverInfo } = + await agent.api.com.atproto.server.describeServer() + if (serverInfo.did !== `did:web:${url.hostname}`) { + // @TODO do bidirectional check once implemented. in the meantime, + // matching did to hostname we're talking to is pretty good. + throw new InvalidRequestError('Invalid pds service in DID doc') + } + const { data: delivery } = await agent.api.com.atproto.admin.sendEmail( + { + subject, + content, + recipientDid, + senderDid: this.cfg.service.did, + }, + { + encoding: 'application/json', + ...(await this.createAuthHeaders(serverInfo.did)), + }, + ) + if (!delivery.sent) { + throw new InvalidRequestError('Email was accepted but not sent') + } + } +} + +const isSafeUrl = (url: URL) => { + if (url.protocol !== 'https:') return false + if (!url.hostname || url.hostname === 'localhost') return false + if (net.isIP(url.hostname) === 0) return false + return true } const TAKEDOWNS = ['pds_takedown' as const, 'appview_takedown' as const] diff --git a/packages/ozone/src/mod-service/views.ts b/packages/ozone/src/mod-service/views.ts index f1188968cbe..498091a8bd0 100644 --- a/packages/ozone/src/mod-service/views.ts +++ b/packages/ozone/src/mod-service/views.ts @@ -26,20 +26,17 @@ import { REASONOTHER } from '../lexicon/types/com/atproto/moderation/defs' import { subjectFromEventRow, subjectFromStatusRow } from './subject' import { formatLabel } from './util' -export type AppviewAuth = () => Promise< - | { - headers: { - authorization: string - } - } - | undefined -> +export type AuthHeaders = { + headers: { + authorization: string + } +} export class ModerationViews { constructor( private db: Database, private appviewAgent: AtpAgent, - private appviewAuth: AppviewAuth, + private appviewAuth: () => Promise, ) {} async getAccoutInfosByDid(dids: string[]): Promise> { @@ -154,6 +151,7 @@ export class ModerationViews { eventView.event = { ...eventView.event, subjectLine: event.meta?.subjectLine ?? '', + content: event.meta?.content, } } diff --git a/packages/ozone/tests/moderation-events.test.ts b/packages/ozone/tests/moderation-events.test.ts index fbe571a8172..1212a6049e6 100644 --- a/packages/ozone/tests/moderation-events.test.ts +++ b/packages/ozone/tests/moderation-events.test.ts @@ -1,4 +1,6 @@ import assert from 'node:assert' +import EventEmitter, { once } from 'node:events' +import Mail from 'nodemailer/lib/mailer' import { TestNetwork, SeedClient, basicSeed } from '@atproto/dev-env' import AtpAgent, { ComAtprotoAdminDefs, @@ -422,4 +424,51 @@ describe('moderation-events', () => { }) }) }) + + describe('email event', () => { + let sendMailOriginal + const mailCatcher = new EventEmitter() + const getMailFrom = async (promise): Promise => { + const result = await Promise.all([once(mailCatcher, 'mail'), promise]) + return result[0][0] + } + + beforeAll(() => { + const mailer = network.pds.ctx.moderationMailer + // Catch emails for use in tests + sendMailOriginal = mailer.transporter.sendMail + mailer.transporter.sendMail = async (opts) => { + const result = await sendMailOriginal.call(mailer.transporter, opts) + mailCatcher.emit('mail', opts) + return result + } + }) + + afterAll(() => { + network.pds.ctx.moderationMailer.transporter.sendMail = sendMailOriginal + }) + + it('sends email via pds.', async () => { + const mail = await getMailFrom( + emitModerationEvent({ + event: { + $type: 'com.atproto.admin.defs#modEventEmail', + comment: 'Reaching out to Alice', + subjectLine: 'Hello', + content: 'Hey Alice, how are you?', + }, + subject: { + $type: 'com.atproto.admin.defs#repoRef', + did: sc.dids.alice, + }, + createdBy: sc.dids.bob, + }), + ) + expect(mail).toEqual({ + to: 'alice@test.com', + subject: 'Hello', + html: 'Hey Alice, how are you?', + }) + }) + }) }) diff --git a/packages/pds/src/api/com/atproto/admin/sendEmail.ts b/packages/pds/src/api/com/atproto/admin/sendEmail.ts index e23d6bea5c1..f6d8cce8d19 100644 --- a/packages/pds/src/api/com/atproto/admin/sendEmail.ts +++ b/packages/pds/src/api/com/atproto/admin/sendEmail.ts @@ -1,23 +1,23 @@ +import assert from 'node:assert' import { AuthRequiredError, InvalidRequestError } from '@atproto/xrpc-server' import { Server } from '../../../../lexicon' import AppContext from '../../../../context' -import { authPassthru, resultPassthru } from '../../../proxy' +import { resultPassthru } from '../../../proxy' export default function (server: Server, ctx: AppContext) { server.com.atproto.admin.sendEmail({ - auth: ctx.authVerifier.role, - handler: async ({ req, input, auth }) => { - if (!auth.credentials.admin && !auth.credentials.moderator) { + auth: ctx.authVerifier.roleOrAdminService, + handler: async ({ input, auth }) => { + if (auth.credentials.type === 'role' && !auth.credentials.moderator) { throw new AuthRequiredError('Insufficient privileges') } const { content, recipientDid, - senderDid, - subject = 'Message from Bluesky moderator', - comment, + subject = 'Message via your PDS', } = input.body + const account = await ctx.accountManager.getAccount(recipientDid, { includeDeactivated: true, includeTakenDown: true, @@ -27,11 +27,15 @@ export default function (server: Server, ctx: AppContext) { } if (ctx.entrywayAgent) { + assert(ctx.cfg.entryway) return resultPassthru( - await ctx.entrywayAgent.com.atproto.admin.sendEmail( - input.body, - authPassthru(req, true), - ), + await ctx.entrywayAgent.com.atproto.admin.sendEmail(input.body, { + encoding: 'application/json', + ...(await ctx.serviceAuthHeaders( + recipientDid, + ctx.cfg.entryway?.did, + )), + }), ) } @@ -44,24 +48,6 @@ export default function (server: Server, ctx: AppContext) { { subject, to: account.email }, ) - if (ctx.moderationAgent) { - await ctx.moderationAgent.api.com.atproto.admin.emitModerationEvent( - { - event: { - $type: 'com.atproto.admin.defs#modEventEmail', - subjectLine: subject, - comment, - }, - subject: { - $type: 'com.atproto.admin.defs#repoRef', - did: recipientDid, - }, - createdBy: senderDid, - }, - { ...authPassthru(req), encoding: 'application/json' }, - ) - } - return { encoding: 'application/json', body: { sent: true }, diff --git a/packages/pds/src/lexicon/lexicons.ts b/packages/pds/src/lexicon/lexicons.ts index 14e4c1cb81e..72ac23230f4 100644 --- a/packages/pds/src/lexicon/lexicons.ts +++ b/packages/pds/src/lexicon/lexicons.ts @@ -897,6 +897,10 @@ export const schemaDict = { type: 'string', description: 'The subject line of the email sent to the user.', }, + content: { + type: 'string', + description: 'The content of the email sent to the user.', + }, comment: { type: 'string', description: 'Additional comment about the outgoing comm.', diff --git a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts index a713a635635..acfda37abbc 100644 --- a/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts +++ b/packages/pds/src/lexicon/types/com/atproto/admin/defs.ts @@ -704,6 +704,8 @@ export function validateModEventUnmute(v: unknown): ValidationResult { export interface ModEventEmail { /** The subject line of the email sent to the user. */ subjectLine: string + /** The content of the email sent to the user. */ + content?: string /** Additional comment about the outgoing comm. */ comment?: string [k: string]: unknown