Merge branch 'main' of github.com:bluesky-social/atproto into nullabl…

…e-review-state

.github/workflows/build-and-push-ozone-aws.yaml

@@ -3,6 +3,7 @@ on:
   push:
     branches:
       - main
+      - ozone-cdn-invalidation
 env:
   REGISTRY: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_REGISTRY }}
   USERNAME: ${{ secrets.AWS_ECR_REGISTRY_USEAST2_PACKAGES_USERNAME }}

lexicons/com/atproto/admin/defs.json

@@ -594,6 +594,10 @@
           "type": "string",
           "description": "The subject line of the email sent to the user."
         },
+        "content": {
+          "type": "string",
+          "description": "The content of the email sent to the user."
+        },
         "comment": {
           "type": "string",
           "description": "Additional comment about the outgoing comm."

packages/api/CHANGELOG.md

@@ -1,5 +1,42 @@
 # @atproto/api
 
+## 0.10.4
+
+### Patch Changes
+
+- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Export regex from rich text detection
+
+- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Disallow rare unicode whitespace characters from tags
+
+- [#2260](https://github.com/bluesky-social/atproto/pull/2260) [`6ec885992`](https://github.com/bluesky-social/atproto/commit/6ec8859929a16f9725319cc398b716acf913b01f) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Allow tags to lead with numbers
+
+## 0.10.3
+
+### Patch Changes
+
+- [#2247](https://github.com/bluesky-social/atproto/pull/2247) [`2a0ceb818`](https://github.com/bluesky-social/atproto/commit/2a0ceb8180faa17de8061d4fa6c361b57a2005ed) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Fix double sanitization bug when editing muted words.
+
+- [#2247](https://github.com/bluesky-social/atproto/pull/2247) [`2a0ceb818`](https://github.com/bluesky-social/atproto/commit/2a0ceb8180faa17de8061d4fa6c361b57a2005ed) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - More sanitization of muted words, including newlines and leading/trailing whitespace
+
+- [#2247](https://github.com/bluesky-social/atproto/pull/2247) [`2a0ceb818`](https://github.com/bluesky-social/atproto/commit/2a0ceb8180faa17de8061d4fa6c361b57a2005ed) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Add `sanitizeMutedWordValue` util
+
+- [#2247](https://github.com/bluesky-social/atproto/pull/2247) [`2a0ceb818`](https://github.com/bluesky-social/atproto/commit/2a0ceb8180faa17de8061d4fa6c361b57a2005ed) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Handle hash emoji in mute words
+
+## 0.10.2
+
+### Patch Changes
+
+- [#2245](https://github.com/bluesky-social/atproto/pull/2245) [`61b3d2525`](https://github.com/bluesky-social/atproto/commit/61b3d25253353db2da1336004f94e7dc5adb0410) Thanks [@mary-ext](https://github.com/mary-ext)! - Prevent hashtag emoji from being parsed as a tag
+
+- [#2218](https://github.com/bluesky-social/atproto/pull/2218) [`43531905c`](https://github.com/bluesky-social/atproto/commit/43531905ce1aec6d36d9be5943782811ecca6e6d) Thanks [@estrattonbailey](https://github.com/estrattonbailey)! - Fix mute word upsert logic by ensuring we're comparing sanitized word values
+
+- [#2245](https://github.com/bluesky-social/atproto/pull/2245) [`61b3d2525`](https://github.com/bluesky-social/atproto/commit/61b3d25253353db2da1336004f94e7dc5adb0410) Thanks [@mary-ext](https://github.com/mary-ext)! - Properly calculate length of tag
+
+- Updated dependencies [[`0c815b964`](https://github.com/bluesky-social/atproto/commit/0c815b964c030aa0f277c40bf9786f130dc320f4)]:
+  - @atproto/syntax@0.2.0
+  - @atproto/lexicon@0.3.2
+  - @atproto/xrpc@0.4.2
+
 ## 0.10.1
 
 ### Patch Changes

packages/api/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@atproto/api",
-  "version": "0.10.1",
+  "version": "0.10.4",
   "license": "MIT",
   "description": "Client library for atproto and Bluesky",
   "keywords": [

packages/api/src/bsky-agent.ts

@@ -13,6 +13,7 @@ import {
   BskyThreadViewPreference,
   BskyInterestsPreference,
 } from './types'
+import { sanitizeMutedWordValue } from './util'
 
 const FEED_VIEW_PREF_DEFAULTS = {
   hideReplies: false,
@@ -565,16 +566,108 @@ export class BskyAgent extends AtpAgent {
     })
   }
 
-  async upsertMutedWords(mutedWords: AppBskyActorDefs.MutedWord[]) {
-    await updateMutedWords(this, mutedWords, 'upsert')
+  async upsertMutedWords(newMutedWords: AppBskyActorDefs.MutedWord[]) {
+    await updatePreferences(this, (prefs: AppBskyActorDefs.Preferences) => {
+      let mutedWordsPref = prefs.findLast(
+        (pref) =>
+          AppBskyActorDefs.isMutedWordsPref(pref) &&
+          AppBskyActorDefs.validateMutedWordsPref(pref).success,
+      )
+
+      if (mutedWordsPref && AppBskyActorDefs.isMutedWordsPref(mutedWordsPref)) {
+        for (const updatedWord of newMutedWords) {
+          let foundMatch = false
+          const sanitizedUpdatedValue = sanitizeMutedWordValue(
+            updatedWord.value,
+          )
+
+          // was trimmed down to an empty string e.g. single `#`
+          if (!sanitizedUpdatedValue) continue
+
+          for (const existingItem of mutedWordsPref.items) {
+            if (existingItem.value === sanitizedUpdatedValue) {
+              existingItem.targets = Array.from(
+                new Set([...existingItem.targets, ...updatedWord.targets]),
+              )
+              foundMatch = true
+              break
+            }
+          }
+
+          if (!foundMatch) {
+            mutedWordsPref.items.push({
+              ...updatedWord,
+              value: sanitizedUpdatedValue,
+            })
+          }
+        }
+      } else {
+        // if the pref doesn't exist, create it
+        mutedWordsPref = {
+          items: newMutedWords.map((w) => ({
+            ...w,
+            value: sanitizeMutedWordValue(w.value),
+          })),
+        }
+      }
+
+      return prefs
+        .filter((p) => !AppBskyActorDefs.isMutedWordsPref(p))
+        .concat([
+          { ...mutedWordsPref, $type: 'app.bsky.actor.defs#mutedWordsPref' },
+        ])
+    })
   }
 
   async updateMutedWord(mutedWord: AppBskyActorDefs.MutedWord) {
-    await updateMutedWords(this, [mutedWord], 'update')
+    await updatePreferences(this, (prefs: AppBskyActorDefs.Preferences) => {
+      let mutedWordsPref = prefs.findLast(
+        (pref) =>
+          AppBskyActorDefs.isMutedWordsPref(pref) &&
+          AppBskyActorDefs.validateMutedWordsPref(pref).success,
+      )
+
+      if (mutedWordsPref && AppBskyActorDefs.isMutedWordsPref(mutedWordsPref)) {
+        for (const existingItem of mutedWordsPref.items) {
+          if (existingItem.value === mutedWord.value) {
+            existingItem.targets = mutedWord.targets
+            break
+          }
+        }
+      }
+
+      return prefs
+        .filter((p) => !AppBskyActorDefs.isMutedWordsPref(p))
+        .concat([
+          { ...mutedWordsPref, $type: 'app.bsky.actor.defs#mutedWordsPref' },
+        ])
+    })
   }
 
   async removeMutedWord(mutedWord: AppBskyActorDefs.MutedWord) {
-    await updateMutedWords(this, [mutedWord], 'remove')
+    await updatePreferences(this, (prefs: AppBskyActorDefs.Preferences) => {
+      let mutedWordsPref = prefs.findLast(
+        (pref) =>
+          AppBskyActorDefs.isMutedWordsPref(pref) &&
+          AppBskyActorDefs.validateMutedWordsPref(pref).success,
+      )
+
+      if (mutedWordsPref && AppBskyActorDefs.isMutedWordsPref(mutedWordsPref)) {
+        for (let i = 0; i < mutedWordsPref.items.length; i++) {
+          const existing = mutedWordsPref.items[i]
+          if (existing.value === mutedWord.value) {
+            mutedWordsPref.items.splice(i, 1)
+            break
+          }
+        }
+      }
+
+      return prefs
+        .filter((p) => !AppBskyActorDefs.isMutedWordsPref(p))
+        .concat([
+          { ...mutedWordsPref, $type: 'app.bsky.actor.defs#mutedWordsPref' },
+        ])
+    })
   }
 
   async hidePost(postUri: string) {
@@ -646,76 +739,6 @@ async function updateFeedPreferences(
   return res
 }
 
-/**
- * A helper specifically for updating muted words preferences
- */
-async function updateMutedWords(
-  agent: BskyAgent,
-  mutedWords: AppBskyActorDefs.MutedWord[],
-  action: 'upsert' | 'update' | 'remove',
-) {
-  const sanitizeMutedWord = (word: AppBskyActorDefs.MutedWord) => ({
-    value: word.value.replace(/^#/, ''),
-    targets: word.targets,
-  })
-
-  await updatePreferences(agent, (prefs: AppBskyActorDefs.Preferences) => {
-    let mutedWordsPref = prefs.findLast(
-      (pref) =>
-        AppBskyActorDefs.isMutedWordsPref(pref) &&
-        AppBskyActorDefs.validateMutedWordsPref(pref).success,
-    )
-
-    if (mutedWordsPref && AppBskyActorDefs.isMutedWordsPref(mutedWordsPref)) {
-      if (action === 'upsert' || action === 'update') {
-        for (const word of mutedWords) {
-          let foundMatch = false
-
-          for (const existingItem of mutedWordsPref.items) {
-            if (existingItem.value === sanitizeMutedWord(word).value) {
-              existingItem.targets =
-                action === 'upsert'
-                  ? Array.from(
-                      new Set([...existingItem.targets, ...word.targets]),
-                    )
-                  : word.targets
-              foundMatch = true
-              break
-            }
-          }
-
-          if (action === 'upsert' && !foundMatch) {
-            mutedWordsPref.items.push(sanitizeMutedWord(word))
-          }
-        }
-      } else if (action === 'remove') {
-        for (const word of mutedWords) {
-          for (let i = 0; i < mutedWordsPref.items.length; i++) {
-            const existing = mutedWordsPref.items[i]
-            if (existing.value === sanitizeMutedWord(word).value) {
-              mutedWordsPref.items.splice(i, 1)
-              break
-            }
-          }
-        }
-      }
-    } else {
-      // if the pref doesn't exist, create it
-      if (action === 'upsert') {
-        mutedWordsPref = {
-          items: mutedWords.map(sanitizeMutedWord),
-        }
-      }
-    }
-
-    return prefs
-      .filter((p) => !AppBskyActorDefs.isMutedWordsPref(p))
-      .concat([
-        { ...mutedWordsPref, $type: 'app.bsky.actor.defs#mutedWordsPref' },
-      ])
-  })
-}
-
 async function updateHiddenPost(
   agent: BskyAgent,
   postUri: string,

packages/api/src/client/lexicons.ts

@@ -903,6 +903,10 @@ export const schemaDict = {
             type: 'string',
             description: 'The subject line of the email sent to the user.',
           },
+          content: {
+            type: 'string',
+            description: 'The content of the email sent to the user.',
+          },
           comment: {
             type: 'string',
             description: 'Additional comment about the outgoing comm.',

packages/api/src/client/types/com/atproto/admin/defs.ts

@@ -707,6 +707,8 @@ export function validateModEventUnmute(v: unknown): ValidationResult {
 export interface ModEventEmail {
   /** The subject line of the email sent to the user. */
   subjectLine: string
+  /** The content of the email sent to the user. */
+  content?: string
   /** Additional comment about the outgoing comm. */
   comment?: string
   [k: string]: unknown

packages/api/src/index.ts

@@ -8,11 +8,13 @@ export {
 } from '@atproto/lexicon'
 export { parseLanguage } from '@atproto/common-web'
 export * from './types'
+export * from './util'
 export * from './client'
 export * from './agent'
 export * from './rich-text/rich-text'
 export * from './rich-text/sanitization'
 export * from './rich-text/unicode'
+export * from './rich-text/util'
 export * from './moderation'
 export * from './moderation/types'
 export { LABELS } from './moderation/const/labels'

packages/api/src/rich-text/detection.ts

@@ -1,6 +1,12 @@
 import TLDs from 'tlds'
 import { AppBskyRichtextFacet } from '../client'
 import { UnicodeString } from './unicode'
+import {
+  URL_REGEX,
+  MENTION_REGEX,
+  TAG_REGEX,
+  TRAILING_PUNCTUATION_REGEX,
+} from './util'
 
 export type Facet = AppBskyRichtextFacet.Main
 
@@ -9,7 +15,7 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined {
   const facets: Facet[] = []
   {
     // mentions
-    const re = /(^|\s|\()(@)([a-zA-Z0-9.-]+)(\b)/g
+    const re = MENTION_REGEX
     while ((match = re.exec(text.utf16))) {
       if (!isValidDomain(match[3]) && !match[3].endsWith('.test')) {
         continue // probably not a handle
@@ -33,8 +39,7 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined {
   }
   {
     // links
-    const re =
-      /(^|\s|\()((https?:\/\/[\S]+)|((?<domain>[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*))/gim
+    const re = URL_REGEX
     while ((match = re.exec(text.utf16))) {
       let uri = match[2]
       if (!uri.startsWith('http')) {
@@ -70,27 +75,28 @@ export function detectFacets(text: UnicodeString): Facet[] | undefined {
     }
   }
   {
-    const re = /(?:^|\s)(#[^\d\s]\S*)(?=\s)?/g
+    const re = TAG_REGEX
     while ((match = re.exec(text.utf16))) {
-      let [tag] = match
-      const hasLeadingSpace = /^\s/.test(tag)
+      let [, leading, tag] = match
 
-      tag = tag.trim().replace(/\p{P}+$/gu, '') // strip ending punctuation
+      if (!tag) continue
 
-      // inclusive of #, max of 64 chars
-      if (tag.length > 66) continue
+      // strip ending punctuation and any spaces
+      tag = tag.trim().replace(TRAILING_PUNCTUATION_REGEX, '')
 
-      const index = match.index + (hasLeadingSpace ? 1 : 0)
+      if (tag.length === 0 || tag.length > 64) continue
+
+      const index = match.index + leading.length
 
       facets.push({
         index: {
           byteStart: text.utf16IndexToUtf8Index(index),
-          byteEnd: text.utf16IndexToUtf8Index(index + tag.length), // inclusive of last char
+          byteEnd: text.utf16IndexToUtf8Index(index + 1 + tag.length),
         },
         features: [
           {
             $type: 'app.bsky.richtext.facet#tag',
-            tag: tag.replace(/^#/, ''),
+            tag: tag,
           },
         ],
       })

packages/api/src/rich-text/util.ts

@@ -0,0 +1,11 @@
+export const MENTION_REGEX = /(^|\s|\()(@)([a-zA-Z0-9.-]+)(\b)/g
+export const URL_REGEX =
+  /(^|\s|\()((https?:\/\/[\S]+)|((?<domain>[a-z][a-z0-9]*(\.[a-z0-9]+)+)[\S]*))/gim
+export const TRAILING_PUNCTUATION_REGEX = /\p{P}+$/gu
+
+/**
+ * `\ufe0f` emoji modifier
+ * `\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2` zero-width spaces (likely incomplete)
+ */
+export const TAG_REGEX =
+  /(^|\s)[#＃]((?!\ufe0f)[^\s\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]*[^\d\s\p{P}\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]+[^\s\u00AD\u2060\u200A\u200B\u200C\u200D\u20e2]*)?/gu