From d18524132a9a245f02e08e276a287026915d6bcc Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Wed, 10 Apr 2024 15:10:52 +0300 Subject: [PATCH 1/6] TW-1364 Implement returning different ads data depending on web extension version --- src/advertising/external-ads.ts | 212 +++++++++ src/advertising/slise.ts | 162 ------- src/index.ts | 10 +- src/routers/slise-ad-rules/ad-places.ts | 588 ++++++++++++++++-------- src/routers/slise-ad-rules/index.ts | 10 +- src/routers/slise-ad-rules/providers.ts | 157 +++++-- src/utils/express-helpers.ts | 59 ++- src/utils/migrations.ts | 34 ++ src/utils/schemas.ts | 73 +-- 9 files changed, 855 insertions(+), 450 deletions(-) create mode 100644 src/advertising/external-ads.ts delete mode 100644 src/advertising/slise.ts create mode 100644 src/utils/migrations.ts diff --git a/src/advertising/external-ads.ts b/src/advertising/external-ads.ts new file mode 100644 index 0000000..fa765c6 --- /dev/null +++ b/src/advertising/external-ads.ts @@ -0,0 +1,212 @@ +import { objectStorageMethodsFactory, redisClient } from '../redis'; +import { isDefined } from '../utils/helpers'; + +/** Style properties names that are likely to be unnecessary for banners are skipped */ +export const stylePropsNames = [ + 'align-content', + 'align-items', + 'align-self', + 'alignment-baseline', + 'aspect-ratio', + 'background', + 'border', + 'border-radius', + 'bottom', + 'box-shadow', + 'box-sizing', + 'display', + 'flex', + 'flex-basis', + 'flex-direction', + 'flex-flow', + 'flex-grow', + 'flex-shrink', + 'flex-wrap', + 'float', + 'height', + 'justify-content', + 'justify-items', + 'justify-self', + 'left', + 'margin', + 'margin-block', + 'margin-block-end', + 'margin-block-start', + 'margin-bottom', + 'margin-inline', + 'margin-inline-end', + 'margin-inline-start', + 'margin-left', + 'margin-right', + 'margin-top', + 'max-block-size', + 'max-height', + 'max-inline-size', + 'max-width', + 'min-block-size', + 'min-height', + 'min-inline-size', + 'min-width', + 'opacity', + 'overflow', + 'overflow-anchor', + 'overflow-wrap', + 'overflow-x', + 'overflow-y', + 'padding', + 'padding-block', + 'padding-block-end', + 'padding-block-start', + 'padding-bottom', + 'padding-inline', + 'padding-inline-end', + 'padding-inline-start', + 'padding-left', + 'padding-right', + 'padding-top', + 'position', + 'right', + 'text-align', + 'top', + 'vertical-align', + 'visibility', + 'width', + 'z-index', + 'background-image', + 'background-clip', + 'background-color' +]; +export type StylePropName = (typeof stylePropsNames)[number]; + +interface AdStylesOverrides { + parentDepth: number; + style: Record; +} + +interface ExtVersionConstraints { + firstExtVersion?: string; + lastExtVersion?: string; +} + +export interface AdPlacesRule extends ExtVersionConstraints { + urlRegexes: string[]; + selector: { + isMultiple: boolean; + cssString: string; + parentDepth: number; + shouldUseDivWrapper: boolean; + divWrapperStyle?: Record; + }; + stylesOverrides?: AdStylesOverrides[]; + shouldHideOriginal?: boolean; +} + +export interface PermanentAdPlacesRule extends ExtVersionConstraints { + urlRegexes: string[]; + adSelector: { + isMultiple: boolean; + cssString: string; + parentDepth: number; + }; + parentSelector: { + isMultiple: boolean; + cssString: string; + parentDepth: number; + }; + insertionIndex?: number; + insertBeforeSelector?: string; + insertAfterSelector?: string; + insertionsCount?: number; + shouldUseDivWrapper: boolean; + elementStyle?: Record; + divWrapperStyle?: Record; + elementToMeasureSelector?: string; + stylesOverrides?: AdStylesOverrides[]; + shouldHideOriginal?: boolean; +} + +export interface AdProvidersByDomainRule extends ExtVersionConstraints { + urlRegexes: string[]; + providers: string[]; +} + +export interface AdProviderSelectorsRule extends ExtVersionConstraints { + selectors: string[]; +} + +export interface AdProviderForAllSitesRule extends ExtVersionConstraints { + providers: string[]; +} + +const AD_PLACES_RULES_KEY = 'slise_ad_places_rules'; +const AD_PROVIDERS_BY_SITES_KEY = 'slise_ad_providers_by_sites'; +const AD_PROVIDERS_ALL_SITES_KEY = 'slise_ad_providers_all_sites'; +const SLISE_AD_PROVIDERS_LIST_KEY = 'slise_ad_providers_list'; +const AD_PROVIDERS_LIST_KEY = 'ad_providers_list'; +const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_slise_ad_places_rules'; +const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules'; + +export const adPlacesRulesMethods = objectStorageMethodsFactory(AD_PLACES_RULES_KEY, []); + +export const adProvidersByDomainRulesMethods = objectStorageMethodsFactory( + AD_PROVIDERS_BY_SITES_KEY, + [] +); + +/** @deprecated */ +export const sliseAdProvidersMethodsLegacy = objectStorageMethodsFactory(SLISE_AD_PROVIDERS_LIST_KEY, []); + +export const adProvidersMethods = objectStorageMethodsFactory(AD_PROVIDERS_LIST_KEY, []); + +export const permanentAdPlacesMethods = objectStorageMethodsFactory( + PERMANENT_AD_PLACES_RULES_KEY, + [] +); + +export const permanentNativeAdPlacesMethods = objectStorageMethodsFactory( + PERMANENT_NATIVE_AD_PLACES_RULES_KEY, + [] +); + +export const getAdProvidersForAllSites = async () => redisClient.smembers(AD_PROVIDERS_ALL_SITES_KEY); + +export const addAdProvidersForAllSites = async (providers: string[]) => + redisClient.sadd(AD_PROVIDERS_ALL_SITES_KEY, ...providers); + +export const removeAdProvidersForAllSites = async (providers: string[]) => + redisClient.srem(AD_PROVIDERS_ALL_SITES_KEY, ...providers); + +const compareVersions = (a: string, b: string) => { + const [aMajor = 0, aMinor = 0, aPatch = 0] = a.split('.').map(Number); + const [bMajor = 0, bMinor = 0, bPatch = 0] = b.split('.').map(Number); + + if (aMajor !== bMajor) { + return aMajor - bMajor; + } + + if (aMinor !== bMinor) { + return aMinor - bMinor; + } + + return aPatch - bPatch; +}; + +export const filterByVersion = (rules: T[], version?: string) => { + if (!isDefined(version)) { + return rules.filter( + ({ firstExtVersion, lastExtVersion }) => !isDefined(firstExtVersion) && !isDefined(lastExtVersion) + ); + } + + return rules.filter(({ firstExtVersion, lastExtVersion }) => { + if (isDefined(firstExtVersion) && compareVersions(firstExtVersion, version) > 0) { + return false; + } + + if (isDefined(lastExtVersion) && compareVersions(lastExtVersion, version) < 0) { + return false; + } + + return true; + }); +}; diff --git a/src/advertising/slise.ts b/src/advertising/slise.ts deleted file mode 100644 index 3e935e2..0000000 --- a/src/advertising/slise.ts +++ /dev/null @@ -1,162 +0,0 @@ -import { objectStorageMethodsFactory, redisClient } from '../redis'; - -/** Style properties names that are likely to be unnecessary for banners are skipped */ -export const stylePropsNames = [ - 'align-content', - 'align-items', - 'align-self', - 'alignment-baseline', - 'aspect-ratio', - 'background', - 'border', - 'border-radius', - 'bottom', - 'box-shadow', - 'box-sizing', - 'display', - 'flex', - 'flex-basis', - 'flex-direction', - 'flex-flow', - 'flex-grow', - 'flex-shrink', - 'flex-wrap', - 'float', - 'height', - 'justify-content', - 'justify-items', - 'justify-self', - 'left', - 'margin', - 'margin-block', - 'margin-block-end', - 'margin-block-start', - 'margin-bottom', - 'margin-inline', - 'margin-inline-end', - 'margin-inline-start', - 'margin-left', - 'margin-right', - 'margin-top', - 'max-block-size', - 'max-height', - 'max-inline-size', - 'max-width', - 'min-block-size', - 'min-height', - 'min-inline-size', - 'min-width', - 'opacity', - 'overflow', - 'overflow-anchor', - 'overflow-wrap', - 'overflow-x', - 'overflow-y', - 'padding', - 'padding-block', - 'padding-block-end', - 'padding-block-start', - 'padding-bottom', - 'padding-inline', - 'padding-inline-end', - 'padding-inline-start', - 'padding-left', - 'padding-right', - 'padding-top', - 'position', - 'right', - 'text-align', - 'top', - 'vertical-align', - 'visibility', - 'width', - 'z-index', - 'background-image', - 'background-clip', - 'background-color' -]; -export type StylePropName = (typeof stylePropsNames)[number]; - -interface SliseAdStylesOverrides { - parentDepth: number; - style: Record; -} - -export interface SliseAdPlacesRule { - urlRegexes: string[]; - selector: { - isMultiple: boolean; - cssString: string; - parentDepth: number; - shouldUseDivWrapper: boolean; - divWrapperStyle?: Record; - }; - stylesOverrides?: SliseAdStylesOverrides[]; - shouldHideOriginal?: boolean; -} - -export interface PermanentSliseAdPlacesRule { - urlRegexes: string[]; - adSelector: { - isMultiple: boolean; - cssString: string; - parentDepth: number; - }; - parentSelector: { - isMultiple: boolean; - cssString: string; - parentDepth: number; - }; - insertionIndex?: number; - insertBeforeSelector?: string; - insertAfterSelector?: string; - insertionsCount?: number; - shouldUseDivWrapper: boolean; - elementStyle?: Record; - divWrapperStyle?: Record; - elementToMeasureSelector?: string; - stylesOverrides?: SliseAdStylesOverrides[]; - shouldHideOriginal?: boolean; -} - -export interface SliseAdProvidersByDomainRule { - urlRegexes: string[]; - providers: string[]; -} - -const SLISE_AD_PLACES_RULES_KEY = 'slise_ad_places_rules'; -const SLISE_AD_PROVIDERS_BY_SITES_KEY = 'slise_ad_providers_by_sites'; -const SLISE_AD_PROVIDERS_ALL_SITES_KEY = 'slise_ad_providers_all_sites'; -const SLISE_AD_PROVIDERS_LIST_KEY = 'slise_ad_providers_list'; -const PERMANENT_SLISE_AD_PLACES_RULES_KEY = 'permanent_slise_ad_places_rules'; -const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules'; - -export const sliseAdPlacesRulesMethods = objectStorageMethodsFactory( - SLISE_AD_PLACES_RULES_KEY, - [] -); - -export const sliseAdProvidersByDomainRulesMethods = objectStorageMethodsFactory( - SLISE_AD_PROVIDERS_BY_SITES_KEY, - [] -); - -export const sliseAdProvidersMethods = objectStorageMethodsFactory(SLISE_AD_PROVIDERS_LIST_KEY, []); - -export const permanentSliseAdPlacesMethods = objectStorageMethodsFactory( - PERMANENT_SLISE_AD_PLACES_RULES_KEY, - [] -); - -export const permanentNativeAdPlacesMethods = objectStorageMethodsFactory( - PERMANENT_NATIVE_AD_PLACES_RULES_KEY, - [] -); - -export const getSliseAdProvidersForAllSites = async () => redisClient.smembers(SLISE_AD_PROVIDERS_ALL_SITES_KEY); - -export const addSliseAdProvidersForAllSites = async (providers: string[]) => - redisClient.sadd(SLISE_AD_PROVIDERS_ALL_SITES_KEY, ...providers); - -export const removeSliseAdProvidersForAllSites = async (providers: string[]) => - redisClient.srem(SLISE_AD_PROVIDERS_ALL_SITES_KEY, ...providers); diff --git a/src/index.ts b/src/index.ts index 19ca0cd..d9149ac 100644 --- a/src/index.ts +++ b/src/index.ts @@ -20,7 +20,7 @@ import { getNotifications } from './notifications/utils/get-notifications.util'; import { getParsedContent } from './notifications/utils/get-parsed-content.util'; import { getPlatforms } from './notifications/utils/get-platforms.util'; import { redisClient } from './redis'; -import { sliseRulesRouter } from './routers/slise-ad-rules'; +import { adRulesRouter } from './routers/slise-ad-rules'; import { getABData } from './utils/ab-test'; import { cancelAliceBobOrder } from './utils/alice-bob/cancel-alice-bob-order'; import { createAliceBobOrder } from './utils/alice-bob/create-alice-bob-order'; @@ -33,6 +33,7 @@ import { CodedError } from './utils/errors'; import { coinGeckoTokens } from './utils/gecko-tokens'; import { getExternalApiErrorPayload, isDefined, isNonEmptyString } from './utils/helpers'; import logger from './utils/logger'; +import { doMigrations } from './utils/migrations'; import { getSignedMoonPayUrl } from './utils/moonpay/get-signed-moonpay-url'; import { getSigningNonce } from './utils/signing-nonce'; import SingleQueryDataProvider from './utils/SingleQueryDataProvider'; @@ -102,6 +103,11 @@ const makeProviderDataRequestHandler = (provider: SingleQueryDataProvider< }; }; +doMigrations().catch(error => { + console.error(error); + process.exit(1); +}); + app.get('/api/top-coins', (_req, res) => { res.status(200).send(coinGeckoTokens); }); @@ -328,7 +334,7 @@ app.get('/api/advertising-info', (_req, res) => { } }); -app.use('/api/slise-ad-rules', sliseRulesRouter); +app.use('/api/slise-ad-rules', adRulesRouter); app.post('/api/magic-square-quest/start', async (req, res) => { try { diff --git a/src/routers/slise-ad-rules/ad-places.ts b/src/routers/slise-ad-rules/ad-places.ts index c3803b8..91d6392 100644 --- a/src/routers/slise-ad-rules/ad-places.ts +++ b/src/routers/slise-ad-rules/ad-places.ts @@ -1,24 +1,25 @@ import { Router } from 'express'; import { + filterByVersion, permanentNativeAdPlacesMethods, - permanentSliseAdPlacesMethods, - sliseAdPlacesRulesMethods -} from '../../advertising/slise'; + permanentAdPlacesMethods, + adPlacesRulesMethods +} from '../../advertising/external-ads'; import { addObjectStorageMethodsToRouter } from '../../utils/express-helpers'; import { hostnamesListSchema, - permanentSliseAdPlacesRulesDictionarySchema, - sliseAdPlacesRulesDictionarySchema + permanentAdPlacesRulesDictionarySchema, + adPlacesRulesDictionarySchema } from '../../utils/schemas'; /** * @swagger * tags: - * name: Slise ad places + * name: Ad places * components: * schemas: - * SliseAdPlacesRuleSelector: + * AdPlacesRuleSelector: * type: object * required: * - isMultiple @@ -42,13 +43,13 @@ import { * children and so on. * shouldUseDivWrapper: * type: boolean - * description: Whether the Slise ads banner should be wrapped in a div + * description: Whether the ads banner should be wrapped in a div * divWrapperStyle: * type: object * description: Style of the div wrapper * additionalProperties: * type: string - * SliseAdStylesOverrides: + * AdStylesOverrides: * type: object * required: * - parentDepth @@ -65,41 +66,59 @@ import { * description: New style of the parent element * additionalProperties: * type: string - * SliseAdPlacesRule: + * ExtVersionConstraints: * type: object - * required: - * - urlRegexes - * - selector * properties: - * urlRegexes: - * type: array - * items: - * type: string - * format: regex - * selector: - * $ref: '#/components/schemas/SliseAdPlacesRuleSelector' - * stylesOverrides: - * type: array - * items: - * $ref: '#/components/schemas/SliseAdStylesOverrides' - * shouldHideOriginal: - * type: boolean - * description: Whether original ads banners should be hidden but not removed - * default: false - * example: - * urlRegexes: - * - '^https://goerli\.etherscan\.io/?$' - * selector: - * isMultiple: false - * cssString: 'main > section div.row > div:nth-child(2) > div' - * parentDepth: 0 - * shouldUseDivWrapper: false - * SliseAdPlacesRulesDictionary: + * firstExtVersion: + * type: string + * format: version + * description: > + * The first extension version where the rule is applicable. If not specified, the rule is applicable + * for all extension versions not greater than the `lastExtVersion` value. If neither of them is specified, + * the rule is applicable for all extension versions. + * lastExtVersion: + * type: string + * format: version + * description: > + * The last extension version where the rule is applicable. If not specified, the rule is applicable + * for all extension versions not less than the `firstExtVersion` value. + * AdPlacesRule: + * allOf: + * - $ref: '#/components/schemas/ExtVersionConstraints' + * - type: object + * required: + * - urlRegexes + * - selector + * properties: + * urlRegexes: + * type: array + * items: + * type: string + * format: regex + * selector: + * $ref: '#/components/schemas/AdPlacesRuleSelector' + * stylesOverrides: + * type: array + * items: + * $ref: '#/components/schemas/AdStylesOverrides' + * shouldHideOriginal: + * type: boolean + * description: Whether original ads banners should be hidden but not removed + * default: false + * example: + * urlRegexes: + * - '^https://goerli\.etherscan\.io/?$' + * selector: + * isMultiple: false + * cssString: 'main > section div.row > div:nth-child(2) > div' + * parentDepth: 0 + * shouldUseDivWrapper: false + * AdPlacesRulesDictionary: * type: object * additionalProperties: * type: array * items: - * $ref: '#/components/schemas/SliseAdPlacesRule' + * $ref: '#/components/schemas/AdPlacesRule' * example: * goerli.etherscan.io: * - urlRegexes: @@ -124,131 +143,133 @@ import { * cssString: 'div.left-container > app-pe-banner:nth-child(2)' * parentDepth: 0 * shouldUseDivWrapper: true - * PermanentSliseAdPlacesRule: - * type: object - * description: > - * This object describes rules of replacing ads banners if they are found and inserting new ads banners if - * they are not found. Exactly one of `insertionIndex`, `insertBeforeSelector` and `insertAfterSelector` - * properties must be specified. - * required: - * - urlRegexes - * - adSelector - * - parentSelector - * - shouldUseDivWrapper - * properties: - * urlRegexes: - * type: array - * items: - * type: string - * format: regex - * adSelector: - * type: object + * PermanentAdPlacesRule: + * allOf: + * - $ref: '#/components/schemas/ExtVersionConstraints' + * - type: object * description: > - * This object describes rules of selecting ads banners in the parents of new ads banners selected - * according to the rules described in the `parentSelector` property. + * This object describes rules of replacing ads banners if they are found and inserting new ads banners if + * they are not found. Exactly one of `insertionIndex`, `insertBeforeSelector` and `insertAfterSelector` + * properties must be specified. * required: - * - isMultiple - * - cssString - * - parentDepth + * - urlRegexes + * - adSelector + * - parentSelector + * - shouldUseDivWrapper * properties: - * isMultiple: - * type: boolean - * description: Whether the selector should return multiple elements - * cssString: - * type: string - * description: CSS selector - * parentDepth: + * urlRegexes: + * type: array + * items: + * type: string + * format: regex + * adSelector: + * type: object + * description: > + * This object describes rules of selecting ads banners in the parents of new ads banners selected + * according to the rules described in the `parentSelector` property. + * required: + * - isMultiple + * - cssString + * - parentDepth + * properties: + * isMultiple: + * type: boolean + * description: Whether the selector should return multiple elements + * cssString: + * type: string + * description: CSS selector + * parentDepth: + * type: number + * min: 0 + * integer: true + * description: > + * Indicates the depth of the parent element of the selected element, i. e. 0 means that the selected + * elements are ads banners themselves, 1 means that the selected elements are ads banners' direct + * children and so on. + * parentSelector: + * type: object + * required: + * - isMultiple + * - cssString + * - parentDepth + * properties: + * isMultiple: + * type: boolean + * description: Whether the selector should return multiple elements + * cssString: + * type: string + * description: CSS selector + * parentDepth: + * type: number + * min: 0 + * integer: true + * description: > + * Indicates the depth of the parent element of the selected element, i. e. 0 means that the selected + * elements are parents of new ads banners themselves, 1 means that the selected elements are their + * direct children and so on. + * insertionIndex: * type: number - * min: 0 * integer: true * description: > - * Indicates the depth of the parent element of the selected element, i. e. 0 means that the selected - * elements are ads banners themselves, 1 means that the selected elements are ads banners' direct - * children and so on. - * parentSelector: - * type: object - * required: - * - isMultiple - * - cssString - * - parentDepth - * properties: - * isMultiple: - * type: boolean - * description: Whether the selector should return multiple elements - * cssString: + * Describes where to insert new ads banners in the selected parents of new ads banners in case if original + * ads banners are not found. If the value is negative, the insertion index will be calculated from the end. + * The counting starts from 0. + * insertBeforeSelector: + * type: string + * description: A selector for the element before which new ads banners should be inserted + * insertAfterSelector: * type: string - * description: CSS selector - * parentDepth: + * description: A selector for the element after which new ads banners should be inserted + * insertionsCount: * type: number - * min: 0 * integer: true + * min: 1 + * default: 1 * description: > - * Indicates the depth of the parent element of the selected element, i. e. 0 means that the selected - * elements are parents of new ads banners themselves, 1 means that the selected elements are their - * direct children and so on. - * insertionIndex: - * type: number - * integer: true - * description: > - * Describes where to insert new ads banners in the selected parents of new ads banners in case if original - * ads banners are not found. If the value is negative, the insertion index will be calculated from the end. - * The counting starts from 0. - * insertBeforeSelector: - * type: string - * description: A selector for the element before which new ads banners should be inserted - * insertAfterSelector: - * type: string - * description: A selector for the element after which new ads banners should be inserted - * insertionsCount: - * type: number - * integer: true - * min: 1 - * default: 1 - * description: > - * Describes how many new ads banners should be inserted in case if original ads banners are not found. - * shouldUseDivWrapper: - * type: boolean - * description: Whether the Slise ads banner should be wrapped in a div - * elementStyle: - * type: object - * description: Style of the new ad banner - * additionalProperties: - * type: string - * divWrapperStyle: - * type: object - * description: Style of the div wrapper - * additionalProperties: - * type: string - * elementToMeasureSelector: - * type: string - * description: A selector of the element which should be measured to define banner size - * stylesOverrides: - * type: array - * items: - * $ref: '#/components/schemas/SliseAdStylesOverrides' - * shouldHideOriginal: - * type: boolean - * description: Whether original ads banners should be hidden but not removed - * default: false - * example: - * urlRegexes: - * - '^https://etherscan\.io/tx/' - * adSelector: - * isMultiple: false - * cssString: '.coinzilla' - * parentDepth: 0 - * parentSelector: - * isMultiple: false - * cssString: '#ContentPlaceHolder1_maintable > * > .row:nth-child(8) > :nth-child(2) > * > *' - * parentDepth: 0 - * insertionIndex: 0 - * shouldUseDivWrapper: false - * PermanentSliseAdPlacesRulesDictionary: + * Describes how many new ads banners should be inserted in case if original ads banners are not found. + * shouldUseDivWrapper: + * type: boolean + * description: Whether the ads banner should be wrapped in a div + * elementStyle: + * type: object + * description: Style of the new ad banner + * additionalProperties: + * type: string + * divWrapperStyle: + * type: object + * description: Style of the div wrapper + * additionalProperties: + * type: string + * elementToMeasureSelector: + * type: string + * description: A selector of the element which should be measured to define banner size + * stylesOverrides: + * type: array + * items: + * $ref: '#/components/schemas/AdStylesOverrides' + * shouldHideOriginal: + * type: boolean + * description: Whether original ads banners should be hidden but not removed + * default: false + * example: + * urlRegexes: + * - '^https://etherscan\.io/tx/' + * adSelector: + * isMultiple: false + * cssString: '.coinzilla' + * parentDepth: 0 + * parentSelector: + * isMultiple: false + * cssString: '#ContentPlaceHolder1_maintable > * > .row:nth-child(8) > :nth-child(2) > * > *' + * parentDepth: 0 + * insertionIndex: 0 + * shouldUseDivWrapper: false + * PermanentAdPlacesRulesDictionary: * type: object * additionalProperties: * type: array * items: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRule' + * $ref: '#/components/schemas/PermanentAdPlacesRule' * example: * etherscan.io: * - urlRegexes: @@ -265,15 +286,53 @@ import { * shouldUseDivWrapper: false */ -export const sliseAdPlacesRulesRouter = Router(); +export const adPlacesRulesRouter = Router(); /** * @swagger + * /api/slise-ad-rules/ad-places/permanent-native/raw/all: + * get: + * summary: Get all rules for permanent native ads places + * tags: + * - Ad places + * responses: + * '200': + * description: Domain - rules list dictionary + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' + * '500': + * $ref: '#/components/responses/ErrorResponse' + * /api/slise-ad-rules/ad-places/permanent-native/{domain}/raw: + * get: + * summary: Get all rules for permanent native ads places for the specified domain + * tags: + * - Ad places + * parameters: + * - in: path + * name: domain + * required: true + * schema: + * type: string + * format: hostname + * example: 'etherscan.io' + * responses: + * '200': + * description: Rules list + * content: + * application/json: + * schema: + * type: array + * items: + * $ref: '#/components/schemas/PermanentAdPlacesRule' + * '500': + * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places/permanent-native/{domain}: * get: - * summary: Get rules for permanent native ads places for the specified domain + * summary: Get rules for permanent native ads places for the specified domain filtered by extension version * tags: - * - Slise ad places + * - Ad places * parameters: * - in: path * name: domain @@ -282,6 +341,13 @@ export const sliseAdPlacesRulesRouter = Router(); * type: string * format: hostname * example: 'etherscan.io' + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Rules list @@ -290,27 +356,35 @@ export const sliseAdPlacesRulesRouter = Router(); * schema: * type: array * items: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRule' + * $ref: '#/components/schemas/PermanentAdPlacesRule' * '500': * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places/permanent-native: * get: - * summary: Get all rules for permanent native ads places + * summary: Get all rules for permanent native ads places filtered by extension version * tags: - * - Slise ad places + * - Ad places + * parameters: + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Domain - rules list dictionary * content: * application/json: * schema: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' * '500': * $ref: '#/components/responses/ErrorResponse' * post: * summary: Add rules for permanent ads places. If rules for a domain already exist, they will be overwritten * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -318,7 +392,7 @@ export const sliseAdPlacesRulesRouter = Router(); * content: * application/json: * schema: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' * responses: * '200': * $ref: '#/components/responses/SuccessResponse' @@ -331,7 +405,7 @@ export const sliseAdPlacesRulesRouter = Router(); * delete: * summary: Remove rules for permanent ads places * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -355,23 +429,61 @@ export const sliseAdPlacesRulesRouter = Router(); * '500': * $ref: '#/components/responses/ErrorResponse' */ -addObjectStorageMethodsToRouter( - sliseAdPlacesRulesRouter, - '/permanent-native', - permanentNativeAdPlacesMethods, - 'domain', - permanentSliseAdPlacesRulesDictionarySchema, - hostnamesListSchema, - entriesCount => `${entriesCount} entries have been removed` -); +addObjectStorageMethodsToRouter(adPlacesRulesRouter, { + path: '/permanent-native', + methods: permanentNativeAdPlacesMethods, + keyName: 'domain', + objectValidationSchema: permanentAdPlacesRulesDictionarySchema, + keysArrayValidationSchema: hostnamesListSchema, + successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`, + transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined) +}); /** * @swagger + * /api/slise-ad-rules/ad-places/permanent/raw/all: + * get: + * summary: Get all rules for permanent ads places + * tags: + * - Ad places + * responses: + * '200': + * description: Domain - rules list dictionary + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' + * '500': + * $ref: '#/components/responses/ErrorResponse' + * /api/slise-ad-rules/ad-places/permanent/{domain}/raw: + * get: + * summary: Get all rules for permanent ads places for the specified domain + * tags: + * - Ad places + * parameters: + * - in: path + * name: domain + * required: true + * schema: + * type: string + * format: hostname + * example: 'etherscan.io' + * responses: + * '200': + * description: Rules list + * content: + * application/json: + * schema: + * type: array + * items: + * $ref: '#/components/schemas/PermanentAdPlacesRule' + * '500': + * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places/permanent/{domain}: * get: - * summary: Get rules for permanent ads places for the specified domain + * summary: Get rules for permanent ads places for the specified domain filtered by extension version * tags: - * - Slise ad places + * - Ad places * parameters: * - in: path * name: domain @@ -380,6 +492,13 @@ addObjectStorageMethodsToRouter( * type: string * format: hostname * example: 'etherscan.io' + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Rules list @@ -388,27 +507,35 @@ addObjectStorageMethodsToRouter( * schema: * type: array * items: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRule' + * $ref: '#/components/schemas/PermanentAdPlacesRule' * '500': * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places/permanent: * get: - * summary: Get all rules for permanent ads places + * summary: Get all rules for permanent ads places filtered by extension version * tags: - * - Slise ad places + * - Ad places + * parameters: + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Domain - rules list dictionary * content: * application/json: * schema: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' * '500': * $ref: '#/components/responses/ErrorResponse' * post: * summary: Add rules for permanent ads places. If rules for a domain already exist, they will be overwritten * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -416,7 +543,7 @@ addObjectStorageMethodsToRouter( * content: * application/json: * schema: - * $ref: '#/components/schemas/PermanentSliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/PermanentAdPlacesRulesDictionary' * responses: * '200': * $ref: '#/components/responses/SuccessResponse' @@ -429,7 +556,7 @@ addObjectStorageMethodsToRouter( * delete: * summary: Remove rules for permanent ads places * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -453,23 +580,61 @@ addObjectStorageMethodsToRouter( * '500': * $ref: '#/components/responses/ErrorResponse' */ -addObjectStorageMethodsToRouter( - sliseAdPlacesRulesRouter, - '/permanent', - permanentSliseAdPlacesMethods, - 'domain', - permanentSliseAdPlacesRulesDictionarySchema, - hostnamesListSchema, - entriesCount => `${entriesCount} entries have been removed` -); +addObjectStorageMethodsToRouter(adPlacesRulesRouter, { + path: '/permanent', + methods: permanentAdPlacesMethods, + keyName: 'domain', + objectValidationSchema: permanentAdPlacesRulesDictionarySchema, + keysArrayValidationSchema: hostnamesListSchema, + successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`, + transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined) +}); /** * @swagger + * /api/slise-ad-rules/ad-places/raw/all: + * get: + * summary: Get all rules for ads places + * tags: + * - Ad places + * responses: + * '200': + * description: Domain - rules list dictionary + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/AdPlacesRulesDictionary' + * '500': + * $ref: '#/components/responses/ErrorResponse' + * /api/slise-ad-rules/ad-places/{domain}/raw: + * get: + * summary: Get all rules for ads places for the specified domain + * tags: + * - Ad places + * parameters: + * - in: path + * name: domain + * required: true + * schema: + * type: string + * format: hostname + * example: 'goerli.etherscan.io' + * responses: + * '200': + * description: Rules list + * content: + * application/json: + * schema: + * type: array + * items: + * $ref: '#/components/schemas/AdPlacesRule' + * '500': + * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places/{domain}: * get: - * summary: Get rules for ads places for the specified domain + * summary: Get rules for ads places for the specified domain filtered by extension version * tags: - * - Slise ad places + * - Ad places * parameters: * - in: path * name: domain @@ -478,6 +643,13 @@ addObjectStorageMethodsToRouter( * type: string * format: hostname * example: 'goerli.etherscan.io' + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Rules list @@ -486,27 +658,35 @@ addObjectStorageMethodsToRouter( * schema: * type: array * items: - * $ref: '#/components/schemas/SliseAdPlacesRule' + * $ref: '#/components/schemas/AdPlacesRule' * '500': * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/ad-places: * get: - * summary: Get all rules for ads places + * summary: Get all rules for ads places filtered by extension version * tags: - * - Slise ad places + * - Ad places + * parameters: + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Domain - rules list dictionary * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/AdPlacesRulesDictionary' * '500': * $ref: '#/components/responses/ErrorResponse' * post: * summary: Add rules for ads places. If rules for a domain already exist, they will be overwritten * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -514,7 +694,7 @@ addObjectStorageMethodsToRouter( * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdPlacesRulesDictionary' + * $ref: '#/components/schemas/AdPlacesRulesDictionary' * responses: * '200': * $ref: '#/components/responses/SuccessResponse' @@ -527,7 +707,7 @@ addObjectStorageMethodsToRouter( * delete: * summary: Remove rules for ads places * tags: - * - Slise ad places + * - Ad places * security: * - basicAuth: [] * requestBody: @@ -551,12 +731,12 @@ addObjectStorageMethodsToRouter( * '500': * $ref: '#/components/responses/ErrorResponse' */ -addObjectStorageMethodsToRouter( - sliseAdPlacesRulesRouter, - '/', - sliseAdPlacesRulesMethods, - 'domain', - sliseAdPlacesRulesDictionarySchema, - hostnamesListSchema, - entriesCount => `${entriesCount} entries have been removed` -); +addObjectStorageMethodsToRouter(adPlacesRulesRouter, { + path: '/', + methods: adPlacesRulesMethods, + keyName: 'domain', + objectValidationSchema: adPlacesRulesDictionarySchema, + keysArrayValidationSchema: hostnamesListSchema, + successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed`, + transformGotValueFn: (value, req) => filterByVersion(value, req.query.extVersion as string | undefined) +}); diff --git a/src/routers/slise-ad-rules/index.ts b/src/routers/slise-ad-rules/index.ts index 0dca519..a56c849 100644 --- a/src/routers/slise-ad-rules/index.ts +++ b/src/routers/slise-ad-rules/index.ts @@ -1,7 +1,7 @@ import { Router } from 'express'; -import { sliseAdPlacesRulesRouter } from './ad-places'; -import { sliseAdProvidersRouter } from './providers'; +import { adPlacesRulesRouter } from './ad-places'; +import { adProvidersRouter } from './providers'; /** * @swagger @@ -37,7 +37,7 @@ import { sliseAdProvidersRouter } from './providers'; * type: string */ -export const sliseRulesRouter = Router(); +export const adRulesRouter = Router(); -sliseRulesRouter.use('/ad-places', sliseAdPlacesRulesRouter); -sliseRulesRouter.use('/providers', sliseAdProvidersRouter); +adRulesRouter.use('/ad-places', adPlacesRulesRouter); +adRulesRouter.use('/providers', adProvidersRouter); diff --git a/src/routers/slise-ad-rules/providers.ts b/src/routers/slise-ad-rules/providers.ts index 8c95059..71098c4 100644 --- a/src/routers/slise-ad-rules/providers.ts +++ b/src/routers/slise-ad-rules/providers.ts @@ -1,19 +1,21 @@ import { Router } from 'express'; import { - addSliseAdProvidersForAllSites, - getSliseAdProvidersForAllSites, - removeSliseAdProvidersForAllSites, - sliseAdProvidersMethods, - sliseAdProvidersByDomainRulesMethods -} from '../../advertising/slise'; + addAdProvidersForAllSites, + getAdProvidersForAllSites, + removeAdProvidersForAllSites, + adProvidersMethods, + adProvidersByDomainRulesMethods, + AdProviderSelectorsRule, + filterByVersion +} from '../../advertising/external-ads'; import { basicAuth } from '../../middlewares/basic-auth.middleware'; import { addObjectStorageMethodsToRouter, withBodyValidation, withExceptionHandler } from '../../utils/express-helpers'; import { adTypesListSchema, hostnamesListSchema, - sliseAdProvidersByDomainsRulesDictionarySchema, - sliseAdProvidersDictionarySchema + adProvidersByDomainsRulesDictionarySchema, + adProvidersDictionarySchema } from '../../utils/schemas'; /** @@ -48,7 +50,7 @@ import { * additionalProperties: * type: array * items: - * $ref: '#/components/schemas/SliseAdProvidersByDomainRule' + * $ref: '#/components/schemas/AdProvidersByDomainRule' * example: * polygonscan.com: * - urlRegexes: @@ -56,7 +58,18 @@ import { * providers: * - 'coinzilla' * - 'bitmedia' - * SliseAdProvidersDictionary: + * AdProvidersInputValue: + * allOf: + * - $ref: '#/components/schemas/ExtVersionConstraints' + * - type: object + * required: + * - selectors + * properties: + * selectors: + * type: array + * items: + * type: string + * AdProvidersDictionary: * type: object * additionalProperties: * type: array @@ -67,9 +80,15 @@ import { * - '#Ads_google_bottom_wide' * - '.GoogleAdInfo' * - 'a[href^="https://googleads.g.doubleclick.net/pcs/click"]' + * AdProvidersInputsDictionary: + * type: object + * additionalProperties: + * type: array + * items: + * $ref: '#/components/schemas/AdProvidersInputValue' */ -export const sliseAdProvidersRouter = Router(); +export const adProvidersRouter = Router(); /** * @swagger @@ -147,11 +166,11 @@ export const sliseAdProvidersRouter = Router(); * '500': * $ref: '#/components/responses/ErrorResponse' */ -sliseAdProvidersRouter +adProvidersRouter .route('/all-sites') .get( withExceptionHandler(async (_req, res) => { - const providers = await getSliseAdProvidersForAllSites(); + const providers = await getAdProvidersForAllSites(); res.status(200).send(providers); }) @@ -160,7 +179,7 @@ sliseAdProvidersRouter basicAuth, withExceptionHandler( withBodyValidation(adTypesListSchema, async (req, res) => { - const providersAddedCount = await addSliseAdProvidersForAllSites(req.body); + const providersAddedCount = await addAdProvidersForAllSites(req.body); res.status(200).send({ message: `${providersAddedCount} providers have been added` }); }) @@ -170,7 +189,7 @@ sliseAdProvidersRouter basicAuth, withExceptionHandler( withBodyValidation(adTypesListSchema, async (req, res) => { - const providersRemovedCount = await removeSliseAdProvidersForAllSites(req.body); + const providersRemovedCount = await removeAdProvidersForAllSites(req.body); res.status(200).send({ message: `${providersRemovedCount} providers have been removed` }); }) @@ -200,7 +219,7 @@ sliseAdProvidersRouter * schema: * type: array * items: - * $ref: '#/components/schemas/SliseAdProvidersByDomainRule' + * $ref: '#/components/schemas/AdProvidersByDomainRule' * '500': * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/providers/by-sites: @@ -214,7 +233,7 @@ sliseAdProvidersRouter * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdProvidersByDomainsRulesDictionary' + * $ref: '#/components/schemas/AdProvidersByDomainsRulesDictionary' * '500': * $ref: '#/components/responses/ErrorResponse' * post: @@ -230,7 +249,7 @@ sliseAdProvidersRouter * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdProvidersByDomainsRulesDictionary' + * $ref: '#/components/schemas/AdProvidersByDomainsRulesDictionary' * responses: * '200': * $ref: '#/components/responses/SuccessResponse' @@ -268,21 +287,57 @@ sliseAdProvidersRouter * '500': * $ref: '#/components/responses/ErrorResponse' */ -addObjectStorageMethodsToRouter( - sliseAdProvidersRouter, - '/by-sites', - sliseAdProvidersByDomainRulesMethods, - 'domain', - sliseAdProvidersByDomainsRulesDictionarySchema, - hostnamesListSchema, - entriesCount => `${entriesCount} entries have been removed` -); +addObjectStorageMethodsToRouter(adProvidersRouter, { + path: '/by-sites', + methods: adProvidersByDomainRulesMethods, + keyName: 'domain', + objectValidationSchema: adProvidersByDomainsRulesDictionarySchema, + keysArrayValidationSchema: hostnamesListSchema, + successfulRemovalMessage: entriesCount => `${entriesCount} entries have been removed` +}); /** * @swagger + * /api/slise-ad-rules/providers/raw/all: + * get: + * summary: Get selectors for all providers and all extensions versions + * tags: + * - Known ads providers + * responses: + * '200': + * description: Provider - selectors dictionary + * content: + * application/json: + * schema: + * $ref: '#/components/schemas/AdProvidersInputsDictionary' + * '500': + * $ref: '#/components/responses/ErrorResponse' + * /api/slise-ad-rules/providers/{providerId}/raw: + * get: + * summary: Get selectors for a provider for all extensions versions + * tags: + * - Known ads providers + * parameters: + * - in: path + * name: providerId + * required: true + * schema: + * type: string + * example: 'google' + * responses: + * '200': + * description: Lists of CSS selectors for all extension versions + * content: + * application/json: + * schema: + * type: array + * items: + * $ref: '#/components/schemas/AdProvidersInputValue' + * '500': + * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/providers/{providerId}: * get: - * summary: Get selectors for a provider + * summary: Get selectors for a provider filtered by extension version * tags: * - Known ads providers * parameters: @@ -292,6 +347,13 @@ addObjectStorageMethodsToRouter( * schema: * type: string * example: 'google' + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: List of CSS selectors @@ -309,16 +371,24 @@ addObjectStorageMethodsToRouter( * $ref: '#/components/responses/ErrorResponse' * /api/slise-ad-rules/providers: * get: - * summary: Get all providers + * summary: Get selectors for all providers filtered by extension version * tags: * - Known ads providers + * parameters: + * - in: query + * name: extVersion + * schema: + * type: string + * description: > + * The extension version for which the rules should be returned. If not specified, the rules that are + * applicable for all extension versions will be returned. * responses: * '200': * description: Provider - selectors dictionary * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdProvidersDictionary' + * $ref: '#/components/schemas/AdProvidersDictionary' * '500': * $ref: '#/components/responses/ErrorResponse' * post: @@ -332,7 +402,7 @@ addObjectStorageMethodsToRouter( * content: * application/json: * schema: - * $ref: '#/components/schemas/SliseAdProvidersDictionary' + * $ref: '#/components/schemas/AdProvidersInputsDictionary' * responses: * '200': * $ref: '#/components/responses/SuccessResponse' @@ -369,12 +439,19 @@ addObjectStorageMethodsToRouter( * '500': * $ref: '#/components/responses/ErrorResponse' */ -addObjectStorageMethodsToRouter( - sliseAdProvidersRouter, - '/', - sliseAdProvidersMethods, - 'providerId', - sliseAdProvidersDictionarySchema, - adTypesListSchema, - entriesCount => `${entriesCount} providers have been removed` -); +addObjectStorageMethodsToRouter(adProvidersRouter, { + path: '/', + methods: adProvidersMethods, + keyName: 'providerId', + objectValidationSchema: adProvidersDictionarySchema, + keysArrayValidationSchema: adTypesListSchema, + successfulRemovalMessage: entriesCount => `${entriesCount} providers have been removed`, + transformGotValueFn: (rules, req) => + Array.from( + new Set( + filterByVersion(rules, req.query.extVersion as string | undefined) + .map(({ selectors }) => selectors) + .flat() + ) + ) +}); diff --git a/src/utils/express-helpers.ts b/src/utils/express-helpers.ts index 8a6e05b..09e94fe 100644 --- a/src/utils/express-helpers.ts +++ b/src/utils/express-helpers.ts @@ -44,17 +44,41 @@ export const withExceptionHandler = } }; -export const addObjectStorageMethodsToRouter = ( +interface ObjectStorageMethodsEntrypointsConfig { + path: string; + methods: ObjectStorageMethods; + keyName: string; + objectValidationSchema: IObjectSchema>; + keysArrayValidationSchema: IArraySchema; + successfulRemovalMessage: (removedEntriesCount: number) => string; + transformGotValueFn?: (value: U, req: Request) => V; +} + +export const addObjectStorageMethodsToRouter = ( router: Router, - path: string, - methods: ObjectStorageMethods, - keyName: string, - objectValidationSchema: IObjectSchema>, - keysArrayValidationSchema: IArraySchema, - successfulRemovalMessage: (removedEntriesCount: number) => string + config: ObjectStorageMethodsEntrypointsConfig ) => { + const { + path, + methods, + keyName, + objectValidationSchema, + keysArrayValidationSchema, + successfulRemovalMessage, + transformGotValueFn = value => value as unknown as V + } = config; + router.get( - path === '/' ? `/:${keyName}` : `${path}/:${keyName}`, + path === '/' ? '/raw/all' : `${path}/raw/all`, + withExceptionHandler(async (_req, res) => { + const values = await methods.getAllValues(); + + res.status(200).send(values); + }) + ); + + router.get( + path === '/' ? `/:${keyName}/raw` : `${path}/:${keyName}/raw`, withExceptionHandler(async (req, res) => { const { [keyName]: key } = req.params; @@ -64,13 +88,28 @@ export const addObjectStorageMethodsToRouter = ( }) ); + router.get( + path === '/' ? `/:${keyName}` : `${path}/:${keyName}`, + withExceptionHandler(async (req, res) => { + const { [keyName]: key } = req.params; + + const value = await methods.getByKey(key); + + res.status(200).send(transformGotValueFn(value, req)); + }) + ); + router .route(path) .get( - withExceptionHandler(async (_req, res) => { + withExceptionHandler(async (req, res) => { const values = await methods.getAllValues(); - res.status(200).send(values); + res + .status(200) + .send( + Object.fromEntries(Object.entries(values).map(([key, value]) => [key, transformGotValueFn(value, req)])) + ); }) ) .post( diff --git a/src/utils/migrations.ts b/src/utils/migrations.ts new file mode 100644 index 0000000..0ec240a --- /dev/null +++ b/src/utils/migrations.ts @@ -0,0 +1,34 @@ +import { adProvidersMethods, sliseAdProvidersMethodsLegacy } from '../advertising/external-ads'; +import { redisClient } from '../redis'; +import { isDefined } from './helpers'; +import logger from './logger'; + +const DATA_VERSION_STORAGE_KEY = 'data_version'; + +const migrations = { + '1': async () => { + const oldSliseAdsProviders = await sliseAdProvidersMethodsLegacy.getAllValues(); + await adProvidersMethods.upsertValues( + Object.fromEntries( + Object.entries(oldSliseAdsProviders).map(([providerName, selectors]) => [providerName, [{ selectors }]]) + ) + ); + } +}; + +export const doMigrations = async () => { + let currentVersion = await redisClient.get(DATA_VERSION_STORAGE_KEY); + + if (!isDefined(currentVersion)) { + currentVersion = '0'; + await redisClient.set(DATA_VERSION_STORAGE_KEY, '0'); + } + + logger.info(`Current data version: ${currentVersion}`); + + for (let i = Number(currentVersion) + 1; i <= Object.keys(migrations).length; i++) { + await migrations[i](); + await redisClient.set(DATA_VERSION_STORAGE_KEY, String(i)); + logger.info(`Migration to version ${i} completed`); + } +}; diff --git a/src/utils/schemas.ts b/src/utils/schemas.ts index d8bff7e..03e1e0f 100644 --- a/src/utils/schemas.ts +++ b/src/utils/schemas.ts @@ -11,12 +11,13 @@ import { } from 'yup'; import { - SliseAdPlacesRule, - PermanentSliseAdPlacesRule, - SliseAdProvidersByDomainRule, + AdPlacesRule, + PermanentAdPlacesRule, + AdProvidersByDomainRule, StylePropName, - stylePropsNames -} from '../advertising/slise'; + stylePropsNames, + AdProviderSelectorsRule +} from '../advertising/external-ads'; import { isValidSelectorsGroup } from '../utils/selectors.min.js'; import { isDefined } from './helpers'; @@ -46,7 +47,7 @@ const makeDictionarySchema = (keySchema: IStringSchema, valueSchema: Schema> = makeDictionary stringSchema().required() ); -const sliseAdStylesOverridesSchema = objectSchema().shape({ +const adStylesOverridesSchema = objectSchema().shape({ parentDepth: numberSchema().integer().min(0).required(), style: styleSchema.clone().required() }); -const sliseAdPlacesRulesSchema = arraySchema() +const adPlacesRulesSchema = arraySchema() .of( objectSchema() .shape({ @@ -115,17 +120,21 @@ const sliseAdPlacesRulesSchema = arraySchema() divWrapperStyle: styleSchema }) .required(), - stylesOverrides: arraySchema().of(sliseAdStylesOverridesSchema.clone().required()), - shouldHideOriginal: booleanSchema().default(false) + stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), + shouldHideOriginal: booleanSchema().default(false), + firstExtVersion: versionSchema, + lastExtVersion: versionSchema }) .required() ) .required(); -export const sliseAdPlacesRulesDictionarySchema: IObjectSchema> = - makeDictionarySchema(hostnameSchema, sliseAdPlacesRulesSchema).required(); +export const adPlacesRulesDictionarySchema: IObjectSchema> = makeDictionarySchema( + hostnameSchema, + adPlacesRulesSchema +).required(); -const permanentSliseAdPlacesRulesSchema = arraySchema() +const permanentAdPlacesRulesSchema = arraySchema() .of( objectSchema() .shape({ @@ -152,10 +161,12 @@ const permanentSliseAdPlacesRulesSchema = arraySchema() elementStyle: styleSchema, divWrapperStyle: styleSchema, elementToMeasureSelector: cssSelectorSchema, - stylesOverrides: arraySchema().of(sliseAdStylesOverridesSchema.clone().required()), - shouldHideOriginal: booleanSchema().default(false) + stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), + shouldHideOriginal: booleanSchema().default(false), + firstExtVersion: versionSchema, + lastExtVersion: versionSchema }) - .test('insertion-place-specified', (value: PermanentSliseAdPlacesRule | undefined) => { + .test('insertion-place-specified', (value: PermanentAdPlacesRule | undefined) => { if (!value) { return true; } @@ -175,25 +186,33 @@ const permanentSliseAdPlacesRulesSchema = arraySchema() ) .required(); -export const permanentSliseAdPlacesRulesDictionarySchema: IObjectSchema> = - makeDictionarySchema(hostnameSchema, permanentSliseAdPlacesRulesSchema).required(); +export const permanentAdPlacesRulesDictionarySchema: IObjectSchema> = + makeDictionarySchema(hostnameSchema, permanentAdPlacesRulesSchema).required(); -const sliseAdProvidersByDomainRulesSchema = arraySchema() +const adProvidersByDomainRulesSchema = arraySchema() .of( objectSchema() .shape({ urlRegexes: arraySchema().of(regexStringSchema.clone().required()).required(), - providers: arraySchema().of(stringSchema().required()).required() + providers: arraySchema().of(stringSchema().required()).required(), + firstExtVersion: versionSchema, + lastExtVersion: versionSchema }) .required() ) .required(); -export const sliseAdProvidersByDomainsRulesDictionarySchema: IObjectSchema< - Record -> = makeDictionarySchema(hostnameSchema, sliseAdProvidersByDomainRulesSchema).required(); +export const adProvidersByDomainsRulesDictionarySchema: IObjectSchema> = + makeDictionarySchema(hostnameSchema, adProvidersByDomainRulesSchema).required(); -export const sliseAdProvidersDictionarySchema: IObjectSchema> = makeDictionarySchema( - adTypeSchema.clone().required(), - cssSelectorsListSchema.clone().required() -).required(); +const adProvidersSelectorsRuleSchema = objectSchema().shape({ + selectors: cssSelectorsListSchema.clone().required(), + firstExtVersion: versionSchema, + lastExtVersion: versionSchema +}); + +export const adProvidersDictionarySchema: IObjectSchema> = + makeDictionarySchema( + adTypeSchema.clone().required(), + arraySchema().of(adProvidersSelectorsRuleSchema.clone().required()).required() + ).required(); From 3395e07962512a42fc73d07d7a332b9e7083239b Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Wed, 10 Apr 2024 15:14:43 +0300 Subject: [PATCH 2/6] TW-1364 Fix some typos --- src/advertising/external-ads.ts | 2 +- src/routers/slise-ad-rules/providers.ts | 4 ++-- src/utils/migrations.ts | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/src/advertising/external-ads.ts b/src/advertising/external-ads.ts index fa765c6..28dac69 100644 --- a/src/advertising/external-ads.ts +++ b/src/advertising/external-ads.ts @@ -154,7 +154,7 @@ export const adProvidersByDomainRulesMethods = objectStorageMethodsFactory(SLISE_AD_PROVIDERS_LIST_KEY, []); +export const adProvidersMethodsLegacy = objectStorageMethodsFactory(SLISE_AD_PROVIDERS_LIST_KEY, []); export const adProvidersMethods = objectStorageMethodsFactory(AD_PROVIDERS_LIST_KEY, []); diff --git a/src/routers/slise-ad-rules/providers.ts b/src/routers/slise-ad-rules/providers.ts index 71098c4..2a04039 100644 --- a/src/routers/slise-ad-rules/providers.ts +++ b/src/routers/slise-ad-rules/providers.ts @@ -24,7 +24,7 @@ import { * name: Known ads providers * components: * schemas: - * SliseAdProvidersByDomainRule: + * AdProvidersByDomainRule: * type: object * required: * - urlRegexes @@ -45,7 +45,7 @@ import { * providers: * - 'coinzilla' * - 'bitmedia' - * SliseAdProvidersByDomainsRulesDictionary: + * AdProvidersByDomainsRulesDictionary: * type: object * additionalProperties: * type: array diff --git a/src/utils/migrations.ts b/src/utils/migrations.ts index 0ec240a..33aa95c 100644 --- a/src/utils/migrations.ts +++ b/src/utils/migrations.ts @@ -1,4 +1,4 @@ -import { adProvidersMethods, sliseAdProvidersMethodsLegacy } from '../advertising/external-ads'; +import { adProvidersMethods, adProvidersMethodsLegacy } from '../advertising/external-ads'; import { redisClient } from '../redis'; import { isDefined } from './helpers'; import logger from './logger'; @@ -7,7 +7,7 @@ const DATA_VERSION_STORAGE_KEY = 'data_version'; const migrations = { '1': async () => { - const oldSliseAdsProviders = await sliseAdProvidersMethodsLegacy.getAllValues(); + const oldSliseAdsProviders = await adProvidersMethodsLegacy.getAllValues(); await adProvidersMethods.upsertValues( Object.fromEntries( Object.entries(oldSliseAdsProviders).map(([providerName, selectors]) => [providerName, [{ selectors }]]) From 1a80cd42f8f4b1398dcb538f5047fbc8ec973a1e Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Wed, 10 Apr 2024 15:17:43 +0300 Subject: [PATCH 3/6] TW-1364 Minor refactoring --- src/utils/migrations.ts | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/utils/migrations.ts b/src/utils/migrations.ts index 33aa95c..0d8524d 100644 --- a/src/utils/migrations.ts +++ b/src/utils/migrations.ts @@ -7,10 +7,10 @@ const DATA_VERSION_STORAGE_KEY = 'data_version'; const migrations = { '1': async () => { - const oldSliseAdsProviders = await adProvidersMethodsLegacy.getAllValues(); + const oldAdsProviders = await adProvidersMethodsLegacy.getAllValues(); await adProvidersMethods.upsertValues( Object.fromEntries( - Object.entries(oldSliseAdsProviders).map(([providerName, selectors]) => [providerName, [{ selectors }]]) + Object.entries(oldAdsProviders).map(([providerName, selectors]) => [providerName, [{ selectors }]]) ) ); } From 2c96fa0d59339650d8c69f6536982c51f8ae407e Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Wed, 10 Apr 2024 18:44:34 +0300 Subject: [PATCH 4/6] TW-1364 Change logic for the case extension version is not specified --- src/advertising/external-ads.ts | 8 +------ src/routers/slise-ad-rules/ad-places.ts | 30 ++++++++++--------------- src/routers/slise-ad-rules/providers.ts | 10 ++++----- 3 files changed, 17 insertions(+), 31 deletions(-) diff --git a/src/advertising/external-ads.ts b/src/advertising/external-ads.ts index 28dac69..4ee8418 100644 --- a/src/advertising/external-ads.ts +++ b/src/advertising/external-ads.ts @@ -191,13 +191,7 @@ const compareVersions = (a: string, b: string) => { return aPatch - bPatch; }; -export const filterByVersion = (rules: T[], version?: string) => { - if (!isDefined(version)) { - return rules.filter( - ({ firstExtVersion, lastExtVersion }) => !isDefined(firstExtVersion) && !isDefined(lastExtVersion) - ); - } - +export const filterByVersion = (rules: T[], version = '1.20.1') => { return rules.filter(({ firstExtVersion, lastExtVersion }) => { if (isDefined(firstExtVersion) && compareVersions(firstExtVersion, version) > 0) { return false; diff --git a/src/routers/slise-ad-rules/ad-places.ts b/src/routers/slise-ad-rules/ad-places.ts index 91d6392..ad0a562 100644 --- a/src/routers/slise-ad-rules/ad-places.ts +++ b/src/routers/slise-ad-rules/ad-places.ts @@ -345,9 +345,8 @@ export const adPlacesRulesRouter = Router(); * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Rules list @@ -369,9 +368,8 @@ export const adPlacesRulesRouter = Router(); * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Domain - rules list dictionary @@ -496,9 +494,8 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Rules list @@ -520,9 +517,8 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Domain - rules list dictionary @@ -647,9 +643,8 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Rules list @@ -671,9 +666,8 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Domain - rules list dictionary diff --git a/src/routers/slise-ad-rules/providers.ts b/src/routers/slise-ad-rules/providers.ts index 2a04039..e24cbd0 100644 --- a/src/routers/slise-ad-rules/providers.ts +++ b/src/routers/slise-ad-rules/providers.ts @@ -351,9 +351,8 @@ addObjectStorageMethodsToRouter(adProvidersRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: List of CSS selectors @@ -379,9 +378,8 @@ addObjectStorageMethodsToRouter(adProvidersRouter, { * name: extVersion * schema: * type: string - * description: > - * The extension version for which the rules should be returned. If not specified, the rules that are - * applicable for all extension versions will be returned. + * default: '1.20.1' + * description: The extension version for which the rules should be returned * responses: * '200': * description: Provider - selectors dictionary From 16c25e3a800afbd114a2695a6c31100f692c080a Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Thu, 11 Apr 2024 11:44:53 +0300 Subject: [PATCH 5/6] TW-1364 Use 'semver' module for versions comparison --- package.json | 1 + src/advertising/external-ads.ts | 35 ++++--------------------- src/routers/slise-ad-rules/ad-places.ts | 14 +++------- src/utils/schemas.ts | 17 +++++------- yarn.lock | 7 +++++ 5 files changed, 23 insertions(+), 51 deletions(-) diff --git a/package.json b/package.json index d44a9c9..c5ce48f 100644 --- a/package.json +++ b/package.json @@ -32,6 +32,7 @@ "pino-pretty": "^4.7.1", "qs": "^6.10.3", "semaphore": "^1.1.0", + "semver": "^7.6.0", "swagger-jsdoc": "^6.2.8", "swagger-ui-express": "^5.0.0", "yup": "^1.3.2" diff --git a/src/advertising/external-ads.ts b/src/advertising/external-ads.ts index 4ee8418..fa08539 100644 --- a/src/advertising/external-ads.ts +++ b/src/advertising/external-ads.ts @@ -1,3 +1,5 @@ +import { satisfies as versionSatisfiesRange } from 'semver'; + import { objectStorageMethodsFactory, redisClient } from '../redis'; import { isDefined } from '../utils/helpers'; @@ -84,8 +86,7 @@ interface AdStylesOverrides { } interface ExtVersionConstraints { - firstExtVersion?: string; - lastExtVersion?: string; + extVersion?: string; } export interface AdPlacesRule extends ExtVersionConstraints { @@ -176,31 +177,5 @@ export const addAdProvidersForAllSites = async (providers: string[]) => export const removeAdProvidersForAllSites = async (providers: string[]) => redisClient.srem(AD_PROVIDERS_ALL_SITES_KEY, ...providers); -const compareVersions = (a: string, b: string) => { - const [aMajor = 0, aMinor = 0, aPatch = 0] = a.split('.').map(Number); - const [bMajor = 0, bMinor = 0, bPatch = 0] = b.split('.').map(Number); - - if (aMajor !== bMajor) { - return aMajor - bMajor; - } - - if (aMinor !== bMinor) { - return aMinor - bMinor; - } - - return aPatch - bPatch; -}; - -export const filterByVersion = (rules: T[], version = '1.20.1') => { - return rules.filter(({ firstExtVersion, lastExtVersion }) => { - if (isDefined(firstExtVersion) && compareVersions(firstExtVersion, version) > 0) { - return false; - } - - if (isDefined(lastExtVersion) && compareVersions(lastExtVersion, version) < 0) { - return false; - } - - return true; - }); -}; +export const filterByVersion = (rules: T[], version = '1.20.1') => + rules.filter(({ extVersion }) => !isDefined(extVersion) || versionSatisfiesRange(version, extVersion)); diff --git a/src/routers/slise-ad-rules/ad-places.ts b/src/routers/slise-ad-rules/ad-places.ts index ad0a562..422e671 100644 --- a/src/routers/slise-ad-rules/ad-places.ts +++ b/src/routers/slise-ad-rules/ad-places.ts @@ -69,19 +69,11 @@ import { * ExtVersionConstraints: * type: object * properties: - * firstExtVersion: + * extVersion: * type: string - * format: version * description: > - * The first extension version where the rule is applicable. If not specified, the rule is applicable - * for all extension versions not greater than the `lastExtVersion` value. If neither of them is specified, - * the rule is applicable for all extension versions. - * lastExtVersion: - * type: string - * format: version - * description: > - * The last extension version where the rule is applicable. If not specified, the rule is applicable - * for all extension versions not less than the `firstExtVersion` value. + * A range of versions where the rule is applicable. If not specified, the rule is applicable + * for all versions. See the [ranges format](https://www.npmjs.com/package/semver#ranges) * AdPlacesRule: * allOf: * - $ref: '#/components/schemas/ExtVersionConstraints' diff --git a/src/utils/schemas.ts b/src/utils/schemas.ts index 03e1e0f..2ad6a6b 100644 --- a/src/utils/schemas.ts +++ b/src/utils/schemas.ts @@ -1,3 +1,4 @@ +import { validRange as getValidatedRange } from 'semver'; import { array as arraySchema, ArraySchema as IArraySchema, @@ -60,8 +61,8 @@ const regexStringSchema = stringSchema().test('is-regex', function (value: strin export const regexStringListSchema = arraySchema().of(regexStringSchema.clone().required()); -const versionSchema = stringSchema().test('is-version', function (value: string | undefined) { - return !isDefined(value) || /^\d+\.\d+\.\d+$/.test(value); +const versionRangeSchema = stringSchema().test('is-version-range', function (value: string | undefined) { + return !isDefined(value) || isDefined(getValidatedRange(value)); }); const cssSelectorSchema = stringSchema().test('is-css-selector', function (value: string | undefined) { @@ -122,8 +123,7 @@ const adPlacesRulesSchema = arraySchema() .required(), stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), shouldHideOriginal: booleanSchema().default(false), - firstExtVersion: versionSchema, - lastExtVersion: versionSchema + extVersion: versionRangeSchema }) .required() ) @@ -163,8 +163,7 @@ const permanentAdPlacesRulesSchema = arraySchema() elementToMeasureSelector: cssSelectorSchema, stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), shouldHideOriginal: booleanSchema().default(false), - firstExtVersion: versionSchema, - lastExtVersion: versionSchema + extVersion: versionRangeSchema }) .test('insertion-place-specified', (value: PermanentAdPlacesRule | undefined) => { if (!value) { @@ -195,8 +194,7 @@ const adProvidersByDomainRulesSchema = arraySchema() .shape({ urlRegexes: arraySchema().of(regexStringSchema.clone().required()).required(), providers: arraySchema().of(stringSchema().required()).required(), - firstExtVersion: versionSchema, - lastExtVersion: versionSchema + extVersion: versionRangeSchema }) .required() ) @@ -207,8 +205,7 @@ export const adProvidersByDomainsRulesDictionarySchema: IObjectSchema> = diff --git a/yarn.lock b/yarn.lock index 4649e53..44c2709 100644 --- a/yarn.lock +++ b/yarn.lock @@ -4014,6 +4014,13 @@ semver@^7.2.1, semver@^7.3.7, semver@^7.3.8: dependencies: lru-cache "^6.0.0" +semver@^7.6.0: + version "7.6.0" + resolved "https://registry.yarnpkg.com/semver/-/semver-7.6.0.tgz#1a46a4db4bffcccd97b743b5005c8325f23d4e2d" + integrity sha512-EnwXhrlwXMk9gKu5/flx5sv/an57AkRplG3hTK68W7FRDN+k+OWBj65M7719OkA82XLBxrcX0KSHj+X5COhOVg== + dependencies: + lru-cache "^6.0.0" + send@0.18.0: version "0.18.0" resolved "https://registry.yarnpkg.com/send/-/send-0.18.0.tgz#670167cc654b05f5aa4a767f9113bb371bc706be" From 7e0139bcf4a87b469d808f3016bd043a8e1528a4 Mon Sep 17 00:00:00 2001 From: Inokentii Mazhara Date: Thu, 11 Apr 2024 17:12:33 +0300 Subject: [PATCH 6/6] TW-1364 Remove migrations --- src/advertising/external-ads.ts | 21 +++++++-------- src/index.ts | 6 ----- src/routers/slise-ad-rules/ad-places.ts | 13 +++++----- src/routers/slise-ad-rules/providers.ts | 4 +-- src/utils/express-helpers.ts | 1 + src/utils/migrations.ts | 34 ------------------------- src/utils/schemas.ts | 14 +++++----- 7 files changed, 26 insertions(+), 67 deletions(-) delete mode 100644 src/utils/migrations.ts diff --git a/src/advertising/external-ads.ts b/src/advertising/external-ads.ts index fa08539..a1bc5a2 100644 --- a/src/advertising/external-ads.ts +++ b/src/advertising/external-ads.ts @@ -1,7 +1,6 @@ import { satisfies as versionSatisfiesRange } from 'semver'; import { objectStorageMethodsFactory, redisClient } from '../redis'; -import { isDefined } from '../utils/helpers'; /** Style properties names that are likely to be unnecessary for banners are skipped */ export const stylePropsNames = [ @@ -86,7 +85,7 @@ interface AdStylesOverrides { } interface ExtVersionConstraints { - extVersion?: string; + extVersion: string; } export interface AdPlacesRule extends ExtVersionConstraints { @@ -139,12 +138,11 @@ export interface AdProviderForAllSitesRule extends ExtVersionConstraints { providers: string[]; } -const AD_PLACES_RULES_KEY = 'slise_ad_places_rules'; -const AD_PROVIDERS_BY_SITES_KEY = 'slise_ad_providers_by_sites'; -const AD_PROVIDERS_ALL_SITES_KEY = 'slise_ad_providers_all_sites'; -const SLISE_AD_PROVIDERS_LIST_KEY = 'slise_ad_providers_list'; +const AD_PLACES_RULES_KEY = 'ad_places_rules'; +const AD_PROVIDERS_BY_SITES_KEY = 'ad_providers_by_sites'; +const AD_PROVIDERS_ALL_SITES_KEY = 'ad_providers_all_sites'; const AD_PROVIDERS_LIST_KEY = 'ad_providers_list'; -const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_slise_ad_places_rules'; +const PERMANENT_AD_PLACES_RULES_KEY = 'permanent_ad_places_rules'; const PERMANENT_NATIVE_AD_PLACES_RULES_KEY = 'permanent_native_ad_places_rules'; export const adPlacesRulesMethods = objectStorageMethodsFactory(AD_PLACES_RULES_KEY, []); @@ -154,9 +152,6 @@ export const adProvidersByDomainRulesMethods = objectStorageMethodsFactory(SLISE_AD_PROVIDERS_LIST_KEY, []); - export const adProvidersMethods = objectStorageMethodsFactory(AD_PROVIDERS_LIST_KEY, []); export const permanentAdPlacesMethods = objectStorageMethodsFactory( @@ -177,5 +172,7 @@ export const addAdProvidersForAllSites = async (providers: string[]) => export const removeAdProvidersForAllSites = async (providers: string[]) => redisClient.srem(AD_PROVIDERS_ALL_SITES_KEY, ...providers); -export const filterByVersion = (rules: T[], version = '1.20.1') => - rules.filter(({ extVersion }) => !isDefined(extVersion) || versionSatisfiesRange(version, extVersion)); +const FALLBACK_VERSION = '0.0.0'; + +export const filterByVersion = (rules: T[], version?: string) => + rules.filter(({ extVersion }) => versionSatisfiesRange(version ?? FALLBACK_VERSION, extVersion)); diff --git a/src/index.ts b/src/index.ts index d9149ac..9938eb9 100644 --- a/src/index.ts +++ b/src/index.ts @@ -33,7 +33,6 @@ import { CodedError } from './utils/errors'; import { coinGeckoTokens } from './utils/gecko-tokens'; import { getExternalApiErrorPayload, isDefined, isNonEmptyString } from './utils/helpers'; import logger from './utils/logger'; -import { doMigrations } from './utils/migrations'; import { getSignedMoonPayUrl } from './utils/moonpay/get-signed-moonpay-url'; import { getSigningNonce } from './utils/signing-nonce'; import SingleQueryDataProvider from './utils/SingleQueryDataProvider'; @@ -103,11 +102,6 @@ const makeProviderDataRequestHandler = (provider: SingleQueryDataProvider< }; }; -doMigrations().catch(error => { - console.error(error); - process.exit(1); -}); - app.get('/api/top-coins', (_req, res) => { res.status(200).send(coinGeckoTokens); }); diff --git a/src/routers/slise-ad-rules/ad-places.ts b/src/routers/slise-ad-rules/ad-places.ts index 422e671..22cd924 100644 --- a/src/routers/slise-ad-rules/ad-places.ts +++ b/src/routers/slise-ad-rules/ad-places.ts @@ -74,6 +74,7 @@ import { * description: > * A range of versions where the rule is applicable. If not specified, the rule is applicable * for all versions. See the [ranges format](https://www.npmjs.com/package/semver#ranges) + * default: '*' * AdPlacesRule: * allOf: * - $ref: '#/components/schemas/ExtVersionConstraints' @@ -337,7 +338,7 @@ export const adPlacesRulesRouter = Router(); * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -360,7 +361,7 @@ export const adPlacesRulesRouter = Router(); * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -486,7 +487,7 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -509,7 +510,7 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -635,7 +636,7 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -658,7 +659,7 @@ addObjectStorageMethodsToRouter(adPlacesRulesRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': diff --git a/src/routers/slise-ad-rules/providers.ts b/src/routers/slise-ad-rules/providers.ts index e24cbd0..e281600 100644 --- a/src/routers/slise-ad-rules/providers.ts +++ b/src/routers/slise-ad-rules/providers.ts @@ -351,7 +351,7 @@ addObjectStorageMethodsToRouter(adProvidersRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': @@ -378,7 +378,7 @@ addObjectStorageMethodsToRouter(adProvidersRouter, { * name: extVersion * schema: * type: string - * default: '1.20.1' + * default: '0.0.0' * description: The extension version for which the rules should be returned * responses: * '200': diff --git a/src/utils/express-helpers.ts b/src/utils/express-helpers.ts index 09e94fe..c514977 100644 --- a/src/utils/express-helpers.ts +++ b/src/utils/express-helpers.ts @@ -21,6 +21,7 @@ export const withBodyValidation = (schema: Schema, handler: TypedBodyRequestHandler): RequestHandler => async (req, res, next) => { try { + console.log('oy vey 1', JSON.stringify(req.body), JSON.stringify(await schema.validate(req.body))); req.body = await schema.validate(req.body); } catch (error) { if (error instanceof ValidationError) { diff --git a/src/utils/migrations.ts b/src/utils/migrations.ts deleted file mode 100644 index 0d8524d..0000000 --- a/src/utils/migrations.ts +++ /dev/null @@ -1,34 +0,0 @@ -import { adProvidersMethods, adProvidersMethodsLegacy } from '../advertising/external-ads'; -import { redisClient } from '../redis'; -import { isDefined } from './helpers'; -import logger from './logger'; - -const DATA_VERSION_STORAGE_KEY = 'data_version'; - -const migrations = { - '1': async () => { - const oldAdsProviders = await adProvidersMethodsLegacy.getAllValues(); - await adProvidersMethods.upsertValues( - Object.fromEntries( - Object.entries(oldAdsProviders).map(([providerName, selectors]) => [providerName, [{ selectors }]]) - ) - ); - } -}; - -export const doMigrations = async () => { - let currentVersion = await redisClient.get(DATA_VERSION_STORAGE_KEY); - - if (!isDefined(currentVersion)) { - currentVersion = '0'; - await redisClient.set(DATA_VERSION_STORAGE_KEY, '0'); - } - - logger.info(`Current data version: ${currentVersion}`); - - for (let i = Number(currentVersion) + 1; i <= Object.keys(migrations).length; i++) { - await migrations[i](); - await redisClient.set(DATA_VERSION_STORAGE_KEY, String(i)); - logger.info(`Migration to version ${i} completed`); - } -}; diff --git a/src/utils/schemas.ts b/src/utils/schemas.ts index 2ad6a6b..7533c51 100644 --- a/src/utils/schemas.ts +++ b/src/utils/schemas.ts @@ -122,8 +122,8 @@ const adPlacesRulesSchema = arraySchema() }) .required(), stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), - shouldHideOriginal: booleanSchema().default(false), - extVersion: versionRangeSchema + shouldHideOriginal: booleanSchema(), + extVersion: versionRangeSchema.clone().required() }) .required() ) @@ -156,14 +156,14 @@ const permanentAdPlacesRulesSchema = arraySchema() insertionIndex: numberSchema().integer(), insertBeforeSelector: cssSelectorSchema, insertAfterSelector: cssSelectorSchema, - insertionsCount: numberSchema().integer().min(1).default(1), + insertionsCount: numberSchema().integer().min(1), shouldUseDivWrapper: booleanSchema().required(), elementStyle: styleSchema, divWrapperStyle: styleSchema, elementToMeasureSelector: cssSelectorSchema, stylesOverrides: arraySchema().of(adStylesOverridesSchema.clone().required()), - shouldHideOriginal: booleanSchema().default(false), - extVersion: versionRangeSchema + shouldHideOriginal: booleanSchema(), + extVersion: versionRangeSchema.clone().required() }) .test('insertion-place-specified', (value: PermanentAdPlacesRule | undefined) => { if (!value) { @@ -194,7 +194,7 @@ const adProvidersByDomainRulesSchema = arraySchema() .shape({ urlRegexes: arraySchema().of(regexStringSchema.clone().required()).required(), providers: arraySchema().of(stringSchema().required()).required(), - extVersion: versionRangeSchema + extVersion: versionRangeSchema.clone().required() }) .required() ) @@ -205,7 +205,7 @@ export const adProvidersByDomainsRulesDictionarySchema: IObjectSchema> =