diff --git a/.github/workflows/publish.yml b/.github/workflows/publish.yml index f95aa33029..8886a26884 100644 --- a/.github/workflows/publish.yml +++ b/.github/workflows/publish.yml @@ -556,7 +556,9 @@ jobs: # Build and push the Docker image echo "Building Docker image..." - docker build --file ./docker/images/provider.dockerfile . -t prosopo/provider:${{ steps.next_version.outputs.version }} + echo '{ "features": { "containerd-snapshotter": true } }' > /etc/docker/daemon.json + sudo apt-get install -y qemu qemu-user-static + docker buildx build --platform linux/amd64,linux/arm64 -f ./docker/images/provider.dockerfile . t prosopo/provider:${{ steps.next_version.outputs.version }} echo "Pushing Docker image..." docker push prosopo/provider:${{ steps.next_version.outputs.version }} diff --git a/demos/client-bundle-example/src/jsBundleTest.html b/demos/client-bundle-example/src/jsBundleTest.html index b856d56670..5ef37bdc11 100644 --- a/demos/client-bundle-example/src/jsBundleTest.html +++ b/demos/client-bundle-example/src/jsBundleTest.html @@ -15,7 +15,7 @@
diff --git a/dev/config/src/vite/NodejsPolarsNativeFilePlugin.ts b/dev/config/src/vite/NodejsPolarsNativeFilePlugin.ts index 71a819fa15..79d7eef31a 100644 --- a/dev/config/src/vite/NodejsPolarsNativeFilePlugin.ts +++ b/dev/config/src/vite/NodejsPolarsNativeFilePlugin.ts @@ -42,7 +42,7 @@ export const nodejsPolarsNativeFilePlugin = (logger: Logger, nodeFiles: string[] const customRequire = createRequire(import.meta.url) // load the .node file expecting it to be in the same directory as the output bundle - const content = customRequire('./nodejs-polars.linux-x64-gnu.node') + const content = customRequire('./${file}') // export the content straight back out again export default content diff --git a/dev/config/src/vite/vite.backend.config.ts b/dev/config/src/vite/vite.backend.config.ts index ff3e0e3818..0fdab62149 100644 --- a/dev/config/src/vite/vite.backend.config.ts +++ b/dev/config/src/vite/vite.backend.config.ts @@ -18,8 +18,6 @@ import { builtinModules } from 'module' import { filterDependencies, getDependencies } from '../dependencies.js' import { getLogger } from '@prosopo/common' import { nodeResolve } from '@rollup/plugin-node-resolve' -import { nodejsPolarsDirnamePlugin } from './NodejsPolarsDirnamePlugin.js' -import { nodejsPolarsNativeFilePlugin } from './NodejsPolarsNativeFilePlugin.js' import { wasm } from '@rollup/plugin-wasm' import VitePluginFixAbsoluteImports from './vite-plugin-fix-absolute-imports.js' import css from 'rollup-plugin-import-css' @@ -42,10 +40,6 @@ export default async function ( // Get all dependencies of the current package const { dependencies: deps, optionalPeerDependencies } = await getDependencies(packageName, true) - // Assuming node_modules are at the root of the workspace - const baseDir = path.resolve(optionalBaseDir) - const nodeModulesDir = path.resolve(baseDir, 'node_modules') - // Output directory is relative to directory of the package const outDir = path.resolve(packageDir, 'dist/bundle') @@ -62,12 +56,6 @@ export default async function ( logger.info(`Bundling. ${JSON.stringify(internal.slice(0, 10), null, 2)}... ${internal.length} deps`) - const nodeJsNodeFileToCopy = path.resolve( - nodeModulesDir, - './nodejs-polars-linux-x64-gnu/nodejs-polars.linux-x64-gnu.node' - ) - logger.info(`.node files to copy ${nodeJsNodeFileToCopy}`) - const define = { 'process.env.WS_NO_BUFFER_UTIL': 'true', 'process.env.WS_NO_UTF_8_VALIDATE': 'true', @@ -93,9 +81,6 @@ export default async function ( // drop console logs if in production mode const drop: Drop[] | undefined = mode === 'production' ? ['console', 'debugger'] : undefined - // a list of the node files to be handled. Starts from root dir - const nodeFiles = [nodeJsNodeFileToCopy] - return { ssr: { noExternal: internal, @@ -135,13 +120,7 @@ export default async function ( output: { entryFileNames: `${bundleName}.[name].bundle.js`, }, - plugins: [ - css(), - wasm(), - nodeResolve(), - nodejsPolarsDirnamePlugin(logger), - nodejsPolarsNativeFilePlugin(logger, nodeFiles, outDir), - ], + plugins: [css(), wasm(), nodeResolve()], }, }, plugins: [ diff --git a/docker/README.md b/docker/README.md index 5e6f202010..8238fbddf8 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,3 +1,8 @@ # Docker Various compose files for running Prosopo services + +## ARM64 + +Mongo dropped support for ARM < 8 so we need to use unofficial releases +from [here](https://github.com/themattman/mongodb-raspberrypi-docker?tab=readme-ov-file) for Raspberry Pi 4 and below. diff --git a/docker/docker-compose.provider.arm64.yml b/docker/docker-compose.provider.arm64.yml new file mode 100644 index 0000000000..24ef05a3f5 --- /dev/null +++ b/docker/docker-compose.provider.arm64.yml @@ -0,0 +1,50 @@ +version: '3.8' +services: + provider: + image: prosopo/provider:${PROVIDER_IMAGE_VERSION} + build: + context: .. + dockerfile: ./docker/images/provider.dockerfile + env_file: + - ../.env.production + ports: + - "9229:9229" + tty: true + depends_on: + - database + networks: + - internal + - external + + database: + image: mongodb-raspberrypi4-unofficial-r6.0.10 + volumes: + - /data/db:/data/db + ports: + - "27017:27017" + env_file: + - ../.env.production + networks: + - internal + caddy: + image: caddy:2 + restart: unless-stopped + ports: + - "80:80" + - "443:443" + - "443:443/udp" + volumes: + - ./provider.Caddyfile:/etc/caddy/Caddyfile + - caddy_data:/data + - caddy_config:/config + networks: + - external +networks: + internal: + name: internal + internal: true + external: + name: external +volumes: + caddy_data: + caddy_config: diff --git a/docker/docker-compose.provider.yml b/docker/docker-compose.provider.yml index 9100ac0a74..dedd1fc282 100644 --- a/docker/docker-compose.provider.yml +++ b/docker/docker-compose.provider.yml @@ -6,7 +6,7 @@ services: context: .. dockerfile: ./docker/images/provider.dockerfile env_file: - - ../.env.rococo + - ../.env.production ports: - "9229:9229" tty: true @@ -23,7 +23,7 @@ services: ports: - "27017:27017" env_file: - - ../.env.rococo + - ../.env.production networks: - internal caddy: @@ -34,7 +34,7 @@ services: - "443:443" - "443:443/udp" volumes: - - ./docker/provider.Caddyfile:/etc/caddy/Caddyfile + - ./provider.Caddyfile:/etc/caddy/Caddyfile - caddy_data:/data - caddy_config:/config networks: diff --git a/package-lock.json b/package-lock.json index 04c9d439b7..20ab196c31 100644 --- a/package-lock.json +++ b/package-lock.json @@ -9490,6 +9490,15 @@ "integrity": "sha512-Klz949h02Gz2uZCMGwDUSDS1YBlTdDDgbWHi+81l29tQALUtvz4rAYi5uoVhE5Lagoq6DeqAUlbrHvW/mXDgdQ==", "dev": true }, + "node_modules/@types/morgan": { + "version": "1.9.9", + "resolved": "https://registry.npmjs.org/@types/morgan/-/morgan-1.9.9.tgz", + "integrity": "sha512-iRYSDKVaC6FkGSpEVVIvrRGw0DfJMiQzIn3qr2G5B3C//AWkulhXgaBd7tS9/J79GWSYMTHGs7PfI5b3Y8m+RQ==", + "dev": true, + "dependencies": { + "@types/node": "*" + } + }, "node_modules/@types/node": { "version": "20.11.29", "resolved": "https://registry.npmjs.org/@types/node/-/node-20.11.29.tgz", @@ -11697,6 +11706,22 @@ } ] }, + "node_modules/basic-auth": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/basic-auth/-/basic-auth-2.0.1.tgz", + "integrity": "sha512-NF+epuEdnUYVlGuhaxbbq+dvJttwLnGY+YixlXlME5KpQ5W3CnXA5cVTneY3SPbPDRkcjMbifrwmFYcClgOZeg==", + "dependencies": { + "safe-buffer": "5.1.2" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/basic-auth/node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, "node_modules/batch": { "version": "0.6.1", "resolved": "https://registry.npmjs.org/batch/-/batch-0.6.1.tgz", @@ -22719,6 +22744,45 @@ "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.3.tgz", "integrity": "sha512-6FlzubTLZG3J2a/NVCAleEhjzq5oxgHyaCU9yYXvcLsvoVaHJq/s5xXI6/XXP6tz7R9xAOtHnSO/tXtF3WRTlA==" }, + "node_modules/morgan": { + "version": "1.10.0", + "resolved": "https://registry.npmjs.org/morgan/-/morgan-1.10.0.tgz", + "integrity": "sha512-AbegBVI4sh6El+1gNwvD5YIck7nSA36weD7xvIxG4in80j/UoK8AEGaWnnz8v1GxonMCltmlNs5ZKbGvl9b1XQ==", + "dependencies": { + "basic-auth": "~2.0.1", + "debug": "2.6.9", + "depd": "~2.0.0", + "on-finished": "~2.3.0", + "on-headers": "~1.0.2" + }, + "engines": { + "node": ">= 0.8.0" + } + }, + "node_modules/morgan/node_modules/debug": { + "version": "2.6.9", + "resolved": "https://registry.npmjs.org/debug/-/debug-2.6.9.tgz", + "integrity": "sha512-bC7ElrdJaJnPbAP+1EotYvqZsb3ecl5wi6Bfi6BJTUcNowp6cvspg0jXznRTKDjm/E7AdgFBVeAPVMNcKGsHMA==", + "dependencies": { + "ms": "2.0.0" + } + }, + "node_modules/morgan/node_modules/ms": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.0.0.tgz", + "integrity": "sha512-Tpp60P6IUJDTuOq/5Z8cdskzJujfwqfOTkrwIwj7IRISpnkJnT6SyJ4PCPnGMoFjC9ddhal5KVIYtAt97ix05A==" + }, + "node_modules/morgan/node_modules/on-finished": { + "version": "2.3.0", + "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", + "integrity": "sha512-ikqdkGAAyf/X/gPhXGvfgAytDZtDbr+bkNUJ0N9h5MI/dmdgCs3l6hoHrcUv41sRKew3jIwrp4qQDXiK99Utww==", + "dependencies": { + "ee-first": "1.1.1" + }, + "engines": { + "node": ">= 0.8" + } + }, "node_modules/mpath": { "version": "0.9.0", "resolved": "https://registry.npmjs.org/mpath/-/mpath-0.9.0.tgz", @@ -23377,129 +23441,6 @@ "which-typed-array": "^1.1.2" } }, - "node_modules/nodejs-polars": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars/-/nodejs-polars-0.6.0.tgz", - "integrity": "sha512-6+VMVjUguKrHZKt3mUTImQy8B31YKuxD0IYsYA6ESnFI/Fr8RSpIb1iVqveTAcjb6iTyQvrFClWItCZIKgz60A==", - "engines": { - "node": ">= 16" - }, - "optionalDependencies": { - "nodejs-polars-android-arm64": "0.6.0", - "nodejs-polars-darwin-arm64": "0.6.0", - "nodejs-polars-darwin-x64": "0.6.0", - "nodejs-polars-linux-arm-gnueabihf": "0.6.0", - "nodejs-polars-linux-arm64-gnu": "0.6.0", - "nodejs-polars-linux-x64-gnu": "0.6.0", - "nodejs-polars-win32-ia32-msvc": "0.6.0", - "nodejs-polars-win32-x64-msvc": "0.6.0" - } - }, - "node_modules/nodejs-polars-darwin-arm64": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-darwin-arm64/-/nodejs-polars-darwin-arm64-0.6.0.tgz", - "integrity": "sha512-LSd9zOpw1t7JeDTqutPWsHMRU4JK0eQ3gdkhzPMvgfGY7FGA1ZSMox2ObTi6rVHL3cUPyDsw9fKXyO3SHCRs0g==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-darwin-x64": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-darwin-x64/-/nodejs-polars-darwin-x64-0.6.0.tgz", - "integrity": "sha512-6xaGi2EQqxtnpiBAeJnYjtR3iSXW7duOxl+nJpoKCxirDbdtpQ+E3ChgEX2TNbDllYAm3OoGrqXpBNVMG2BCgA==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "darwin" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-linux-arm-gnueabihf": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-linux-arm-gnueabihf/-/nodejs-polars-linux-arm-gnueabihf-0.6.0.tgz", - "integrity": "sha512-B25RWn7neamKfnJH07rz+LFAI9v0Yu3Bm0ZYL3JwZvI00zAVEnqPEKn9zbSdgajnK8bImInu8FMDp0XJK531Zw==", - "cpu": [ - "arm" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-linux-arm64-gnu": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-linux-arm64-gnu/-/nodejs-polars-linux-arm64-gnu-0.6.0.tgz", - "integrity": "sha512-561E3SaGb/6bW4NNqmYP6y1opyAgyNP0hp9rIS8D9O0efhXu3Tq2mN14g/ZjylJFqKb+Znc5fvfVtEDPld+d0Q==", - "cpu": [ - "arm64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-linux-x64-gnu": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-linux-x64-gnu/-/nodejs-polars-linux-x64-gnu-0.6.0.tgz", - "integrity": "sha512-L36qBR1nEIhozTn09ZfDap713zIOpnm0sOkr9W8NHk/jUP0Ljl3WL1i4Q62nGsFnl1QwR7qHaYqqYIZUQ+3qsw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "linux" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-win32-ia32-msvc": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-win32-ia32-msvc/-/nodejs-polars-win32-ia32-msvc-0.6.0.tgz", - "integrity": "sha512-kqwSdX2F4wwQg2PSfVqHKsqIGoRBLGd1K0DM9eNldDc5wrvkmzR8joPT0NDSOQrKOplzzKIrNAFzJgKgEs6QlQ==", - "cpu": [ - "ia32" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, - "node_modules/nodejs-polars-win32-x64-msvc": { - "version": "0.6.0", - "resolved": "https://registry.npmjs.org/nodejs-polars-win32-x64-msvc/-/nodejs-polars-win32-x64-msvc-0.6.0.tgz", - "integrity": "sha512-7ZJH3Ylcj0RrPv8bmj9tyJ3TB762mhfQuRLqb2FcdYGV1klT/MgP/RCIuMBkid2cRHVKeqHJG4dGLDyBDn8dGw==", - "cpu": [ - "x64" - ], - "optional": true, - "os": [ - "win32" - ], - "engines": { - "node": ">= 10" - } - }, "node_modules/nodemon": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/nodemon/-/nodemon-3.1.0.tgz", @@ -32038,12 +31979,14 @@ "cors": "^2.8.5", "cron-parser": "^4.9.0", "dotenv": "^16.0.1", + "morgan": "^1.10.0", "yargs": "^17.7.2", "zod": "^3.22.4" }, "devDependencies": { "@prosopo/config": "0.3.5", "@types/cors": "^2.8.14", + "@types/morgan": "^1.9.9", "es-main": "^1.2.0", "express": "^4.18.2", "tslib": "2.6.2", @@ -32469,7 +32412,6 @@ "cron-parser": "^4.5.0", "express": "^4.18.1", "jsonwebtoken": "^9.0.1", - "nodejs-polars": "^0.6.0", "yargs": "^17.5.1", "yargs-parser": "^21.0.1" }, diff --git a/packages/cli/src/argv.ts b/packages/cli/src/argv.ts index 95c3186cfb..2e0fb28272 100644 --- a/packages/cli/src/argv.ts +++ b/packages/cli/src/argv.ts @@ -16,7 +16,6 @@ import { LogLevel, getLogger } from '@prosopo/common' import { ProsopoConfigOutput } from '@prosopo/types' import { commandBatchCommit, - commandCalculateCaptchaSolutions, commandDappAccounts, commandDappDetails, commandDappRegister, @@ -52,7 +51,6 @@ export function processArgs(args: string[], pair: KeyringPair, config: ProsopoCo .command(commandProviderDetails(pair, config, { logger })) .command(commandProviderDataset(pair, config, { logger })) .command(commandDappDetails(pair, config, { logger })) - .command(commandCalculateCaptchaSolutions(pair, config, { logger })) .command(commandBatchCommit(pair, config, { logger })) .command(commandVersion(pair, config, { logger })) .parse() diff --git a/packages/cli/src/commands/calculateCaptchaSolutions.ts b/packages/cli/src/commands/calculateCaptchaSolutions.ts deleted file mode 100644 index 758bc55d18..0000000000 --- a/packages/cli/src/commands/calculateCaptchaSolutions.ts +++ /dev/null @@ -1,48 +0,0 @@ -// Copyright 2021-2024 Prosopo (UK) Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -import { ArgumentsCamelCase, Argv } from 'yargs' -import { CalculateSolutionsTask } from '@prosopo/provider' -import { KeyringPair } from '@polkadot/keyring/types' -import { LogLevel, Logger, ProsopoEnvError, getLogger } from '@prosopo/common' -import { ProsopoConfigOutput } from '@prosopo/types' -import { ProviderEnvironment } from '@prosopo/env' -import { validateScheduleExpression } from './validators.js' - -export default (pair: KeyringPair, config: ProsopoConfigOutput, cmdArgs?: { logger?: Logger }) => { - const logger = cmdArgs?.logger || getLogger(LogLevel.enum.info, 'cli.calculate_captcha_solutions') - - return { - command: 'calculate_captcha_solutions', - describe: 'Calculate captcha solutions', - builder: (yargs: Argv) => { - return yargs.option('schedule', { - type: 'string' as const, - demand: false, - desc: 'A Recurring schedule expression', - } as const) - }, - handler: async (argv: ArgumentsCamelCase) => { - const env = new ProviderEnvironment(config, pair) - await env.isReady() - if (argv.schedule) { - throw new ProsopoEnvError('GENERAL.NOT_IMPLEMENTED') - } else { - const calculateSolutionsTask = new CalculateSolutionsTask(env) - const result = await calculateSolutionsTask.run() - logger.info(`Updated ${result} captcha solutions`) - } - }, - middlewares: [validateScheduleExpression], - } -} diff --git a/packages/cli/src/commands/index.ts b/packages/cli/src/commands/index.ts index 033a88e40c..c82b114f51 100644 --- a/packages/cli/src/commands/index.ts +++ b/packages/cli/src/commands/index.ts @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. export { default as commandBatchCommit } from './batchCommit.js' -export { default as commandCalculateCaptchaSolutions } from './calculateCaptchaSolutions.js' export { default as commandDappAccounts } from './dappAccounts.js' export { default as commandDappDetails } from './dappDetails.js' export { default as commandDappUpdate } from './dappUpdate.js' diff --git a/packages/cli/src/start.ts b/packages/cli/src/start.ts index e818003c9f..070c1e6bb3 100644 --- a/packages/cli/src/start.ts +++ b/packages/cli/src/start.ts @@ -51,11 +51,12 @@ function startApi(env: ProviderEnvironment, admin = false): Server { apiApp.use(express.json({ limit: '50mb' })) apiApp.use(i18nMiddleware({})) apiApp.use(prosopoRouter(env)) + apiApp.use(handleErrors) + if (admin) { apiApp.use(prosopoAdminRouter(env)) } - apiApp.use(handleErrors) return apiApp.listen(apiPort, () => { env.logger.info(`Prosopo app listening at http://localhost:${apiPort}`) }) diff --git a/packages/provider/package.json b/packages/provider/package.json index eb197bf6ed..51ee267151 100644 --- a/packages/provider/package.json +++ b/packages/provider/package.json @@ -51,7 +51,6 @@ "cron-parser": "^4.5.0", "express": "^4.18.1", "jsonwebtoken": "^9.0.1", - "nodejs-polars": "^0.6.0", "yargs": "^17.5.1", "yargs-parser": "^21.0.1" }, diff --git a/packages/provider/src/scheduler.ts b/packages/provider/src/scheduler.ts index c84f84fbbc..21b8ec69a8 100644 --- a/packages/provider/src/scheduler.ts +++ b/packages/provider/src/scheduler.ts @@ -12,7 +12,6 @@ // See the License for the specific language governing permissions and // limitations under the License. import { BatchCommitmentsTask } from './batch/commitments.js' -import { CalculateSolutionsTask } from './tasks/calculateSolutions.js' import { CronJob } from 'cron' import { KeyringPair } from '@polkadot/keyring/types' import { ProsopoConfigOutput } from '@prosopo/types' @@ -20,20 +19,6 @@ import { ProsopoEnvError } from '@prosopo/common' import { ProviderEnvironment } from '@prosopo/env' import { at } from '@prosopo/util' -export async function calculateSolutionsScheduler(pair: KeyringPair, config: ProsopoConfigOutput) { - const env = new ProviderEnvironment(config, pair) - await env.isReady() - const tasks = new CalculateSolutionsTask(env) - const job = new CronJob(at(process.argv, 2), () => { - env.logger.debug('CalculateSolutionsTask....') - tasks.run().catch((err) => { - env.logger.error(err) - }) - }) - - job.start() -} - export async function batchCommitScheduler(pair: KeyringPair, config: ProsopoConfigOutput) { const env = new ProviderEnvironment(config, pair) await env.isReady() diff --git a/packages/provider/src/tasks/calculateSolutions.ts b/packages/provider/src/tasks/calculateSolutions.ts deleted file mode 100644 index 9b844070f7..0000000000 --- a/packages/provider/src/tasks/calculateSolutions.ts +++ /dev/null @@ -1,98 +0,0 @@ -// Copyright 2021-2024 Prosopo (UK) Ltd. -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. -import { CaptchaStates } from '@prosopo/types' -import { ProsopoEnvError, getLogger } from '@prosopo/common' -import { ProviderEnvironment } from '@prosopo/types-env' -import { ScheduledTaskNames } from '@prosopo/types' -import { Tasks } from './tasks.js' -import { calculateNewSolutions, checkIfTaskIsRunning, updateSolutions } from '../util.js' -import { captchaSort } from '@prosopo/datasets' -export class CalculateSolutionsTask extends Tasks { - constructor(env: ProviderEnvironment) { - super(env) - this.logger = getLogger(env.config.logLevel, 'CalculateSolutionsTask') - } - - /** - * Apply new captcha solutions to captcha dataset and recalculate merkle tree - */ - async run(): Promise { - try { - const taskRunning = await checkIfTaskIsRunning(ScheduledTaskNames.CalculateSolution, this.db) - if (!taskRunning) { - // Get the current datasetId from the contract - const provider = (await this.contract.methods.getProvider(this.contract.pair.address, {})).value - .unwrap() - .unwrap() - - // Get any unsolved CAPTCHA challenges from the database for this datasetId - const unsolvedCaptchas = await this.db.getAllCaptchasByDatasetId( - provider.datasetId.toString(), - CaptchaStates.Unsolved - ) - - // edge case when a captcha dataset contains no unsolved CAPTCHA challenges - if (!unsolvedCaptchas) { - return 0 - } - - // Sort the unsolved CAPTCHA challenges by their captchaId - const unsolvedSorted = unsolvedCaptchas.sort(captchaSort) - this.logger.info(`There are ${unsolvedSorted.length} unsolved CAPTCHA challenges`) - - // Get the solution configuration from the config file - const requiredNumberOfSolutions = this.captchaSolutionConfig.requiredNumberOfSolutions - const winningPercentage = this.captchaSolutionConfig.solutionWinningPercentage - const winningNumberOfSolutions = Math.round(requiredNumberOfSolutions * (winningPercentage / 100)) - if (unsolvedSorted && unsolvedSorted.length > 0) { - const captchaIds = unsolvedSorted.map((captcha) => captcha.captchaId) - const solutions = (await this.db.getAllDappUserSolutions(captchaIds)) || [] - const solutionsToUpdate = calculateNewSolutions(solutions, winningNumberOfSolutions) - if (solutionsToUpdate.rows().length > 0) { - this.logger.info( - `There are ${solutionsToUpdate.rows().length} CAPTCHA challenges to update with solutions` - ) - try { - // TODO polars doesn't have the captchaId field in the type - const captchaIdsToUpdate = [...(solutionsToUpdate as any)['captchaId'].values()] - const commitmentIds = solutions - .filter((s) => captchaIdsToUpdate.indexOf(s.captchaId) > -1) - .map((s) => s.commitmentId) - const dataset = await this.db.getDataset(provider.datasetId.toString()) - dataset.captchas = updateSolutions(solutionsToUpdate, dataset.captchas, this.logger) - // store new solutions in database - await this.providerSetDataset(dataset) - // mark user solutions as used to calculate new solutions - await this.db.flagProcessedDappUserSolutions(captchaIdsToUpdate) - // mark user commitments as used to calculate new solutions - await this.db.flagProcessedDappUserCommitments(commitmentIds) - // remove old captcha challenges from database - await this.db.removeCaptchas(captchaIdsToUpdate) - return solutionsToUpdate.rows().length - } catch (error) { - this.logger.error(error) - } - } - return 0 - } else { - this.logger.info(`There are no CAPTCHA challenges that require their solutions to be updated`) - return 0 - } - } - return 0 - } catch (error) { - throw new ProsopoEnvError('GENERAL.CALCULATE_CAPTCHA_SOLUTION', { context: { error } }) - } - } -} diff --git a/packages/provider/src/tasks/index.ts b/packages/provider/src/tasks/index.ts index b92976e5cb..b78ed8329a 100644 --- a/packages/provider/src/tasks/index.ts +++ b/packages/provider/src/tasks/index.ts @@ -12,4 +12,3 @@ // See the License for the specific language governing permissions and // limitations under the License. export * from './tasks.js' -export * from './calculateSolutions.js' diff --git a/packages/provider/src/tests/tasks/tasks.test.ts b/packages/provider/src/tests/tasks/tasks.test.ts index a13569083f..af0089506f 100644 --- a/packages/provider/src/tests/tasks/tasks.test.ts +++ b/packages/provider/src/tests/tasks/tasks.test.ts @@ -1090,63 +1090,4 @@ describe.sequential('CONTRACT TASKS', async function (): Promise { const isError = (await tasks.contract.tx.providerDeregister()).result?.isError expect(isError).to.be.false }) - - // TODO find out what is making this fail occasionally - // test('Calculate captcha solution on the basis of Dapp users provided solutions', async ({env}): Promise => { - // const providerAccount = await getUser(env, AccountKey.providersWithStakeAndDataset) - // const providerTasks = await getSignedTasks(env, providerAccount) - // const providerDetails = await providerTasks.contractApi.getProvider(accountAddress(providerAccount)) - // const dappAccount = await getUser(env, AccountKey.dapps) - // - // const randomCaptchasResult = await providerTasks.db.getRandomCaptcha(false, providerDetails.datasetId) - // if (randomCaptchasResult) { - // const unsolvedCaptcha = randomCaptchasResult[0] - // const solution = [ - // unsolvedCaptcha.items[0].hash || '', - // unsolvedCaptcha.items[2].hash || '', - // unsolvedCaptcha.items[3].hash || '', - // ] - // const captchaSolution: CaptchaSolution = { ...unsolvedCaptcha, solution, salt: 'blah' } - // const commitments: string[] = [] - // for (let count = 0; count < 10; count++) { - // const commitmentId = hexHash(`test${count}`) - // commitments.push(commitmentId) - // await providerTasks.db.storeDappUserSolution( - // [captchaSolution], - // commitmentId, - // randomAsHex(), - // accountContract(dappAccount), - // providerDetails.datasetId.toString() - // ) - // const userSolutions = await providerTasks.db.getDappUserSolutionById(commitmentId) - // expect(userSolutions).to.be.not.empty - // } - // - // const result = await providerTasks.calculateCaptchaSolutions() - // expect(result).to.equal(1) - // - // for (const commitment of commitments) { - // const userSolution = await providerTasks.db.getDappUserSolutionById(commitment) - // expect(userSolution?.processed).to.be.true - // } - // - // const providerDetailsNew = await providerTasks.contractApi.getProvider( - // accountAddress(providerAccount) - // ) - // - // const captchas = await providerTasks.db.getAllCaptchasByDatasetId(providerDetailsNew.datasetId.toString()) - // expect(captchas?.every((captcha) => captcha.datasetId === providerDetailsNew.datasetId.toString())).to.be - // .true - // - // expect(providerDetails.datasetId).to.not.equal(providerDetailsNew.datasetId) - // - // expect(Promise.resolve(providerTasks.db.getCaptchaById([unsolvedCaptcha.captchaId]))).to.be.rejected.then( - // (error) => { - // expect(error.message).to.equal('Failed to get captcha') - // } - // ) - // } else { - // throw new ProsopoEnvError('DATABASE.CAPTCHA_GET_FAILED') - // } - // }) }) diff --git a/packages/provider/src/util.ts b/packages/provider/src/util.ts index 4b38e609ed..99e7c57ac0 100644 --- a/packages/provider/src/util.ts +++ b/packages/provider/src/util.ts @@ -11,15 +11,13 @@ // WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. // See the License for the specific language governing permissions and // limitations under the License. -import { Captcha, CaptchaSolution, ScheduledTaskNames, ScheduledTaskStatus } from '@prosopo/types' import { Database } from '@prosopo/types-database' -import { Logger, ProsopoContractError } from '@prosopo/common' -import { arrayJoin } from '@prosopo/common' +import { ProsopoContractError } from '@prosopo/common' +import { ScheduledTaskNames, ScheduledTaskStatus } from '@prosopo/types' import { at } from '@prosopo/util' import { decodeAddress, encodeAddress } from '@polkadot/util-crypto/address' import { hexToU8a } from '@polkadot/util/hex' import { isHex } from '@polkadot/util/is' -import pl from 'nodejs-polars' export function encodeStringAddress(address: string) { try { @@ -74,47 +72,6 @@ export function parseBlockNumber(blockNumberString: string) { return parseInt(blockNumberString.replace(/,/g, '')) } -export function calculateNewSolutions(solutions: CaptchaSolution[], winningNumberOfSolutions: number) { - if (solutions.length === 0) { - return pl.DataFrame([]) - } - const solutionsNoEmptyArrays = solutions.map(({ solution, ...otherAttrs }) => { - return { solutionKey: arrayJoin(solution, ','), ...otherAttrs } - }) - let df = pl.readRecords(solutionsNoEmptyArrays) - df = df.drop('salt') - const group = df.groupBy(['captchaId', 'solutionKey']).agg(pl.count('captchaContentId').alias('count')) - const filtered: pl.DataFrame = group.filter(pl.col('count').gt(winningNumberOfSolutions)) - // TODO is below correct? 'solutionKey' does not exist in the type - const key = (filtered as any)['solutionKey'] - return filtered.withColumn(key.str.split(',').rename('solution')) -} - -export function updateSolutions(solutions: pl.DataFrame, captchas: Captcha[], logger: Logger): Captcha[] { - // Note - loading the dataset in nodejs-polars doesn't work because of nested objects, which is why this is done in - // a map instead of a join - return captchas.map((captcha: Captcha) => { - // try to find the solution in the solutions dataframe - if (!captcha.solution) { - try { - const captchaSolutions = [ - // TODO is below correct? 'solution' is not in the type - ...(solutions.filter(pl.col('captchaId').eq(pl.lit(captcha.captchaId))) as any)[ - 'solution' - ].values(), - ] - if (captchaSolutions.length > 0) { - captcha.solution = captchaSolutions[0] - captcha.solved = true - } - } catch { - logger.debug('No solution found for captchaId', captcha.captchaId) - } - } - return captcha - }) -} - /** * Check if there is a batch running. * If the batch task is running and not completed, return true.