diff --git a/.github/workflows/e2e_ios.yml b/.github/workflows/e2e_ios.yml index 853bc9bc7..fc252c9ec 100644 --- a/.github/workflows/e2e_ios.yml +++ b/.github/workflows/e2e_ios.yml @@ -51,6 +51,11 @@ jobs: with: node-version: 18 + - name: Setup Ruby version according to .ruby-version with cached gems + uses: ruby/setup-ruby@v1 + with: + bundler-cache: true + - name: Cache node modules uses: actions/cache@v3 id: cache @@ -98,8 +103,13 @@ jobs: run: echo "MOCK_MODE=e2e" >> "$GITHUB_ENV" # Install prerequisites for detox and build app, and test - - run: brew tap wix/brew - - run: brew install applesimutils + - name: Install macOS dependencies + env: + HOMEBREW_NO_AUTO_UPDATE: 1 + HOMEBREW_NO_INSTALL_CLEANUP: 1 + run: | + brew tap wix/brew + brew install applesimutils - name: Build test app run: npm run e2e:build:ios diff --git a/README.md b/README.md index cff7933f1..51fc9e5c0 100644 --- a/README.md +++ b/README.md @@ -9,7 +9,7 @@ See [CONTRIBUTING](CONTRIBUTING.md) for guidelines on contributing to this proje ### Requirements -* Xcode 13 or above +* Xcode 15 or above * [Android and iOS environment setup](https://reactnative.dev/docs/environment-setup) described in the RN docs ### Install packages and pods diff --git a/babel.config.js b/babel.config.js index 491dd6dfc..de668d2ea 100644 --- a/babel.config.js +++ b/babel.config.js @@ -22,14 +22,11 @@ module.exports = { tests: "./tests" } }], - // Reanimated 2 plugin has to be listed last https://docs.swmansion.com/react-native-reanimated/docs/fundamentals/installation/ - // react-native-vision-camera v3 - // "react-native-reanimated/plugin", - // react-native-vision-camera v2 + // Reanimated plugin has to be listed last https://docs.swmansion.com/react-native-reanimated/docs/fundamentals/installation/ [ "react-native-reanimated/plugin", { - globals: ["__inatVision"] + processNestedWorklets: true } ] ], diff --git a/e2e/signedIn.e2e.js b/e2e/signedIn.e2e.js index f091a7824..2b7610cfd 100644 --- a/e2e/signedIn.e2e.js +++ b/e2e/signedIn.e2e.js @@ -30,7 +30,7 @@ describe( "Signed in user", () => { await expect( uploadNowButton ).toBeVisible(); await uploadNowButton.tap(); } else { - // Press Upload now button + // Press Save button const saveButton = element( by.id( "ObsEdit.saveButton" ) ); await expect( saveButton ).toBeVisible(); await saveButton.tap(); diff --git a/ios/Podfile.lock b/ios/Podfile.lock index b3fc328aa..19b952c67 100644 --- a/ios/Podfile.lock +++ b/ios/Podfile.lock @@ -317,7 +317,7 @@ PODS: - React-Core - react-native-webview (11.23.1): - React-Core - - react-native-worklets-core (0.2.0): + - react-native-worklets-core (0.4.0): - React - React-callinvoker - React-Core @@ -435,33 +435,10 @@ PODS: - React-Core - RNPermissions (3.10.0): - React-Core - - RNReanimated (2.17.0): - - DoubleConversion - - FBLazyVector - - FBReactNativeSpec - - glog - - RCT-Folly - - RCTRequired - - RCTTypeSafety - - React-callinvoker + - RNReanimated (3.7.1): + - RCT-Folly (= 2021.07.22.00) - React-Core - - React-Core/DevSupport - - React-Core/RCTWebSocket - - React-CoreModules - - React-cxxreact - - React-jsi - - React-jsiexecutor - - React-jsinspector - - React-RCTActionSheet - - React-RCTAnimation - - React-RCTBlob - - React-RCTImage - - React-RCTLinking - - React-RCTNetwork - - React-RCTSettings - - React-RCTText - ReactCommon/turbomodule/core - - Yoga - RNScreens (3.21.1): - React-Core - React-RCTImage @@ -479,11 +456,12 @@ PODS: - SDWebImageWebPCoder (0.8.5): - libwebp (~> 1.0) - SDWebImage/Core (~> 5.10) - - VisionCamera (2.15.6): + - VisionCamera (3.9.1): - React - React-callinvoker - React-Core - - VisionCameraPluginInatVision (2.2.0): + - react-native-worklets-core + - VisionCameraPluginInatVision (3.0.0): - React-Core - Yoga (1.14.0) @@ -768,7 +746,7 @@ SPEC CHECKSUMS: react-native-sensitive-info: d44e909d065f9c0e15734245e5dd6a24b82e3dcd react-native-slider: cc89964e1432fa31aa9db7a0fa9b21e26b5d5152 react-native-webview: d33e2db8925d090871ffeb232dfa50cb3a727581 - react-native-worklets-core: 7ad416a8965086b98b07964f7f6932560a54a14c + react-native-worklets-core: 2efe80a3ee87fe5e6fefa814e0e20c2708d3ad25 React-perflogger: c944b06edad34f5ecded0f29a6e66290a005d365 React-RCTActionSheet: fa467f37777dacba2c72da4be7ae065da4482d7d React-RCTAnimation: 0591ee5f9e3d8c864a0937edea2165fe968e7099 @@ -796,7 +774,7 @@ SPEC CHECKSUMS: RNGestureHandler: 6e4dc6b7ab3a385386d4e36228bd065e5a611394 RNLocalize: d4b8af4e442d4bcca54e68fc687a2129b4d71a81 RNPermissions: 0332875c444efe864dd97071dc848529bd7cc692 - RNReanimated: f186e85d9f28c9383d05ca39e11dd194f59093ec + RNReanimated: e626b8c31f56bf320fd27ee5ca0d5349792f2a8d RNScreens: d3675ab2878704de70c9dae57fa5d024802404cc RNShareMenu: cb9dac548c8bf147d06f0bf07296ad51ea9f5fc3 RNStoreReview: 31dbfd0dac2eea9675f0b84f1dd3261c2110c337 @@ -804,8 +782,8 @@ SPEC CHECKSUMS: RNVectorIcons: fcc2f6cb32f5735b586e66d14103a74ce6ad61f8 SDWebImage: a7f831e1a65eb5e285e3fb046a23fcfbf08e696d SDWebImageWebPCoder: 908b83b6adda48effe7667cd2b7f78c897e5111d - VisionCamera: 523b49054bee9dace64189ab6631cb41e8b83fe0 - VisionCameraPluginInatVision: 7e09a4ca0b34dd81afd4b68aa26a27eff5bb8fd4 + VisionCamera: 609b194489f336792caa5eda305a3fdd4ba5d44c + VisionCameraPluginInatVision: edd58cf80291675d1a1523a3d8d3b2c2f1bff26a Yoga: e29645ec5a66fb00934fad85338742d1c247d4cb PODFILE CHECKSUM: 77ed9526d4011b245ce5afa1ea331dea4c67d753 diff --git a/package-lock.json b/package-lock.json index 2a95e45d5..fa0c8b01b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -81,7 +81,7 @@ "react-native-paper": "^5.10.5", "react-native-permissions": "^3.10.0", "react-native-picker-select": "8.0.4", - "react-native-reanimated": "^2.17.0", + "react-native-reanimated": "^3.7.0", "react-native-reanimated-carousel": "^3.4.0", "react-native-render-html": "^6.3.4", "react-native-safe-area-context": "^4.7.4", @@ -94,14 +94,14 @@ "react-native-url-polyfill": "^2.0.0", "react-native-uuid": "^2.0.1", "react-native-vector-icons": "^9.1.0", - "react-native-vision-camera": "github:inaturalist/react-native-vision-camera#our-main-fork-2", + "react-native-vision-camera": "3.9.1", "react-native-webview": "11.23.1", - "react-native-worklets-core": "^0.2.0", + "react-native-worklets-core": "0.4.0", "realm": "^12.6.0", "reassure": "^0.10.1", "sanitize-html": "^2.11.0", "use-debounce": "^9.0.4", - "vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision", + "vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision#version-3", "zustand": "^4.4.7" }, "devDependencies": { @@ -6305,9 +6305,9 @@ } }, "node_modules/@react-navigation/elements": { - "version": "1.3.18", - "resolved": "https://registry.npmjs.org/@react-navigation/elements/-/elements-1.3.18.tgz", - "integrity": "sha512-/0hwnJkrr415yP0Hf4PjUKgGyfshrvNUKFXN85Mrt1gY49hy9IwxZgrrxlh0THXkPeq8q4VWw44eHDfAcQf20Q==", + "version": "1.3.21", + "resolved": "https://registry.npmjs.org/@react-navigation/elements/-/elements-1.3.21.tgz", + "integrity": "sha512-eyS2C6McNR8ihUoYfc166O1D8VYVh9KIl0UQPI8/ZJVsStlfSTgeEEh+WXge6+7SFPnZ4ewzEJdSAHH+jzcEfg==", "peerDependencies": { "@react-navigation/native": "^6.0.0", "react": "*", @@ -23067,19 +23067,22 @@ } }, "node_modules/react-native-reanimated": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-2.17.0.tgz", - "integrity": "sha512-bVy+FUEaHXq4i+aPPqzGeor1rG4scgVNBbBz21ohvC7iMpB9IIgvGsmy1FAoodZhZ5sa3EPF67Rcec76F1PXlQ==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.7.1.tgz", + "integrity": "sha512-bapCxhnS58+GZynQmA/f5U8vRlmhXlI/WhYg0dqnNAGXHNIc+38ahRWcG8iK8e0R2v9M8Ky2ZWObEC6bmweofg==", "dependencies": { "@babel/plugin-transform-object-assign": "^7.16.7", "@babel/preset-typescript": "^7.16.7", - "invariant": "^2.2.4", - "lodash.isequal": "^4.5.0", - "setimmediate": "^1.0.5", - "string-hash-64": "^1.0.3" + "convert-source-map": "^2.0.0", + "invariant": "^2.2.4" }, "peerDependencies": { "@babel/core": "^7.0.0-0", + "@babel/plugin-proposal-nullish-coalescing-operator": "^7.0.0-0", + "@babel/plugin-proposal-optional-chaining": "^7.0.0-0", + "@babel/plugin-transform-arrow-functions": "^7.0.0-0", + "@babel/plugin-transform-shorthand-properties": "^7.0.0-0", + "@babel/plugin-transform-template-literals": "^7.0.0-0", "react": "*", "react-native": "*" } @@ -23095,6 +23098,11 @@ "react-native-reanimated": ">=2.7.0" } }, + "node_modules/react-native-reanimated/node_modules/convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + }, "node_modules/react-native-redash": { "version": "18.1.0", "resolved": "https://registry.npmjs.org/react-native-redash/-/react-native-redash-18.1.0.tgz", @@ -23268,12 +23276,18 @@ } }, "node_modules/react-native-vision-camera": { - "version": "2.15.6", - "resolved": "git+ssh://git@github.com/inaturalist/react-native-vision-camera.git#e5a8a91759843e1255742ce44e5638e8fab19fe1", - "license": "MIT", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/react-native-vision-camera/-/react-native-vision-camera-3.9.1.tgz", + "integrity": "sha512-Pi9ikguJlN1ydVZOyRaMfUij1raUY93rVuPM92BsGnXEfxSLbvRYXW4ll1DRtVtjS0kZq4IW7Oavg8syRPc/xQ==", "peerDependencies": { "react": "*", - "react-native": "*" + "react-native": "*", + "react-native-worklets-core": "*" + }, + "peerDependenciesMeta": { + "react-native-worklets-core": { + "optional": true + } } }, "node_modules/react-native-webview": { @@ -23298,9 +23312,12 @@ } }, "node_modules/react-native-worklets-core": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/react-native-worklets-core/-/react-native-worklets-core-0.2.0.tgz", - "integrity": "sha512-gxpfl3KnoRmDtBBVs08K7Ru3xaOrBTGXKQtcjwDzgIcaiNhPfKiJdUz1AIa3SX+M2I/f/7fgWK2W9OFYae/AzA==", + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/react-native-worklets-core/-/react-native-worklets-core-0.4.0.tgz", + "integrity": "sha512-0rYCwxnG6L85mih+3xe1r2RhhwKqjgVN9jikVh0iMPKDf9L4OZMJnl6Kh4zjXiW9rkyI5lBghfM4XIcLMfeU6w==", + "dependencies": { + "string-hash-64": "^1.0.3" + }, "engines": { "node": ">= 16.0.0" }, @@ -24279,11 +24296,6 @@ "node": ">= 0.4" } }, - "node_modules/setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" - }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -25850,8 +25862,8 @@ } }, "node_modules/vision-camera-plugin-inatvision": { - "version": "2.2.0", - "resolved": "git+ssh://git@github.com/inaturalist/vision-camera-plugin-inatvision.git#34b9269b9491c1f667bdbbf3c3edfe7288a977cf", + "version": "3.0.0", + "resolved": "git+ssh://git@github.com/inaturalist/vision-camera-plugin-inatvision.git#d48dc24cea1b204fae3ee8500c0fe781f2146b8a", "license": "MIT", "engines": { "node": ">= 16.0.0" @@ -25859,8 +25871,7 @@ "peerDependencies": { "react": "*", "react-native": "*", - "react-native-reanimated": ">=2.1.0", - "react-native-vision-camera": ">=2.0.0" + "react-native-vision-camera": ">=3.6.3" } }, "node_modules/vlq": { @@ -30741,9 +30752,9 @@ } }, "@react-navigation/elements": { - "version": "1.3.18", - "resolved": "https://registry.npmjs.org/@react-navigation/elements/-/elements-1.3.18.tgz", - "integrity": "sha512-/0hwnJkrr415yP0Hf4PjUKgGyfshrvNUKFXN85Mrt1gY49hy9IwxZgrrxlh0THXkPeq8q4VWw44eHDfAcQf20Q==", + "version": "1.3.21", + "resolved": "https://registry.npmjs.org/@react-navigation/elements/-/elements-1.3.21.tgz", + "integrity": "sha512-eyS2C6McNR8ihUoYfc166O1D8VYVh9KIl0UQPI8/ZJVsStlfSTgeEEh+WXge6+7SFPnZ4ewzEJdSAHH+jzcEfg==", "requires": {} }, "@react-navigation/native": { @@ -43069,16 +43080,21 @@ } }, "react-native-reanimated": { - "version": "2.17.0", - "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-2.17.0.tgz", - "integrity": "sha512-bVy+FUEaHXq4i+aPPqzGeor1rG4scgVNBbBz21ohvC7iMpB9IIgvGsmy1FAoodZhZ5sa3EPF67Rcec76F1PXlQ==", + "version": "3.7.1", + "resolved": "https://registry.npmjs.org/react-native-reanimated/-/react-native-reanimated-3.7.1.tgz", + "integrity": "sha512-bapCxhnS58+GZynQmA/f5U8vRlmhXlI/WhYg0dqnNAGXHNIc+38ahRWcG8iK8e0R2v9M8Ky2ZWObEC6bmweofg==", "requires": { "@babel/plugin-transform-object-assign": "^7.16.7", "@babel/preset-typescript": "^7.16.7", - "invariant": "^2.2.4", - "lodash.isequal": "^4.5.0", - "setimmediate": "^1.0.5", - "string-hash-64": "^1.0.3" + "convert-source-map": "^2.0.0", + "invariant": "^2.2.4" + }, + "dependencies": { + "convert-source-map": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/convert-source-map/-/convert-source-map-2.0.0.tgz", + "integrity": "sha512-Kvp459HrV2FEJ1CAsi1Ku+MY3kasH19TFykTz2xWmMeq6bk2NU3XXvfJ+Q61m0xktWwt+1HSYf3JZsTms3aRJg==" + } } }, "react-native-reanimated-carousel": { @@ -43217,8 +43233,9 @@ } }, "react-native-vision-camera": { - "version": "git+ssh://git@github.com/inaturalist/react-native-vision-camera.git#e5a8a91759843e1255742ce44e5638e8fab19fe1", - "from": "react-native-vision-camera@github:inaturalist/react-native-vision-camera#our-main-fork-2", + "version": "3.9.1", + "resolved": "https://registry.npmjs.org/react-native-vision-camera/-/react-native-vision-camera-3.9.1.tgz", + "integrity": "sha512-Pi9ikguJlN1ydVZOyRaMfUij1raUY93rVuPM92BsGnXEfxSLbvRYXW4ll1DRtVtjS0kZq4IW7Oavg8syRPc/xQ==", "requires": {} }, "react-native-webview": { @@ -43238,10 +43255,12 @@ } }, "react-native-worklets-core": { - "version": "0.2.0", - "resolved": "https://registry.npmjs.org/react-native-worklets-core/-/react-native-worklets-core-0.2.0.tgz", - "integrity": "sha512-gxpfl3KnoRmDtBBVs08K7Ru3xaOrBTGXKQtcjwDzgIcaiNhPfKiJdUz1AIa3SX+M2I/f/7fgWK2W9OFYae/AzA==", - "requires": {} + "version": "0.4.0", + "resolved": "https://registry.npmjs.org/react-native-worklets-core/-/react-native-worklets-core-0.4.0.tgz", + "integrity": "sha512-0rYCwxnG6L85mih+3xe1r2RhhwKqjgVN9jikVh0iMPKDf9L4OZMJnl6Kh4zjXiW9rkyI5lBghfM4XIcLMfeU6w==", + "requires": { + "string-hash-64": "^1.0.3" + } }, "react-refresh": { "version": "0.4.3", @@ -43830,11 +43849,6 @@ "has-property-descriptors": "^1.0.0" } }, - "setimmediate": { - "version": "1.0.5", - "resolved": "https://registry.npmjs.org/setimmediate/-/setimmediate-1.0.5.tgz", - "integrity": "sha512-MATJdZp8sLqDl/68LfQmbP8zKPLQNV6BIZoIgrscFDQ+RsvK/BxeDQOgyxKKoh0y/8h3BqVFnCqQ/gd+reiIXA==" - }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -45036,8 +45050,8 @@ "integrity": "sha512-BNGbWLfd0eUPabhkXUVm0j8uuvREyTh5ovRa/dyow/BqAbZJyC+5fU+IzQOzmAKzYqYRAISoRhdQr3eIZ/PXqg==" }, "vision-camera-plugin-inatvision": { - "version": "git+ssh://git@github.com/inaturalist/vision-camera-plugin-inatvision.git#34b9269b9491c1f667bdbbf3c3edfe7288a977cf", - "from": "vision-camera-plugin-inatvision@github:inaturalist/vision-camera-plugin-inatvision", + "version": "git+ssh://git@github.com/inaturalist/vision-camera-plugin-inatvision.git#d48dc24cea1b204fae3ee8500c0fe781f2146b8a", + "from": "vision-camera-plugin-inatvision@github:inaturalist/vision-camera-plugin-inatvision#version-3", "requires": {} }, "vlq": { diff --git a/package.json b/package.json index 44dd608f1..dcb50b088 100644 --- a/package.json +++ b/package.json @@ -106,7 +106,7 @@ "react-native-paper": "^5.10.5", "react-native-permissions": "^3.10.0", "react-native-picker-select": "8.0.4", - "react-native-reanimated": "^2.17.0", + "react-native-reanimated": "^3.7.0", "react-native-reanimated-carousel": "^3.4.0", "react-native-render-html": "^6.3.4", "react-native-safe-area-context": "^4.7.4", @@ -119,14 +119,14 @@ "react-native-url-polyfill": "^2.0.0", "react-native-uuid": "^2.0.1", "react-native-vector-icons": "^9.1.0", - "react-native-vision-camera": "github:inaturalist/react-native-vision-camera#our-main-fork-2", + "react-native-vision-camera": "3.9.1", "react-native-webview": "11.23.1", - "react-native-worklets-core": "^0.2.0", + "react-native-worklets-core": "0.4.0", "realm": "^12.6.0", "reassure": "^0.10.1", "sanitize-html": "^2.11.0", "use-debounce": "^9.0.4", - "vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision", + "vision-camera-plugin-inatvision": "github:inaturalist/vision-camera-plugin-inatvision#version-3", "zustand": "^4.4.7" }, "devDependencies": { diff --git a/patches/react-native-worklets-core+0.4.0.patch b/patches/react-native-worklets-core+0.4.0.patch new file mode 100644 index 000000000..330850c75 --- /dev/null +++ b/patches/react-native-worklets-core+0.4.0.patch @@ -0,0 +1,30 @@ +diff --git a/node_modules/react-native-worklets-core/cpp/wrappers/WKTJsiArrayWrapper.h b/node_modules/react-native-worklets-core/cpp/wrappers/WKTJsiArrayWrapper.h +index aea3ee9..72e2191 100644 +--- a/node_modules/react-native-worklets-core/cpp/wrappers/WKTJsiArrayWrapper.h ++++ b/node_modules/react-native-worklets-core/cpp/wrappers/WKTJsiArrayWrapper.h +@@ -86,6 +86,17 @@ public: + return lastEl->unwrapAsProxyOrValue(runtime); + }; + ++ JSI_HOST_FUNCTION(shift) { ++ // Shift first element from array ++ if (_array.empty()) { ++ return jsi::Value::undefined(); ++ } ++ auto firstEl = _array.at(0); ++ _array.erase(_array.begin()); ++ notify(); ++ return firstEl->unwrapAsProxyOrValue(runtime); ++ }; ++ + JSI_HOST_FUNCTION(forEach) { + auto callbackFn = arguments[0].asObject(runtime).asFunction(runtime); + for (size_t i = 0; i < _array.size(); i++) { +@@ -275,6 +286,7 @@ public: + JSI_EXPORT_FUNCTIONS( + JSI_EXPORT_FUNC(JsiArrayWrapper, push), + JSI_EXPORT_FUNC(JsiArrayWrapper, pop), ++ JSI_EXPORT_FUNC(JsiArrayWrapper, shift), + JSI_EXPORT_FUNC(JsiArrayWrapper, forEach), + JSI_EXPORT_FUNC(JsiArrayWrapper, map), + JSI_EXPORT_FUNC(JsiArrayWrapper, filter), diff --git a/src/components/Camera/ARCamera/ARCamera.js b/src/components/Camera/ARCamera/ARCamera.js index 4e85ae385..009f1afea 100644 --- a/src/components/Camera/ARCamera/ARCamera.js +++ b/src/components/Camera/ARCamera/ARCamera.js @@ -13,7 +13,7 @@ import DeviceInfo from "react-native-device-info"; import LinearGradient from "react-native-linear-gradient"; import { useTheme } from "react-native-paper"; import { convertOfflineScoreToConfidence } from "sharedHelpers/convertScores"; -import { useTranslation } from "sharedHooks"; +import { useDebugMode, useTranslation } from "sharedHooks"; import { handleCameraError, @@ -52,6 +52,7 @@ const ARCamera = ( { isLandscapeMode }: Props ): Node => { const hasFlash = device?.hasFlash; + const { isDebug } = useDebugMode( ); const { animatedProps, changeZoom, @@ -161,6 +162,11 @@ const ARCamera = ( { : t( "Loading-iNaturalists-AR-Camera" )} )} + {isDebug && result && ( + + {`Age of result: ${Date.now() - result.timestamp}ms`} + + )} {!modelLoaded && ( diff --git a/src/components/Camera/ARCamera/FrameProcessorCamera.js b/src/components/Camera/ARCamera/FrameProcessorCamera.js index 4063dc806..9529e790f 100644 --- a/src/components/Camera/ARCamera/FrameProcessorCamera.js +++ b/src/components/Camera/ARCamera/FrameProcessorCamera.js @@ -2,18 +2,19 @@ import CameraView from "components/Camera/CameraView"; import type { Node } from "react"; import React, { - useEffect + useEffect, + useState } from "react"; import { Platform } from "react-native"; -import * as REA from "react-native-reanimated"; import { - // react-native-vision-camera v3 - // runAtTargetFps, useFrameProcessor } from "react-native-vision-camera"; -// react-native-vision-camera v3 -// import { Worklets } from "react-native-worklets-core"; +import { Worklets } from "react-native-worklets-core"; import { modelPath, modelVersion, taxonomyPath } from "sharedHelpers/cvModel.ts"; +import { + orientationPatchFrameProcessor, + usePatchedRunAsync +} from "sharedHelpers/visionCameraPatches"; import { useDeviceOrientation } from "sharedHooks"; import * as InatVision from "vision-camera-plugin-inatvision"; @@ -36,16 +37,19 @@ type Props = { takingPhoto: boolean }; +const DEFAULT_FPS = 1; const DEFAULT_CONFIDENCE_THRESHOLD = 0.5; const DEFAULT_NUM_STORED_RESULTS = 4; const DEFAULT_CROP_RATIO = 1.0; +let framesProcessingTime = []; + const FrameProcessorCamera = ( { animatedProps, cameraRef, confidenceThreshold = DEFAULT_CONFIDENCE_THRESHOLD, device, - fps, + fps = DEFAULT_FPS, numStoredResults = DEFAULT_NUM_STORED_RESULTS, cropRatio = DEFAULT_CROP_RATIO, onCameraError, @@ -59,6 +63,7 @@ const FrameProcessorCamera = ( { takingPhoto }: Props ): Node => { const { deviceOrientation } = useDeviceOrientation(); + const [lastTimestamp, setLastTimestamp] = useState( Date.now() ); useEffect( () => { // This registers a listener for the frame processor plugin's log events @@ -75,14 +80,33 @@ const FrameProcessorCamera = ( { }; }, [onLog] ); - // react-native-vision-camera v3 - // const handleResults = Worklets.createRunInJsFn( predictions => { - // onTaxaDetected( predictions ); - // } ); - // const handleError = Worklets.createRunInJsFn( error => { - // onClassifierError( error ); - // } ); + const handleResults = Worklets.createRunInJsFn( ( result, timeTaken ) => { + // I don't know if it is a temporary thing but as of vision-camera@3.9.1 + // and react-native-woklets-core@0.4.0 the Array in the worklet does not have all + // the methods of a normal array, so we need to convert it to a normal array here + // getPredictionsForImage is fine + let { predictions } = result; + if ( !Array.isArray( predictions ) ) { + predictions = Object.keys( predictions ).map( key => predictions[key] ); + } + const handledResult = { predictions, timestamp: result.timestamp }; + // TODO: using current time here now, for some reason result.timestamp is not working + setLastTimestamp( Date.now() ); + framesProcessingTime.push( timeTaken ); + if ( framesProcessingTime.length === 10 ) { + const avgTime = framesProcessingTime.reduce( ( a, b ) => a + b, 0 ) / 10; + onLog( { log: `Average frame processing time over 10 frames: ${avgTime}ms` } ); + framesProcessingTime = []; + } + onTaxaDetected( handledResult ); + } ); + + const handleError = Worklets.createRunInJsFn( error => { + onClassifierError( error ); + } ); + const patchedOrientationAndroid = orientationPatchFrameProcessor( deviceOrientation ); + const patchedRunAsync = usePatchedRunAsync( ); const frameProcessor = useFrameProcessor( frame => { "worklet"; @@ -90,56 +114,54 @@ const FrameProcessorCamera = ( { if ( takingPhoto ) { return; } - - // react-native-vision-camera v2 - // Reminder: this is a worklet, running on the UI thread. - try { - const result = InatVision.inatVision( frame, { - version: modelVersion, - modelPath, - taxonomyPath, - // Johannes: when I copied over the native code from the legacy - // react-native-camera on Android this value had to be a string. On - // iOS I changed the API to also accept a string (was number). - // Maybe, the intention would look clearer if we refactor to use a - // number here. - confidenceThreshold: confidenceThreshold.toString( ), - numStoredResults, - cropRatio - } ); - REA.runOnJS( onTaxaDetected )( result ); - } catch ( classifierError ) { - console.log( `Error: ${classifierError.message}` ); - REA.runOnJS( onClassifierError )( classifierError ); + const timestamp = Date.now(); + const timeSinceLastFrame = timestamp - lastTimestamp; + if ( timeSinceLastFrame < ( 1000 / fps ) ) { + return; } - // react-native-vision-camera v3 - // runAtTargetFps( 1, () => { - // "worklet"; - // // Reminder: this is a worklet, running on the UI thread. - // try { - // const results = InatVision.inatVision( frame, { - // version, - // modelPath, - // taxonomyPath, - // confidenceThreshold, - // patchedOrientationAndroid: deviceOrientation - // } ); - // handleResults( results ); - // } catch ( classifierError ) { - // console.log( `Error: ${classifierError.message}` ); - // handleError( classifierError ); - // } - // } ); + patchedRunAsync( frame, () => { + "worklet"; + + // Reminder: this is a worklet, running on a C++ thread. Make sure to check the + // react-native-worklets-core documentation for what is supported in those worklets. + const timeBefore = Date.now(); + try { + const result = InatVision.inatVision( frame, { + version: modelVersion, + modelPath, + taxonomyPath, + confidenceThreshold, + numStoredResults, + cropRatio, + patchedOrientationAndroid + } ); + const timeAfter = Date.now(); + const timeTaken = timeAfter - timeBefore; + handleResults( result, timeTaken ); + } catch ( classifierError ) { + console.log( `Error: ${classifierError.message}` ); + handleError( classifierError ); + } + } ); }, - [modelVersion, confidenceThreshold, takingPhoto, deviceOrientation, numStoredResults, cropRatio] + [ + patchedRunAsync, + modelVersion, + confidenceThreshold, + takingPhoto, + patchedOrientationAndroid, + numStoredResults, + cropRatio, + lastTimestamp, + fps + ] ); return ( { const [result, setResult] = useState( null ); + const [resultTimestamp, setResultTimestamp] = useState( undefined ); const [modelLoaded, setModelLoaded] = useState( false ); const [confidenceThreshold, setConfidenceThreshold] = useState( 0.5 ); const [fps, setFPS] = useState( 1 ); @@ -18,6 +19,7 @@ const usePredictions = ( ): Object => { if ( cvResult && !modelLoaded ) { setModelLoaded( true ); } + setResultTimestamp( cvResult.timestamp ); let prediction = null; const { predictions: branch } = cvResult; branch.sort( ( a, b ) => a.rank_level - b.rank_level ); @@ -34,7 +36,8 @@ const usePredictions = ( ): Object => { name: finestPrediction.name, iconic_taxon_name: iconicTaxon?.name }, - score: finestPrediction.score + score: finestPrediction.score, + timestamp: cvResult.timestamp }; } setResult( prediction ); @@ -48,6 +51,7 @@ const usePredictions = ( ): Object => { numStoredResults, cropRatio, result, + resultTimestamp, setConfidenceThreshold, setFPS, setNumStoredResults, diff --git a/src/components/Camera/CameraContainer.js b/src/components/Camera/CameraContainer.js index cf941ad48..7e27e5c09 100644 --- a/src/components/Camera/CameraContainer.js +++ b/src/components/Camera/CameraContainer.js @@ -5,13 +5,8 @@ import type { Node } from "react"; import React, { useState } from "react"; -// Temporarily using a fork so this is to avoid that eslint error. Need to -// remove if/when we return to the main repo import { - // react-native-vision-camera v3 - // useCameraDevice - // react-native-vision-camera v2 - useCameraDevices + useCameraDevice } from "react-native-vision-camera"; import CameraWithDevice from "./CameraWithDevice"; @@ -21,11 +16,7 @@ const CameraContainer = ( ): Node => { const addEvidence = params?.addEvidence; const cameraType = params?.camera; const [cameraPosition, setCameraPosition] = useState( "back" ); - // react-native-vision-camera v3 - // const device = useCameraDevice( cameraPosition ); - // react-native-vision-camera v2 - const devices = useCameraDevices( ); - const device = devices[cameraPosition]; + const device = useCameraDevice( cameraPosition ); if ( !device ) { return null; diff --git a/src/components/Camera/CameraView.js b/src/components/Camera/CameraView.js index 12bda4bbc..4d3cec893 100644 --- a/src/components/Camera/CameraView.js +++ b/src/components/Camera/CameraView.js @@ -25,7 +25,6 @@ Reanimated.addWhitelistedNativeProps( { type Props = { cameraRef: Object, device: Object, - fps?: number, onClassifierError?: Function, onDeviceNotSupported?: Function, onCaptureError?: Function, @@ -37,14 +36,11 @@ type Props = { resizeMode?: string }; -const DEFAULT_FPS = 1; - // A container for the Camera component // that has logic that applies to both use cases in StandardCamera and ARCamera const CameraView = ( { cameraRef, device, - fps = DEFAULT_FPS, onClassifierError, onDeviceNotSupported, onCaptureError, @@ -159,7 +155,7 @@ const CameraView = ( { onZoomChange?.( e.scale ); } ); - // react-native-vision-camera v3.3.1: + // react-native-vision-camera v3.9.0: // iPad camera preview is wrong in anything else than portrait, hence the // VeryBadIpadRotator, which will rotate its contents us a style transform // and adjust position accordingly @@ -172,23 +168,22 @@ const CameraView = ( { onError( e )} - // react-native-vision-camera v3.3.1: This prop is undocumented, but does work on iOS + // react-native-vision-camera v3.9.0: This prop is undocumented, but does work on iOS // it does nothing on Android so we set it to null there orientation={orientationPatch( deviceOrientation )} - ref={cameraRef} - device={device} enableHighQualityPhotos // Props for ARCamera only frameProcessor={frameProcessor} pixelFormat={pixelFormatPatch()} animatedProps={animatedProps} resizeMode={resizeMode || "cover"} - frameProcessorFps={fps} /> { }; const handleLog = (event: Event) => { - logger.info(`${JSON.stringify(event)}`); + logger.info(event.log); }; export { diff --git a/src/components/SharedComponents/PermissionGate.js b/src/components/SharedComponents/PermissionGate.js index 9f24eea41..a08a9acda 100644 --- a/src/components/SharedComponents/PermissionGate.js +++ b/src/components/SharedComponents/PermissionGate.js @@ -54,12 +54,12 @@ const PermissionGate = ( { )} > onClose( )} className="absolute top-2 right-2 z-10" accessibilityLabel={t( "Close-permission-request-screen" )} - testID="close-permission-gate" /> { @@ -18,8 +18,7 @@ const VeryBadIpadRotator = ( { children } ) => { // courtesy of // https://github.com/mrousavy/react-native-vision-camera/issues/1891#issuecomment-1746222690 if ( - visionCameraMajorVersion >= 3 - && innerStyle.transform + innerStyle.transform && dims.width && dims.height ) { diff --git a/src/sharedHelpers/cvModel.ts b/src/sharedHelpers/cvModel.ts index ef66452c5..c9218fa74 100644 --- a/src/sharedHelpers/cvModel.ts +++ b/src/sharedHelpers/cvModel.ts @@ -37,7 +37,7 @@ export const predictImage = ( uri: string ) => getPredictionsForImage( { modelPath, taxonomyPath, version: modelVersion, - confidenceThreshold: "0.2" + confidenceThreshold: 0.2 } ); const addCameraFilesAndroid = () => { diff --git a/src/sharedHelpers/visionCameraPatches.js b/src/sharedHelpers/visionCameraPatches.js index 00c2c5fb1..820a63fe8 100644 --- a/src/sharedHelpers/visionCameraPatches.js +++ b/src/sharedHelpers/visionCameraPatches.js @@ -5,6 +5,11 @@ import ImageResizer from "@bam.tech/react-native-image-resizer"; import { Platform } from "react-native"; import { isTablet } from "react-native-device-info"; import RNFS from "react-native-fs"; +import { + useSharedValue as useWorkletSharedValue, + useWorklet, + Worklets +} from "react-native-worklets-core"; import { LANDSCAPE_LEFT, LANDSCAPE_RIGHT, @@ -12,16 +17,7 @@ import { PORTRAIT_UPSIDE_DOWN } from "sharedHooks/useDeviceOrientation"; -import { dependencies } from "../../package.json"; - -export const visionCameraMajorVersion = dependencies["react-native-vision-camera"].match( /inaturalist/ ) - // Our custom fork is forked from v2 - ? 2 - : Number( - dependencies["react-native-vision-camera"].replace( /[^\d.]/, "" ).split( "." )[0] - ); - -// Needed for react-native-vision-camera v3.3.1 +// Needed for react-native-vision-camera v3.9.0 // This patch is used to set the pixelFormat prop which should not be needed because the default // value would be fine for both platforms. // However, on Android for the "native" pixelFormat I could not find any method or properties to @@ -31,19 +27,24 @@ export const pixelFormatPatch = () => ( Platform.OS === "ios" ? "native" : "yuv" ); -// Needed for react-native-vision-camera v3.3.1 +// Needed for react-native-vision-camera v3.9.0 // This patch is used to determine the orientation prop for the Camera component. // On Android, the orientation prop is not used, so we return null. // On iOS, the orientation prop is undocumented, but it does get used in a sense that the // photo metadata shows the correct Orientation only if this prop is set. -export const orientationPatch = deviceOrientation => { - if ( visionCameraMajorVersion < 3 ) return deviceOrientation; - return Platform.OS === "android" - ? null - : deviceOrientation; -}; +export const orientationPatch = deviceOrientation => ( Platform.OS === "android" + ? null + : deviceOrientation ); + +// Needed for react-native-vision-camera v3.9.0 in combination +// with our vision-camera-plugin-inatvision +// This patch is used to determine the orientation prop for the FrameProcessor. +// This is only needed for Android, so on iOS we return null. +export const orientationPatchFrameProcessor = deviceOrientation => ( Platform.OS === "android" + ? deviceOrientation + : null ); -// Needed for react-native-vision-camera v2 and v3.3.1 +// Needed for react-native-vision-camera v3.9.0 // As of this version the photo from takePhoto is not oriented coming from the native side. // E.g. if you take a photo in landscape-right and save it to camera roll directly from the // vision camera, it will be tilted in the native photo app. So, on iOS, depending on the @@ -86,7 +87,7 @@ export const rotationTempPhotoPatch = ( photo, deviceOrientation ) => { return photoRotation; }; -// Needed for react-native-vision-camera v3.3.1 +// Needed for react-native-vision-camera v3.9.0 // This patch is used to rotate the photo taken with the vision camera. // Because the photos coming from the vision camera are not oriented correctly, we // rotate them with image-resizer as a first step, replacing the original photo. @@ -111,7 +112,7 @@ export const rotatePhotoPatch = async ( photo, rotation ) => { await RNFS.moveFile( tempUri, photo.path ); }; -// Needed for react-native-vision-camera v3.3.1 +// Needed for react-native-vision-camera v3.9.0 // This patch is here to remember to replace the rotation used when resizing the original // photo to a smaller local copy we keep in the app cache. Previously we had a flow where // we would resize the original photo to a smaller version including rotation. Now, we @@ -134,9 +135,6 @@ export const iPadStylePatch = deviceOrientation => { if ( !isTablet() ) { return {}; } - if ( visionCameraMajorVersion < 3 ) { - return {}; - } if ( deviceOrientation === LANDSCAPE_RIGHT ) { return { transform: [{ rotate: "90deg" }] @@ -152,3 +150,48 @@ export const iPadStylePatch = deviceOrientation => { } return {}; }; + +// This patch is currently required because we are using react-native-vision-camera v3.9.1 +// together wit react-native-reanimated. The problem is that the runAsync function +// from react-native-vision-camera does not work in release mode with this reanimated. +// Uses this workaround: https://gist.github.com/nonam4/7a6409cd1273e8ed7466ba3a48dd1ecc +// Posted on this currently open issue: https://github.com/mrousavy/react-native-vision-camera/issues/2589 +export const usePatchedRunAsync = ( ) => { + /** + * Print worklets logs/errors on js thread + */ + const logOnJs = Worklets.createRunInJsFn( ( log, error ) => { + console.log( "logOnJs - ", log, " - error?:", error?.message ?? "no error" ); + } ); + const isAsyncContextBusy = useWorkletSharedValue( false ); + const customRunOnAsyncContext = useWorklet( + "CustomVisionCamera.async", + ( frame, func ) => { + "worklet"; + + try { + func( frame ); + } catch ( e ) { + logOnJs( "customRunOnAsyncContext error", e ); + } finally { + frame.decrementRefCount(); + isAsyncContextBusy.value = false; + } + }, + [] + ); + + function customRunAsync( frame, func ) { + "worklet"; + + if ( isAsyncContextBusy.value ) { + return; + } + isAsyncContextBusy.value = true; + const internal = frame; + internal.incrementRefCount(); + customRunOnAsyncContext( internal, func ); + } + + return customRunAsync; +}; diff --git a/src/sharedHooks/useDeviceOrientation.js b/src/sharedHooks/useDeviceOrientation.js index a49d510ef..2fdc809d0 100644 --- a/src/sharedHooks/useDeviceOrientation.js +++ b/src/sharedHooks/useDeviceOrientation.js @@ -6,16 +6,10 @@ import { Dimensions } from "react-native"; import DeviceInfo from "react-native-device-info"; import Orientation from "react-native-orientation-locker"; -// react-native-vision-camera v3 -// export const LANDSCAPE_LEFT = "landscape-left"; -// export const LANDSCAPE_RIGHT = "landscape-right"; -// export const PORTRAIT = "portrait"; -// export const PORTRAIT_UPSIDE_DOWN = "portrait-upside-down"; -// react-native-vision-camera v2 -export const LANDSCAPE_LEFT = "landscapeLeft"; -export const LANDSCAPE_RIGHT = "landscapeRight"; +export const LANDSCAPE_LEFT = "landscape-left"; +export const LANDSCAPE_RIGHT = "landscape-right"; export const PORTRAIT = "portrait"; -export const PORTRAIT_UPSIDE_DOWN = "portraitUpsideDown"; +export const PORTRAIT_UPSIDE_DOWN = "portrait-upside-down"; export function orientationLockerToIosOrientation( orientation: string ): string { // react-native-orientation-locker and react-native-vision-camera different diff --git a/tests/jest.setup.js b/tests/jest.setup.js index 5c278ffa1..19d30772a 100644 --- a/tests/jest.setup.js +++ b/tests/jest.setup.js @@ -17,8 +17,7 @@ import factory, { makeResponse } from "./factory"; import { mockCamera, mockSortDevices, - mockUseCameraDevice, - mockUseCameraDevices + mockUseCameraDevice } from "./vision-camera/vision-camera"; // Mock the react-native-logs config because it has a dependency on AuthenticationService @@ -56,17 +55,14 @@ jest.mock( () => require( "@react-native-async-storage/async-storage/jest/async-storage-mock" ) ); -require( "react-native-reanimated/lib/reanimated2/jestUtils" ).setUpTests(); +require( "react-native-reanimated" ).setUpTests(); jest.mock( "react-native-vision-camera", ( ) => ( { Camera: mockCamera, sortDevices: mockSortDevices, - // react-native-vision-camera v2 - useCameraDevices: mockUseCameraDevices, - // react-native-vision-camera v3 useCameraDevice: mockUseCameraDevice, VisionCameraProxy: { - getFrameProcessorPlugin: jest.fn( ) + initFrameProcessorPlugin: jest.fn( ) } } ) ); diff --git a/tests/unit/components/Camera/__snapshots__/PhotoCarousel.test.js.snap b/tests/unit/components/Camera/__snapshots__/PhotoCarousel.test.js.snap index 1c0e1a50d..c68f8fcea 100644 --- a/tests/unit/components/Camera/__snapshots__/PhotoCarousel.test.js.snap +++ b/tests/unit/components/Camera/__snapshots__/PhotoCarousel.test.js.snap @@ -51,7 +51,8 @@ exports[`PhotoCarousel renders correctly 1`] = ` } > 1; -// react-native-vision-camera v2 -export const mockUseCameraDevices = _deviceType => { - const devices = { - back: { - position: "back", - hasFlash: true - }, - front: { - devices: ["wide-angle-camera"], - hasFlash: true, - hasTorch: true, - id: "1", - isMultiCam: true, - maxZoom: 12.931958198547363, - minZoom: 1, - name: "front (1)", - neutralZoom: 1, - position: "front", - supportsDepthCapture: false, - supportsFocus: true, - supportsLowLightBoost: false, - supportsParallelVideoProcessing: true, - supportsRawCapture: true - } - }; - return devices; -}; - -// react-native-vision-camera v3 export const mockUseCameraDevice = _deviceType => { const device = { devices: ["wide-angle-camera"],