Rendering using Android hardware buffers

example/src/Examples/Stickers/Stickers.tsx

@@ -33,10 +33,12 @@ export const Stickers = () => {
       <GestureHandler
         matrix={helloMatrix}
         dimensions={HelloStickerDimensions}
+        debug
       />
       <GestureHandler
         matrix={locationMatrix}
         dimensions={LocationStickerDimensions}
+        debug
       />
     </View>
   );

package/android/cpp/rnskia-android/RNSkAndroidView.h

@@ -56,8 +56,6 @@ class RNSkAndroidView : public T, public RNSkBaseAndroidView {
   void surfaceSizeChanged(int width, int height) override {
     std::static_pointer_cast<RNSkOpenGLCanvasProvider>(T::getCanvasProvider())
         ->surfaceSizeChanged(width, height);
-    // This is only need for the first time to frame, this renderImmediate call
-    // will invoke updateTexImage for the previous frame
     RNSkView::renderImmediate();
   }
 

package/android/cpp/rnskia-android/RNSkOpenGLCanvasProvider.cpp

@@ -39,7 +39,6 @@ bool RNSkOpenGLCanvasProvider::renderToCanvas(
       if (!_surfaceHolder->makeCurrent()) {
         return false;
       }
-      _surfaceHolder->updateTexImage();
 
       // Draw into canvas using callback
       cb(surface->getCanvas());

package/android/cpp/rnskia-android/SkiaOpenGLSurfaceFactory.h

@@ -44,32 +44,13 @@ class ThreadContextHolder {
  */
 class WindowSurfaceHolder {
 public:
-  WindowSurfaceHolder(jobject jSurfaceTexture, int width, int height)
+  WindowSurfaceHolder(jobject jSurface, int width, int height)
       : _width(width), _height(height) {
     JNIEnv *env = facebook::jni::Environment::current();
-    _jSurfaceTexture = env->NewGlobalRef(jSurfaceTexture);
-    jclass surfaceClass = env->FindClass("android/view/Surface");
-    jmethodID surfaceConstructor = env->GetMethodID(
-        surfaceClass, "<init>", "(Landroid/graphics/SurfaceTexture;)V");
-    // Create a new Surface instance
-    jobject jSurface =
-        env->NewObject(surfaceClass, surfaceConstructor, jSurfaceTexture);
-
-    jclass surfaceTextureClass = env->GetObjectClass(_jSurfaceTexture);
-    _updateTexImageMethod =
-        env->GetMethodID(surfaceTextureClass, "updateTexImage", "()V");
-
-    // Acquire the native window from the Surface
     _window = ANativeWindow_fromSurface(env, jSurface);
-    // Clean up local references
-    env->DeleteLocalRef(jSurface);
-    env->DeleteLocalRef(surfaceClass);
-    env->DeleteLocalRef(surfaceTextureClass);
   }
 
   ~WindowSurfaceHolder() {
-    JNIEnv *env = facebook::jni::Environment::current();
-    env->DeleteGlobalRef(_jSurfaceTexture);
     ANativeWindow_release(_window);
   }
 
@@ -81,20 +62,6 @@ class WindowSurfaceHolder {
    */
   sk_sp<SkSurface> getSurface();
 
-  void updateTexImage() {
-    JNIEnv *env = facebook::jni::Environment::current();
-
-    // Call updateTexImage on the SurfaceTexture object
-    env->CallVoidMethod(_jSurfaceTexture, _updateTexImageMethod);
-
-    // Check for exceptions
-    if (env->ExceptionCheck()) {
-      RNSkLogger::logToConsole("updateAndRelease() failed. The exception above "
-                               "can safely be ignored");
-      env->ExceptionClear();
-    }
-  }
-
   /**
    * Resizes the surface
    * @param width
@@ -132,9 +99,7 @@ class WindowSurfaceHolder {
 private:
   ANativeWindow *_window;
   sk_sp<SkSurface> _skSurface = nullptr;
-  jobject _jSurfaceTexture = nullptr;
   EGLSurface _glSurface = EGL_NO_SURFACE;
-  jmethodID _updateTexImageMethod = nullptr;
   int _width = 0;
   int _height = 0;
 };

package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java

@@ -49,19 +49,19 @@ private byte[] getStreamAsBytes(InputStream is) throws IOException {
     }
 
     private void postFrameLoop() {
-        Choreographer.FrameCallback frameCallback = new Choreographer.FrameCallback() {
-            @Override
-            public void doFrame(long frameTimeNanos) {
-                if (_drawLoopActive) {
-                    Choreographer.getInstance().postFrameCallback(this);
-                }
-                if (_isPaused) {
-                    return;
-                }
-                notifyDrawLoop();
-            }
-        };
-        Choreographer.getInstance().postFrameCallback(frameCallback);
+//        Choreographer.FrameCallback frameCallback = new Choreographer.FrameCallback() {
+//            @Override
+//            public void doFrame(long frameTimeNanos) {
+//                if (_drawLoopActive) {
+//                    Choreographer.getInstance().postFrameCallback(this);
+//                }
+//                if (_isPaused) {
+//                    return;
+//                }
+//                notifyDrawLoop();
+//            }
+//        };
+//        Choreographer.getInstance().postFrameCallback(frameCallback);
     }
 
 

package/android/src/main/java/com/shopify/reactnative/skia/SkiaBaseView.java

@@ -1,54 +1,98 @@
 package com.shopify.reactnative.skia;
 
+import android.annotation.SuppressLint;
 import android.content.Context;
-import android.graphics.SurfaceTexture;
+import android.graphics.Bitmap;
+import android.graphics.Canvas;
+import android.graphics.Matrix;
+import android.graphics.Paint;
+import android.graphics.PixelFormat;
+import android.media.Image;
+import android.media.ImageReader;
+import android.os.Build;
 import android.util.Log;
+import android.view.Choreographer;
 import android.view.MotionEvent;
-import android.view.TextureView;
+import android.view.ViewGroup;
+import android.hardware.HardwareBuffer;
 
-import com.facebook.react.views.view.ReactViewGroup;
+import androidx.annotation.RequiresApi;
 
-public abstract class SkiaBaseView extends ReactViewGroup implements TextureView.SurfaceTextureListener {
-    private TextureView mTexture;
+import com.facebook.react.views.view.ReactViewGroup;
 
+@RequiresApi(api = Build.VERSION_CODES.Q)
+public abstract class SkiaBaseView extends ReactViewGroup implements Choreographer.FrameCallback {
+    private ImageReader mImageReader = null;
+    private Bitmap mBitmap = null;
     private String tag = "SkiaView";
+    private Choreographer choreographer;
 
+    private Paint paint = new Paint();
+    private Matrix matrix = new Matrix();
+
+    @SuppressLint("WrongConstant")
     public SkiaBaseView(Context context) {
         super(context);
-        mTexture = new TextureView(context);
-        mTexture.setSurfaceTextureListener(this);
-        mTexture.setOpaque(false);
-        addView(mTexture);
+        LayoutParams params = new LayoutParams(ViewGroup.LayoutParams.MATCH_PARENT, ViewGroup.LayoutParams.MATCH_PARENT);
+        setLayoutParams(params);
+        setWillNotDraw(false);
+    }
+
+    @Override
+    protected void onDraw(Canvas canvas) {
+        super.onDraw(canvas);
+        if (mBitmap != null) {
+            end = System.nanoTime();
+            Log.i(tag, "render time: " + (end - start) / 1000000 + "ms");
+            canvas.drawBitmap(mBitmap, matrix, paint);
+        }
     }
 
     public void destroySurface() {
         Log.i(tag, "destroySurface");
         surfaceDestroyed();
     }
 
-    private void createSurfaceTexture() {
-        // This API Level is >= 26, we created our own SurfaceTexture to have a faster time to first frame
-        if (android.os.Build.VERSION.SDK_INT >= android.os.Build.VERSION_CODES.O) {
-            Log.i(tag, "Create SurfaceTexture");
-            SurfaceTexture surface = new SurfaceTexture(false);
-            mTexture.setSurfaceTexture(surface);
-            this.onSurfaceTextureAvailable(surface, this.getMeasuredWidth(), this.getMeasuredHeight());
-        }
-    }
-
+    @SuppressLint("WrongConstant")
     @Override
-    protected void onAttachedToWindow() {
-        super.onAttachedToWindow();
-        if (this.getMeasuredWidth() == 0) {
-            createSurfaceTexture();
+    protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
+        Log.i(tag, "onLayout " + this.getMeasuredWidth() + "/" + this.getMeasuredHeight());
+        super.onLayout(changed, left, top, right, bottom);
+        // TODO: fix lifecycle there
+        if (mImageReader != null) {
+          //  mImageReader.close();
+          //  surfaceSizeChanged(getMeasuredWidth(), getMeasuredHeight());
+        } else {
+            long usage = HardwareBuffer.USAGE_CPU_READ_RARELY |
+                    HardwareBuffer.USAGE_CPU_WRITE_RARELY |
+                    HardwareBuffer.USAGE_GPU_COLOR_OUTPUT |
+                    HardwareBuffer.USAGE_GPU_SAMPLED_IMAGE;
+            mImageReader = ImageReader.newInstance(getMeasuredWidth(), getMeasuredHeight(), PixelFormat.RGBA_8888, 2, usage);
+            surfaceAvailable(mImageReader.getSurface(), getMeasuredWidth(), getMeasuredHeight());
+            choreographer = Choreographer.getInstance();
+            choreographer.postFrameCallback(this);
         }
     }
 
+    long start;
+    long end;
+
     @Override
-    protected void onLayout(boolean changed, int left, int top, int right, int bottom) {
-        Log.i(tag, "onLayout " + this.getMeasuredWidth() + "/" + this.getMeasuredHeight());
-        super.onLayout(changed, left, top, right, bottom);
-        mTexture.layout(0, 0, this.getMeasuredWidth(), this.getMeasuredHeight());
+    public void doFrame(long frameTimeNanos) {
+        choreographer.postFrameCallback(this);
+        if (mImageReader.getSurface() != null) {
+            // TODO: use drawFrame() instead
+            surfaceSizeChanged(getMeasuredWidth(), getMeasuredHeight());
+            try (Image image = mImageReader.acquireLatestImage()) {
+                if (image != null) {
+                    start = System.nanoTime();
+                    HardwareBuffer hb = image.getHardwareBuffer();
+                    mBitmap = Bitmap.wrapHardwareBuffer(hb, null);
+                    hb.close();
+                    invalidate();
+                }
+            }
+        }
     }
 
     @Override
@@ -121,40 +165,6 @@ private static int motionActionToType(int action) {
         return actionType;
     }
 
-    @Override
-    public void onSurfaceTextureAvailable(SurfaceTexture surface, int width, int height) {
-        Log.i(tag, "onSurfaceTextureAvailable " + width + "/" + height);
-        surfaceAvailable(surface, width, height);
-    }
-
-    @Override
-    public void onSurfaceTextureSizeChanged(SurfaceTexture surface, int width, int height) {
-        Log.i(tag, "onSurfaceTextureSizeChanged " + width + "/" + height);
-        surfaceSizeChanged(width, height);
-    }
-
-    @Override
-    public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) {
-        Log.i(tag, "onSurfaceTextureDestroyed");
-        // https://developer.android.com/reference/android/view/TextureView.SurfaceTextureListener#onSurfaceTextureDestroyed(android.graphics.SurfaceTexture)
-        destroySurface();
-        // Because of React Native Screens (which dettach the view), we always keep the surface alive.
-        // If not, Texture view will recreate the texture surface by itself and
-        // we will lose the fast first time to frame.
-        // We only delete the surface when the view is dropped (destroySurface invoked by SkiaBaseViewManager);
-        createSurfaceTexture();
-        return false;
-    }
-
-    //private long _prevTimestamp = 0;
-    @Override
-    public void onSurfaceTextureUpdated(SurfaceTexture surface) {
-//        long timestamp = surface.getTimestamp();
-//        long frameDuration = (timestamp - _prevTimestamp)/1000000;
-//        Log.i(tag, "onSurfaceTextureUpdated "+frameDuration+"ms");
-//        _prevTimestamp = timestamp;
-    }
-
     protected abstract void surfaceAvailable(Object surface, int width, int height);
 
     protected abstract void surfaceSizeChanged(int width, int height);