From 8bb28d12d422f6180b961d9369e0f39ff51d9921 Mon Sep 17 00:00:00 2001 From: Hans Date: Mon, 20 Nov 2023 18:40:51 +0800 Subject: [PATCH] Support readPixels for Image and Canvas (#1860) --------- Co-authored-by: William Candillon --- docs/docs/getting-started/installation.md | 12 +- docs/docs/image.md | 2 + docs/docs/tutorials.md | 1 + .../reactnative/skia/PlatformContext.java | 19 ++-- .../reactnative/skia/SkiaBaseView.java | 6 +- package/cpp/api/JsiSkCanvas.h | 39 ++++++- package/cpp/api/JsiSkImage.h | 50 +++++++++ package/cpp/api/JsiSkImageInfo.h | 19 ++++ package/cpp/rnskia/dom/nodes/JsiImageNode.h | 7 +- .../props/{ImageProps.h => SkImageProps.h} | 0 package/cpp/utils/RNSkTypedArray.h | 41 +++++++ package/jestEnv.mjs | 4 +- package/jestSetup.mjs | 5 +- package/package.json | 5 +- .../src/renderer/__tests__/e2e/Image.spec.tsx | 105 +++++++++++++++++- package/src/renderer/__tests__/setup.tsx | 21 ++-- package/src/skia/__tests__/Enums.spec.ts | 3 + package/src/skia/types/Canvas.ts | 15 ++- package/src/skia/types/Image/Image.ts | 20 ++++ package/src/skia/types/Image/ImageFactory.ts | 55 ++++----- package/src/skia/web/Host.ts | 4 +- package/src/skia/web/JsiSkCanvas.ts | 14 ++- package/src/skia/web/JsiSkFontMgrFactory.ts | 2 - package/src/skia/web/JsiSkImage.ts | 59 ++++++++-- .../src/skia/web/JsiSkTypefaceFontProvider.ts | 4 - package/yarn.lock | 15 ++- scripts/build-npm-package.ts | 2 +- 27 files changed, 439 insertions(+), 90 deletions(-) rename package/cpp/rnskia/dom/props/{ImageProps.h => SkImageProps.h} (100%) create mode 100644 package/cpp/utils/RNSkTypedArray.h diff --git a/docs/docs/getting-started/installation.md b/docs/docs/getting-started/installation.md index 7fac57d129..6e059558e5 100644 --- a/docs/docs/getting-started/installation.md +++ b/docs/docs/getting-started/installation.md @@ -84,9 +84,17 @@ There is also an [React Native VSCode extension](https://marketplace.visualstudi ## Testing with Jest -React Native Skia test mocks use a web implementation that depends on loading CanvasKit. Before using the mocks, some setup actions are required. +React Native Skia test mocks use a web implementation that depends on loading CanvasKit. -We recommend using [ESM](https://jestjs.io/docs/ecmascript-modules). To enable ESM support, you need to update your `jest` command to `node --experimental-vm-modules node_modules/.bin/jest`. +The very first step is to make sure that your Skia files are not being transformed by jest, for instance, we can add it the `transformIgnorePatterns` directive: +```js +"transformIgnorePatterns": [ + "node_modules/(?!(react-native|react-native.*|@react-native.*|@?react-navigation.*|@shopify/react-native-skia)/)" +] +``` + +Next, we recommend using [ESM](https://jestjs.io/docs/ecmascript-modules). To enable ESM support, you need to update your `jest` command to `node --experimental-vm-modules node_modules/.bin/jest`. +But we also support [CommonJS](#commonjs-setup). ### ESM Setup diff --git a/docs/docs/image.md b/docs/docs/image.md index 29da967f0d..fc5fc24cc7 100644 --- a/docs/docs/image.md +++ b/docs/docs/image.md @@ -133,5 +133,7 @@ const ImageDemo = () => { | :-------------- | :-------------------------------------------------------------------- | | `height` | Returns the possibly scaled height of the image. | | `width` | Returns the possibly scaled width of the image. | +| `getImageInfo` | Returns the image info for the image. | | `encodeToBytes` | Encodes the image pixels, returning the result as a `UInt8Array`. | | `encodeToBase64`| Encodes the image pixels, returning the result as a base64-encoded string. | +| `readPixels` | Reads the image pixels, returning result as UInt8Array or Float32Array | diff --git a/docs/docs/tutorials.md b/docs/docs/tutorials.md index d74e947734..0354ff4937 100644 --- a/docs/docs/tutorials.md +++ b/docs/docs/tutorials.md @@ -56,6 +56,7 @@ Please [make a PR](https://github.com/Shopify/react-native-skia/edit/main/docs/d * [Gradient along Path (youtu.be)](https://www.youtube.com/watch?v=7SCzL-XnfUU) * [Headspace Player - “Can it be done in React Native?” (youtu.be)](https://www.youtube.com/watch?v=pErnuAx5GjE) * [Make an Animated Wave Slider Effect with React-Native Skia (youtu.be)](https://www.youtube.com/watch?v=I6elFawLceY) +* [Liquid Wave Progress Indicator. Skia, Reanimated, D3. (youtu.be)](https://youtu.be/CGcLDoZWciA) ## Vertices * [Song of Bloom - “Can it be done in React Native?” (youtu.be)](https://www.youtube.com/watch?v=PfCQEA72ljU) diff --git a/package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java b/package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java index 0e7c7df2a1..00ebf26bf6 100644 --- a/package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java +++ b/package/android/src/main/java/com/shopify/reactnative/skia/PlatformContext.java @@ -1,7 +1,5 @@ package com.shopify.reactnative.skia; -import android.app.Application; -import android.graphics.Bitmap; import android.os.Handler; import android.os.Looper; import android.util.Log; @@ -10,7 +8,6 @@ import com.facebook.jni.HybridData; import com.facebook.proguard.annotations.DoNotStrip; import com.facebook.react.bridge.ReactContext; -import com.facebook.react.turbomodule.core.CallInvokerHolderImpl; import java.io.BufferedInputStream; import java.io.ByteArrayOutputStream; @@ -21,8 +18,6 @@ import java.net.URISyntaxException; import java.net.URL; import java.net.URLConnection; -import java.util.concurrent.ExecutorService; -import java.util.concurrent.Executors; public class PlatformContext { @DoNotStrip @@ -35,6 +30,9 @@ public class PlatformContext { private final String TAG = "PlatformContext"; + private final Handler mainHandler = new Handler(Looper.getMainLooper()); + + public PlatformContext(ReactContext reactContext) { mContext = reactContext; mHybridData = initHybrid(reactContext.getResources().getDisplayMetrics().density); @@ -66,9 +64,10 @@ public void doFrame(long frameTimeNanos) { Choreographer.getInstance().postFrameCallback(frameCallback); } + @DoNotStrip public void notifyTaskReadyOnMainThread() { - new Handler(Looper.getMainLooper()).post(new Runnable() { + mainHandler.post(new Runnable() { @Override public void run() { notifyTaskReady(); @@ -83,7 +82,7 @@ Object takeScreenshotFromViewTag(int tag) { @DoNotStrip public void raise(final String message) { - new Handler(Looper.getMainLooper()).post(new Runnable() { + mainHandler.post(new Runnable() { @Override public void run() { mContext.handleException(new Exception(message)); @@ -97,7 +96,7 @@ public void beginDrawLoop() { return; } _drawLoopActive = true; - new Handler(Looper.getMainLooper()).post(new Runnable() { + mainHandler.post(new Runnable() { @Override public void run() { postFrameLoop(); @@ -169,7 +168,7 @@ void onResume() { Log.i(TAG, "Resume"); if(_drawLoopActive) { // Restart draw loop - new Handler(Looper.getMainLooper()).post(new Runnable() { + mainHandler.post(new Runnable() { @Override public void run() { postFrameLoop(); @@ -188,4 +187,4 @@ protected void finalize() throws Throwable { private native HybridData initHybrid(float pixelDensity); private native void notifyDrawLoop(); private native void notifyTaskReady(); -} +} \ No newline at end of file diff --git a/package/android/src/main/java/com/shopify/reactnative/skia/SkiaBaseView.java b/package/android/src/main/java/com/shopify/reactnative/skia/SkiaBaseView.java index aa97bdd2fd..c6dc1eed60 100644 --- a/package/android/src/main/java/com/shopify/reactnative/skia/SkiaBaseView.java +++ b/package/android/src/main/java/com/shopify/reactnative/skia/SkiaBaseView.java @@ -157,9 +157,13 @@ public boolean onSurfaceTextureDestroyed(SurfaceTexture surface) { return false; } + //private long _prevTimestamp = 0; @Override public void onSurfaceTextureUpdated(SurfaceTexture surface) { - // Nothing special to do here +// long timestamp = surface.getTimestamp(); +// long frameDuration = (timestamp - _prevTimestamp)/1000000; +// Log.i(tag, "onSurfaceTextureUpdated "+frameDuration+"ms"); +// _prevTimestamp = timestamp; } protected abstract void surfaceAvailable(Object surface, int width, int height); diff --git a/package/cpp/api/JsiSkCanvas.h b/package/cpp/api/JsiSkCanvas.h index 3be658bc8b..ca5e801ad3 100644 --- a/package/cpp/api/JsiSkCanvas.h +++ b/package/cpp/api/JsiSkCanvas.h @@ -7,6 +7,7 @@ #include "JsiSkFont.h" #include "JsiSkHostObjects.h" #include "JsiSkImage.h" +#include "JsiSkImageInfo.h" #include "JsiSkMatrix.h" #include "JsiSkPaint.h" #include "JsiSkPath.h" @@ -17,6 +18,8 @@ #include "JsiSkTextBlob.h" #include "JsiSkVertices.h" +#include "RNSkTypedArray.h" + #include #pragma clang diagnostic push @@ -491,6 +494,39 @@ class JsiSkCanvas : public JsiSkHostObject { return jsi::Value::undefined(); } + JSI_HOST_FUNCTION(readPixels) { + auto srcX = static_cast(arguments[0].asNumber()); + auto srcY = static_cast(arguments[1].asNumber()); + auto info = JsiSkImageInfo::fromValue(runtime, arguments[2]); + if (!info) { + return jsi::Value::null(); + } + size_t bytesPerRow = 0; + if (count > 4 && !arguments[4].isUndefined()) { + bytesPerRow = static_cast(arguments[4].asNumber()); + } else { + bytesPerRow = info->minRowBytes(); + } + auto dest = + count > 3 + ? RNSkTypedArray::getTypedArray(runtime, arguments[3], *info) + : RNSkTypedArray::getTypedArray(runtime, jsi::Value::null(), *info); + if (!dest.isObject()) { + return jsi::Value::null(); + } + jsi::ArrayBuffer buffer = + dest.asObject(runtime) + .getProperty(runtime, jsi::PropNameID::forAscii(runtime, "buffer")) + .asObject(runtime) + .getArrayBuffer(runtime); + auto bfrPtr = reinterpret_cast(buffer.data(runtime)); + + if (!_canvas->readPixels(*info, bfrPtr, bytesPerRow, srcX, srcY)) { + return jsi::Value::null(); + } + return std::move(dest); + } + JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkCanvas, drawPaint), JSI_EXPORT_FUNC(JsiSkCanvas, drawLine), JSI_EXPORT_FUNC(JsiSkCanvas, drawRect), @@ -529,7 +565,8 @@ class JsiSkCanvas : public JsiSkHostObject { JSI_EXPORT_FUNC(JsiSkCanvas, drawColor), JSI_EXPORT_FUNC(JsiSkCanvas, clear), JSI_EXPORT_FUNC(JsiSkCanvas, concat), - JSI_EXPORT_FUNC(JsiSkCanvas, drawPicture)) + JSI_EXPORT_FUNC(JsiSkCanvas, drawPicture), + JSI_EXPORT_FUNC(JsiSkCanvas, readPixels)) explicit JsiSkCanvas(std::shared_ptr context) : JsiSkHostObject(std::move(context)) {} diff --git a/package/cpp/api/JsiSkImage.h b/package/cpp/api/JsiSkImage.h index 90581e9eb7..3d6d00e7b4 100644 --- a/package/cpp/api/JsiSkImage.h +++ b/package/cpp/api/JsiSkImage.h @@ -5,9 +5,12 @@ #include #include "JsiSkHostObjects.h" +#include "JsiSkImageInfo.h" #include "JsiSkMatrix.h" #include "JsiSkShader.h" +#include "RNSkTypedArray.h" + #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdocumentation" @@ -34,6 +37,11 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { return static_cast(getObject()->height()); } + JSI_HOST_FUNCTION(getImageInfo) { + return JsiSkImageInfo::toValue(runtime, getContext(), + getObject()->imageInfo()); + } + JSI_HOST_FUNCTION(makeShaderOptions) { auto tmx = (SkTileMode)arguments[0].asNumber(); auto tmy = (SkTileMode)arguments[1].asNumber(); @@ -117,6 +125,46 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { return jsi::String::createFromAscii(runtime, buffer); } + JSI_HOST_FUNCTION(readPixels) { + int srcX = 0; + int srcY = 0; + if (count > 0 && !arguments[0].isUndefined()) { + srcX = static_cast(arguments[0].asNumber()); + } + if (count > 1 && !arguments[1].isUndefined()) { + srcY = static_cast(arguments[1].asNumber()); + } + SkImageInfo info = + (count > 2 && !arguments[2].isUndefined()) + ? *JsiSkImageInfo::fromValue(runtime, arguments[2]) + : SkImageInfo::MakeN32(getObject()->width(), getObject()->height(), + getObject()->imageInfo().alphaType()); + size_t bytesPerRow = 0; + if (count > 4 && !arguments[4].isUndefined()) { + bytesPerRow = static_cast(arguments[4].asNumber()); + } else { + bytesPerRow = info.minRowBytes(); + } + auto dest = + count > 3 + ? RNSkTypedArray::getTypedArray(runtime, arguments[3], info) + : RNSkTypedArray::getTypedArray(runtime, jsi::Value::null(), info); + if (!dest.isObject()) { + return jsi::Value::null(); + } + jsi::ArrayBuffer buffer = + dest.asObject(runtime) + .getProperty(runtime, jsi::PropNameID::forAscii(runtime, "buffer")) + .asObject(runtime) + .getArrayBuffer(runtime); + auto bfrPtr = reinterpret_cast(buffer.data(runtime)); + + if (!getObject()->readPixels(info, bfrPtr, bytesPerRow, srcX, srcY)) { + return jsi::Value::null(); + } + return std::move(dest); + } + JSI_HOST_FUNCTION(makeNonTextureImage) { auto image = getObject()->makeNonTextureImage(); return jsi::Object::createFromHostObject( @@ -127,10 +175,12 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkImage, width), JSI_EXPORT_FUNC(JsiSkImage, height), + JSI_EXPORT_FUNC(JsiSkImage, getImageInfo), JSI_EXPORT_FUNC(JsiSkImage, makeShaderOptions), JSI_EXPORT_FUNC(JsiSkImage, makeShaderCubic), JSI_EXPORT_FUNC(JsiSkImage, encodeToBytes), JSI_EXPORT_FUNC(JsiSkImage, encodeToBase64), + JSI_EXPORT_FUNC(JsiSkImage, readPixels), JSI_EXPORT_FUNC(JsiSkImage, makeNonTextureImage), JSI_EXPORT_FUNC(JsiSkImage, dispose)) diff --git a/package/cpp/api/JsiSkImageInfo.h b/package/cpp/api/JsiSkImageInfo.h index 150700da3e..632be2ac91 100644 --- a/package/cpp/api/JsiSkImageInfo.h +++ b/package/cpp/api/JsiSkImageInfo.h @@ -56,5 +56,24 @@ class JsiSkImageInfo : public JsiSkWrappingSharedPtrHostObject { runtime, std::make_shared(std::move(context), imageInfo)); } + + JSI_PROPERTY_GET(width) { return static_cast(getObject()->width()); } + JSI_PROPERTY_GET(height) { + return static_cast(getObject()->height()); + } + JSI_PROPERTY_GET(colorType) { + return static_cast(getObject()->colorType()); + } + JSI_PROPERTY_GET(alphaType) { + return static_cast(getObject()->alphaType()); + } + + JSI_API_TYPENAME(ImageInfo); + + JSI_EXPORT_PROPERTY_GETTERS(JSI_EXPORT_PROP_GET(JsiSkImageInfo, width), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, height), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, colorType), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, alphaType), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, __typename__)) }; } // namespace RNSkia diff --git a/package/cpp/rnskia/dom/nodes/JsiImageNode.h b/package/cpp/rnskia/dom/nodes/JsiImageNode.h index 987add6706..bcdcf93b23 100644 --- a/package/cpp/rnskia/dom/nodes/JsiImageNode.h +++ b/package/cpp/rnskia/dom/nodes/JsiImageNode.h @@ -1,12 +1,7 @@ #pragma once -#ifdef TARGET_OS_IPHONE -#include -#else -#include "ImageProps.h" -#endif - #include "JsiDomDrawingNode.h" +#include "SkImageProps.h" #include diff --git a/package/cpp/rnskia/dom/props/ImageProps.h b/package/cpp/rnskia/dom/props/SkImageProps.h similarity index 100% rename from package/cpp/rnskia/dom/props/ImageProps.h rename to package/cpp/rnskia/dom/props/SkImageProps.h diff --git a/package/cpp/utils/RNSkTypedArray.h b/package/cpp/utils/RNSkTypedArray.h new file mode 100644 index 0000000000..9df7977bc5 --- /dev/null +++ b/package/cpp/utils/RNSkTypedArray.h @@ -0,0 +1,41 @@ +#pragma once + +#include "SkImage.h" +#include + +namespace RNSkia { + +namespace jsi = facebook::jsi; + +class RNSkTypedArray { +public: + static jsi::Value getTypedArray(jsi::Runtime &runtime, + const jsi::Value &value, SkImageInfo &info) { + auto reqSize = info.computeMinByteSize(); + if (reqSize > 0) { + if (value.isObject()) { + auto typedArray = value.asObject(runtime); + auto size = static_cast( + typedArray.getProperty(runtime, "byteLength").asNumber()); + if (size >= reqSize) { + return typedArray; + } + } else { + if (info.colorType() == kRGBA_F32_SkColorType) { + auto arrayCtor = + runtime.global().getPropertyAsFunction(runtime, "Float32Array"); + return arrayCtor.callAsConstructor(runtime, + static_cast(reqSize / 4)); + } else { + auto arrayCtor = + runtime.global().getPropertyAsFunction(runtime, "Uint8Array"); + return arrayCtor.callAsConstructor(runtime, + static_cast(reqSize)); + } + } + } + return jsi::Value::null(); + } +}; + +} // namespace RNSkia diff --git a/package/jestEnv.mjs b/package/jestEnv.mjs index 12182acfcd..286daf0196 100644 --- a/package/jestEnv.mjs +++ b/package/jestEnv.mjs @@ -1,9 +1,9 @@ /* eslint-disable import/no-default-export */ // eslint-disable-next-line import/no-extraneous-dependencies import { TestEnvironment } from "jest-environment-node"; -import { LoadSkiaWeb } from "@shopify/react-native-skia/lib/commonjs/web/LoadSkiaWeb"; +import CanvasKitInit from "canvaskit-wasm/bin/full/canvaskit"; -const CanvasKit = await LoadSkiaWeb(); +const CanvasKit = await CanvasKitInit({}); export default class SkiaEnvironment extends TestEnvironment { constructor(config, context) { diff --git a/package/jestSetup.mjs b/package/jestSetup.mjs index f6254d3aa6..49e4ec2e41 100644 --- a/package/jestSetup.mjs +++ b/package/jestSetup.mjs @@ -1,8 +1,7 @@ /* eslint-disable import/no-extraneous-dependencies */ import { jest } from "@jest/globals"; import CanvasKitInit from "canvaskit-wasm/bin/full/canvaskit"; - -import Mock from "./src/mock"; +import { Mock } from "@shopify/react-native-skia/lib/module/mock"; global.CanvasKit = await CanvasKitInit({}); @@ -19,5 +18,5 @@ jest.mock("@shopify/react-native-skia", () => { View: Noop, }; }); - return Mock.Mock(global.CanvasKit); + return Mock(global.CanvasKit); }); diff --git a/package/package.json b/package/package.json index 9314f14595..53b5c8ce8a 100644 --- a/package/package.json +++ b/package/package.json @@ -25,7 +25,8 @@ "libs/android/**", "index.js", "jestSetup.js", - "globalJestSetup.js", + "jestSetup.mjs", + "jestEnv.mjs", "cpp/**/*.{h,cpp}", "ios", "libs/ios/libskia.xcframework", @@ -105,7 +106,7 @@ "ws": "^8.11.0" }, "dependencies": { - "canvaskit-wasm": "0.38.2", + "canvaskit-wasm": "0.39.1", "react-reconciler": "^0.27.0" }, "eslintIgnore": [ diff --git a/package/src/renderer/__tests__/e2e/Image.spec.tsx b/package/src/renderer/__tests__/e2e/Image.spec.tsx index 86206bbbf0..ee520d1d73 100644 --- a/package/src/renderer/__tests__/e2e/Image.spec.tsx +++ b/package/src/renderer/__tests__/e2e/Image.spec.tsx @@ -1,8 +1,9 @@ import React from "react"; import { checkImage } from "../../../__tests__/setup"; -import { images, surface } from "../setup"; +import { images, loadImage, surface } from "../setup"; import { Fill, Image as SkiaImage } from "../../components"; +import { AlphaType, ColorType } from "../../../skia/types"; describe("Image loading from bundles", () => { it("should render png, jpg from bundle", async () => { @@ -24,6 +25,108 @@ describe("Image loading from bundles", () => { ); checkImage(image, `snapshots/images/bundle-${surface.OS}.png`); }); + + it("should read pixels from an image", async () => { + const pixels = await surface.eval( + (Skia, { data }) => { + const image = Skia.Image.MakeImageFromEncoded( + Skia.Data.fromBytes(new Uint8Array(data)) + )!; + return Array.from( + image.readPixels(0, 0, { + width: 2, + height: 2, + colorType: image.getImageInfo().colorType, + alphaType: image.getImageInfo().alphaType, + })! + ); + }, + { + data: Array.from( + loadImage("skia/__tests__/assets/oslo.jpg").encodeToBytes() + ), + } + ); + expect(pixels).toBeDefined(); + expect(pixels).toEqual([ + 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, + 255, + ]); + }); + + // it("should read pixels from an image using a preallocated buffer", async () => { + // const pixels = await surface.eval( + // (Skia, { colorType, alphaType, data }) => { + // const image = Skia.Image.MakeImageFromEncoded( + // Skia.Data.fromBytes(new Uint8Array(data)) + // )!; + // const result = new Uint8Array(16); + // image.readPixels( + // 0, + // 0, + // { + // width: 2, + // height: 2, + // colorType, + // alphaType, + // }, + // result + // ); + // return result; + // }, + // { + // colorType: ColorType.RGBA_8888, + // alphaType: AlphaType.Unpremul, + // data: Array.from( + // loadImage("skia/__tests__/assets/oslo.jpg").encodeToBytes() + // ), + // } + // ); + // expect(pixels).toBeDefined(); + // expect(Array.from(pixels!)).toEqual([ + // 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, + // 255, + // ]); + // }); + it("should read pixels from a canvas", async () => { + const pixels = await surface.eval( + (Skia, { colorType, alphaType }) => { + const offscreen = Skia.Surface.MakeOffscreen(10, 10)!; + const canvas = offscreen.getCanvas(); + canvas.drawColor(Skia.Color("red")); + return Array.from( + canvas.readPixels(0, 0, { + width: 1, + height: 1, + colorType, + alphaType, + })! + ); + }, + { colorType: ColorType.RGBA_8888, alphaType: AlphaType.Unpremul } + ); + expect(pixels).toBeDefined(); + expect(Array.from(pixels!)).toEqual([255, 0, 0, 255]); + }); + // it("should read pixels from a canvas using a preallocated buffer", async () => { + // const pixels = await surface.eval( + // (Skia, { colorType, alphaType }) => { + // const offscreen = Skia.Surface.MakeOffscreen(10, 10)!; + // const canvas = offscreen.getCanvas(); + // canvas.drawColor(Skia.Color("red")); + // const result = new Uint8Array(4); + // canvas.readPixels(0, 0, { + // width: 1, + // height: 1, + // colorType, + // alphaType, + // }, result); + // }, + // { colorType: ColorType.RGBA_8888, alphaType: AlphaType.Unpremul } + // ); + // expect(pixels).toBeDefined(); + // expect(Array.from(pixels!)).toEqual([255, 0, 0, 255]); + // }); // This test should only run on CI because it will trigger a redbox. // While this is fine on CI, it is undesirable on local dev. // it("should not crash with an invalid viewTag", async () => { diff --git a/package/src/renderer/__tests__/setup.tsx b/package/src/renderer/__tests__/setup.tsx index 39b5f944eb..5416f75eb3 100644 --- a/package/src/renderer/__tests__/setup.tsx +++ b/package/src/renderer/__tests__/setup.tsx @@ -8,12 +8,7 @@ import type { Server, WebSocket } from "ws"; import { DependencyManager } from "../DependencyManager"; import { ValueApi } from "../../values/web"; -import type * as SkiaExports from "../../skia"; -import type * as AnimationExports from "../../animation"; -import type * as ValuesExports from "../../values"; -import type * as RendererExports from "../index"; -import type * as OffscreenExports from "../Offscreen"; -import type * as TouchHandlerExports from "../../views/useTouchHandler"; +import type * as SkiaExports from "../../index"; import { JsiSkApi } from "../../skia/web/JsiSkia"; import type { Node } from "../../dom/nodes"; import { JsiSkDOM } from "../../dom/nodes"; @@ -162,15 +157,15 @@ export const loadFont = (uri: string, ftSize?: number) => { return Skia.Font(tf!, ftSize ?? fontSize); }; -export const importSkia = () => { +export const importSkia = (): typeof SkiaExports => { //const core = require("../../skia/core"); - const skia: typeof SkiaExports = require("../../skia"); - const renderer: typeof RendererExports = require("../../renderer"); - const offscreen: typeof OffscreenExports = require("../Offscreen"); + const skia = require("../../skia"); + const renderer = require("../../renderer"); + const offscreen = require("../Offscreen"); // TODO: to remove - const animation: typeof AnimationExports = require("../../animation"); - const values: typeof ValuesExports = require("../../values"); - const useTouchHandler: typeof TouchHandlerExports = require("../../views/useTouchHandler"); + const animation = require("../../animation"); + const values = require("../../values"); + const useTouchHandler = require("../../views/useTouchHandler"); return { ...skia, ...renderer, diff --git a/package/src/skia/__tests__/Enums.spec.ts b/package/src/skia/__tests__/Enums.spec.ts index 96e27b71a5..c02229b634 100644 --- a/package/src/skia/__tests__/Enums.spec.ts +++ b/package/src/skia/__tests__/Enums.spec.ts @@ -37,6 +37,9 @@ const checkEnum = (skiaEnum: T, canvasKitEnum: EmbindEnum) => { // eslint-disable-next-line @typescript-eslint/ban-ts-comment // @ts-expect-error const selectedEnum = canvasKitEnum[namedKey]; + if (namedKey === undefined || selectedEnum === undefined) { + console.log({ skiaEnum, canvasKitEnum, key, namedKey, expected }); + } expect(selectedEnum).toBeDefined(); expect(expected).toBe(selectedEnum.value); }); diff --git a/package/src/skia/types/Canvas.ts b/package/src/skia/types/Canvas.ts index 5705e8d395..0798e22f9e 100644 --- a/package/src/skia/types/Canvas.ts +++ b/package/src/skia/types/Canvas.ts @@ -2,7 +2,7 @@ import type { SkPaint } from "./Paint"; import type { SkRect } from "./Rect"; import type { SkFont } from "./Font"; import type { SkPath } from "./Path"; -import type { SkImage, MipmapMode, FilterMode } from "./Image"; +import type { SkImage, MipmapMode, FilterMode, ImageInfo } from "./Image"; import type { SkSVG } from "./SVG"; import type { SkColor } from "./Color"; import type { SkRRect } from "./RRect"; @@ -492,4 +492,17 @@ export interface SkCanvas { * @param skp */ drawPicture(skp: SkPicture): void; + + /** Read Image pixels + * + * @param srcX - x-axis upper left corner of the rectangle to read from + * @param srcY - y-axis upper left corner of the rectangle to read from + * @param imageInfo - describes the pixel format and dimensions of the data to read into + * @return Float32Array or Uint8Array with data or null if the read failed. + */ + readPixels( + srcX: number, + srcY: number, + imageInfo: ImageInfo + ): Float32Array | Uint8Array | null; } diff --git a/package/src/skia/types/Image/Image.ts b/package/src/skia/types/Image/Image.ts index a6a1ef5321..4406885f6f 100644 --- a/package/src/skia/types/Image/Image.ts +++ b/package/src/skia/types/Image/Image.ts @@ -3,6 +3,8 @@ import type { SkJSIInstance } from "../JsiInstance"; import type { TileMode } from "../ImageFilter"; import type { SkShader } from "../Shader"; +import type { ImageInfo } from "./ImageFactory"; + export enum FilterMode { Nearest, Linear, @@ -31,6 +33,11 @@ export interface SkImage extends SkJSIInstance<"Image"> { */ width(): number; + /** + * Returns the ImageInfo describing the image. + */ + getImageInfo(): ImageInfo; + /** * Returns this image as a shader with the specified tiling. It will use cubic sampling. * @param tx - tile mode in the x direction. @@ -94,6 +101,19 @@ export interface SkImage extends SkJSIInstance<"Image"> { */ encodeToBase64(fmt?: ImageFormat, quality?: number): string; + /** Read Image pixels + * + * @param srcX - optional x-axis upper left corner of the rectangle to read from + * @param srcY - optional y-axis upper left corner of the rectangle to read from + * @param imageInfo - optional describes the pixel format and dimensions of the data to read into + * @return Float32Array or Uint8Array with data or null if the read failed. + */ + readPixels( + srcX?: number, + srcY?: number, + imageInfo?: ImageInfo + ): Float32Array | Uint8Array | null; + /** * Returns raster image or lazy image. Copies SkImage backed by GPU texture * into CPU memory if needed. Returns original SkImage if decoded in raster diff --git a/package/src/skia/types/Image/ImageFactory.ts b/package/src/skia/types/Image/ImageFactory.ts index a93dea5c2f..4a98d3cd49 100644 --- a/package/src/skia/types/Image/ImageFactory.ts +++ b/package/src/skia/types/Image/ImageFactory.ts @@ -10,36 +10,41 @@ export enum AlphaType { } export enum ColorType { - Unknown, //!< uninitialized - Alpha_8, //!< pixel with alpha in 8-bit byte - RGB_565, //!< pixel with 5 bits red, 6 bits green, 5 bits blue, in 16-bit word - ARGB_4444, //!< pixel with 4 bits for alpha, red, green, blue; in 16-bit word - RGBA_8888, //!< pixel with 8 bits for red, green, blue, alpha; in 32-bit word - RGB_888x, //!< pixel with 8 bits each for red, green, blue; in 32-bit word - BGRA_8888, //!< pixel with 8 bits for blue, green, red, alpha; in 32-bit word - RGBA_1010102, //!< 10 bits for red, green, blue; 2 bits for alpha; in 32-bit word - BGRA_1010102, //!< 10 bits for blue, green, red; 2 bits for alpha; in 32-bit word - RGB_101010x, //!< pixel with 10 bits each for red, green, blue; in 32-bit word - BGR_101010x, //!< pixel with 10 bits each for blue, green, red; in 32-bit word - BGR_101010x_XR, //!< pixel with 10 bits each for blue, green, red; in 32-bit word, extended range - Gray_8, //!< pixel with grayscale level in 8-bit byte - RGBA_F16Norm, //!< pixel with half floats in [0,1] for red, green, blue, alpha; - // in 64-bit word - RGBA_F16, //!< pixel with half floats for red, green, blue, alpha; - // in 64-bit word - RGBA_F32, //!< pixel using C float for red, green, blue, alpha; in 128-bit word + Unknown, // uninitialized + Alpha_8, // pixel with alpha in 8-bit byte + RGB_565, // pixel with 5 bits red, 6 bits green, 5 bits blue, in 16-bit word + ARGB_4444, // pixel with 4 bits for alpha, red, green, blue; in 16-bit word + RGBA_8888, // pixel with 8 bits for red, green, blue, alpha; in 32-bit word + RGB_888x, // pixel with 8 bits each for red, green, blue; in 32-bit word + BGRA_8888, // pixel with 8 bits for blue, green, red, alpha; in 32-bit word + RGBA_1010102, // 10 bits for red, green, blue; 2 bits for alpha; in 32-bit word + BGRA_1010102, // 10 bits for blue, green, red; 2 bits for alpha; in 32-bit word + RGB_101010x, // pixel with 10 bits each for red, green, blue; in 32-bit word + BGR_101010x, // pixel with 10 bits each for blue, green, red; in 32-bit word + BGR_101010x_XR, // pixel with 10 bits each for blue, green, red; in 32-bit word, extended range + RGBA_10x6, // pixel with 10 used bits (most significant) followed by 6 unused + Gray_8, // pixel with grayscale level in 8-bit byte + RGBA_F16Norm, // pixel with half floats in [0,1] for red, green, blue, alpha; in 64-bit word + RGBA_F16, // pixel with half floats for red, green, blue, alpha; in 64-bit word + RGBA_F32, // pixel using C float for red, green, blue, alpha; in 128-bit word // The following 6 colortypes are just for reading from - not for rendering to - R8G8_unorm, //!< pixel with a uint8_t for red and green + R8G8_unorm, // pixel with a uint8_t for red and green - A16_float, //!< pixel with a half float for alpha - R16G16_float, //!< pixel with a half float for red and green + A16_float, // pixel with a half float for alpha + R16G16_float, // pixel with a half float for red and green + + A16_unorm, // pixel with a little endian uint16_t for alpha + R16G16_unorm, // pixel with a little endian uint16_t for red and green + R16G16B16A16_unorm, // pixel with a little endian uint16_t for red, green, blue, and alpha - A16_unorm, //!< pixel with a little endian uint16_t for alpha - R16G16_unorm, //!< pixel with a little endian uint16_t for red and green - R16G16B16A16_unorm, //!< pixel with a little endian uint16_t for red, green, blue - // and alpha SRGBA_8888, + R8_unorm, + + // The `kN32_SkColorType` is platform dependent in the original enum, + // and TypeScript doesn't support conditional compilation natively. + // You might need to handle it differently based on your use case. + N32_SkColorType, // either BGRA_8888 or RGBA_8888 based on the platform } export interface ImageInfo { diff --git a/package/src/skia/web/Host.ts b/package/src/skia/web/Host.ts index d2f118f554..97d3ff97f4 100644 --- a/package/src/skia/web/Host.ts +++ b/package/src/skia/web/Host.ts @@ -1,4 +1,4 @@ -import type { CanvasKit, EmbindEnumEntity } from "canvaskit-wasm"; +import type { CanvasKit, EmbindEnumEntity, EmbindEnum } from "canvaskit-wasm"; import type { SkJSIInstance } from "../types"; @@ -41,6 +41,8 @@ export abstract class HostObject extends BaseHostObject< } } +export const getCkEnum = (e: EmbindEnum, v: number): EmbindEnumEntity => + Object.values(e).find(({ value }) => value === v); export const ckEnum = (value: number): EmbindEnumEntity => ({ value }); export const optEnum = ( value: number | undefined diff --git a/package/src/skia/web/JsiSkCanvas.ts b/package/src/skia/web/JsiSkCanvas.ts index b9ddaba3f1..3979c13902 100644 --- a/package/src/skia/web/JsiSkCanvas.ts +++ b/package/src/skia/web/JsiSkCanvas.ts @@ -7,6 +7,7 @@ import type { MipmapMode, PointMode, SaveLayerFlag, + ImageInfo, SkCanvas, SkColor, SkFont, @@ -24,7 +25,7 @@ import type { SkVertices, } from "../types"; -import { ckEnum, HostObject } from "./Host"; +import { ckEnum, getCkEnum, HostObject } from "./Host"; import { JsiSkPaint } from "./JsiSkPaint"; import { JsiSkRect } from "./JsiSkRect"; import { JsiSkRRect } from "./JsiSkRRect"; @@ -374,4 +375,15 @@ export class JsiSkCanvas drawPicture(skp: SkPicture) { this.ref.drawPicture(JsiSkPicture.fromValue(skp)); } + + readPixels(srcX: number, srcY: number, imageInfo: ImageInfo) { + const pxInfo = { + width: imageInfo.width, + height: imageInfo.height, + colorSpace: this.CanvasKit.ColorSpace.SRGB, + alphaType: getCkEnum(this.CanvasKit.AlphaType, imageInfo.alphaType), + colorType: getCkEnum(this.CanvasKit.ColorType, imageInfo.colorType), + }; + return this.ref.readPixels(srcX, srcY, pxInfo); + } } diff --git a/package/src/skia/web/JsiSkFontMgrFactory.ts b/package/src/skia/web/JsiSkFontMgrFactory.ts index e07447d8ab..461da91c0c 100644 --- a/package/src/skia/web/JsiSkFontMgrFactory.ts +++ b/package/src/skia/web/JsiSkFontMgrFactory.ts @@ -15,8 +15,6 @@ export class JsiSkFontMgrFactory extends Host implements FontMgrFactory { if (!fontMgr) { throw new Error("Couldn't create system font manager"); } - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error return new JsiSkFontMgr(this.CanvasKit, fontMgr); } } diff --git a/package/src/skia/web/JsiSkImage.ts b/package/src/skia/web/JsiSkImage.ts index dcb7403d66..a06b6fde28 100644 --- a/package/src/skia/web/JsiSkImage.ts +++ b/package/src/skia/web/JsiSkImage.ts @@ -1,16 +1,21 @@ -import type { CanvasKit, Image } from "canvaskit-wasm"; - import type { - ImageFormat, - FilterMode, - MipmapMode, - SkImage, - SkMatrix, - SkShader, - TileMode, + CanvasKit, + Image, + ImageInfo as CKImageInfo, +} from "canvaskit-wasm"; + +import { + type ImageFormat, + type ImageInfo, + type FilterMode, + type MipmapMode, + type SkImage, + type SkMatrix, + type SkShader, + type TileMode, } from "../types"; -import { ckEnum, HostObject } from "./Host"; +import { ckEnum, getCkEnum, HostObject } from "./Host"; import { JsiSkMatrix } from "./JsiSkMatrix"; import { JsiSkShader } from "./JsiSkShader"; @@ -52,6 +57,16 @@ export class JsiSkImage extends HostObject implements SkImage { return this.ref.width(); } + getImageInfo(): ImageInfo { + const info = this.ref.getImageInfo(); + return { + width: info.width, + height: info.height, + colorType: info.colorType.value, + alphaType: info.alphaType.value, + }; + } + makeShaderOptions( tx: TileMode, ty: TileMode, @@ -110,6 +125,30 @@ export class JsiSkImage extends HostObject implements SkImage { return toBase64String(bytes); } + readPixels(srcX?: number, srcY?: number, imageInfo?: ImageInfo) { + const info = this.getImageInfo(); + console.log({ + alphaType: ckEnum(info.alphaType), + colorType: ckEnum(info.colorType), + realAlphaType: this.CanvasKit.AlphaType.Opaque.value, + realColorType: this.CanvasKit.ColorType.RGBA_8888.value, + }); + const pxInfo: CKImageInfo = { + colorSpace: this.CanvasKit.ColorSpace.SRGB, + width: imageInfo?.width ?? info.width, + height: imageInfo?.height ?? info.height, + alphaType: getCkEnum( + this.CanvasKit.AlphaType, + (imageInfo ?? info).alphaType + ), + colorType: getCkEnum( + this.CanvasKit.ColorType, + (imageInfo ?? info).colorType + ), + }; + return this.ref.readPixels(srcX ?? 0, srcY ?? 0, pxInfo); + } + dispose = () => { this.ref.delete(); }; diff --git a/package/src/skia/web/JsiSkTypefaceFontProvider.ts b/package/src/skia/web/JsiSkTypefaceFontProvider.ts index 47d3515d40..940a09833d 100644 --- a/package/src/skia/web/JsiSkTypefaceFontProvider.ts +++ b/package/src/skia/web/JsiSkTypefaceFontProvider.ts @@ -19,13 +19,9 @@ export class JsiSkTypefaceFontProvider throw new NotImplementedOnRNWeb(); } countFamilies() { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error return this.ref.countFamilies(); } getFamilyName(index: number) { - // eslint-disable-next-line @typescript-eslint/ban-ts-comment - // @ts-expect-error return this.ref.getFamilyName(index); } registerFont(typeface: SkTypeface, familyName: string) { diff --git a/package/yarn.lock b/package/yarn.lock index a728f4cab2..b3bcdcdb13 100644 --- a/package/yarn.lock +++ b/package/yarn.lock @@ -2582,6 +2582,11 @@ "@typescript-eslint/types" "6.10.0" eslint-visitor-keys "^3.4.1" +"@webgpu/types@0.1.21": + version "0.1.21" + resolved "https://registry.yarnpkg.com/@webgpu/types/-/types-0.1.21.tgz#b181202daec30d66ccd67264de23814cfd176d3a" + integrity sha512-pUrWq3V5PiSGFLeLxoGqReTZmiiXwY3jRkIG5sLLKjyqNxrwm/04b4nw7LSmGWJcKk59XOM/YRTUwOzo4MMlow== + abort-controller@^3.0.0: version "3.0.0" resolved "https://registry.yarnpkg.com/abort-controller/-/abort-controller-3.0.0.tgz#eaf54d53b62bae4138e809ca225c8439a6efb392" @@ -3110,10 +3115,12 @@ caniuse-lite@^1.0.30001400: resolved "https://registry.yarnpkg.com/caniuse-lite/-/caniuse-lite-1.0.30001400.tgz#3038bee70d8b875604cd8833cb0e5e254ee0281a" integrity sha512-Mv659Hn65Z4LgZdJ7ge5JTVbE3rqbJaaXgW5LEI9/tOaXclfIZ8DW7D7FCWWWmWiiPS7AC48S8kf3DApSxQdgA== -canvaskit-wasm@0.38.2: - version "0.38.2" - resolved "https://registry.yarnpkg.com/canvaskit-wasm/-/canvaskit-wasm-0.38.2.tgz#b6c2be236670fd0f18977b9026652b2c0e201fee" - integrity sha512-ieRb6DO4yL91qUfyRgmyhp2Hi1KmQ9lIMfKacxHVlfp/CpKCkzgAxRGUbCsJFzwLKjs9fufGrIyvnzEYRwm1XQ== +canvaskit-wasm@0.39.1: + version "0.39.1" + resolved "https://registry.yarnpkg.com/canvaskit-wasm/-/canvaskit-wasm-0.39.1.tgz#c3c8f3962cbabbedf246f7bcf90e859013c7eae9" + integrity sha512-Gy3lCmhUdKq+8bvDrs9t8+qf7RvcjuQn+we7vTVVyqgOVO1UVfHpsnBxkTZw+R4ApEJ3D5fKySl9TU11hmjl/A== + dependencies: + "@webgpu/types" "0.1.21" chalk@^1.0.0: version "1.1.3" diff --git a/scripts/build-npm-package.ts b/scripts/build-npm-package.ts index fa0d6a1580..8a6442c220 100644 --- a/scripts/build-npm-package.ts +++ b/scripts/build-npm-package.ts @@ -83,7 +83,7 @@ pck.version = nextVersion; pck.types = "lib/typescript/index.d.ts"; pck.main = "lib/module/index.js"; pck.module = "lib/module/index.js"; -pck["react-native"] = "lib/module/index.js"; +pck["react-native"] = "src/index.ts"; console.log("Building version:", nextVersion); // Overwrite the package.json file