diff --git a/docs/docs/image.md b/docs/docs/image.md index 29da967f0d..fc5fc24cc7 100644 --- a/docs/docs/image.md +++ b/docs/docs/image.md @@ -133,5 +133,7 @@ const ImageDemo = () => { | :-------------- | :-------------------------------------------------------------------- | | `height` | Returns the possibly scaled height of the image. | | `width` | Returns the possibly scaled width of the image. | +| `getImageInfo` | Returns the image info for the image. | | `encodeToBytes` | Encodes the image pixels, returning the result as a `UInt8Array`. | | `encodeToBase64`| Encodes the image pixels, returning the result as a base64-encoded string. | +| `readPixels` | Reads the image pixels, returning result as UInt8Array or Float32Array | diff --git a/package/cpp/api/JsiSkCanvas.h b/package/cpp/api/JsiSkCanvas.h index 3be658bc8b..ca5e801ad3 100644 --- a/package/cpp/api/JsiSkCanvas.h +++ b/package/cpp/api/JsiSkCanvas.h @@ -7,6 +7,7 @@ #include "JsiSkFont.h" #include "JsiSkHostObjects.h" #include "JsiSkImage.h" +#include "JsiSkImageInfo.h" #include "JsiSkMatrix.h" #include "JsiSkPaint.h" #include "JsiSkPath.h" @@ -17,6 +18,8 @@ #include "JsiSkTextBlob.h" #include "JsiSkVertices.h" +#include "RNSkTypedArray.h" + #include #pragma clang diagnostic push @@ -491,6 +494,39 @@ class JsiSkCanvas : public JsiSkHostObject { return jsi::Value::undefined(); } + JSI_HOST_FUNCTION(readPixels) { + auto srcX = static_cast(arguments[0].asNumber()); + auto srcY = static_cast(arguments[1].asNumber()); + auto info = JsiSkImageInfo::fromValue(runtime, arguments[2]); + if (!info) { + return jsi::Value::null(); + } + size_t bytesPerRow = 0; + if (count > 4 && !arguments[4].isUndefined()) { + bytesPerRow = static_cast(arguments[4].asNumber()); + } else { + bytesPerRow = info->minRowBytes(); + } + auto dest = + count > 3 + ? RNSkTypedArray::getTypedArray(runtime, arguments[3], *info) + : RNSkTypedArray::getTypedArray(runtime, jsi::Value::null(), *info); + if (!dest.isObject()) { + return jsi::Value::null(); + } + jsi::ArrayBuffer buffer = + dest.asObject(runtime) + .getProperty(runtime, jsi::PropNameID::forAscii(runtime, "buffer")) + .asObject(runtime) + .getArrayBuffer(runtime); + auto bfrPtr = reinterpret_cast(buffer.data(runtime)); + + if (!_canvas->readPixels(*info, bfrPtr, bytesPerRow, srcX, srcY)) { + return jsi::Value::null(); + } + return std::move(dest); + } + JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkCanvas, drawPaint), JSI_EXPORT_FUNC(JsiSkCanvas, drawLine), JSI_EXPORT_FUNC(JsiSkCanvas, drawRect), @@ -529,7 +565,8 @@ class JsiSkCanvas : public JsiSkHostObject { JSI_EXPORT_FUNC(JsiSkCanvas, drawColor), JSI_EXPORT_FUNC(JsiSkCanvas, clear), JSI_EXPORT_FUNC(JsiSkCanvas, concat), - JSI_EXPORT_FUNC(JsiSkCanvas, drawPicture)) + JSI_EXPORT_FUNC(JsiSkCanvas, drawPicture), + JSI_EXPORT_FUNC(JsiSkCanvas, readPixels)) explicit JsiSkCanvas(std::shared_ptr context) : JsiSkHostObject(std::move(context)) {} diff --git a/package/cpp/api/JsiSkImage.h b/package/cpp/api/JsiSkImage.h index 90581e9eb7..3d6d00e7b4 100644 --- a/package/cpp/api/JsiSkImage.h +++ b/package/cpp/api/JsiSkImage.h @@ -5,9 +5,12 @@ #include #include "JsiSkHostObjects.h" +#include "JsiSkImageInfo.h" #include "JsiSkMatrix.h" #include "JsiSkShader.h" +#include "RNSkTypedArray.h" + #pragma clang diagnostic push #pragma clang diagnostic ignored "-Wdocumentation" @@ -34,6 +37,11 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { return static_cast(getObject()->height()); } + JSI_HOST_FUNCTION(getImageInfo) { + return JsiSkImageInfo::toValue(runtime, getContext(), + getObject()->imageInfo()); + } + JSI_HOST_FUNCTION(makeShaderOptions) { auto tmx = (SkTileMode)arguments[0].asNumber(); auto tmy = (SkTileMode)arguments[1].asNumber(); @@ -117,6 +125,46 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { return jsi::String::createFromAscii(runtime, buffer); } + JSI_HOST_FUNCTION(readPixels) { + int srcX = 0; + int srcY = 0; + if (count > 0 && !arguments[0].isUndefined()) { + srcX = static_cast(arguments[0].asNumber()); + } + if (count > 1 && !arguments[1].isUndefined()) { + srcY = static_cast(arguments[1].asNumber()); + } + SkImageInfo info = + (count > 2 && !arguments[2].isUndefined()) + ? *JsiSkImageInfo::fromValue(runtime, arguments[2]) + : SkImageInfo::MakeN32(getObject()->width(), getObject()->height(), + getObject()->imageInfo().alphaType()); + size_t bytesPerRow = 0; + if (count > 4 && !arguments[4].isUndefined()) { + bytesPerRow = static_cast(arguments[4].asNumber()); + } else { + bytesPerRow = info.minRowBytes(); + } + auto dest = + count > 3 + ? RNSkTypedArray::getTypedArray(runtime, arguments[3], info) + : RNSkTypedArray::getTypedArray(runtime, jsi::Value::null(), info); + if (!dest.isObject()) { + return jsi::Value::null(); + } + jsi::ArrayBuffer buffer = + dest.asObject(runtime) + .getProperty(runtime, jsi::PropNameID::forAscii(runtime, "buffer")) + .asObject(runtime) + .getArrayBuffer(runtime); + auto bfrPtr = reinterpret_cast(buffer.data(runtime)); + + if (!getObject()->readPixels(info, bfrPtr, bytesPerRow, srcX, srcY)) { + return jsi::Value::null(); + } + return std::move(dest); + } + JSI_HOST_FUNCTION(makeNonTextureImage) { auto image = getObject()->makeNonTextureImage(); return jsi::Object::createFromHostObject( @@ -127,10 +175,12 @@ class JsiSkImage : public JsiSkWrappingSkPtrHostObject { JSI_EXPORT_FUNCTIONS(JSI_EXPORT_FUNC(JsiSkImage, width), JSI_EXPORT_FUNC(JsiSkImage, height), + JSI_EXPORT_FUNC(JsiSkImage, getImageInfo), JSI_EXPORT_FUNC(JsiSkImage, makeShaderOptions), JSI_EXPORT_FUNC(JsiSkImage, makeShaderCubic), JSI_EXPORT_FUNC(JsiSkImage, encodeToBytes), JSI_EXPORT_FUNC(JsiSkImage, encodeToBase64), + JSI_EXPORT_FUNC(JsiSkImage, readPixels), JSI_EXPORT_FUNC(JsiSkImage, makeNonTextureImage), JSI_EXPORT_FUNC(JsiSkImage, dispose)) diff --git a/package/cpp/api/JsiSkImageInfo.h b/package/cpp/api/JsiSkImageInfo.h index 150700da3e..632be2ac91 100644 --- a/package/cpp/api/JsiSkImageInfo.h +++ b/package/cpp/api/JsiSkImageInfo.h @@ -56,5 +56,24 @@ class JsiSkImageInfo : public JsiSkWrappingSharedPtrHostObject { runtime, std::make_shared(std::move(context), imageInfo)); } + + JSI_PROPERTY_GET(width) { return static_cast(getObject()->width()); } + JSI_PROPERTY_GET(height) { + return static_cast(getObject()->height()); + } + JSI_PROPERTY_GET(colorType) { + return static_cast(getObject()->colorType()); + } + JSI_PROPERTY_GET(alphaType) { + return static_cast(getObject()->alphaType()); + } + + JSI_API_TYPENAME(ImageInfo); + + JSI_EXPORT_PROPERTY_GETTERS(JSI_EXPORT_PROP_GET(JsiSkImageInfo, width), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, height), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, colorType), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, alphaType), + JSI_EXPORT_PROP_GET(JsiSkImageInfo, __typename__)) }; } // namespace RNSkia diff --git a/package/cpp/utils/RNSkTypedArray.h b/package/cpp/utils/RNSkTypedArray.h new file mode 100644 index 0000000000..9df7977bc5 --- /dev/null +++ b/package/cpp/utils/RNSkTypedArray.h @@ -0,0 +1,41 @@ +#pragma once + +#include "SkImage.h" +#include + +namespace RNSkia { + +namespace jsi = facebook::jsi; + +class RNSkTypedArray { +public: + static jsi::Value getTypedArray(jsi::Runtime &runtime, + const jsi::Value &value, SkImageInfo &info) { + auto reqSize = info.computeMinByteSize(); + if (reqSize > 0) { + if (value.isObject()) { + auto typedArray = value.asObject(runtime); + auto size = static_cast( + typedArray.getProperty(runtime, "byteLength").asNumber()); + if (size >= reqSize) { + return typedArray; + } + } else { + if (info.colorType() == kRGBA_F32_SkColorType) { + auto arrayCtor = + runtime.global().getPropertyAsFunction(runtime, "Float32Array"); + return arrayCtor.callAsConstructor(runtime, + static_cast(reqSize / 4)); + } else { + auto arrayCtor = + runtime.global().getPropertyAsFunction(runtime, "Uint8Array"); + return arrayCtor.callAsConstructor(runtime, + static_cast(reqSize)); + } + } + } + return jsi::Value::null(); + } +}; + +} // namespace RNSkia diff --git a/package/src/renderer/__tests__/e2e/Image.spec.tsx b/package/src/renderer/__tests__/e2e/Image.spec.tsx index 86206bbbf0..ee520d1d73 100644 --- a/package/src/renderer/__tests__/e2e/Image.spec.tsx +++ b/package/src/renderer/__tests__/e2e/Image.spec.tsx @@ -1,8 +1,9 @@ import React from "react"; import { checkImage } from "../../../__tests__/setup"; -import { images, surface } from "../setup"; +import { images, loadImage, surface } from "../setup"; import { Fill, Image as SkiaImage } from "../../components"; +import { AlphaType, ColorType } from "../../../skia/types"; describe("Image loading from bundles", () => { it("should render png, jpg from bundle", async () => { @@ -24,6 +25,108 @@ describe("Image loading from bundles", () => { ); checkImage(image, `snapshots/images/bundle-${surface.OS}.png`); }); + + it("should read pixels from an image", async () => { + const pixels = await surface.eval( + (Skia, { data }) => { + const image = Skia.Image.MakeImageFromEncoded( + Skia.Data.fromBytes(new Uint8Array(data)) + )!; + return Array.from( + image.readPixels(0, 0, { + width: 2, + height: 2, + colorType: image.getImageInfo().colorType, + alphaType: image.getImageInfo().alphaType, + })! + ); + }, + { + data: Array.from( + loadImage("skia/__tests__/assets/oslo.jpg").encodeToBytes() + ), + } + ); + expect(pixels).toBeDefined(); + expect(pixels).toEqual([ + 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, + 255, + ]); + }); + + // it("should read pixels from an image using a preallocated buffer", async () => { + // const pixels = await surface.eval( + // (Skia, { colorType, alphaType, data }) => { + // const image = Skia.Image.MakeImageFromEncoded( + // Skia.Data.fromBytes(new Uint8Array(data)) + // )!; + // const result = new Uint8Array(16); + // image.readPixels( + // 0, + // 0, + // { + // width: 2, + // height: 2, + // colorType, + // alphaType, + // }, + // result + // ); + // return result; + // }, + // { + // colorType: ColorType.RGBA_8888, + // alphaType: AlphaType.Unpremul, + // data: Array.from( + // loadImage("skia/__tests__/assets/oslo.jpg").encodeToBytes() + // ), + // } + // ); + // expect(pixels).toBeDefined(); + // expect(Array.from(pixels!)).toEqual([ + // 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, 255, 170, 186, 199, + // 255, + // ]); + // }); + it("should read pixels from a canvas", async () => { + const pixels = await surface.eval( + (Skia, { colorType, alphaType }) => { + const offscreen = Skia.Surface.MakeOffscreen(10, 10)!; + const canvas = offscreen.getCanvas(); + canvas.drawColor(Skia.Color("red")); + return Array.from( + canvas.readPixels(0, 0, { + width: 1, + height: 1, + colorType, + alphaType, + })! + ); + }, + { colorType: ColorType.RGBA_8888, alphaType: AlphaType.Unpremul } + ); + expect(pixels).toBeDefined(); + expect(Array.from(pixels!)).toEqual([255, 0, 0, 255]); + }); + // it("should read pixels from a canvas using a preallocated buffer", async () => { + // const pixels = await surface.eval( + // (Skia, { colorType, alphaType }) => { + // const offscreen = Skia.Surface.MakeOffscreen(10, 10)!; + // const canvas = offscreen.getCanvas(); + // canvas.drawColor(Skia.Color("red")); + // const result = new Uint8Array(4); + // canvas.readPixels(0, 0, { + // width: 1, + // height: 1, + // colorType, + // alphaType, + // }, result); + // }, + // { colorType: ColorType.RGBA_8888, alphaType: AlphaType.Unpremul } + // ); + // expect(pixels).toBeDefined(); + // expect(Array.from(pixels!)).toEqual([255, 0, 0, 255]); + // }); // This test should only run on CI because it will trigger a redbox. // While this is fine on CI, it is undesirable on local dev. // it("should not crash with an invalid viewTag", async () => { diff --git a/package/src/skia/types/Canvas.ts b/package/src/skia/types/Canvas.ts index 5705e8d395..0798e22f9e 100644 --- a/package/src/skia/types/Canvas.ts +++ b/package/src/skia/types/Canvas.ts @@ -2,7 +2,7 @@ import type { SkPaint } from "./Paint"; import type { SkRect } from "./Rect"; import type { SkFont } from "./Font"; import type { SkPath } from "./Path"; -import type { SkImage, MipmapMode, FilterMode } from "./Image"; +import type { SkImage, MipmapMode, FilterMode, ImageInfo } from "./Image"; import type { SkSVG } from "./SVG"; import type { SkColor } from "./Color"; import type { SkRRect } from "./RRect"; @@ -492,4 +492,17 @@ export interface SkCanvas { * @param skp */ drawPicture(skp: SkPicture): void; + + /** Read Image pixels + * + * @param srcX - x-axis upper left corner of the rectangle to read from + * @param srcY - y-axis upper left corner of the rectangle to read from + * @param imageInfo - describes the pixel format and dimensions of the data to read into + * @return Float32Array or Uint8Array with data or null if the read failed. + */ + readPixels( + srcX: number, + srcY: number, + imageInfo: ImageInfo + ): Float32Array | Uint8Array | null; } diff --git a/package/src/skia/types/Image/Image.ts b/package/src/skia/types/Image/Image.ts index a6a1ef5321..4406885f6f 100644 --- a/package/src/skia/types/Image/Image.ts +++ b/package/src/skia/types/Image/Image.ts @@ -3,6 +3,8 @@ import type { SkJSIInstance } from "../JsiInstance"; import type { TileMode } from "../ImageFilter"; import type { SkShader } from "../Shader"; +import type { ImageInfo } from "./ImageFactory"; + export enum FilterMode { Nearest, Linear, @@ -31,6 +33,11 @@ export interface SkImage extends SkJSIInstance<"Image"> { */ width(): number; + /** + * Returns the ImageInfo describing the image. + */ + getImageInfo(): ImageInfo; + /** * Returns this image as a shader with the specified tiling. It will use cubic sampling. * @param tx - tile mode in the x direction. @@ -94,6 +101,19 @@ export interface SkImage extends SkJSIInstance<"Image"> { */ encodeToBase64(fmt?: ImageFormat, quality?: number): string; + /** Read Image pixels + * + * @param srcX - optional x-axis upper left corner of the rectangle to read from + * @param srcY - optional y-axis upper left corner of the rectangle to read from + * @param imageInfo - optional describes the pixel format and dimensions of the data to read into + * @return Float32Array or Uint8Array with data or null if the read failed. + */ + readPixels( + srcX?: number, + srcY?: number, + imageInfo?: ImageInfo + ): Float32Array | Uint8Array | null; + /** * Returns raster image or lazy image. Copies SkImage backed by GPU texture * into CPU memory if needed. Returns original SkImage if decoded in raster diff --git a/package/src/skia/web/Host.ts b/package/src/skia/web/Host.ts index d2f118f554..97d3ff97f4 100644 --- a/package/src/skia/web/Host.ts +++ b/package/src/skia/web/Host.ts @@ -1,4 +1,4 @@ -import type { CanvasKit, EmbindEnumEntity } from "canvaskit-wasm"; +import type { CanvasKit, EmbindEnumEntity, EmbindEnum } from "canvaskit-wasm"; import type { SkJSIInstance } from "../types"; @@ -41,6 +41,8 @@ export abstract class HostObject extends BaseHostObject< } } +export const getCkEnum = (e: EmbindEnum, v: number): EmbindEnumEntity => + Object.values(e).find(({ value }) => value === v); export const ckEnum = (value: number): EmbindEnumEntity => ({ value }); export const optEnum = ( value: number | undefined diff --git a/package/src/skia/web/JsiSkCanvas.ts b/package/src/skia/web/JsiSkCanvas.ts index b9ddaba3f1..3979c13902 100644 --- a/package/src/skia/web/JsiSkCanvas.ts +++ b/package/src/skia/web/JsiSkCanvas.ts @@ -7,6 +7,7 @@ import type { MipmapMode, PointMode, SaveLayerFlag, + ImageInfo, SkCanvas, SkColor, SkFont, @@ -24,7 +25,7 @@ import type { SkVertices, } from "../types"; -import { ckEnum, HostObject } from "./Host"; +import { ckEnum, getCkEnum, HostObject } from "./Host"; import { JsiSkPaint } from "./JsiSkPaint"; import { JsiSkRect } from "./JsiSkRect"; import { JsiSkRRect } from "./JsiSkRRect"; @@ -374,4 +375,15 @@ export class JsiSkCanvas drawPicture(skp: SkPicture) { this.ref.drawPicture(JsiSkPicture.fromValue(skp)); } + + readPixels(srcX: number, srcY: number, imageInfo: ImageInfo) { + const pxInfo = { + width: imageInfo.width, + height: imageInfo.height, + colorSpace: this.CanvasKit.ColorSpace.SRGB, + alphaType: getCkEnum(this.CanvasKit.AlphaType, imageInfo.alphaType), + colorType: getCkEnum(this.CanvasKit.ColorType, imageInfo.colorType), + }; + return this.ref.readPixels(srcX, srcY, pxInfo); + } } diff --git a/package/src/skia/web/JsiSkImage.ts b/package/src/skia/web/JsiSkImage.ts index dcb7403d66..a06b6fde28 100644 --- a/package/src/skia/web/JsiSkImage.ts +++ b/package/src/skia/web/JsiSkImage.ts @@ -1,16 +1,21 @@ -import type { CanvasKit, Image } from "canvaskit-wasm"; - import type { - ImageFormat, - FilterMode, - MipmapMode, - SkImage, - SkMatrix, - SkShader, - TileMode, + CanvasKit, + Image, + ImageInfo as CKImageInfo, +} from "canvaskit-wasm"; + +import { + type ImageFormat, + type ImageInfo, + type FilterMode, + type MipmapMode, + type SkImage, + type SkMatrix, + type SkShader, + type TileMode, } from "../types"; -import { ckEnum, HostObject } from "./Host"; +import { ckEnum, getCkEnum, HostObject } from "./Host"; import { JsiSkMatrix } from "./JsiSkMatrix"; import { JsiSkShader } from "./JsiSkShader"; @@ -52,6 +57,16 @@ export class JsiSkImage extends HostObject implements SkImage { return this.ref.width(); } + getImageInfo(): ImageInfo { + const info = this.ref.getImageInfo(); + return { + width: info.width, + height: info.height, + colorType: info.colorType.value, + alphaType: info.alphaType.value, + }; + } + makeShaderOptions( tx: TileMode, ty: TileMode, @@ -110,6 +125,30 @@ export class JsiSkImage extends HostObject implements SkImage { return toBase64String(bytes); } + readPixels(srcX?: number, srcY?: number, imageInfo?: ImageInfo) { + const info = this.getImageInfo(); + console.log({ + alphaType: ckEnum(info.alphaType), + colorType: ckEnum(info.colorType), + realAlphaType: this.CanvasKit.AlphaType.Opaque.value, + realColorType: this.CanvasKit.ColorType.RGBA_8888.value, + }); + const pxInfo: CKImageInfo = { + colorSpace: this.CanvasKit.ColorSpace.SRGB, + width: imageInfo?.width ?? info.width, + height: imageInfo?.height ?? info.height, + alphaType: getCkEnum( + this.CanvasKit.AlphaType, + (imageInfo ?? info).alphaType + ), + colorType: getCkEnum( + this.CanvasKit.ColorType, + (imageInfo ?? info).colorType + ), + }; + return this.ref.readPixels(srcX ?? 0, srcY ?? 0, pxInfo); + } + dispose = () => { this.ref.delete(); };