diff --git a/App.tsx b/App.tsx index 9d8b636..e7b6d50 100644 --- a/App.tsx +++ b/App.tsx @@ -22,7 +22,6 @@ import Section from './components/form/Section'; import SelectField from './components/form/SelectField'; import Progress from './components/Progress'; import Models from './components/models'; -import { GCanvasView } from '@flyskywhy/react-native-gcanvas'; import * as logger from './utils/logger'; const tasks = Object.keys(Models); @@ -75,7 +74,7 @@ function App(): JSX.Element { logger.log('Result:', result); return result; } catch (e) { - console.error(e); + console.error(e.stack); await pipe?.dispose(); throw e; } @@ -91,20 +90,6 @@ function App(): JSX.Element { barStyle={isDarkMode ? 'light-content' : 'dark-content'} backgroundColor={backgroundColor} /> - console.log('Off-screen canvas is ready')} - devicePixelRatio={1} - isGestureResponsible={false} - /> diff --git a/android/app/proguard-rules.pro b/android/app/proguard-rules.pro index 11b0257..420171c 100644 --- a/android/app/proguard-rules.pro +++ b/android/app/proguard-rules.pro @@ -8,3 +8,4 @@ # http://developer.android.com/guide/developing/tools/proguard.html # Add any project specific keep options here: +-keep class com.shopify.reactnative.skia.** { *; } \ No newline at end of file diff --git a/android/settings.gradle b/android/settings.gradle index e9f85ef..bec01fb 100644 --- a/android/settings.gradle +++ b/android/settings.gradle @@ -2,12 +2,3 @@ rootProject.name = 'transformers_example' apply from: file("../node_modules/@react-native-community/cli-platform-android/native_modules.gradle"); applyNativeModulesSettingsGradle(settings) include ':app' includeBuild('../node_modules/react-native-gradle-plugin') - -include ':android:gcanvas_library' -project(':android:gcanvas_library').projectDir = new File(rootProject.projectDir, '../node_modules/@flyskywhy/react-native-gcanvas/android/gcanvas_library') -include ':android:bridge_spec' -project(':android:bridge_spec').projectDir = new File(rootProject.projectDir, '../node_modules/@flyskywhy/react-native-gcanvas/android/bridge_spec') -include ':android:adapters:gcanvas_imageloader_fresco' -project(':android:adapters:gcanvas_imageloader_fresco').projectDir = new File(rootProject.projectDir, '../node_modules/@flyskywhy/react-native-gcanvas/android/adapters/gcanvas_imageloader_fresco') -include ':android:adapters:bridge_adapter' -project(':android:adapters:bridge_adapter').projectDir = new File(rootProject.projectDir, '../node_modules/@flyskywhy/react-native-gcanvas/android/adapters/bridge_adapter') \ No newline at end of file diff --git a/components/Canvas.tsx b/components/Canvas.tsx deleted file mode 100644 index 71022c6..0000000 --- a/components/Canvas.tsx +++ /dev/null @@ -1,39 +0,0 @@ -import React, { useImperativeHandle, useRef, useCallback } from 'react'; -import { GCanvasView } from '@flyskywhy/react-native-gcanvas'; - -export interface CanvasProps { - width: number | string; - height: number | string; -} - -export interface CanvasRef { - getContext(name: string): CanvasRenderingContext2D | null; -} - -export default React.forwardRef((props, ref) => { - const { width, height, ...rest } = props; - const canvasRef = useRef(null); - - useImperativeHandle(ref, () => ({ - getContext: (name) => canvasRef.current?.getContext(name) ?? null, - get width() { - return canvasRef.current?.width ?? 0; - }, - get height() { - return canvasRef.current?.height ?? 0; - }, - })); - - const canvasInit = useCallback((canvas: HTMLCanvasElement) => { - canvasRef.current = canvas; - }, []); - - return ( - - ); -}); diff --git a/components/models/ImageSegmentation.tsx b/components/models/ImageSegmentation.tsx index 5ce46eb..0921404 100644 --- a/components/models/ImageSegmentation.tsx +++ b/components/models/ImageSegmentation.tsx @@ -1,13 +1,23 @@ -import React, { useState, useEffect, useCallback, useRef } from 'react'; -import { StyleSheet } from 'react-native'; -import { GCanvasView } from '@flyskywhy/react-native-gcanvas'; +import React, { useState, useEffect, useCallback, useMemo, useRef } from 'react'; +import { StyleSheet, Text } from 'react-native'; +import { + Skia, + Canvas, + Image, + useImage, + Group, + Mask, + Fill, + ColorType, + AlphaType, +} from '@shopify/react-native-skia'; import { RawImage } from '@xenova/transformers/src/utils/image'; import uniqolor from 'uniqolor'; +import parseColor from 'color-parse'; import SelectField from '../form/SelectField'; import TextField from '../form/TextField'; import Button from '../form/Button'; import Progress from '../Progress'; -import Canvas from '../Canvas'; import { getImageData, createRawImage } from '../../utils/image'; import { usePhoto } from '../../utils/photo'; @@ -32,54 +42,46 @@ interface Segment { export function Interact({ settings: { model }, runPipe }: InteractProps): JSX.Element { const [results, setResults] = useState([]); const [isWIP, setWIP] = useState(false); - const canvasRef = useRef(null); - const inferImg = useRef(null); + const [input, setInput] = useState(''); + const img = useImage(input); + const [size, setSize] = useState({ width: 0, height: 0 }); const call = useCallback(async (input) => { setWIP(true); try { - inferImg.current = await getImageData(input, canvasRef.current.width); - const predicts = await runPipe('image-segmentation', model, createRawImage(inferImg.current)); + setInput(input); + const data = await getImageData(input); + const predicts = await runPipe('image-segmentation', model, createRawImage(data)); setResults(predicts); } catch {} setWIP(false); }, [model]); + const masks = useRef([]); useEffect(() => { - const canvas = canvasRef.current; - const ctx = canvas?.getContext('2d'); - if (ctx && inferImg.current) { - const width = inferImg.current.width; - const height = inferImg.current.height; - ctx.clearRect(0, 0, canvas.width, canvas.height); - ctx.putImageData(inferImg.current, 0, 0); - results.reduce((p, { label, mask, score }, index) => p.then(async () => { - // mask data to RGBA - const dataRGBA = new Uint8ClampedArray(width * height * 4); - const color = uniqolor(label, { format: 'rgb', lightness: 50 }).color; - const parsedRGB = color.match(/rgb\((\d+),\s*(\d+),\s*(\d+)\)/); - const red = parsedRGB ? Number(parsedRGB[1]) : 0; - const green = parsedRGB ? Number(parsedRGB[2]) : 0; - const blue = parsedRGB ? Number(parsedRGB[3]) : 0; - for (let i = 0; i < dataRGBA.length; i += 4) { - const maskIndex = Math.floor(i / 4); - dataRGBA[i] = red; - dataRGBA[i + 1] = green; - dataRGBA[i + 2] = blue; - dataRGBA[i + 3] = mask.data[maskIndex] * 0.6; - } - ctx.putImageData(new ImageData(dataRGBA, width, height), 0, 0); - ctx.font = '20px serif'; - ctx.fillStyle = color; - ctx.fillText( - `${label} (${score.toFixed(2)})`, - 0, - height + 20 * (index + 1), - ); - }), Promise.resolve()); + if (results?.length) { + masks.current = results.map(({ mask }) => { + const data = Skia.Data.fromBytes(mask.data); + const image = Skia.Image.MakeImage({ + width: mask.width, + height: mask.height, + colorType: ColorType.Gray_8, + alphaType: AlphaType.Unpremul, + }, data, mask.width); + data.dispose(); + return image; + }); } + return () => masks.current.forEach((mask) => mask.dispose()); }, [results]); + const colors = useMemo(() => results?.map(({ label }) => { + const lightness = label.startsWith('LABEL_') ? 40 : 80; + const { color } = uniqolor(label, { lightness }); + const { values } = parseColor(color); + return { text: color, mask: `rgba(${values.join(', ')}, 0.8)` }; + }), [results]); + const { selectPhoto, takePhoto } = usePhoto((uri) => call(uri)); return ( @@ -95,11 +97,56 @@ export function Interact({ settings: { model }, runPipe }: InteractProps): JSX.E disabled={isWIP} /> + style={styles.canvas} + onLayout={(e) => setSize({ + width: e.nativeEvent.layout.width, + height: e.nativeEvent.layout.height, + })} + onSize={(width, height) => setSize({ width, height })} + > + + {results?.map((result, i) => ( + + + )} + > + + + + ))} + + {results?.map(({ label, score }, i) => ( + + {`${label} (${score.toFixed(2)})`} + + ))} ) } + +const styles = StyleSheet.create({ + canvas: { + width: '100%', + height: 512, + }, +}); diff --git a/components/models/ObjectDetection.tsx b/components/models/ObjectDetection.tsx index 857b39b..a9c6490 100644 --- a/components/models/ObjectDetection.tsx +++ b/components/models/ObjectDetection.tsx @@ -1,13 +1,14 @@ -import React, { useState, useEffect, useCallback, useRef } from 'react'; +import React, { useState, useEffect, useCallback } from 'react'; import { StyleSheet } from 'react-native'; import uniqolor from 'uniqolor'; +import { Canvas, Image, useImage, Text, Rect, Group } from '@shopify/react-native-skia'; import SelectField from '../form/SelectField'; import TextField from '../form/TextField'; import Button from '../form/Button'; import Progress from '../Progress'; -import Canvas from '../Canvas'; -import { getImageData, createRawImage } from '../../utils/image'; +import { getImageData, createRawImage, calcPosition } from '../../utils/image'; import { usePhoto } from '../../utils/photo'; +import { log } from '../../utils/logger'; export const title = 'Object Detection'; @@ -28,16 +29,19 @@ interface Result { } export function Interact({ settings: { model }, runPipe }: InteractProps): JSX.Element { - const [results, setResults] = useState(null); + const [results, setResults] = useState(null); const [isWIP, setWIP] = useState(false); - const canvasRef = useRef(null); - const inferImg = useRef(null); + const [input, setInput] = useState(''); + const img = useImage(input); + const [size, setSize] = useState({ width: 0, height: 0 }); const call = useCallback(async (input) => { setWIP(true); try { - inferImg.current = await getImageData(input, model, canvasRef.current.width); - const predicts = await runPipe('object-detection', createRawImage(inferImg.current)); + setResults(null); + setInput(input); + const data = await getImageData(input, 128); + const predicts = await runPipe('object-detection', model, createRawImage(data)); setResults(predicts); } catch {} setWIP(false); @@ -45,37 +49,6 @@ export function Interact({ settings: { model }, runPipe }: InteractProps): JSX.E const { selectPhoto, takePhoto } = usePhoto((uri) => call(uri)); - useEffect(() => { - const ctx = canvasRef.current?.getContext('2d'); - if (ctx && inferImg.current && results) { - const width = inferImg.current.width; - const height = inferImg.current.height; - ctx.clearRect(0, 0, canvasRef.current.width, canvasRef.current.height); - ctx.putImageData(inferImg.current, 0, 0); - ctx.fillStyle = '#FFFFFF'; // Avoid weired bug - results.forEach(({ box: { xmin, ymin, xmax, ymax}, label, score }, i) => { - const color = uniqolor(label, { lightness: 50 }).color; - ctx.beginPath(); - ctx.lineWidth = 2; - ctx.strokeStyle = color; - ctx.rect( - xmin, - ymin, - (xmax - xmin), - (ymax - ymin), - ); - ctx.stroke(); - ctx.font = '16px Arial'; - ctx.fillStyle = color; - ctx.fillText( - `${label} ${score.toFixed(2)}`, - ymin > 10 ? xmin : xmin + 4, - ymin > 10 ? ymin - 5 : ymin + 16, - ); - }); - } - }, [results]); - return ( <>