diff --git a/examples/computer-vision/App.tsx b/examples/computer-vision/App.tsx index cacb0c6..5f6c4ef 100644 --- a/examples/computer-vision/App.tsx +++ b/examples/computer-vision/App.tsx @@ -6,11 +6,12 @@ import { useState } from 'react'; import { StyleTransferScreen } from './screens/StyleTransferScreen'; import { SafeAreaProvider, SafeAreaView } from 'react-native-safe-area-context'; import { View, StyleSheet } from 'react-native'; +import { ClassificationScreen } from './screens/ClassificationScreen'; enum ModelType { STYLE_TRANSFER, OBJECT_DETECTION, - IMAGE_CLASSIFICATION, + CLASSIFICATION, SEMANTIC_SEGMENTATION, } @@ -36,8 +37,10 @@ export default function App() { ); case ModelType.OBJECT_DETECTION: return <>; - case ModelType.IMAGE_CLASSIFICATION: - return <>; + case ModelType.CLASSIFICATION: + return ( + + ); case ModelType.SEMANTIC_SEGMENTATION: return <>; default: @@ -57,17 +60,17 @@ export default function App() { dataSource={[ 'Style Transfer', 'Object Detection', - 'Image Classification', + 'Classification', 'Semantic Segmentation', ]} onValueChange={(_, selectedIndex) => { handleModeChange(selectedIndex); }} - wrapperHeight={135} + wrapperHeight={100} highlightColor={ColorPalette.primary} wrapperBackground="#fff" highlightBorderWidth={3} - itemHeight={60} + itemHeight={40} activeItemTextStyle={styles.activeScrollItem} /> @@ -85,7 +88,7 @@ const styles = StyleSheet.create({ }, topContainer: { marginTop: 5, - height: 150, + height: 145, width: '100%', alignItems: 'center', justifyContent: 'center', @@ -93,7 +96,7 @@ const styles = StyleSheet.create({ }, wheelPickerContainer: { width: '100%', - height: 135, + height: 100, }, activeScrollItem: { color: ColorPalette.primary, diff --git a/examples/computer-vision/assets/classification/android/efficientnet_v2_s_xnnpack.pte b/examples/computer-vision/assets/classification/android/efficientnet_v2_s_xnnpack.pte new file mode 100644 index 0000000..0cb56a2 Binary files /dev/null and b/examples/computer-vision/assets/classification/android/efficientnet_v2_s_xnnpack.pte differ diff --git a/examples/computer-vision/assets/classification/ios/efficientnet_v2_s_coreml_all.pte b/examples/computer-vision/assets/classification/ios/efficientnet_v2_s_coreml_all.pte new file mode 100644 index 0000000..2cf7f9f Binary files /dev/null and b/examples/computer-vision/assets/classification/ios/efficientnet_v2_s_coreml_all.pte differ diff --git a/examples/computer-vision/ios/Podfile.lock b/examples/computer-vision/ios/Podfile.lock index 08e28f5..2b2e442 100644 --- a/examples/computer-vision/ios/Podfile.lock +++ b/examples/computer-vision/ios/Podfile.lock @@ -42,6 +42,7 @@ PODS: - hermes-engine (0.76.3): - hermes-engine/Pre-built (= 0.76.3) - hermes-engine/Pre-built (0.76.3) + - opencv-rne (0.1.0) - RCT-Folly (2024.01.01.00): - boost - DoubleConversion @@ -1277,10 +1278,11 @@ PODS: - ReactCommon/turbomodule/bridging - ReactCommon/turbomodule/core - Yoga - - react-native-executorch (0.1.524): + - react-native-executorch (0.1.100): - DoubleConversion - glog - hermes-engine + - opencv-rne (~> 0.1.0) - RCT-Folly (= 2024.01.01.00) - RCTRequired - RCTTypeSafety @@ -1866,6 +1868,7 @@ DEPENDENCIES: SPEC REPOS: trunk: + - opencv-rne - SocketRocket EXTERNAL SOURCES: @@ -2035,6 +2038,7 @@ SPEC CHECKSUMS: fmt: 10c6e61f4be25dc963c36bd73fc7b1705fe975be glog: 08b301085f15bcbb6ff8632a8ebaf239aae04e6a hermes-engine: 0555a84ea495e8e3b4bde71b597cd87fbb382888 + opencv-rne: 63e933ae2373fc91351f9a348dc46c3f523c2d3f RCT-Folly: bf5c0376ffe4dd2cf438dcf86db385df9fdce648 RCTDeprecation: 2c5e1000b04ab70b53956aa498bf7442c3c6e497 RCTRequired: 5f785a001cf68a551c5f5040fb4c415672dbb481 @@ -2064,7 +2068,7 @@ SPEC CHECKSUMS: React-logger: 26155dc23db5c9038794db915f80bd2044512c2e React-Mapbuffer: ad1ba0205205a16dbff11b8ade6d1b3959451658 React-microtasksnativemodule: e771eb9eb6ace5884ee40a293a0e14a9d7a4343c - react-native-executorch: 9782e20c5bb4ddf7836af8779f887223228833e9 + react-native-executorch: a30dd907f470d5c4f8135e2ba1fa2a3bb65ea47a react-native-image-picker: bfb56e2b39dc63abfcc6de44ee239c6633f47d66 react-native-safe-area-context: d6406c2adbd41b2e09ab1c386781dc1c81a90919 React-nativeconfig: aeed6e2a8ac02b2df54476afcc7c663416c12bf7 diff --git a/examples/computer-vision/ios/computervision.xcodeproj/project.pbxproj b/examples/computer-vision/ios/computervision.xcodeproj/project.pbxproj index edc6d49..8bf0767 100644 --- a/examples/computer-vision/ios/computervision.xcodeproj/project.pbxproj +++ b/examples/computer-vision/ios/computervision.xcodeproj/project.pbxproj @@ -367,7 +367,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_DEBUG"; PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.computervision; - PRODUCT_NAME = "computervision"; + PRODUCT_NAME = computervision; SWIFT_OBJC_BRIDGING_HEADER = "computervision/computervision-Bridging-Header.h"; SWIFT_OPTIMIZATION_LEVEL = "-Onone"; SWIFT_VERSION = 5.0; @@ -399,7 +399,7 @@ ); OTHER_SWIFT_FLAGS = "$(inherited) -D EXPO_CONFIGURATION_RELEASE"; PRODUCT_BUNDLE_IDENTIFIER = com.anonymous.computervision; - PRODUCT_NAME = "computervision"; + PRODUCT_NAME = computervision; SWIFT_OBJC_BRIDGING_HEADER = "computervision/computervision-Bridging-Header.h"; SWIFT_VERSION = 5.0; TARGETED_DEVICE_FAMILY = "1,2"; diff --git a/examples/computer-vision/models/classification.ts b/examples/computer-vision/models/classification.ts new file mode 100644 index 0000000..3d18dfa --- /dev/null +++ b/examples/computer-vision/models/classification.ts @@ -0,0 +1,6 @@ +import { Platform } from 'react-native'; + +export const efficientnet_v2_s = + Platform.OS === 'ios' + ? require('../assets/classification/ios/efficientnet_v2_s_coreml_all.pte') + : require('../assets/classification/android/efficientnet_v2_s_xnnpack.pte'); diff --git a/examples/computer-vision/screens/ClassificationScreen.tsx b/examples/computer-vision/screens/ClassificationScreen.tsx new file mode 100644 index 0000000..6e735a7 --- /dev/null +++ b/examples/computer-vision/screens/ClassificationScreen.tsx @@ -0,0 +1,126 @@ +import { useState } from 'react'; +import Spinner from 'react-native-loading-spinner-overlay'; +import { BottomBar } from '../components/BottomBar'; +import { efficientnet_v2_s } from '../models/classification'; +import { getImageUri } from '../utils'; +import { useClassification } from 'react-native-executorch'; +import { View, StyleSheet, Image, Text, ScrollView } from 'react-native'; + +export const ClassificationScreen = ({ + imageUri, + setImageUri, +}: { + imageUri: string; + setImageUri: (imageUri: string) => void; +}) => { + const [results, setResults] = useState<{ label: string; score: number }[]>( + [] + ); + + const model = useClassification({ + modulePath: efficientnet_v2_s, + }); + + const handleCameraPress = async (isCamera: boolean) => { + const uri = await getImageUri(isCamera); + if (typeof uri === 'string') { + setImageUri(uri as string); + setResults([]); + } + }; + + const runForward = async () => { + if (imageUri) { + try { + const output = await model.forward(imageUri); + const top10 = Object.entries(output) + .sort(([, a], [, b]) => b - a) + .slice(0, 10) + .map(([label, score]) => ({ label, score })); + setResults(top10); + } catch (e) { + console.error(e); + } + } + }; + + if (!model.isModelReady) { + return ( + + ); + } + + return ( + <> + + + {results.length > 0 && ( + + Results Top 10 + + {results.map(({ label, score }) => ( + + {label} + {score.toFixed(3)} + + ))} + + + )} + + + + ); +}; + +const styles = StyleSheet.create({ + imageContainer: { + flex: 6, + width: '100%', + padding: 16, + }, + image: { + flex: 2, + borderRadius: 8, + width: '100%', + }, + results: { + flex: 1, + alignItems: 'center', + justifyContent: 'center', + gap: 4, + padding: 4, + }, + resultHeader: { + fontSize: 18, + color: 'navy', + }, + resultsList: { + flex: 1, + }, + resultRecord: { + flexDirection: 'row', + width: '100%', + justifyContent: 'space-between', + padding: 8, + borderBottomWidth: 1, + }, + resultLabel: { + flex: 1, + marginRight: 4, + }, +}); diff --git a/examples/computer-vision/yarn.lock b/examples/computer-vision/yarn.lock index 17e9b51..5249305 100644 --- a/examples/computer-vision/yarn.lock +++ b/examples/computer-vision/yarn.lock @@ -3352,7 +3352,7 @@ __metadata: metro-config: ^0.81.0 react: 18.3.1 react-native: 0.76.3 - react-native-executorch: /Users/norbertklockiewicz/Desktop/work/react-native-executorch/react-native-executorch-0.1.524.tgz + react-native-executorch: ../../react-native-executorch-0.1.100.tgz react-native-image-picker: ^7.2.2 react-native-loading-spinner-overlay: ^3.0.1 react-native-reanimated: ^3.16.3 @@ -6799,13 +6799,13 @@ __metadata: languageName: node linkType: hard -"react-native-executorch@file:/Users/norbertklockiewicz/Desktop/work/react-native-executorch/react-native-executorch-0.1.524.tgz::locator=computer-vision%40workspace%3A.": - version: 0.1.524 - resolution: "react-native-executorch@file:/Users/norbertklockiewicz/Desktop/work/react-native-executorch/react-native-executorch-0.1.524.tgz::locator=computer-vision%40workspace%3A." +"react-native-executorch@file:../../react-native-executorch-0.1.100.tgz::locator=computer-vision%40workspace%3A.": + version: 0.1.100 + resolution: "react-native-executorch@file:../../react-native-executorch-0.1.100.tgz::locator=computer-vision%40workspace%3A." peerDependencies: react: "*" react-native: "*" - checksum: 4f67dbd81711997e5f890b2d7c7b025777d6bcf71a335c7efa56da3fa59a6d00a4915c8cb63e4362d931885adde4072fad63bfbabb93bc36edf116e4fa98f5b9 + checksum: f258452e2050df59e150938f6482ef8eee5fbd4ef4fc4073a920293ca87d543daddf76c560701d0c2626e6677d964b446dad8e670e978ea4f80d0a1bd17dfa03 languageName: node linkType: hard diff --git a/ios/RnExecutorch/Classification.h b/ios/RnExecutorch/Classification.h new file mode 100644 index 0000000..763a402 --- /dev/null +++ b/ios/RnExecutorch/Classification.h @@ -0,0 +1,5 @@ +#import + +@interface Classification : NSObject + +@end diff --git a/ios/RnExecutorch/Classification.mm b/ios/RnExecutorch/Classification.mm new file mode 100644 index 0000000..96f510c --- /dev/null +++ b/ios/RnExecutorch/Classification.mm @@ -0,0 +1,56 @@ +#import "Classification.h" +#import "utils/Fetcher.h" +#import "models/BaseModel.h" +#import "utils/ETError.h" +#import "ImageProcessor.h" +#import +#import +#import "models/classification/ClassificationModel.h" +#import "opencv2/opencv.hpp" + +@implementation Classification { + ClassificationModel* model; +} + +RCT_EXPORT_MODULE() + +- (void)loadModule:(NSString *)modelSource + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject { + model = [[ClassificationModel alloc] init]; + [model loadModel: [NSURL URLWithString:modelSource] completion:^(BOOL success, NSNumber *errorCode){ + if(success){ + resolve(errorCode); + return; + } + + reject(@"init_module_error", [NSString + stringWithFormat:@"%ld", (long)[errorCode longValue]], nil); + return; + }]; +} + +- (void)forward:(NSString *)input + resolve:(RCTPromiseResolveBlock)resolve + reject:(RCTPromiseRejectBlock)reject { + @try { + cv::Mat image = [ImageProcessor readImage:input]; + NSDictionary *result = [model runModel:image]; + + resolve(result); + return; + } @catch (NSException *exception) { + NSLog(@"An exception occurred: %@, %@", exception.name, exception.reason); + reject(@"forward_error", [NSString stringWithFormat:@"%@", exception.reason], + nil); + return; + } +} + + +- (std::shared_ptr)getTurboModule: +(const facebook::react::ObjCTurboModule::InitParams &)params { + return std::make_shared(params); +} + +@end diff --git a/ios/RnExecutorch/models/StyleTransferModel.h b/ios/RnExecutorch/models/StyleTransferModel.h index e8ad4b0..1fd91d7 100644 --- a/ios/RnExecutorch/models/StyleTransferModel.h +++ b/ios/RnExecutorch/models/StyleTransferModel.h @@ -1,6 +1,5 @@ -#import #import "BaseModel.h" -#import +#import "opencv2/opencv.hpp" @interface StyleTransferModel : BaseModel diff --git a/ios/RnExecutorch/models/classification/ClassificationModel.h b/ios/RnExecutorch/models/classification/ClassificationModel.h new file mode 100644 index 0000000..77c034b --- /dev/null +++ b/ios/RnExecutorch/models/classification/ClassificationModel.h @@ -0,0 +1,10 @@ +#import "BaseModel.h" +#import "opencv2/opencv.hpp" + +@interface ClassificationModel : BaseModel + +- (NSArray *)preprocess:(cv::Mat &)input; +- (NSDictionary *)runModel:(cv::Mat &)input; +- (NSDictionary *)postprocess:(NSArray *)output; + +@end diff --git a/ios/RnExecutorch/models/classification/ClassificationModel.mm b/ios/RnExecutorch/models/classification/ClassificationModel.mm new file mode 100644 index 0000000..f1b3f94 --- /dev/null +++ b/ios/RnExecutorch/models/classification/ClassificationModel.mm @@ -0,0 +1,54 @@ +#import "ClassificationModel.h" +#import "opencv2/opencv.hpp" +#import "Utils.h" +#import "Constants.h" +#import "../../utils/ImageProcessor.h" + +@implementation ClassificationModel + +- (cv::Size)getModelImageSize { + NSArray * inputShape = [module getInputShape: 0]; + NSNumber *widthNumber = inputShape.lastObject; + NSNumber *heightNumber = inputShape[inputShape.count - 2]; + + int height = [heightNumber intValue]; + int width = [widthNumber intValue]; + + return cv::Size(height, width); +} + +- (NSArray *)preprocess:(cv::Mat &)input { + cv::Size modelImageSize = [self getModelImageSize]; + cv::resize(input, input, modelImageSize); + + NSArray *modelInput = [ImageProcessor matToNSArray: input]; + return modelInput; +} + +- (NSDictionary *)postprocess:(NSArray *)output { + output = output[0]; // take the first output tensor + std::vector outputVector(output.count); + + for (NSUInteger i = 0; i < output.count; ++i) { + outputVector[i] = [output[i] doubleValue]; + } + + std::vector probabilities = softmax(outputVector); + NSMutableDictionary *result = [NSMutableDictionary dictionary]; + + for (int i = 0; i < probabilities.size(); ++i) { + NSString *className = @(imagenet1k_v1_labels[i].c_str()); + NSNumber *probability = @(probabilities[i]); + result[className] = probability; + } + + return result; +} + +- (NSDictionary *)runModel:(cv::Mat &)input { + NSArray *modelInput = [self preprocess:input]; + NSArray *modelOutput = [self forward:modelInput]; + return [self postprocess:modelOutput]; +} + +@end diff --git a/ios/RnExecutorch/models/classification/Constants.h b/ios/RnExecutorch/models/classification/Constants.h new file mode 100644 index 0000000..da27401 --- /dev/null +++ b/ios/RnExecutorch/models/classification/Constants.h @@ -0,0 +1,3 @@ +#import + +extern const std::string imagenet1k_v1_labels[]; diff --git a/ios/RnExecutorch/models/classification/Constants.mm b/ios/RnExecutorch/models/classification/Constants.mm new file mode 100644 index 0000000..c83b4d4 --- /dev/null +++ b/ios/RnExecutorch/models/classification/Constants.mm @@ -0,0 +1,1005 @@ +#import "Constants.h" +#import + +const std::string imagenet1k_v1_labels[] = { + "tench, Tinca tinca", + "goldfish, Carassius auratus", + "great white shark, white shark, man-eater, man-eating shark, Carcharodon carcharias", + "tiger shark, Galeocerdo cuvieri", + "hammerhead, hammerhead shark", + "electric ray, crampfish, numbfish, torpedo", + "stingray", + "cock", + "hen", + "ostrich, Struthio camelus", + "brambling, Fringilla montifringilla", + "goldfinch, Carduelis carduelis", + "house finch, linnet, Carpodacus mexicanus", + "junco, snowbird", + "indigo bunting, indigo finch, indigo bird, Passerina cyanea", + "robin, American robin, Turdus migratorius", + "bulbul", + "jay", + "magpie", + "chickadee", + "water ouzel, dipper", + "kite", + "bald eagle, American eagle, Haliaeetus leucocephalus", + "vulture", + "great grey owl, great gray owl, Strix nebulosa", + "European fire salamander, Salamandra salamandra", + "common newt, Triturus vulgaris", + "eft", + "spotted salamander, Ambystoma maculatum", + "axolotl, mud puppy, Ambystoma mexicanum", + "bullfrog, Rana catesbeiana", + "tree frog, tree-frog", + "tailed frog, bell toad, ribbed toad, tailed toad, Ascaphus trui", + "loggerhead, loggerhead turtle, Caretta caretta", + "leatherback turtle, leatherback, leathery turtle, Dermochelys coriacea", + "mud turtle", + "terrapin", + "box turtle, box tortoise", + "banded gecko", + "common iguana, iguana, Iguana iguana", + "American chameleon, anole, Anolis carolinensis", + "whiptail, whiptail lizard", + "agama", + "frilled lizard, Chlamydosaurus kingi", + "alligator lizard", + "Gila monster, Heloderma suspectum", + "green lizard, Lacerta viridis", + "African chameleon, Chamaeleo chamaeleon", + "Komodo dragon, Komodo lizard, dragon lizard, giant lizard, Varanus komodoensis", + "African crocodile, Nile crocodile, Crocodylus niloticus", + "American alligator, Alligator mississipiensis", + "triceratops", + "thunder snake, worm snake, Carphophis amoenus", + "ringneck snake, ring-necked snake, ring snake", + "hognose snake, puff adder, sand viper", + "green snake, grass snake", + "king snake, kingsnake", + "garter snake, grass snake", + "water snake", + "vine snake", + "night snake, Hypsiglena torquata", + "boa constrictor, Constrictor constrictor", + "rock python, rock snake, Python sebae", + "Indian cobra, Naja naja", + "green mamba", + "sea snake", + "horned viper, cerastes, sand viper, horned asp, Cerastes cornutus", + "diamondback, diamondback rattlesnake, Crotalus adamanteus", + "sidewinder, horned rattlesnake, Crotalus cerastes", + "trilobite", + "harvestman, daddy longlegs, Phalangium opilio", + "scorpion", + "black and gold garden spider, Argiope aurantia", + "barn spider, Araneus cavaticus", + "garden spider, Aranea diademata", + "black widow, Latrodectus mactans", + "tarantula", + "wolf spider, hunting spider", + "tick", + "centipede", + "black grouse", + "ptarmigan", + "ruffed grouse, partridge, Bonasa umbellus", + "prairie chicken, prairie grouse, prairie fowl", + "peacock", + "quail", + "partridge", + "African grey, African gray, Psittacus erithacus", + "macaw", + "sulphur-crested cockatoo, Kakatoe galerita, Cacatua galerita", + "lorikeet", + "coucal", + "bee eater", + "hornbill", + "hummingbird", + "jacamar", + "toucan", + "drake", + "red-breasted merganser, Mergus serrator", + "goose", + "black swan, Cygnus atratus", + "tusker", + "echidna, spiny anteater, anteater", + "platypus, duckbill, duckbilled platypus, duck-billed platypus, Ornithorhynchus anatinus", + "wallaby, brush kangaroo", + "koala, koala bear, kangaroo bear, native bear, Phascolarctos cinereus", + "wombat", + "jellyfish", + "sea anemone, anemone", + "brain coral", + "flatworm, platyhelminth", + "nematode, nematode worm, roundworm", + "conch", + "snail", + "slug", + "sea slug, nudibranch", + "chiton, coat-of-mail shell, sea cradle, polyplacophore", + "chambered nautilus, pearly nautilus, nautilus", + "Dungeness crab, Cancer magister", + "rock crab, Cancer irroratus", + "fiddler crab", + "king crab, Alaska crab, Alaskan king crab, Alaska king crab, Paralithodes camtschatica", + "American lobster, Northern lobster, Maine lobster, Homarus americanus", + "spiny lobster, langouste, rock lobster, crawfish, crayfish, sea crawfish", + "crayfish, crawfish, crawdad, crawdaddy", + "hermit crab", + "isopod", + "white stork, Ciconia ciconia", + "black stork, Ciconia nigra", + "spoonbill", + "flamingo", + "little blue heron, Egretta caerulea", + "American egret, great white heron, Egretta albus", + "bittern", + "crane", + "limpkin, Aramus pictus", + "European gallinule, Porphyrio porphyrio", + "American coot, marsh hen, mud hen, water hen, Fulica americana", + "bustard", + "ruddy turnstone, Arenaria interpres", + "red-backed sandpiper, dunlin, Erolia alpina", + "redshank, Tringa totanus", + "dowitcher", + "oystercatcher, oyster catcher", + "pelican", + "king penguin, Aptenodytes patagonica", + "albatross, mollymawk", + "grey whale, gray whale, devilfish, Eschrichtius gibbosus, Eschrichtius robustus", + "killer whale, killer, orca, grampus, sea wolf, Orcinus orca", + "dugong, Dugong dugon", + "sea lion", + "Chihuahua", + "Japanese spaniel", + "Maltese dog, Maltese terrier, Maltese", + "Pekinese, Pekingese, Peke", + "Shih-Tzu", + "Blenheim spaniel", + "papillon", + "toy terrier", + "Rhodesian ridgeback", + "Afghan hound, Afghan", + "basset, basset hound", + "beagle", + "bloodhound, sleuthhound", + "bluetick", + "black-and-tan coonhound", + "Walker hound, Walker foxhound", + "English foxhound", + "redbone", + "borzoi, Russian wolfhound", + "Irish wolfhound", + "Italian greyhound", + "whippet", + "Ibizan hound, Ibizan Podenco", + "Norwegian elkhound, elkhound", + "otterhound, otter hound", + "Saluki, gazelle hound", + "Scottish deerhound, deerhound", + "Weimaraner", + "Staffordshire bullterrier, Staffordshire bull terrier", + "American Staffordshire terrier, Staffordshire terrier, American pit bull terrier, pit bull terrier", + "Bedlington terrier", + "Border terrier", + "Kerry blue terrier", + "Irish terrier", + "Norfolk terrier", + "Norwich terrier", + "Yorkshire terrier", + "wire-haired fox terrier", + "Lakeland terrier", + "Sealyham terrier, Sealyham", + "Airedale, Airedale terrier", + "cairn, cairn terrier", + "Australian terrier", + "Dandie Dinmont, Dandie Dinmont terrier", + "Boston bull, Boston terrier", + "miniature schnauzer", + "giant schnauzer", + "standard schnauzer", + "Scotch terrier, Scottish terrier, Scottie", + "Tibetan terrier, chrysanthemum dog", + "silky terrier, Sydney silky", + "soft-coated wheaten terrier", + "West Highland white terrier", + "Lhasa, Lhasa apso", + "flat-coated retriever", + "curly-coated retriever", + "golden retriever", + "Labrador retriever", + "Chesapeake Bay retriever", + "German short-haired pointer", + "vizsla, Hungarian pointer", + "English setter", + "Irish setter, red setter", + "Gordon setter", + "Brittany spaniel", + "clumber, clumber spaniel", + "English springer, English springer spaniel", + "Welsh springer spaniel", + "cocker spaniel, English cocker spaniel, cocker", + "Sussex spaniel", + "Irish water spaniel", + "kuvasz", + "schipperke", + "groenendael", + "malinois", + "briard", + "kelpie", + "komondor", + "Old English sheepdog, bobtail", + "Shetland sheepdog, Shetland sheep dog, Shetland", + "collie", + "Border collie", + "Bouvier des Flandres, Bouviers des Flandres", + "Rottweiler", + "German shepherd, German shepherd dog, German police dog, alsatian", + "Doberman, Doberman pinscher", + "miniature pinscher", + "Greater Swiss Mountain dog", + "Bernese mountain dog", + "Appenzeller", + "EntleBucher", + "boxer", + "bull mastiff", + "Tibetan mastiff", + "French bulldog", + "Great Dane", + "Saint Bernard, St Bernard", + "Eskimo dog, husky", + "malamute, malemute, Alaskan malamute", + "Siberian husky", + "dalmatian, coach dog, carriage dog", + "affenpinscher, monkey pinscher, monkey dog", + "basenji", + "pug, pug-dog", + "Leonberg", + "Newfoundland, Newfoundland dog", + "Great Pyrenees", + "Samoyed, Samoyede", + "Pomeranian", + "chow, chow chow", + "keeshond", + "Brabancon griffon", + "Pembroke, Pembroke Welsh corgi", + "Cardigan, Cardigan Welsh corgi", + "toy poodle", + "miniature poodle", + "standard poodle", + "Mexican hairless", + "timber wolf, grey wolf, gray wolf, Canis lupus", + "white wolf, Arctic wolf, Canis lupus tundrarum", + "red wolf, maned wolf, Canis rufus, Canis niger", + "coyote, prairie wolf, brush wolf, Canis latrans", + "dingo, warrigal, warragal, Canis dingo", + "dhole, Cuon alpinus", + "African hunting dog, hyena dog, Cape hunting dog, Lycaon pictus", + "hyena, hyaena", + "red fox, Vulpes vulpes", + "kit fox, Vulpes macrotis", + "Arctic fox, white fox, Alopex lagopus", + "grey fox, gray fox, Urocyon cinereoargenteus", + "tabby, tabby cat", + "tiger cat", + "Persian cat", + "Siamese cat, Siamese", + "Egyptian cat", + "cougar, puma, catamount, mountain lion, painter, panther, Felis concolor", + "lynx, catamount", + "leopard, Panthera pardus", + "snow leopard, ounce, Panthera uncia", + "jaguar, panther, Panthera onca, Felis onca", + "lion, king of beasts, Panthera leo", + "tiger, Panthera tigris", + "cheetah, chetah, Acinonyx jubatus", + "brown bear, bruin, Ursus arctos", + "American black bear, black bear, Ursus americanus, Euarctos americanus", + "ice bear, polar bear, Ursus Maritimus, Thalarctos maritimus", + "sloth bear, Melursus ursinus, Ursus ursinus", + "mongoose", + "meerkat, mierkat", + "tiger beetle", + "ladybug, ladybeetle, lady beetle, ladybird, ladybird beetle", + "ground beetle, carabid beetle", + "long-horned beetle, longicorn, longicorn beetle", + "leaf beetle, chrysomelid", + "dung beetle", + "rhinoceros beetle", + "weevil", + "fly", + "bee", + "ant, emmet, pismire", + "grasshopper, hopper", + "cricket", + "walking stick, walkingstick, stick insect", + "cockroach, roach", + "mantis, mantid", + "cicada, cicala", + "leafhopper", + "lacewing, lacewing fly", + "dragonfly, darning needle, devil's darning needle, sewing needle, snake feeder, snake doctor, mosquito hawk, skeeter hawk", + "damselfly", + "admiral", + "ringlet, ringlet butterfly", + "monarch, monarch butterfly, milkweed butterfly, Danaus plexippus", + "cabbage butterfly", + "sulphur butterfly, sulfur butterfly", + "lycaenid, lycaenid butterfly", + "starfish, sea star", + "sea urchin", + "sea cucumber, holothurian", + "wood rabbit, cottontail, cottontail rabbit", + "hare", + "Angora, Angora rabbit", + "hamster", + "porcupine, hedgehog", + "fox squirrel, eastern fox squirrel, Sciurus niger", + "marmot", + "beaver", + "guinea pig, Cavia cobaya", + "sorrel", + "zebra", + "hog, pig, grunter, squealer, Sus scrofa", + "wild boar, boar, Sus scrofa", + "warthog", + "hippopotamus, hippo, river horse, Hippopotamus amphibius", + "ox", + "water buffalo, water ox, Asiatic buffalo, Bubalus bubalis", + "bison", + "ram, tup", + "bighorn, bighorn sheep, cimarron, Rocky Mountain bighorn, Rocky Mountain sheep, Ovis canadensis", + "ibex, Capra ibex", + "hartebeest", + "impala, Aepyceros melampus", + "gazelle", + "Arabian camel, dromedary, Camelus dromedarius", + "llama", + "weasel", + "mink", + "polecat, fitch, foulmart, foumart, Mustela putorius", + "black-footed ferret, ferret, Mustela nigripes", + "otter", + "skunk, polecat, wood pussy", + "badger", + "armadillo", + "three-toed sloth, ai, Bradypus tridactylus", + "orangutan, orang, orangutang, Pongo pygmaeus", + "gorilla, Gorilla gorilla", + "chimpanzee, chimp, Pan troglodytes", + "gibbon, Hylobates lar", + "siamang, Hylobates syndactylus, Symphalangus syndactylus", + "guenon, guenon monkey", + "patas, hussar monkey, Erythrocebus patas", + "baboon", + "macaque", + "langur", + "colobus, colobus monkey", + "proboscis monkey, Nasalis larvatus", + "marmoset", + "capuchin, ringtail, Cebus capucinus", + "howler monkey, howler", + "titi, titi monkey", + "spider monkey, Ateles geoffroyi", + "squirrel monkey, Saimiri sciureus", + "Madagascar cat, ring-tailed lemur, Lemur catta", + "indri, indris, Indri indri, Indri brevicaudatus", + "Indian elephant, Elephas maximus", + "African elephant, Loxodonta africana", + "lesser panda, red panda, panda, bear cat, cat bear, Ailurus fulgens", + "giant panda, panda, panda bear, coon bear, Ailuropoda melanoleuca", + "barracouta, snoek", + "eel", + "coho, cohoe, coho salmon, blue jack, silver salmon, Oncorhynchus kisutch", + "rock beauty, Holocanthus tricolor", + "anemone fish", + "sturgeon", + "gar, garfish, garpike, billfish, Lepisosteus osseus", + "lionfish", + "puffer, pufferfish, blowfish, globefish", + "abacus", + "abaya", + "academic gown, academic robe, judge's robe", + "accordion, piano accordion, squeeze box", + "acoustic guitar", + "aircraft carrier, carrier, flattop, attack aircraft carrier", + "airliner", + "airship, dirigible", + "altar", + "ambulance", + "amphibian, amphibious vehicle", + "analog clock", + "apiary, bee house", + "apron", + "ashcan, trash can, garbage can, wastebin, ash bin, ash-bin, ashbin, dustbin, trash barrel, trash bin", + "assault rifle, assault gun", + "backpack, back pack, knapsack, packsack, rucksack, haversack", + "bakery, bakeshop, bakehouse", + "balance beam, beam", + "balloon", + "ballpoint, ballpoint pen, ballpen, Biro", + "Band Aid", + "banjo", + "bannister, banister, balustrade, balusters, handrail", + "barbell", + "barber chair", + "barbershop", + "barn", + "barometer", + "barrel, cask", + "barrow, garden cart, lawn cart, wheelbarrow", + "baseball", + "basketball", + "bassinet", + "bassoon", + "bathing cap, swimming cap", + "bath towel", + "bathtub, bathing tub, bath, tub", + "beach wagon, station wagon, wagon, estate car, beach waggon, station waggon, waggon", + "beacon, lighthouse, beacon light, pharos", + "beaker", + "bearskin, busby, shako", + "beer bottle", + "beer glass", + "bell cote, bell cot", + "bib", + "bicycle-built-for-two, tandem bicycle, tandem", + "bikini, two-piece", + "binder, ring-binder", + "binoculars, field glasses, opera glasses", + "birdhouse", + "boathouse", + "bobsled, bobsleigh, bob", + "bolo tie, bolo, bola tie, bola", + "bonnet, poke bonnet", + "bookcase", + "bookshop, bookstore, bookstall", + "bottlecap", + "bow", + "bow tie, bow-tie, bowtie", + "brass, memorial tablet, plaque", + "brassiere, bra, bandeau", + "breakwater, groin, groyne, mole, bulwark, seawall, jetty", + "breastplate, aegis, egis", + "broom", + "bucket, pail", + "buckle", + "bulletproof vest", + "bullet train, bullet", + "butcher shop, meat market", + "cab, hack, taxi, taxicab", + "caldron, cauldron", + "candle, taper, wax light", + "cannon", + "canoe", + "can opener, tin opener", + "cardigan", + "car mirror", + "carousel, carrousel, merry-go-round, roundabout, whirligig", + "carpenter's kit, tool kit", + "carton", + "car wheel", + "cash machine, cash dispenser, automated teller machine, automatic teller machine, automated teller, automatic teller, ATM", + "cassette", + "cassette player", + "castle", + "catamaran", + "CD player", + "cello, violoncello", + "cellular telephone, cellular phone, cellphone, cell, mobile phone", + "chain", + "chainlink fence", + "chain mail, ring mail, mail, chain armor, chain armour, ring armor, ring armour", + "chain saw, chainsaw", + "chest", + "chiffonier, commode", + "chime, bell, gong", + "china cabinet, china closet", + "Christmas stocking", + "church, church building", + "cinema, movie theater, movie theatre, movie house, picture palace", + "cleaver, meat cleaver, chopper", + "cliff dwelling", + "cloak", + "clog, geta, patten, sabot", + "cocktail shaker", + "coffee mug", + "coffeepot", + "coil, spiral, volute, whorl, helix", + "combination lock", + "computer keyboard, keypad", + "confectionery, confectionary, candy store", + "container ship, containership, container vessel", + "convertible", + "corkscrew, bottle screw", + "cornet, horn, trumpet, trump", + "cowboy boot", + "cowboy hat, ten-gallon hat", + "cradle", + "crane", + "crash helmet", + "crate", + "crib, cot", + "Crock Pot", + "croquet ball", + "crutch", + "cuirass", + "dam, dike, dyke", + "desk", + "desktop computer", + "dial telephone, dial phone", + "diaper, nappy, napkin", + "digital clock", + "digital watch", + "dining table, board", + "dishrag, dishcloth", + "dishwasher, dish washer, dishwashing machine", + "disk brake, disc brake", + "dock, dockage, docking facility", + "dogsled, dog sled, dog sleigh", + "dome", + "doormat, welcome mat", + "drilling platform, offshore rig", + "drum, membranophone, tympan", + "drumstick", + "dumbbell", + "Dutch oven", + "electric fan, blower", + "electric guitar", + "electric locomotive", + "entertainment center", + "envelope", + "espresso maker", + "face powder", + "feather boa, boa", + "file, file cabinet, filing cabinet", + "fireboat", + "fire engine, fire truck", + "fire screen, fireguard", + "flagpole, flagstaff", + "flute, transverse flute", + "folding chair", + "football helmet", + "forklift", + "fountain", + "fountain pen", + "four-poster", + "freight car", + "French horn, horn", + "frying pan, frypan, skillet", + "fur coat", + "garbage truck, dustcart", + "gasmask, respirator, gas helmet", + "gas pump, gasoline pump, petrol pump, island dispenser", + "goblet", + "go-kart", + "golf ball", + "golfcart, golf cart", + "gondola", + "gong, tam-tam", + "gown", + "grand piano, grand", + "greenhouse, nursery, glasshouse", + "grille, radiator grille", + "grocery store, grocery, food market, market", + "guillotine", + "hair slide", + "hair spray", + "half track", + "hammer", + "hamper", + "hand blower, blow dryer, blow drier, hair dryer, hair drier", + "hand-held computer, hand-held microcomputer", + "handkerchief, hankie, hanky, hankey", + "hard disc, hard disk, fixed disk", + "harmonica, mouth organ, harp, mouth harp", + "harp", + "harvester, reaper", + "hatchet", + "holster", + "home theater, home theatre", + "honeycomb", + "hook, claw", + "hoopskirt, crinoline", + "horizontal bar, high bar", + "horse cart, horse-cart", + "hourglass", + "iPod", + "iron, smoothing iron", + "jack-o'-lantern", + "jean, blue jean, denim", + "jeep, landrover", + "jersey, T-shirt, tee shirt", + "jigsaw puzzle", + "jinrikisha, ricksha, rickshaw", + "joystick", + "kimono", + "knee pad", + "knot", + "lab coat, laboratory coat", + "ladle", + "lampshade, lamp shade", + "laptop, laptop computer", + "lawn mower, mower", + "lens cap, lens cover", + "letter opener, paper knife, paperknife", + "library", + "lifeboat", + "lighter, light, igniter, ignitor", + "limousine, limo", + "liner, ocean liner", + "lipstick, lip rouge", + "Loafer", + "lotion", + "loudspeaker, speaker, speaker unit, loudspeaker system, speaker system", + "loupe, jeweler's loupe", + "lumbermill, sawmill", + "magnetic compass", + "mailbag, postbag", + "mailbox, letter box", + "maillot", + "maillot, tank suit", + "manhole cover", + "maraca", + "marimba, xylophone", + "mask", + "matchstick", + "maypole", + "maze, labyrinth", + "measuring cup", + "medicine chest, medicine cabinet", + "megalith, megalithic structure", + "microphone, mike", + "microwave, microwave oven", + "military uniform", + "milk can", + "minibus", + "miniskirt, mini", + "minivan", + "missile", + "mitten", + "mixing bowl", + "mobile home, manufactured home", + "Model T", + "modem", + "monastery", + "monitor", + "moped", + "mortar", + "mortarboard", + "mosque", + "mosquito net", + "motor scooter, scooter", + "mountain bike, all-terrain bike, off-roader", + "mountain tent", + "mouse, computer mouse", + "mousetrap", + "moving van", + "muzzle", + "nail", + "neck brace", + "necklace", + "nipple", + "notebook, notebook computer", + "obelisk", + "oboe, hautboy, hautbois", + "ocarina, sweet potato", + "odometer, hodometer, mileometer, milometer", + "oil filter", + "organ, pipe organ", + "oscilloscope, scope, cathode-ray oscilloscope, CRO", + "overskirt", + "oxcart", + "oxygen mask", + "packet", + "paddle, boat paddle", + "paddlewheel, paddle wheel", + "padlock", + "paintbrush", + "pajama, pyjama, pj's, jammies", + "palace", + "panpipe, pandean pipe, syrinx", + "paper towel", + "parachute, chute", + "parallel bars, bars", + "park bench", + "parking meter", + "passenger car, coach, carriage", + "patio, terrace", + "pay-phone, pay-station", + "pedestal, plinth, footstall", + "pencil box, pencil case", + "pencil sharpener", + "perfume, essence", + "Petri dish", + "photocopier", + "pick, plectrum, plectron", + "pickelhaube", + "picket fence, paling", + "pickup, pickup truck", + "pier", + "piggy bank, penny bank", + "pill bottle", + "pillow", + "ping-pong ball", + "pinwheel", + "pirate, pirate ship", + "pitcher, ewer", + "plane, carpenter's plane, woodworking plane", + "planetarium", + "plastic bag", + "plate rack", + "plow, plough", + "plunger, plumber's helper", + "Polaroid camera, Polaroid Land camera", + "pole", + "police van, police wagon, paddy wagon, patrol wagon, wagon, black Maria", + "poncho", + "pool table, billiard table, snooker table", + "pop bottle, soda bottle", + "pot, flowerpot", + "potter's wheel", + "power drill", + "prayer rug, prayer mat", + "printer", + "prison, prison house", + "projectile, missile", + "projector", + "puck, hockey puck", + "punching bag, punch bag, punching ball, punchball", + "purse", + "quill, quill pen", + "quilt, comforter, comfort, puff", + "racer, race car, racing car", + "racket, racquet", + "radiator", + "radio, wireless", + "radio telescope, radio reflector", + "rain barrel", + "recreational vehicle, RV, R.V.", + "reel", + "reflex camera", + "refrigerator, icebox", + "remote control, remote", + "restaurant, eating house, eating place, eatery", + "revolver, six-gun, six-shooter", + "rifle", + "rocking chair, rocker", + "rotisserie", + "rubber eraser, rubber, pencil eraser", + "rugby ball", + "rule, ruler", + "running shoe", + "safe", + "safety pin", + "saltshaker, salt shaker", + "sandal", + "sarong", + "sax, saxophone", + "scabbard", + "scale, weighing machine", + "school bus", + "schooner", + "scoreboard", + "screen, CRT screen", + "screw", + "screwdriver", + "seat belt, seatbelt", + "sewing machine", + "shield, buckler", + "shoe shop, shoe-shop, shoe store", + "shoji", + "shopping basket", + "shopping cart", + "shovel", + "shower cap", + "shower curtain", + "ski", + "ski mask", + "sleeping bag", + "slide rule, slipstick", + "sliding door", + "slot, one-armed bandit", + "snorkel", + "snowmobile", + "snowplow, snowplough", + "soap dispenser", + "soccer ball", + "sock", + "solar dish, solar collector, solar furnace", + "sombrero", + "soup bowl", + "space bar", + "space heater", + "space shuttle", + "spatula", + "speedboat", + "spider web, spider's web", + "spindle", + "sports car, sport car", + "spotlight, spot", + "stage", + "steam locomotive", + "steel arch bridge", + "steel drum", + "stethoscope", + "stole", + "stone wall", + "stopwatch, stop watch", + "stove", + "strainer", + "streetcar, tram, tramcar, trolley, trolley car", + "stretcher", + "studio couch, day bed", + "stupa, tope", + "submarine, pigboat, sub, U-boat", + "suit, suit of clothes", + "sundial", + "sunglass", + "sunglasses, dark glasses, shades", + "sunscreen, sunblock, sun blocker", + "suspension bridge", + "swab, swob, mop", + "sweatshirt", + "swimming trunks, bathing trunks", + "swing", + "switch, electric switch, electrical switch", + "syringe", + "table lamp", + "tank, army tank, armored combat vehicle, armoured combat vehicle", + "tape player", + "teapot", + "teddy, teddy bear", + "television, television system", + "tennis ball", + "thatch, thatched roof", + "theater curtain, theatre curtain", + "thimble", + "thresher, thrasher, threshing machine", + "throne", + "tile roof", + "toaster", + "tobacco shop, tobacconist shop, tobacconist", + "toilet seat", + "torch", + "totem pole", + "tow truck, tow car, wrecker", + "toyshop", + "tractor", + "trailer truck, tractor trailer, trucking rig, rig, articulated lorry, semi", + "tray", + "trench coat", + "tricycle, trike, velocipede", + "trimaran", + "tripod", + "triumphal arch", + "trolleybus, trolley coach, trackless trolley", + "trombone", + "tub, vat", + "turnstile", + "typewriter keyboard", + "umbrella", + "unicycle, monocycle", + "upright, upright piano", + "vacuum, vacuum cleaner", + "vase", + "vault", + "velvet", + "vending machine", + "vestment", + "viaduct", + "violin, fiddle", + "volleyball", + "waffle iron", + "wall clock", + "wallet, billfold, notecase, pocketbook", + "wardrobe, closet, press", + "warplane, military plane", + "washbasin, handbasin, washbowl, lavabo, wash-hand basin", + "washer, automatic washer, washing machine", + "water bottle", + "water jug", + "water tower", + "whiskey jug", + "whistle", + "wig", + "window screen", + "window shade", + "Windsor tie", + "wine bottle", + "wing", + "wok", + "wooden spoon", + "wool, woolen, woollen", + "worm fence, snake fence, snake-rail fence, Virginia fence", + "wreck", + "yawl", + "yurt", + "web site, website, internet site, site", + "comic book", + "crossword puzzle, crossword", + "street sign", + "traffic light, traffic signal, stoplight", + "book jacket, dust cover, dust jacket, dust wrapper", + "menu", + "plate", + "guacamole", + "consomme", + "hot pot, hotpot", + "trifle", + "ice cream, icecream", + "ice lolly, lolly, lollipop, popsicle", + "French loaf", + "bagel, beigel", + "pretzel", + "cheeseburger", + "hotdog, hot dog, red hot", + "mashed potato", + "head cabbage", + "broccoli", + "cauliflower", + "zucchini, courgette", + "spaghetti squash", + "acorn squash", + "butternut squash", + "cucumber, cuke", + "artichoke, globe artichoke", + "bell pepper", + "cardoon", + "mushroom", + "Granny Smith", + "strawberry", + "orange", + "lemon", + "fig", + "pineapple, ananas", + "banana", + "jackfruit, jak, jack", + "custard apple", + "pomegranate", + "hay", + "carbonara", + "chocolate sauce, chocolate syrup", + "dough", + "meat loaf, meatloaf", + "pizza, pizza pie", + "potpie", + "burrito", + "red wine", + "espresso", + "cup", + "eggnog", + "alp", + "bubble", + "cliff, drop, drop-off", + "coral reef", + "geyser", + "lakeside, lakeshore", + "promontory, headland, head, foreland", + "sandbar, sand bar", + "seashore, coast, seacoast, sea-coast", + "valley, vale", + "volcano", + "ballplayer, baseball player", + "groom, bridegroom", + "scuba diver", + "rapeseed", + "daisy", + "yellow lady's slipper, yellow lady-slipper, Cypripedium calceolus, Cypripedium parviflorum", + "corn", + "acorn", + "hip, rose hip, rosehip", + "buckeye, horse chestnut, conker", + "coral fungus", + "agaric", + "gyromitra", + "stinkhorn, carrion fungus", + "earthstar", + "hen-of-the-woods, hen of the woods, Polyporus frondosus, Grifola frondosa", + "bolete", + "ear, spike, capitulum", + "toilet tissue, toilet paper, bathroom tissue" +}; diff --git a/ios/RnExecutorch/models/classification/Utils.h b/ios/RnExecutorch/models/classification/Utils.h new file mode 100644 index 0000000..5785a5c --- /dev/null +++ b/ios/RnExecutorch/models/classification/Utils.h @@ -0,0 +1,3 @@ +#include + +std::vector softmax(const std::vector& v); \ No newline at end of file diff --git a/ios/RnExecutorch/models/classification/Utils.mm b/ios/RnExecutorch/models/classification/Utils.mm new file mode 100644 index 0000000..da61301 --- /dev/null +++ b/ios/RnExecutorch/models/classification/Utils.mm @@ -0,0 +1,20 @@ +#include "Utils.h" +#include +#include + +std::vector softmax(const std::vector& v) { + std::vector result(v.size()); + double maxVal = *std::max_element(v.begin(), v.end()); + + double sumExp = 0.0; + for (size_t i = 0; i < v.size(); ++i) { + result[i] = std::exp(v[i] - maxVal); + sumExp += result[i]; + } + + for (size_t i = 0; i < v.size(); ++i) { + result[i] /= sumExp; + } + + return result; +} \ No newline at end of file diff --git a/src/index.tsx b/src/index.tsx index fa96d91..a8d0404 100644 --- a/src/index.tsx +++ b/src/index.tsx @@ -1,6 +1,7 @@ export * from './ETModule'; export * from './LLM'; export * from './StyleTransfer'; +export * from './models/Classification'; export * from './constants/modelUrls'; export * from './models/object_detection/ObjectDetection'; export * from './models/object_detection/types'; diff --git a/src/models/Classification.ts b/src/models/Classification.ts new file mode 100644 index 0000000..bb2fd1e --- /dev/null +++ b/src/models/Classification.ts @@ -0,0 +1,70 @@ +import { useEffect, useState } from 'react'; +import { Image } from 'react-native'; +import { Classification } from '../native/RnExecutorchModules'; +import { ETError, getError } from '../Error'; + +interface Props { + modulePath: string | number; +} + +interface ClassificationModule { + error: string | null; + isModelReady: boolean; + isModelGenerating: boolean; + forward: (input: string) => Promise<{ [category: string]: number }>; +} + +export const useClassification = ({ + modulePath, +}: Props): ClassificationModule => { + const [error, setError] = useState(null); + const [isModelReady, setIsModelReady] = useState(false); + const [isModelGenerating, setIsModelGenerating] = useState(false); + + useEffect(() => { + const loadModel = async () => { + let path = modulePath; + + if (typeof modulePath === 'number') { + path = Image.resolveAssetSource(modulePath).uri; + } + + try { + setIsModelReady(false); + await Classification.loadModule(path); + } catch (e) { + setError(getError(e)); + } finally { + setIsModelReady(true); + } + }; + + loadModel(); + }, [modulePath]); + + const forward = async (input: string) => { + if (!isModelReady) { + throw new Error(getError(ETError.ModuleNotLoaded)); + } + + if (error) { + throw new Error(error); + } + + if (isModelGenerating) { + throw new Error(getError(ETError.ModelGenerating)); + } + + try { + setIsModelGenerating(true); + const output = await Classification.forward(input); + setIsModelGenerating(false); + return output; + } catch (e) { + setIsModelGenerating(false); + throw new Error(getError(e)); + } + }; + + return { error, isModelReady, isModelGenerating, forward }; +}; diff --git a/src/native/NativeClassification.ts b/src/native/NativeClassification.ts new file mode 100644 index 0000000..9aad6de --- /dev/null +++ b/src/native/NativeClassification.ts @@ -0,0 +1,10 @@ +import type { TurboModule } from 'react-native'; +import { TurboModuleRegistry } from 'react-native'; + +export interface Spec extends TurboModule { + loadModule(modelSource: string): Promise; + + forward(input: string): Promise<{ [category: string]: number }>; +} + +export default TurboModuleRegistry.get('Classification'); diff --git a/src/native/RnExecutorchModules.ts b/src/native/RnExecutorchModules.ts index c535f8f..72170e3 100644 --- a/src/native/RnExecutorchModules.ts +++ b/src/native/RnExecutorchModules.ts @@ -7,6 +7,7 @@ const LINKING_ERROR = '- You are not using Expo Go\n'; const LLMSpec = require('./NativeLLM').default; + const LLM = LLMSpec ? LLMSpec : new Proxy( @@ -31,10 +32,10 @@ const ETModule = ETModuleSpec } ); -const StyleTransferSpec = require('./NativeStyleTransfer').default; +const ClassificationSpec = require('./NativeClassification').default; -const StyleTransfer = StyleTransferSpec - ? StyleTransferSpec +const Classification = ClassificationSpec + ? ClassificationSpec : new Proxy( {}, { @@ -57,4 +58,17 @@ const ObjectDetection = ObjectDetectionSpec } ); -export { LLM, ETModule, StyleTransfer, ObjectDetection }; +const StyleTransferSpec = require('./NativeStyleTransfer').default; + +const StyleTransfer = StyleTransferSpec + ? StyleTransferSpec + : new Proxy( + {}, + { + get() { + throw new Error(LINKING_ERROR); + }, + } + ); + +export { LLM, ETModule, Classification, ObjectDetection, StyleTransfer };