From 0d7375745309f8b17e646f6daff01f71c187b066 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 11 Nov 2024 17:53:00 +0100 Subject: [PATCH 01/74] Add genai nodejs bindings --- samples/js/.gitignore | 1 + samples/js/app.js | 28 ++++ samples/js/interactive.js | 56 +++++++ samples/js/package-lock.json | 37 +++++ samples/js/package.json | 12 ++ src/CMakeLists.txt | 4 + src/js/.gitignore | 5 + src/js/.npmignore | 15 ++ src/js/CMakeLists.txt | 92 +++++++++++ src/js/README.md | 31 ++++ src/js/include/addon.hpp | 20 +++ src/js/include/helper.hpp | 23 +++ .../llm_pipeline/finish_chat_worker.hpp | 18 +++ src/js/include/llm_pipeline/init_worker.hpp | 21 +++ .../llm_pipeline/llm_pipeline_wrapper.hpp | 27 ++++ .../llm_pipeline/start_chat_worker.hpp | 18 +++ src/js/lib/bindings.cjs | 1 + src/js/lib/module.js | 118 ++++++++++++++ src/js/package-lock.json | 21 +++ src/js/package.json | 24 +++ src/js/src/addon.cpp | 30 ++++ src/js/src/helper.cpp | 53 ++++++ .../src/llm_pipeline/finish_chat_worker.cpp | 14 ++ src/js/src/llm_pipeline/init_worker.cpp | 18 +++ .../src/llm_pipeline/llm_pipeline_wrapper.cpp | 151 ++++++++++++++++++ src/js/src/llm_pipeline/start_chat_worker.cpp | 14 ++ src/js/tests/bindings.test.js | 71 ++++++++ src/js/tests/module.test.js | 140 ++++++++++++++++ src/js/tests/setup.js | 9 ++ src/js/tests/utils.js | 73 +++++++++ 30 files changed, 1145 insertions(+) create mode 100644 samples/js/.gitignore create mode 100644 samples/js/app.js create mode 100644 samples/js/interactive.js create mode 100644 samples/js/package-lock.json create mode 100644 samples/js/package.json create mode 100644 src/js/.gitignore create mode 100644 src/js/.npmignore create mode 100644 src/js/CMakeLists.txt create mode 100644 src/js/README.md create mode 100644 src/js/include/addon.hpp create mode 100644 src/js/include/helper.hpp create mode 100644 src/js/include/llm_pipeline/finish_chat_worker.hpp create mode 100644 src/js/include/llm_pipeline/init_worker.hpp create mode 100644 src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp create mode 100644 src/js/include/llm_pipeline/start_chat_worker.hpp create mode 100644 src/js/lib/bindings.cjs create mode 100644 src/js/lib/module.js create mode 100644 src/js/package-lock.json create mode 100644 src/js/package.json create mode 100644 src/js/src/addon.cpp create mode 100644 src/js/src/helper.cpp create mode 100644 src/js/src/llm_pipeline/finish_chat_worker.cpp create mode 100644 src/js/src/llm_pipeline/init_worker.cpp create mode 100644 src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp create mode 100644 src/js/src/llm_pipeline/start_chat_worker.cpp create mode 100644 src/js/tests/bindings.test.js create mode 100644 src/js/tests/module.test.js create mode 100644 src/js/tests/setup.js create mode 100644 src/js/tests/utils.js diff --git a/samples/js/.gitignore b/samples/js/.gitignore new file mode 100644 index 0000000000..3c3629e647 --- /dev/null +++ b/samples/js/.gitignore @@ -0,0 +1 @@ +node_modules diff --git a/samples/js/app.js b/samples/js/app.js new file mode 100644 index 0000000000..f57e2fad1d --- /dev/null +++ b/samples/js/app.js @@ -0,0 +1,28 @@ +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +const generationCallback = (chunk) => { + process.stdout.write(chunk); +}; + +const prompt = 'Who are you?'; +console.log(`User Prompt: "${prompt}"\n`); + +const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + +await pipeline.startChat(); +const result = await pipeline.generate( + prompt, + generationCallback, + { temperature: 0 }, +); +await pipeline.finishChat(); + +console.log(`\n\nGeneration result:\n"${result}"`) diff --git a/samples/js/interactive.js b/samples/js/interactive.js new file mode 100644 index 0000000000..466326ced0 --- /dev/null +++ b/samples/js/interactive.js @@ -0,0 +1,56 @@ +import path from 'node:path'; +import readline from 'readline'; + +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +main(); + +async function main() { + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` + + 'Type something and press enter. Type "finish" to exit.'); + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + await pipeline.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('> ', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (input === 'finish') { + console.log('Goodbye!'); + await pipeline.finishChat(); + rl.close(); + process.exit(0); + } + + const result = await pipeline.generate(input, generationCallback); + console.log('\n'); + + // Wait for new input + promptUser(); + } + + function generationCallback(chunk) { + process.stdout.write(chunk); + } +} diff --git a/samples/js/package-lock.json b/samples/js/package-lock.json new file mode 100644 index 0000000000..4b2d14913f --- /dev/null +++ b/samples/js/package-lock.json @@ -0,0 +1,37 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "devDependencies": { + "genai-node": "../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + } + }, + "../../src/js": { + "name": "genai-node", + "version": "2024.5.0-preview", + "dev": true, + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + } + }, + "node_modules/genai-node": { + "resolved": "../../src/js", + "link": true + } + } +} diff --git a/samples/js/package.json b/samples/js/package.json new file mode 100644 index 0000000000..0251bdfa14 --- /dev/null +++ b/samples/js/package.json @@ -0,0 +1,12 @@ +{ + "name": "genai-node-demo", + "version": "1.0.0", + "license": "Apache-2.0", + "type": "module", + "devDependencies": { + "genai-node": "../../src/js/" + }, + "engines": { + "node": ">=21.0.0" + } +} diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d9f3cc64db..432d68cbef 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -7,3 +7,7 @@ add_subdirectory(cpp) if(ENABLE_PYTHON) add_subdirectory(python) endif() + +if(ENABLE_JS) + add_subdirectory(js) +endif() diff --git a/src/js/.gitignore b/src/js/.gitignore new file mode 100644 index 0000000000..e7fffbc37a --- /dev/null +++ b/src/js/.gitignore @@ -0,0 +1,5 @@ +.vscode +bin +bin.* +build +thirdparty diff --git a/src/js/.npmignore b/src/js/.npmignore new file mode 100644 index 0000000000..9bf3e571b1 --- /dev/null +++ b/src/js/.npmignore @@ -0,0 +1,15 @@ +.vscode +bin.* +build +include +src +tests + +.eslintrc.js +CMakeLists.txt +tsconfig.json +TODO.md +build.sh + +**/*.tsbuildinfo +*.tgz diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt new file mode 100644 index 0000000000..96ba0019c1 --- /dev/null +++ b/src/js/CMakeLists.txt @@ -0,0 +1,92 @@ +cmake_minimum_required(VERSION 3.18) + +# Set C++ standard +set(CMAKE_CXX_STANDARD 17) + +set(dist_folder "${CMAKE_SOURCE_DIR}/bin/") + +if(WIN32) + set(CMAKE_SHARED_LINKER_FLAGS /DELAYLOAD:NODE.EXE) + set(CMAKE_JS_LIB ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node.lib) + set(CMAKE_JS_SRC ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/win_delay_load_hook.cc) + + set(CMAKE_JS_NODELIB_DEF ${CMAKE_CURRENT_SOURCE_DIR}/thirdparty/node-lib.def) + set(CMAKE_JS_NODELIB_TARGET ${CMAKE_JS_LIB}) + set(DELAYIMP_LIB delayimp.lib) +endif() + +project(genai_node_addon) + +# Add definitions +add_definitions(-DNAPI_VERSION=8) + +include(FetchContent) + +FetchContent_Declare( + node-api-headers + URL https://github.com/nodejs/node-api-headers/archive/refs/tags/v1.1.0.tar.gz + URL_HASH SHA256=70608bc1e6dddce280285f3462f18a106f687c0720a4b90893e1ecd86e5a8bbf +) +FetchContent_MakeAvailable(node-api-headers) + +FetchContent_Declare( + node-addon-api + URL https://github.com/nodejs/node-addon-api/archive/refs/tags/v8.0.0.tar.gz + URL_HASH SHA256=42424c5206b9d67b41af4fcff5d6e3cb22074168035a03b8467852938a281d47 +) +FetchContent_MakeAvailable(node-addon-api) + +# Create a library +add_library(${PROJECT_NAME} SHARED + ${CMAKE_CURRENT_SOURCE_DIR}/src/addon.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/llm_pipeline_wrapper.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/finish_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/start_chat_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/llm_pipeline/init_worker.cpp + ${CMAKE_CURRENT_SOURCE_DIR}/src/helper.cpp + + ${CMAKE_JS_SRC} +) + +# Include directories +target_include_directories(${PROJECT_NAME} PRIVATE + "${node-api-headers_SOURCE_DIR}/include" + "${node-addon-api_SOURCE_DIR}" + "${CMAKE_CURRENT_SOURCE_DIR}" +) + +target_link_libraries(${PROJECT_NAME} PRIVATE openvino::genai ${DELAYIMP_LIB} ${CMAKE_JS_LIB}) + +if(MSVC AND CMAKE_JS_NODELIB_DEF AND CMAKE_JS_NODELIB_TARGET) # Generate node.lib + execute_process(COMMAND ${CMAKE_AR} /def:${CMAKE_JS_NODELIB_DEF} /out:${CMAKE_JS_NODELIB_TARGET} ${CMAKE_STATIC_LINKER_FLAGS}) +endif() + +if(APPLE) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,-undefined,suppress,-flat_namespace) +elseif(AARCH64 OR ARM) + target_link_options(${PROJECT_NAME} PRIVATE -Wl,--unresolved-symbols=ignore-all) +endif() + +# Set library properties +set_target_properties(${PROJECT_NAME} PROPERTIES + PREFIX "" + SUFFIX ".node" +) + +# setting RPATH / LC_RPATH depending on platform +if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") +elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") +endif() + +if(rpaths) + set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + +install(TARGETS ${PROJECT_NAME} + LIBRARY DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL + RUNTIME DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL +) diff --git a/src/js/README.md b/src/js/README.md new file mode 100644 index 0000000000..77ebc0792c --- /dev/null +++ b/src/js/README.md @@ -0,0 +1,31 @@ +# OpenVINO™ GenAI Node.js bindings (preview) + +## DISCLAIMER + +This is preview version, do not use it in production! + +## Install and Run + +### Requirements + +- Node.js v21+ +- Tested on Ubuntu, another OS didn't tested yet + +### Build Bindings + +TODO: Add instructions + +### Perform Test Run + +- To run sample you should have prepared model. + Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) +- Go to [samples](../../samples/js/) +- Run `node app.js`, you should see: `User Prompt: ...` + +### Using as npm Dependency + +To use this package locally use `npm link` in this directory +and `npm link genai-node` in the folder where you want add this package as dependency + +To extract this package and use it as distributed npm package run `npm package`. +This command creates archive that you may use in your projects. diff --git a/src/js/include/addon.hpp b/src/js/include/addon.hpp new file mode 100644 index 0000000000..35e5cc462e --- /dev/null +++ b/src/js/include/addon.hpp @@ -0,0 +1,20 @@ +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +#pragma once + +#include + +typedef Napi::Function (*Prototype)(Napi::Env); + +struct AddonData { + Napi::FunctionReference core; +}; + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference); + +Napi::Object init_module(Napi::Env env, Napi::Object exports); diff --git a/src/js/include/helper.hpp b/src/js/include/helper.hpp new file mode 100644 index 0000000000..4a010df019 --- /dev/null +++ b/src/js/include/helper.hpp @@ -0,0 +1,23 @@ +#pragma once +#include + +#include "openvino/core/type/element_type.hpp" +#include "openvino/openvino.hpp" + +ov::AnyMap to_anyMap(const Napi::Env&, const Napi::Value&); + +/** + * @brief Template function to convert Javascript data types into C++ data types + * @tparam TargetType destinated C++ data type + * @param info Napi::CallbackInfo contains all arguments passed to a function or method + * @param idx specifies index of a argument inside info. + * @return specified argument converted to a TargetType. + */ +template +TargetType js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +/** @brief A template specialization for TargetType ov::Any */ +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value); + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num); diff --git a/src/js/include/llm_pipeline/finish_chat_worker.hpp b/src/js/include/llm_pipeline/finish_chat_worker.hpp new file mode 100644 index 0000000000..ca80b30aff --- /dev/null +++ b/src/js/include/llm_pipeline/finish_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class FinishChatWorker : public AsyncWorker { + public: + FinishChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~FinishChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/include/llm_pipeline/init_worker.hpp b/src/js/include/llm_pipeline/init_worker.hpp new file mode 100644 index 0000000000..5fc05969fb --- /dev/null +++ b/src/js/include/llm_pipeline/init_worker.hpp @@ -0,0 +1,21 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class InitWorker : public AsyncWorker { + public: + InitWorker(Function& callback, std::shared_ptr& pipe, + const std::string model_path, std::string device); + virtual ~InitWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; + std::string model_path; + std::string device; +}; diff --git a/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp new file mode 100644 index 0000000000..872e9ea023 --- /dev/null +++ b/src/js/include/llm_pipeline/llm_pipeline_wrapper.hpp @@ -0,0 +1,27 @@ +#pragma once + +#include +#include +#include "openvino/genai/llm_pipeline.hpp" + +class LLMPipelineWrapper : public Napi::ObjectWrap { +public: + LLMPipelineWrapper(const Napi::CallbackInfo& info); + + static Napi::Function get_class(Napi::Env env); + + Napi::Value init(const Napi::CallbackInfo& info); + Napi::Value generate(const Napi::CallbackInfo& info); + Napi::Value start_chat(const Napi::CallbackInfo& info); + Napi::Value finish_chat(const Napi::CallbackInfo& info); +private: + bool is_loaded = false; + bool is_initialized = false; + bool is_running = false; + + std::string model_path; + std::string device; + + std::shared_ptr pipe = nullptr; + std::function streamer; +}; diff --git a/src/js/include/llm_pipeline/start_chat_worker.hpp b/src/js/include/llm_pipeline/start_chat_worker.hpp new file mode 100644 index 0000000000..fde0cfaa0a --- /dev/null +++ b/src/js/include/llm_pipeline/start_chat_worker.hpp @@ -0,0 +1,18 @@ +#pragma once + +#include +#include "openvino/genai/llm_pipeline.hpp" + +using namespace Napi; + +class StartChatWorker : public AsyncWorker { + public: + StartChatWorker(Function& callback, std::shared_ptr& pipe); + virtual ~StartChatWorker(){}; + + void Execute(); + void OnOK(); + + private: + std::shared_ptr& pipe; +}; diff --git a/src/js/lib/bindings.cjs b/src/js/lib/bindings.cjs new file mode 100644 index 0000000000..acd9e590b8 --- /dev/null +++ b/src/js/lib/bindings.cjs @@ -0,0 +1 @@ +module.exports = require('../bin/genai_node_addon.node'); diff --git a/src/js/lib/module.js b/src/js/lib/module.js new file mode 100644 index 0000000000..2f3d557711 --- /dev/null +++ b/src/js/lib/module.js @@ -0,0 +1,118 @@ +import util from 'node:util'; + +import addon from './bindings.cjs'; + +class LLMPipeline { + modelPath = null; + device = null; + pipeline = null; + isInitialized = false; + isChatStarted = false; + + constructor(modelPath, device) { + this.modelPath = modelPath; + this.device = device; + } + + async init() { + if (this.isInitialized) + throw new Error('Pipeline is already initialized'); + + this.pipeline = new addon.LLMPipeline(); + + const init = util.promisify(this.pipeline.init.bind(this.pipeline)); + const result = await init(this.modelPath, this.device); + + this.isInitialized = true; + + return result; + } + + async startChat() { + if (this.isChatStarted) + throw new Error('Chat is already started'); + + const startChatPromise = util.promisify( + this.pipeline.startChat.bind(this.pipeline) + ); + const result = await startChatPromise(); + + this.isChatStarted = true; + + return result; + } + async finishChat() { + if (!this.isChatStarted) + throw new Error('Chat is not started'); + + const finishChatPromise = util.promisify( + this.pipeline.finishChat.bind(this.pipeline) + ); + const result = await finishChatPromise(); + + this.isChatStarted = false; + + return result; + } + + async generate(prompt, generationCallback, options = {}) { + if (!this.isInitialized) + throw new Error('Pipeline is not initialized'); + + if (typeof prompt !== 'string') + throw new Error('Prompt must be a string'); + if (typeof generationCallback !== 'function') + throw new Error('Generation callback must be a function'); + if (typeof options !== 'object') + throw new Error('Options must be an object'); + + let result = ''; + const castedOptions = {}; + + for (const key in options) castedOptions[key] = String(options[key]); + + const promise = new Promise((resolve, reject) => { + const generationCallbackDecorator = function(isDone, chunk) { + if (isDone) return resolve(result); + + result += chunk; + + try { + generationCallback(chunk); + } catch (err) { + reject(err); + } + }; + + try { + this.pipeline.generate(prompt, generationCallbackDecorator, castedOptions); + } catch (err) { + reject(err); + } + }); + + return promise; + } +} + +const availablePipelines = { LLMPipeline: LLMPipeline }; + +class Pipeline { + static async create(pipelineType, modelPath, device = 'CPU') { + if (!Object.keys(availablePipelines).includes(pipelineType)) + throw new Error(`Pipeline type: '${pipelineType}' doesn't support`); + + const pipeline = new availablePipelines[pipelineType](modelPath, device); + await pipeline.init(); + + return pipeline; + } +} + +const availablePipelinesKeys = Object.keys(availablePipelines); + +export { + addon, + Pipeline, + availablePipelinesKeys as availablePipelines, +}; diff --git a/src/js/package-lock.json b/src/js/package-lock.json new file mode 100644 index 0000000000..5964072048 --- /dev/null +++ b/src/js/package-lock.json @@ -0,0 +1,21 @@ +{ + "name": "genai-node", + "version": "2024.5.0-preview", + "lockfileVersion": 3, + "requires": true, + "packages": { + "": { + "name": "genai-node", + "version": "2024.5.0-preview", + "license": "Apache-2.0", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + } + } + } +} diff --git a/src/js/package.json b/src/js/package.json new file mode 100644 index 0000000000..89335285cb --- /dev/null +++ b/src/js/package.json @@ -0,0 +1,24 @@ +{ + "name": "genai-node", + "type": "module", + "version": "2024.5.0-preview", + "description": "OpenVINO™ GenAI pipelines for using from Node.js environment", + "license": "Apache-2.0", + "main": "./lib/module.js", + "os": [ + "linux", + "darwin", + "win32" + ], + "engines": { + "node": ">=21.0.0" + }, + "keywords": [ + "OpenVINO", + "OpenVINO GenAI", + "GenAI" + ], + "scripts": { + "test": "node --test ./tests/*.test.js" + } +} diff --git a/src/js/src/addon.cpp b/src/js/src/addon.cpp new file mode 100644 index 0000000000..4bd1da7bb6 --- /dev/null +++ b/src/js/src/addon.cpp @@ -0,0 +1,30 @@ +#include +#include + +#include "include/addon.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" + +void init_class(Napi::Env env, + Napi::Object exports, + std::string class_name, + Prototype func, + Napi::FunctionReference& reference) { + const auto& prototype = func(env); + + reference = Napi::Persistent(prototype); + exports.Set(class_name, prototype); +} + +// Define the addon initialization function +Napi::Object init_module(Napi::Env env, Napi::Object exports) { + auto addon_data = new AddonData(); + env.SetInstanceData(addon_data); + + init_class(env, exports, "LLMPipeline", &LLMPipelineWrapper::get_class, addon_data->core); + + return exports; +} + +// Register the addon with Node.js +NODE_API_MODULE(genai-node, init_module) diff --git a/src/js/src/helper.cpp b/src/js/src/helper.cpp new file mode 100644 index 0000000000..106994603b --- /dev/null +++ b/src/js/src/helper.cpp @@ -0,0 +1,53 @@ +#include "include/helper.hpp" + +ov::AnyMap to_anyMap(const Napi::Env& env, const Napi::Value& val) { + ov::AnyMap properties; + if (!val.IsObject()) { + OPENVINO_THROW("Passed Napi::Value must be an object."); + } + const auto& parameters = val.ToObject(); + const auto& keys = parameters.GetPropertyNames(); + + for (uint32_t i = 0; i < keys.Length(); ++i) { + const auto& property_name = static_cast(keys[i]).ToString().Utf8Value(); + + const auto& any_value = js_to_cpp(env, parameters.Get(property_name)); + + properties.insert(std::make_pair(property_name, any_value)); + } + + return properties; +} + +template <> +ov::Any js_to_cpp(const Napi::Env& env, const Napi::Value& value) { + if (value.IsString()) { + return ov::Any(value.ToString().Utf8Value()); + } else if (value.IsBigInt()) { + Napi::BigInt big_value = value.As(); + bool is_lossless; + int64_t big_num = big_value.Int64Value(&is_lossless); + + if (!is_lossless) { + OPENVINO_THROW("Result of BigInt conversion to int64_t results in a loss of precision"); + } + + return ov::Any(big_num); + } else if (value.IsNumber()) { + Napi::Number num = value.ToNumber(); + + if (is_napi_value_int(env, value)) { + return ov::Any(num.Int32Value()); + } else { + return ov::Any(num.DoubleValue()); + } + } else if (value.IsBoolean()) { + return ov::Any(value.ToBoolean()); + } else { + OPENVINO_THROW("Cannot convert to ov::Any"); + } +} + +bool is_napi_value_int(const Napi::Env& env, const Napi::Value& num) { + return env.Global().Get("Number").ToObject().Get("isInteger").As().Call({num}).ToBoolean().Value(); +} diff --git a/src/js/src/llm_pipeline/finish_chat_worker.cpp b/src/js/src/llm_pipeline/finish_chat_worker.cpp new file mode 100644 index 0000000000..b07284688c --- /dev/null +++ b/src/js/src/llm_pipeline/finish_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include +#include + +FinishChatWorker::FinishChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void FinishChatWorker::Execute() { + this->pipe->finish_chat(); +}; + +void FinishChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/init_worker.cpp b/src/js/src/llm_pipeline/init_worker.cpp new file mode 100644 index 0000000000..87dd1aaf34 --- /dev/null +++ b/src/js/src/llm_pipeline/init_worker.cpp @@ -0,0 +1,18 @@ +#include "include/llm_pipeline/init_worker.hpp" +#include +#include + +InitWorker::InitWorker( + Function& callback, + std::shared_ptr& pipe, + const std::string model_path, + const std::string device +) : AsyncWorker(callback), pipe(pipe), model_path(model_path), device(device) {}; + +void InitWorker::Execute() { + this->pipe = std::make_shared(this->model_path, this->device); +}; + +void InitWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp new file mode 100644 index 0000000000..df1cfceadc --- /dev/null +++ b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp @@ -0,0 +1,151 @@ +#include "include/helper.hpp" + +#include "include/llm_pipeline/llm_pipeline_wrapper.hpp" +#include "include/llm_pipeline/start_chat_worker.hpp" +#include "include/llm_pipeline/finish_chat_worker.hpp" +#include "include/llm_pipeline/init_worker.hpp" + +struct TsfnContext { + TsfnContext(std::string prompt) : prompt(prompt) {}; + ~TsfnContext() { + // std::cout << "Tsfn destructed" << std::endl; + }; + + std::thread native_thread; + Napi::ThreadSafeFunction tsfn; + + std::string prompt; + std::shared_ptr pipe = nullptr; + std::shared_ptr options = nullptr; +}; + +void performInferenceThread(TsfnContext* context) { + auto callback = [](Napi::Env env, Napi::Function js_callback, TsfnContext* context) { + try { + std::function streamer = [env, js_callback](std::string word) { + js_callback.Call({ + Napi::Boolean::New(env, false), + Napi::String::New(env, word) + }); + + // Return flag corresponds whether generation should be stopped. + // false means continue generation. + return false; + }; + + ov::genai::GenerationConfig config; + + config.update_generation_config(*context->options); + + context->pipe->generate(context->prompt, config, streamer); + js_callback.Call({ + Napi::Boolean::New(env, true) + }); + } catch(std::exception& err) { + Napi::Error::Fatal("performInferenceThread callback error. Details:" , err.what()); + } + }; + + try { + napi_status status = context->tsfn.BlockingCall(context, callback); + if (status != napi_ok) { + // Handle error + Napi::Error::Fatal("performInferenceThread error", "napi_status != napi_ok"); + } + + context->tsfn.Release(); + } + catch(std::exception& e) { + Napi::Error::Fatal("performInferenceThread error" , e.what()); + + context->tsfn.Release(); + } +} + +LLMPipelineWrapper::LLMPipelineWrapper(const Napi::CallbackInfo& info) : Napi::ObjectWrap(info) {}; + +Napi::Function LLMPipelineWrapper::get_class(Napi::Env env) { + return DefineClass(env, + "LLMPipeline", + {InstanceMethod("init", &LLMPipelineWrapper::init), + InstanceMethod("generate", &LLMPipelineWrapper::generate), + InstanceMethod("startChat", &LLMPipelineWrapper::start_chat), + InstanceMethod("finishChat", &LLMPipelineWrapper::finish_chat)}); +} + +Napi::Value LLMPipelineWrapper::init(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + const std::string model_path = info[0].ToString(); + const std::string device = info[1].ToString(); + Napi::Function callback = info[2].As(); + + InitWorker* asyncWorker = new InitWorker(callback, this->pipe, model_path, device); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::generate(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + TsfnContext* context = nullptr; + + try { + std::string prompt = info[0].ToString(); + ov::AnyMap options; + if (info.Length() == 3) { + options = to_anyMap(info.Env(), info[2]); + } + + context = new TsfnContext(prompt); + context->pipe = this->pipe; + context->options = std::make_shared(options); + // Create a ThreadSafeFunction + context->tsfn = Napi::ThreadSafeFunction::New( + env, + info[1].As(), // JavaScript function called asynchronously + "TSFN", // Name + 0, // Unlimited queue + 1, // Only one thread will use this initially + [context](Napi::Env) { // Finalizer used to clean threads up + // std::cout << "Finalize TFSN" << std::endl; + context->native_thread.join(); + delete context; + } + ); + context->native_thread = std::thread(performInferenceThread, context); + + return Napi::Boolean::New(env, false); + } catch(Napi::TypeError& type_err) { + throw type_err; + } catch(std::exception& err) { + std::cout << "Catch in the thread: '" << err.what() << "'" << std::endl; + if (context != nullptr) { + context->tsfn.Release(); + } + + throw Napi::Error::New(env, err.what()); + } + + return Napi::Boolean::New(env, true); +} + +Napi::Value LLMPipelineWrapper::start_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + Napi::Function callback = info[0].As(); + + StartChatWorker* asyncWorker = new StartChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} + +Napi::Value LLMPipelineWrapper::finish_chat(const Napi::CallbackInfo& info) { + Napi::Env env = info.Env(); + + Napi::Function callback = info[0].As(); + + FinishChatWorker* asyncWorker = new FinishChatWorker(callback, this->pipe); + asyncWorker->Queue(); + + return info.Env().Undefined(); +} diff --git a/src/js/src/llm_pipeline/start_chat_worker.cpp b/src/js/src/llm_pipeline/start_chat_worker.cpp new file mode 100644 index 0000000000..302c505105 --- /dev/null +++ b/src/js/src/llm_pipeline/start_chat_worker.cpp @@ -0,0 +1,14 @@ +#include "include/llm_pipeline/start_chat_worker.hpp" +#include +#include + +StartChatWorker::StartChatWorker(Function& callback, std::shared_ptr& pipe) + : AsyncWorker(callback), pipe(pipe) {}; + +void StartChatWorker::Execute() { + this->pipe->start_chat(); +}; + +void StartChatWorker::OnOK() { + Callback().Call({ Env().Null() }); +}; diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js new file mode 100644 index 0000000000..2b031124f0 --- /dev/null +++ b/src/js/tests/bindings.test.js @@ -0,0 +1,71 @@ +import addon from '../lib/bindings.cjs'; + +import assert from 'node:assert'; +import { describe, it, before, after } from 'node:test'; + +const MODEL_PATH = process.env.MODEL_PATH; + +describe('bindings', () => { + let pipeline = null; + + before((_, done) => { + pipeline = new addon.LLMPipeline(); + + pipeline.init(MODEL_PATH, 'AUTO', (err) => { + if (err) { + console.error(err); + process.exit(1); + } + + pipeline.startChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + }); + + after((_, done) => { + pipeline.finishChat((err) => { + if (err) { + console.error(err); + process.exit(1); + } + + done(); + }); + }); + + it('should generate string result', (_, done) => { + let output = ''; + + pipeline.generate('Say Hello', (isDone, chunk) => { + if (!isDone) { + output += chunk; + + return; + } + }, { temperature: '0', max_new_tokens: '4' }); + + assert.ok(true); + done(); + }); + + // it('should generate "Hello world"', (_, done) => { + // let output = ''; + + // pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + // if (!isDone) { + // output += chunk; + + // return; + // } + + // assert.strictEqual(output, '"Hello world!"'); + // done(); + // }, { temperature: '0', max_new_tokens: '4' }); + // }); +}); diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js new file mode 100644 index 0000000000..167cf91436 --- /dev/null +++ b/src/js/tests/module.test.js @@ -0,0 +1,140 @@ +import { Pipeline } from '../lib/module.js'; + +import assert from 'node:assert/strict'; +import { describe, it, before, after } from 'node:test'; + +const MODEL_PATH = process.env.MODEL_PATH; + +// describe('module', async () => { +// let pipeline = null; + +// await before(async () => { +// pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + +// await pipeline.startChat(); +// }); + +// await after(async () => { +// await pipeline.finishChat(); +// }); + +// await it('should generate "Hello world"', async () => { +// const result = await pipeline.generate( +// 'Type "Hello world!" in English', +// () => {}, +// { temperature: '0', max_new_tokens: '4' } +// ); + +// assert.strictEqual(result, '"Hello world!"'); +// }); +// }); + +describe('corner cases', async () => { + it('should throw an error if pipeline is already initialized', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await assert.rejects( + async () => await pipeline.init(), + { + name: 'Error', + message: 'Pipeline is already initialized', + }, + ); + }); + + it('should throw an error if chat is already started', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + + await assert.rejects( + () => pipeline.startChat(), + { + name: 'Error', + message: 'Chat is already started', + }, + ); + }); + + it('should throw an error if chat is not started', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await assert.rejects( + () => pipeline.finishChat(), + { + name: 'Error', + message: 'Chat is not started', + }, + ); + }); +}); + +describe('generation parameters validation', () => { + let pipeline = null; + + before(async () => { + pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + }); + + after(async () => { + await pipeline.finishChat(); + }); + + it('should throw an error if temperature is not a number', async () => { + await assert.rejects( + async () => await pipeline.generate(), + { + name: 'Error', + message: 'Prompt must be a string', + }, + ); + }); + + it('should throw an error if generationCallback is not a function', async () => { + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + + await pipeline.startChat(); + + await assert.rejects( + async () => await pipeline.generate('prompt'), + { + name: 'Error', + message: 'Generation callback must be a function', + }, + ); + }); + + it('should throw an error if options specified but not an object', async () => { + await assert.rejects( + async () => await pipeline.generate('prompt', () => {}, 'options'), + { + name: 'Error', + message: 'Options must be an object', + }, + ); + }); + + it('should perform generation with default options', async () => { + try { + await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + } catch (error) { + assert.fail(error); + } + + assert.ok(true); + }); + + it('should return a string as generation result', async () => { + const reply = await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + + assert.strictEqual(typeof reply, 'string'); + }); + + it('should call generationCallback with string chunk', async () => { + await pipeline.generate('prompt', (chunk) => { + assert.strictEqual(typeof chunk, 'string'); + }, { max_new_tokens: 1 }); + }); +}); diff --git a/src/js/tests/setup.js b/src/js/tests/setup.js new file mode 100644 index 0000000000..b39f7c0136 --- /dev/null +++ b/src/js/tests/setup.js @@ -0,0 +1,9 @@ +import { testModels, downloadTestModel } from './utils.js'; + +if (require.main === module) { + main(); +} + +async function main() { + await downloadTestModel(testModels.testModelFP32); +} diff --git a/src/js/tests/utils.js b/src/js/tests/utils.js new file mode 100644 index 0000000000..c2089e30b4 --- /dev/null +++ b/src/js/tests/utils.js @@ -0,0 +1,73 @@ +// -*- coding: utf-8 -*- +// Copyright (C) 2018-2024 Intel Corporation +// SPDX-License-Identifier: Apache-2.0 + +const path = require('path'); +const fs = require('node:fs/promises'); +const { + downloadFile, + checkIfPathExists, +} = require('../../scripts/download_runtime'); + +const modelDir = 'tests/unit/test_models/'; +const testModels = { + testModelFP32: { + xml: 'test_model_fp32.xml', + bin: 'test_model_fp32.bin', + xmlURL: + 'https://raw.githubusercontent.com/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.xml', + binURL: + 'https://media.githubusercontent.com/media/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.bin', + }, +}; + +module.exports = { + getModelPath, + downloadTestModel, + isModelAvailable, + testModels, +}; + +function getModelPath(isFP16 = false) { + const modelName = `test_model_fp${isFP16 ? 16 : 32}`; + + return { + xml: path.join(modelDir, `${modelName}.xml`), + bin: path.join(modelDir, `${modelName}.bin`), + }; +} + +async function downloadTestModel(model) { + const modelsDir = './tests/unit/test_models'; + try { + const ifModelsDirectoryExists = await checkIfPathExists(modelsDir); + if (!ifModelsDirectoryExists) { + await fs.mkdir(modelDir); + } + + const modelPath = path.join(modelsDir, model.xml); + const modelExists = await checkIfPathExists(modelPath); + if (modelExists) return; + + const { env } = process; + const proxyUrl = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + + await downloadFile(model.xmlURL, modelsDir, model.xml, proxyUrl); + await downloadFile(model.binURL, modelsDir, model.bin, proxyUrl); + } catch(error) { + console.error(`Failed to download the model: ${error}.`); + throw error; + } +} + +async function isModelAvailable(model) { + const baseArtifactsDir = './tests/unit/test_models'; + const modelPath = path.join(baseArtifactsDir, model.xml); + const modelExists = await checkIfPathExists(modelPath); + if (modelExists) return; + + console.log( + '\n\nTestModel cannot be found.\nPlease run `npm run test_setup`.\n\n', + ); + process.exit(1); +} From 2d71b6b4ac14191dbd5330305aae22edee7f9962 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 13 Nov 2024 18:36:50 +0100 Subject: [PATCH 02/74] Include js bindings into package build --- src/js/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index 96ba0019c1..d7ea811618 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -87,6 +87,6 @@ if(rpaths) endif() install(TARGETS ${PROJECT_NAME} - LIBRARY DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL - RUNTIME DESTINATION openvino_genai COMPONENT ${PROJECT_NAME} EXCLUDE_FROM_ALL + LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} + RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} ) From 0c0b41ca817b9326f493f9fcb285c7e7237a260c Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Fri, 15 Nov 2024 15:48:07 +0100 Subject: [PATCH 03/74] Put binaries at the top level of NPM package --- src/cpp/CMakeLists.txt | 26 +++++++++++++++++++++----- 1 file changed, 21 insertions(+), 5 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 49b640763d..f689547a69 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -118,13 +118,29 @@ if(MSVC OR APPLE) set(ARCH_DIR ${ARCH_DIR}/${CMAKE_BUILD_TYPE}) endif() +# Put binaries at the top level for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + set(LIBRARY_DESTINATION .) + set(ARCHIVE_DESTINATION .) + set(RUNTIME_DESTINATION .) +else() + set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR}) + set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR}) + set(RUNTIME_DESTINATION runtime/bin/${ARCH_DIR}) +endif() + install(TARGETS ${TARGET_NAME} EXPORT OpenVINOGenAITargets - LIBRARY DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai + LIBRARY DESTINATION ${LIBRARY_DESTINATION} COMPONENT core_genai NAMELINK_COMPONENT core_genai_dev - ARCHIVE DESTINATION runtime/lib/${ARCH_DIR} COMPONENT core_genai_dev - RUNTIME DESTINATION runtime/bin/${ARCH_DIR} COMPONENT core_genai + ARCHIVE DESTINATION ${ARCHIVE_DESTINATION} COMPONENT core_genai_dev + RUNTIME DESTINATION ${RUNTIME_DESTINATION} COMPONENT core_genai INCLUDES DESTINATION runtime/include) +# samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + install(DIRECTORY ${CMAKE_CURRENT_SOURCE_DIR}/include/ DESTINATION runtime/include COMPONENT core_genai_dev) install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake @@ -133,9 +149,9 @@ install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake include(CMakePackageConfigHelpers) configure_package_config_file("${OpenVINOGenAI_SOURCE_DIR}/cmake/templates/OpenVINOGenAIConfig.cmake.in" - "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) + "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) write_basic_package_version_file("${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" - VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) + VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" DESTINATION runtime/cmake COMPONENT core_genai_dev) export(EXPORT OpenVINOGenAITargets FILE "${CMAKE_BINARY_DIR}/OpenVINOGenAITargets.cmake" NAMESPACE openvino::) From 7e80306dacb46ba8ea79fdd455c057e926a52421 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Fri, 15 Nov 2024 15:48:58 +0100 Subject: [PATCH 04/74] Skip samples for NPM package --- samples/CMakeLists.txt | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 229eccb3fe..dbb9639128 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -2,6 +2,11 @@ # SPDX-License-Identifier: Apache-2.0 # +# Samples do not need to be built for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + return() +endif() + add_subdirectory(cpp/beam_search_causal_lm) add_subdirectory(cpp/benchmark_genai) add_subdirectory(cpp/chat_sample) From 67d5a404c2b90ceb82b2f98979a9a189e8d9b23e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 19 Nov 2024 17:56:37 +0100 Subject: [PATCH 05/74] Add model downloading into js unit tests --- src/js/.gitignore | 2 + src/js/package-lock.json | 449 ++++++++++++++++++++++++++++++++++ src/js/package.json | 8 +- src/js/tests/bindings.test.js | 26 +- src/js/tests/models.js | 3 + src/js/tests/module.test.js | 40 +-- src/js/tests/setup.js | 11 +- src/js/tests/utils.js | 96 +++----- 8 files changed, 535 insertions(+), 100 deletions(-) create mode 100644 src/js/tests/models.js diff --git a/src/js/.gitignore b/src/js/.gitignore index e7fffbc37a..638d8a5bb7 100644 --- a/src/js/.gitignore +++ b/src/js/.gitignore @@ -3,3 +3,5 @@ bin bin.* build thirdparty +node_modules +tests/models diff --git a/src/js/package-lock.json b/src/js/package-lock.json index 5964072048..4da5b57ea7 100644 --- a/src/js/package-lock.json +++ b/src/js/package-lock.json @@ -13,9 +13,458 @@ "darwin", "win32" ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, "engines": { "node": ">=21.0.0" } + }, + "node_modules/@huggingface/hub": { + "version": "0.21.0", + "resolved": "https://registry.npmjs.org/@huggingface/hub/-/hub-0.21.0.tgz", + "integrity": "sha512-DpitNhqobMJLTv8dUq/EMtrz1dpfs3UrSVCxe1aKpjLAdOs6Gm6rqrinUFNvC9G88RIRzIYzojUtYUqlkKwKnA==", + "dev": true, + "license": "MIT", + "dependencies": { + "@huggingface/tasks": "^0.13.3" + }, + "engines": { + "node": ">=18" + } + }, + "node_modules/@huggingface/tasks": { + "version": "0.13.4", + "resolved": "https://registry.npmjs.org/@huggingface/tasks/-/tasks-0.13.4.tgz", + "integrity": "sha512-LETHbMSK3gHBFU0D09ziEJm6t1Pcgii4SFwHw+d+8MFGfkAryxaDl2qaHY+PxiTkZEeaTLd6G8/239SJuVxyWg==", + "dev": true, + "license": "MIT" + }, + "node_modules/boolean": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/boolean/-/boolean-3.2.0.tgz", + "integrity": "sha512-d0II/GO9uf9lfUHH2BQsjxzRJZBdsjgsBiW4BvhWk/3qoKwQFjIDVN19PfX8F2D/r9PCMTtLWjYVCFrpeYUzsw==", + "deprecated": "Package no longer supported. Contact Support at https://www.npmjs.com/support for more info.", + "dev": true, + "license": "MIT" + }, + "node_modules/data-uri-to-buffer": { + "version": "4.0.1", + "resolved": "https://registry.npmjs.org/data-uri-to-buffer/-/data-uri-to-buffer-4.0.1.tgz", + "integrity": "sha512-0R9ikRb668HB7QDxT1vkpuUBtqc53YyAwMwGeUFKRojY/NWKvdZ+9UYtRfGmhqNbRkTSVpMbmyhXipFFv2cb/A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 12" + } + }, + "node_modules/define-data-property": { + "version": "1.1.4", + "resolved": "https://registry.npmjs.org/define-data-property/-/define-data-property-1.1.4.tgz", + "integrity": "sha512-rBMvIzlpA8v6E+SJZoo++HAYqsLrkg7MSfIinMPFhmkorw7X+dOXVJQs+QT69zGkzMyfDnIMN2Wid1+NbL3T+A==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0", + "es-errors": "^1.3.0", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/define-properties": { + "version": "1.2.1", + "resolved": "https://registry.npmjs.org/define-properties/-/define-properties-1.2.1.tgz", + "integrity": "sha512-8QmQKqEASLd5nx0U1B1okLElbUuuttJ/AnYmRXbbbGDWh6uS208EjD4Xqq/I9wK7u0v6O08XhTWnt5XtEbR6Dg==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-data-property": "^1.0.1", + "has-property-descriptors": "^1.0.0", + "object-keys": "^1.1.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/detect-node": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/detect-node/-/detect-node-2.1.0.tgz", + "integrity": "sha512-T0NIuQpnTvFDATNuHN5roPwSBG83rFsuO+MXXH9/3N1eFbn4wcPjttvjMLEPWJ0RGUYgQE7cGgS3tNxbqCGM7g==", + "dev": true, + "license": "MIT" + }, + "node_modules/es-define-property": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/es-define-property/-/es-define-property-1.0.0.tgz", + "integrity": "sha512-jxayLKShrEqqzJ0eumQbVhTYQM27CfT1T35+gCgDFoL82JLsXqTJ76zv6A0YLOgEnLUMvLzsDsGIrl8NFpT2gQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.2.4" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es-errors": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/es-errors/-/es-errors-1.3.0.tgz", + "integrity": "sha512-Zf5H2Kxt2xjTvbJvP2ZWLEICxA6j+hAmMzIlypy4xcBg1vKVnx89Wy0GbS+kf5cwCVFFzdCFh2XSCFNULS6csw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/es6-error": { + "version": "4.1.1", + "resolved": "https://registry.npmjs.org/es6-error/-/es6-error-4.1.1.tgz", + "integrity": "sha512-Um/+FxMr9CISWh0bi5Zv0iOD+4cFh5qLeks1qhAopKVAJw3drgKbKySikp7wGhDL0HPeaja0P5ULZrxLkniUVg==", + "dev": true, + "license": "MIT" + }, + "node_modules/escape-string-regexp": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/escape-string-regexp/-/escape-string-regexp-4.0.0.tgz", + "integrity": "sha512-TtpcNJ3XAzx3Gq8sWRzJaVajRs0uVxA2YAkdb1jm2YkPz4G6egUFAyA3n5vtEIZefPk5Wa4UXbKuS5fKkJWdgA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/fetch-blob": { + "version": "3.2.0", + "resolved": "https://registry.npmjs.org/fetch-blob/-/fetch-blob-3.2.0.tgz", + "integrity": "sha512-7yAQpD2UMJzLi1Dqv7qFYnPbaPx7ZfFK6PiIxQ4PfkGPyNyl2Ugx+a/umUonmKqjhM4DnfbMvdX6otXq83soQQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "paypal", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "dependencies": { + "node-domexception": "^1.0.0", + "web-streams-polyfill": "^3.0.3" + }, + "engines": { + "node": "^12.20 || >= 14.13" + } + }, + "node_modules/formdata-polyfill": { + "version": "4.0.10", + "resolved": "https://registry.npmjs.org/formdata-polyfill/-/formdata-polyfill-4.0.10.tgz", + "integrity": "sha512-buewHzMvYL29jdeQTVILecSaZKnt/RJWjoZCF5OW60Z67/GmSLBkOFM7qh1PI3zFNtJbaZL5eQu1vLfazOwj4g==", + "dev": true, + "license": "MIT", + "dependencies": { + "fetch-blob": "^3.1.2" + }, + "engines": { + "node": ">=12.20.0" + } + }, + "node_modules/function-bind": { + "version": "1.1.2", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.2.tgz", + "integrity": "sha512-7XHNxH7qX9xG5mIwxkhumTox/MIRNcOgDrxWsMt2pAr23WHp6MrRlN7FBSFpCpr+oVO0F744iUgR82nJMfG2SA==", + "dev": true, + "license": "MIT", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/get-intrinsic": { + "version": "1.2.4", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.2.4.tgz", + "integrity": "sha512-5uYhsJH8VJBTv7oslg4BznJYhDoRI6waYCxMmCdnTrcCrHA/fCFKoTFz2JKKE0HdDFUF7/oQuhzumXJK7paBRQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-errors": "^1.3.0", + "function-bind": "^1.1.2", + "has-proto": "^1.0.1", + "has-symbols": "^1.0.3", + "hasown": "^2.0.0" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/global-agent": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/global-agent/-/global-agent-3.0.0.tgz", + "integrity": "sha512-PT6XReJ+D07JvGoxQMkT6qji/jVNfX/h364XHZOWeRzy64sSFr+xJ5OX7LI3b4MPQzdL4H8Y8M0xzPpsVMwA8Q==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "es6-error": "^4.1.1", + "matcher": "^3.0.0", + "roarr": "^2.15.3", + "semver": "^7.3.2", + "serialize-error": "^7.0.1" + }, + "engines": { + "node": ">=10.0" + } + }, + "node_modules/globalthis": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/globalthis/-/globalthis-1.0.4.tgz", + "integrity": "sha512-DpLKbNU4WylpxJykQujfCcwYWiV/Jhm50Goo0wrVILAv5jOr9d+H+UR3PhSCD2rCCEIg0uc+G+muBTwD54JhDQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "define-properties": "^1.2.1", + "gopd": "^1.0.1" + }, + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/gopd": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/gopd/-/gopd-1.0.1.tgz", + "integrity": "sha512-d65bNlIadxvpb/A2abVdlqKqV563juRnZ1Wtk6s1sIR8uNsXR70xqIzVqxVf1eTqDunwT2MkczEeaezCKTZhwA==", + "dev": true, + "license": "MIT", + "dependencies": { + "get-intrinsic": "^1.1.3" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-property-descriptors": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/has-property-descriptors/-/has-property-descriptors-1.0.2.tgz", + "integrity": "sha512-55JNKuIW+vq4Ke1BjOTjM2YctQIvCT7GFzHwmfZPGo5wnrgkid0YQtnAleFSqumZm4az3n2BS+erby5ipJdgrg==", + "dev": true, + "license": "MIT", + "dependencies": { + "es-define-property": "^1.0.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-proto": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-proto/-/has-proto-1.0.3.tgz", + "integrity": "sha512-SJ1amZAJUiZS+PhsVLf5tGydlaVB8EdFpaSO4gmiUKUOxk8qzn5AIy4ZeJUmh22znIdk/uMAUT2pl3FxzVUH+Q==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/has-symbols": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", + "integrity": "sha512-l3LCuF6MgDNwTDKkdYGEihYjt5pRPbEg46rtlmnSPlUbgmB8LOIrKJbYYFBSbnPaJexMKtiPO8hmeRjRz2Td+A==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/hasown": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/hasown/-/hasown-2.0.2.tgz", + "integrity": "sha512-0hJU9SCPvmMzIBdZFqNPXWa6dqh7WdH0cII9y+CyS8rG3nL48Bclra9HmKhVVUHyPWNH5Y7xDwAB7bfgSjkUMQ==", + "dev": true, + "license": "MIT", + "dependencies": { + "function-bind": "^1.1.2" + }, + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/json-stringify-safe": { + "version": "5.0.1", + "resolved": "https://registry.npmjs.org/json-stringify-safe/-/json-stringify-safe-5.0.1.tgz", + "integrity": "sha512-ZClg6AaYvamvYEE82d3Iyd3vSSIjQ+odgjaTzRuO3s7toCdFKczob2i0zCh7JE8kWn17yvAWhUVxvqGwUalsRA==", + "dev": true, + "license": "ISC" + }, + "node_modules/matcher": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/matcher/-/matcher-3.0.0.tgz", + "integrity": "sha512-OkeDaAZ/bQCxeFAozM55PKcKU0yJMPGifLwV4Qgjitu+5MoAfSQN4lsLJeXZ1b8w0x+/Emda6MZgXS1jvsapng==", + "dev": true, + "license": "MIT", + "dependencies": { + "escape-string-regexp": "^4.0.0" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/node-domexception": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/node-domexception/-/node-domexception-1.0.0.tgz", + "integrity": "sha512-/jKZoMpw0F8GRwl4/eLROPA3cfcXtLApP0QzLmUT/HuPCZWyB7IY9ZrMeKw2O/nFIqPQB3PVM9aYm0F312AXDQ==", + "dev": true, + "funding": [ + { + "type": "github", + "url": "https://github.com/sponsors/jimmywarting" + }, + { + "type": "github", + "url": "https://paypal.me/jimmywarting" + } + ], + "license": "MIT", + "engines": { + "node": ">=10.5.0" + } + }, + "node_modules/node-fetch": { + "version": "3.3.2", + "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-3.3.2.tgz", + "integrity": "sha512-dRB78srN/l6gqWulah9SrxeYnxeddIG30+GOqK/9OlLVyLg3HPnr6SqOWTWOXKRwC2eGYCkZ59NNuSgvSrpgOA==", + "dev": true, + "license": "MIT", + "dependencies": { + "data-uri-to-buffer": "^4.0.0", + "fetch-blob": "^3.1.4", + "formdata-polyfill": "^4.0.10" + }, + "engines": { + "node": "^12.20.0 || ^14.13.1 || >=16.0.0" + }, + "funding": { + "type": "opencollective", + "url": "https://opencollective.com/node-fetch" + } + }, + "node_modules/object-keys": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/object-keys/-/object-keys-1.1.1.tgz", + "integrity": "sha512-NuAESUOUMrlIXOfHKzD6bpPu3tYt3xvjNdRIQ+FeT0lNb4K8WR70CaDxhuNguS2XG+GjkyMwOzsN5ZktImfhLA==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 0.4" + } + }, + "node_modules/roarr": { + "version": "2.15.4", + "resolved": "https://registry.npmjs.org/roarr/-/roarr-2.15.4.tgz", + "integrity": "sha512-CHhPh+UNHD2GTXNYhPWLnU8ONHdI+5DI+4EYIAOaiD63rHeYlZvyh8P+in5999TTSFgUYuKUAjzRI4mdh/p+2A==", + "dev": true, + "license": "BSD-3-Clause", + "dependencies": { + "boolean": "^3.0.1", + "detect-node": "^2.0.4", + "globalthis": "^1.0.1", + "json-stringify-safe": "^5.0.1", + "semver-compare": "^1.0.0", + "sprintf-js": "^1.1.2" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/semver": { + "version": "7.6.3", + "resolved": "https://registry.npmjs.org/semver/-/semver-7.6.3.tgz", + "integrity": "sha512-oVekP1cKtI+CTDvHWYFUcMtsK/00wmAEfyqKfNdARm8u1wNVhSgaX7A8d4UuIlUI5e84iEwOhs7ZPYRmzU9U6A==", + "dev": true, + "license": "ISC", + "bin": { + "semver": "bin/semver.js" + }, + "engines": { + "node": ">=10" + } + }, + "node_modules/semver-compare": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/semver-compare/-/semver-compare-1.0.0.tgz", + "integrity": "sha512-YM3/ITh2MJ5MtzaM429anh+x2jiLVjqILF4m4oyQB18W7Ggea7BfqdH/wGMK7dDiMghv/6WG7znWMwUDzJiXow==", + "dev": true, + "license": "MIT" + }, + "node_modules/serialize-error": { + "version": "7.0.1", + "resolved": "https://registry.npmjs.org/serialize-error/-/serialize-error-7.0.1.tgz", + "integrity": "sha512-8I8TjW5KMOKsZQTvoxjuSIa7foAwPWGOts+6o7sgjz41/qMD9VQHEDxi6PBvK2l0MXUmqZyNpUK+T2tQaaElvw==", + "dev": true, + "license": "MIT", + "dependencies": { + "type-fest": "^0.13.1" + }, + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/sprintf-js": { + "version": "1.1.3", + "resolved": "https://registry.npmjs.org/sprintf-js/-/sprintf-js-1.1.3.tgz", + "integrity": "sha512-Oo+0REFV59/rz3gfJNKQiBlwfHaSESl1pcGyABQsnnIfWOFt6JNj5gCog2U6MLZ//IGYD+nA8nI+mTShREReaA==", + "dev": true, + "license": "BSD-3-Clause" + }, + "node_modules/type-fest": { + "version": "0.13.1", + "resolved": "https://registry.npmjs.org/type-fest/-/type-fest-0.13.1.tgz", + "integrity": "sha512-34R7HTnG0XIJcBSn5XhDd7nNFPRcXYRZrBB2O2jdKqYODldSzBAqzsWoZYYvduky73toYS/ESqxPvkDf/F0XMg==", + "dev": true, + "license": "(MIT OR CC0-1.0)", + "engines": { + "node": ">=10" + }, + "funding": { + "url": "https://github.com/sponsors/sindresorhus" + } + }, + "node_modules/web-streams-polyfill": { + "version": "3.3.3", + "resolved": "https://registry.npmjs.org/web-streams-polyfill/-/web-streams-polyfill-3.3.3.tgz", + "integrity": "sha512-d2JWLCivmZYTSIoge9MsgFCZrt571BikcWGYkjC1khllbTeDlGqZ2D8vD8E/lJa8WGWbb7Plm8/XJYV7IJHZZw==", + "dev": true, + "license": "MIT", + "engines": { + "node": ">= 8" + } } } } diff --git a/src/js/package.json b/src/js/package.json index 89335285cb..5b069fb01f 100644 --- a/src/js/package.json +++ b/src/js/package.json @@ -19,6 +19,12 @@ "GenAI" ], "scripts": { - "test": "node --test ./tests/*.test.js" + "test_setup": "node ./tests/setup.js", + "test": "npm run test_setup && node --test ./tests/*.test.js" + }, + "devDependencies": { + "node-fetch": "^3.3.2", + "global-agent": "^3.0.0", + "@huggingface/hub": "^0.21.0" } } diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index 2b031124f0..3e852e5307 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -2,8 +2,10 @@ import addon from '../lib/bindings.cjs'; import assert from 'node:assert'; import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; -const MODEL_PATH = process.env.MODEL_PATH; +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; describe('bindings', () => { let pipeline = null; @@ -54,18 +56,18 @@ describe('bindings', () => { done(); }); - // it('should generate "Hello world"', (_, done) => { - // let output = ''; + it('should generate "Hello world"', (_, done) => { + let output = ''; - // pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { - // if (!isDone) { - // output += chunk; + pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + if (!isDone) { + output += chunk; - // return; - // } + return; + } - // assert.strictEqual(output, '"Hello world!"'); - // done(); - // }, { temperature: '0', max_new_tokens: '4' }); - // }); + assert.strictEqual(output, 'Hello world!'); + done(); + }, { temperature: '0', max_new_tokens: '4' }); + }); }); diff --git a/src/js/tests/models.js b/src/js/tests/models.js new file mode 100644 index 0000000000..b7f7505464 --- /dev/null +++ b/src/js/tests/models.js @@ -0,0 +1,3 @@ +export const models = [ + 'AIFunOver/Llama-3.2-3B-Instruct-openvino-8bit', +]; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 167cf91436..90b2f5266d 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -2,32 +2,34 @@ import { Pipeline } from '../lib/module.js'; import assert from 'node:assert/strict'; import { describe, it, before, after } from 'node:test'; +import { models } from './models.js'; -const MODEL_PATH = process.env.MODEL_PATH; +const MODEL_PATH = process.env.MODEL_PATH + || `./tests/models/${models[0].split('/')[1]}`; -// describe('module', async () => { -// let pipeline = null; +describe('module', async () => { + let pipeline = null; -// await before(async () => { -// pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + await before(async () => { + pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); -// await pipeline.startChat(); -// }); + await pipeline.startChat(); + }); -// await after(async () => { -// await pipeline.finishChat(); -// }); + await after(async () => { + await pipeline.finishChat(); + }); -// await it('should generate "Hello world"', async () => { -// const result = await pipeline.generate( -// 'Type "Hello world!" in English', -// () => {}, -// { temperature: '0', max_new_tokens: '4' } -// ); + await it('should generate "Hello world"', async () => { + const result = await pipeline.generate( + 'Type "Hello world!" in English', + () => {}, + { temperature: '0', max_new_tokens: '4' } + ); -// assert.strictEqual(result, '"Hello world!"'); -// }); -// }); + assert.strictEqual(result, 'Hello world!'); + }); +}); describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { diff --git a/src/js/tests/setup.js b/src/js/tests/setup.js index b39f7c0136..3b52651719 100644 --- a/src/js/tests/setup.js +++ b/src/js/tests/setup.js @@ -1,9 +1,6 @@ -import { testModels, downloadTestModel } from './utils.js'; +import { dowloadModel } from './utils.js'; +import { models } from './models.js'; -if (require.main === module) { - main(); -} - -async function main() { - await downloadTestModel(testModels.testModelFP32); +for (const model of models) { + await dowloadModel(model); } diff --git a/src/js/tests/utils.js b/src/js/tests/utils.js index c2089e30b4..504782d8d1 100644 --- a/src/js/tests/utils.js +++ b/src/js/tests/utils.js @@ -1,73 +1,47 @@ -// -*- coding: utf-8 -*- -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 +import { bootstrap } from 'global-agent'; +import { promises as fs } from 'node:fs'; +import { listFiles, downloadFile } from '@huggingface/hub'; -const path = require('path'); -const fs = require('node:fs/promises'); -const { - downloadFile, - checkIfPathExists, -} = require('../../scripts/download_runtime'); +const BASE_DIR = './tests/models/'; -const modelDir = 'tests/unit/test_models/'; -const testModels = { - testModelFP32: { - xml: 'test_model_fp32.xml', - bin: 'test_model_fp32.bin', - xmlURL: - 'https://raw.githubusercontent.com/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.xml', - binURL: - 'https://media.githubusercontent.com/media/openvinotoolkit/testdata/master/models/test_model/test_model_fp32.bin', - }, -}; +bootstrap(); -module.exports = { - getModelPath, - downloadTestModel, - isModelAvailable, - testModels, -}; +export async function dowloadModel(repo) { + console.log(`Downloading model '${repo}'`); -function getModelPath(isFP16 = false) { - const modelName = `test_model_fp${isFP16 ? 16 : 32}`; + const fetch = await import('node-fetch'); + const modelName = repo.split('/')[1]; + const destDir = `${BASE_DIR}${modelName}`; - return { - xml: path.join(modelDir, `${modelName}.xml`), - bin: path.join(modelDir, `${modelName}.bin`), - }; -} - -async function downloadTestModel(model) { - const modelsDir = './tests/unit/test_models'; - try { - const ifModelsDirectoryExists = await checkIfPathExists(modelsDir); - if (!ifModelsDirectoryExists) { - await fs.mkdir(modelDir); - } - - const modelPath = path.join(modelsDir, model.xml); - const modelExists = await checkIfPathExists(modelPath); - if (modelExists) return; + await fs.mkdir(destDir, { recursive: true }); - const { env } = process; - const proxyUrl = env.http_proxy || env.HTTP_PROXY || env.npm_config_proxy; + const fileList = await listFiles({ + repo, + fetch: fetch.default, + }); + const fileNames = []; + for await (const file of fileList) { + fileNames.push(file.path); + } - await downloadFile(model.xmlURL, modelsDir, model.xml, proxyUrl); - await downloadFile(model.binURL, modelsDir, model.bin, proxyUrl); - } catch(error) { - console.error(`Failed to download the model: ${error}.`); - throw error; + for (const path of fileNames) { + console.log(`Downloading file '${path}'`); + const response = await downloadFile({ + repo, + path, + fetch: fetch.default, + }); + const filename = `${destDir}/${path}`; + + await saveFile(filename, response); + console.log(`File '${path}' downloaded`); } + + console.log(`Model '${repo}' downloaded`); } -async function isModelAvailable(model) { - const baseArtifactsDir = './tests/unit/test_models'; - const modelPath = path.join(baseArtifactsDir, model.xml); - const modelExists = await checkIfPathExists(modelPath); - if (modelExists) return; +async function saveFile(file, response) { + const arrayBuffer = await response.arrayBuffer(); - console.log( - '\n\nTestModel cannot be found.\nPlease run `npm run test_setup`.\n\n', - ); - process.exit(1); + await fs.writeFile(file, Buffer.from(arrayBuffer)); } From 685344611228f11dca98194c11ae5eae10fcef87 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 20 Nov 2024 15:01:14 +0100 Subject: [PATCH 06/74] Set openvino_tokenizers on latest master --- thirdparty/openvino_tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers index 306dcd8dae..aeb1bc2cb1 160000 --- a/thirdparty/openvino_tokenizers +++ b/thirdparty/openvino_tokenizers @@ -1 +1 @@ -Subproject commit 306dcd8daec36bbc680c50c68de1e954f42b0ab8 +Subproject commit aeb1bc2cb1646650d6ff0fa258ca775bd20796d6 From ae806da5d252e90d1fc72076194bac1dbe59e7e7 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 15:55:04 +0100 Subject: [PATCH 07/74] Temporary disable most of gha wf to debug --- .github/workflows/assign_issue.yml | 10 ++++----- .github/workflows/bandit.yml | 10 ++++----- .github/workflows/causal_lm_cpp.yml | 16 +++++++------- .github/workflows/job_vlm_sample_llava.yml | 18 +++++++-------- .github/workflows/labeler.yml | 4 ++-- .github/workflows/lcm_dreamshaper_cpp.yml | 22 +++++++++---------- .github/workflows/llm_bench-python.yml | 22 +++++++++---------- .github/workflows/mac.yml | 16 +++++++------- .../workflows/stable_diffusion_1_5_cpp.yml | 14 ++++++------ .github/workflows/windows.yml | 16 +++++++------- 10 files changed, 74 insertions(+), 74 deletions(-) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index 4a4579e2c7..68bfb6cf2d 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -1,10 +1,10 @@ name: Take Issue -on: - issue_comment: - types: - - created - - edited +# on: +# issue_comment: +# types: +# - created +# - edited permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index fce770d101..356e66e6f7 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,9 +1,9 @@ name: python -m bandit --recursive --configfile bandit.yml . -on: - pull_request: - paths-ignore: - - 'thirdparty' - - '**.md' +# on: +# pull_request: +# paths-ignore: +# - 'thirdparty' +# - '**.md' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions jobs: bandit: diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml index c75ac3214c..9d1b5564b5 100644 --- a/.github/workflows/causal_lm_cpp.yml +++ b/.github/workflows/causal_lm_cpp.yml @@ -1,12 +1,12 @@ name: causal_lm_cpp -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/job_vlm_sample_llava.yml b/.github/workflows/job_vlm_sample_llava.yml index 7394464026..5d1ad4f59b 100644 --- a/.github/workflows/job_vlm_sample_llava.yml +++ b/.github/workflows/job_vlm_sample_llava.yml @@ -1,14 +1,14 @@ name: visual_language_chat sample - LLaVA -on: - workflow_call: - inputs: - model_id: - required: true - type: string - model_dir: - required: true - type: string +# on: +# workflow_call: +# inputs: +# model_id: +# required: true +# type: string +# model_dir: +# required: true +# type: string env: l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 102dee120b..1e17202ec0 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,6 @@ name: "Pull Request Labeler" -on: -- pull_request_target +# on: +# - pull_request_target permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml index 6bd25cbdfe..f6c903187a 100644 --- a/.github/workflows/lcm_dreamshaper_cpp.yml +++ b/.github/workflows/lcm_dreamshaper_cpp.yml @@ -1,13 +1,13 @@ name: lcm_dreamshaper -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -46,7 +46,7 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Build app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh @@ -106,10 +106,10 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Create virtual environment run: python -m venv openvino_lcm_cpp - + - name: Build app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" diff --git a/.github/workflows/llm_bench-python.yml b/.github/workflows/llm_bench-python.yml index 0ac47d1aa0..02e836349f 100644 --- a/.github/workflows/llm_bench-python.yml +++ b/.github/workflows/llm_bench-python.yml @@ -3,17 +3,17 @@ name: llm_bench Python Test -on: - push: - branches: [ "master" ] - paths: - - tools/llm_bench/** - - tools/who_what_benchmark/** - pull_request: - paths: - - tools/llm_bench/** - - tools/who_what_benchmark/** - - .github/workflows/llm_bench-python.yml +# on: +# push: +# branches: [ "master" ] +# paths: +# - tools/llm_bench/** +# - tools/who_what_benchmark/** +# pull_request: +# paths: +# - tools/llm_bench/** +# - tools/who_what_benchmark/** +# - .github/workflows/llm_bench-python.yml permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 935d6556b3..09e14f2b20 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,12 +1,12 @@ name: macOS (12, Python 3.9) -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index f36ac43839..0c4212bfa6 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' + # workflow_dispatch: + # pull_request: + # merge_group: + # push: + # branches: + # - master + # - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 1e4164aa0b..f48e80413e 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,12 +1,12 @@ name: Windows (VS 2019, Python 3.11) -on: - workflow_dispatch: - pull_request: - merge_group: - push: - branches: - - master - - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions From 8632882eaa4d305b7fc05938c72f5e04e507cde8 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 15:55:20 +0100 Subject: [PATCH 08/74] Minify linux wf --- .github/workflows/linux.yml | 458 ++++++++++++++++++------------------ 1 file changed, 229 insertions(+), 229 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3c3e0347e7..4b3914cdb5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -182,183 +182,183 @@ jobs: path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' - genai_python_lib: - name: OpenVINO genai extension (cmake + wheel) - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 120 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-16-cores - env: - CMAKE_GENERATOR: Unix Makefiles - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - # Should save cache only if run in the master branch of the base repo - # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - cmake --build ./build/ --config Release -j - - - name: Test bindings - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template - env: - PYTHONPATH: "./build/:$PYTHONPATH" - - - name: Test bindings (wheel) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" - - - run: > - source ${OV_INSTALL_DIR}/setupvars.sh - && python -m pytest -v ./tests/python_tests/test_vlm_api.py - - genai_python_lib_whisper: - name: OpenVINO genai extension whisper tests (cmake + wheel) - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 90 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-16-cores - env: - CMAKE_GENERATOR: Unix Makefiles - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - # Should save cache only if run in the master branch of the base repo - # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - cmake --build ./build/ --config Release --target py_openvino_genai -j - - - name: Test bindings - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke - env: - PYTHONPATH: "./build/:$PYTHONPATH" - - - name: Test bindings (wheel) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" + # genai_python_lib: + # name: OpenVINO genai extension (cmake + wheel) + # needs: [ openvino_download, openvino_build ] + # if: | + # always() && + # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + # timeout-minutes: 120 + # defaults: + # run: + # shell: bash + # runs-on: ubuntu-20.04-16-cores + # env: + # CMAKE_GENERATOR: Unix Makefiles + # CMAKE_BUILD_PARALLEL_LEVEL: null + # OV_INSTALL_DIR: ${{ github.workspace }}/ov + # CCACHE_DIR: ${{ github.workspace }}/ccache + # CCACHE_MAXSIZE: 500Mi + # CMAKE_CXX_COMPILER_LAUNCHER: ccache + # CMAKE_C_COMPILER_LAUNCHER: ccache + + # steps: + # - name: Clone openvino.genai + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' + + # - name: Download OpenVINO package + # uses: actions/download-artifact@v4 + # with: + # name: openvino_package + # path: ${{ env.OV_INSTALL_DIR }} + + # - name: Extract OpenVINO packages + # run: | + # pushd ${OV_INSTALL_DIR} + # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + # popd + + # - name: Set apt + # run: | + # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + # - name: Install build dependencies + # run: | + # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + # sudo apt-get install ccache + + # - name: Setup ccache + # uses: actions/cache@v4 + # with: + # # Should save cache only if run in the master branch of the base repo + # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + # path: ${{ env.CCACHE_DIR }} + # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + # - name: Build genai + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + # cmake --build ./build/ --config Release -j + + # - name: Test bindings + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + # python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template + # env: + # PYTHONPATH: "./build/:$PYTHONPATH" + + # - name: Test bindings (wheel) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + # python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" + + # - run: > + # source ${OV_INSTALL_DIR}/setupvars.sh + # && python -m pytest -v ./tests/python_tests/test_vlm_api.py + + # genai_python_lib_whisper: + # name: OpenVINO genai extension whisper tests (cmake + wheel) + # needs: [ openvino_download, openvino_build ] + # if: | + # always() && + # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + # timeout-minutes: 90 + # defaults: + # run: + # shell: bash + # runs-on: ubuntu-20.04-16-cores + # env: + # CMAKE_GENERATOR: Unix Makefiles + # CMAKE_BUILD_PARALLEL_LEVEL: null + # OV_INSTALL_DIR: ${{ github.workspace }}/ov + # CCACHE_DIR: ${{ github.workspace }}/ccache + # CCACHE_MAXSIZE: 500Mi + # CMAKE_CXX_COMPILER_LAUNCHER: ccache + # CMAKE_C_COMPILER_LAUNCHER: ccache + + # steps: + # - name: Clone openvino.genai + # uses: actions/checkout@v4 + # with: + # submodules: recursive + + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' + + # - name: Download OpenVINO package + # uses: actions/download-artifact@v4 + # with: + # name: openvino_package + # path: ${{ env.OV_INSTALL_DIR }} + + # - name: Extract OpenVINO packages + # run: | + # pushd ${OV_INSTALL_DIR} + # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + # popd + + # - name: Set apt + # run: | + # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + # - name: Install build dependencies + # run: | + # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + # sudo apt-get install ccache + + # - name: Setup ccache + # uses: actions/cache@v4 + # with: + # # Should save cache only if run in the master branch of the base repo + # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + # path: ${{ env.CCACHE_DIR }} + # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + # restore-keys: | + # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + # - name: Build genai + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + # cmake --build ./build/ --config Release --target py_openvino_genai -j + + # - name: Test bindings + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke + # env: + # PYTHONPATH: "./build/:$PYTHONPATH" + + # - name: Test bindings (wheel) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_package: name: OpenVINO genai extension (install to OpenVINO package) @@ -388,11 +388,11 @@ jobs: with: submodules: recursive - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' + # - name: Setup Python ${{ env.PYTHON_VERSION }} + # uses: actions/setup-python@v5 + # with: + # python-version: ${{ env.PYTHON_VERSION }} + # cache: 'pip' - name: Download OpenVINO package uses: actions/download-artifact@v4 @@ -433,53 +433,53 @@ jobs: cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j - - name: Build and Install dependencies - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels - optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny - - - name: Install samples - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} - - - name: Build samples (Release) - if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build - run: | - ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace - - - name: Build samples (Debug) - if: ${{ 'Release' != matrix.build-type }} - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j - cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace - - - name: Test C++ samples (greedy_causal_lm) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" - - - name: Test C++ samples (whisper_speech_recognition) - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav - timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav - - - name: Test python samples (multinomial_causal_lm) - if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 - - - name: Test python samples (whisper_speech_recognition) - if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + # - name: Build and Install dependencies + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels + # python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + # optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 + # optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny + + # - name: Install samples + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} + + # - name: Build samples (Release) + # if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build + # run: | + # ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace + + # - name: Build samples (Debug) + # if: ${{ 'Release' != matrix.build-type }} + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j + # cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace + + # - name: Test C++ samples (greedy_causal_lm) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" + + # - name: Test C++ samples (whisper_speech_recognition) + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav + # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav + + # - name: Test python samples (multinomial_causal_lm) + # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 + + # - name: Test python samples (whisper_speech_recognition) + # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + # run: | + # source ${OV_INSTALL_DIR}/setupvars.sh + # timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav Overall_Status: name: ci/gha_overall_status_linux From 9ceb3b598a1061c6cb17351d101b9baed3f77bef Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:02:26 +0100 Subject: [PATCH 09/74] Fix wf file --- .github/workflows/stable_diffusion_1_5_cpp.yml | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index 0c4212bfa6..f9357902dc 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp -on: - # workflow_dispatch: - # pull_request: - # merge_group: - # push: - # branches: - # - master - # - 'releases/**' +# on: +# workflow_dispatch: +# pull_request: +# merge_group: +# push: +# branches: +# - master +# - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions From 599713b25d0b4afad4cf0313ccd0f8d9e711849b Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:05:36 +0100 Subject: [PATCH 10/74] Fix wf file --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4b3914cdb5..d68b07ea34 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -483,7 +483,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] + needs: [openvino_download, openvino_build, genai_package] if: ${{ always() }} runs-on: ubuntu-latest steps: From 222940d59df23fd3235512c9eaa8dd3a121891a9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 16:25:28 +0100 Subject: [PATCH 11/74] Set cpack generator as NPM --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d68b07ea34..3a720c8f41 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -133,7 +133,7 @@ jobs: -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ + -DCPACK_GENERATOR=NPM \ -DENABLE_JS=OFF \ -DENABLE_SAMPLES=ON \ -DENABLE_OV_ONNX_FRONTEND=OFF \ From 901dd90e4e530e5293d0353da73e959fc32ec3d9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 17:31:50 +0100 Subject: [PATCH 12/74] Set ENABLE_JS on --- .github/workflows/linux.yml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3a720c8f41..41e98a77ce 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -133,8 +133,8 @@ jobs: -DENABLE_STRICT_DEPENDENCIES=OFF \ -DENABLE_SYSTEM_OPENCL=ON \ -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=NPM \ - -DENABLE_JS=OFF \ + -DCPACK_GENERATOR=TGZ \ + -DENABLE_JS=ON \ -DENABLE_SAMPLES=ON \ -DENABLE_OV_ONNX_FRONTEND=OFF \ -DENABLE_OV_PADDLE_FRONTEND=OFF \ @@ -430,7 +430,7 @@ jobs: - name: Build genai run: | source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ./ -B ./build/ cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j # - name: Build and Install dependencies From 09356efaf528ac608d8acfdf43f2bfd79432dade Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 26 Nov 2024 17:44:59 +0100 Subject: [PATCH 13/74] Add genai_node_addon component if ENABLE_JS is ON --- CMakeLists.txt | 3 +++ 1 file changed, 3 insertions(+) diff --git a/CMakeLists.txt b/CMakeLists.txt index 35ca895abc..5ffe7fbdf1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -102,6 +102,9 @@ set(CPACK_COMPONENTS_ALL core_genai core_genai_dev cpp_samples_genai licensing_g if(ENABLE_PYTHON) list(APPEND CPACK_COMPONENTS_ALL pygenai_${Python3_VERSION_MAJOR}_${Python3_VERSION_MINOR}) endif() +if(ENABLE_JS) + list(APPEND CPACK_COMPONENTS_ALL genai_node_addon) +endif() if(WIN32 AND NOT DEFINED CPACK_GENERATOR) set(CPACK_GENERATOR "ZIP") endif() From 96d53b2aed1bb978f2ebbf82b63a4ab75c9b244f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 14:04:09 +0100 Subject: [PATCH 14/74] Add wf to check genai js bindings compilation --- .github/workflows/linux.yml | 388 ++---------------------------------- 1 file changed, 17 insertions(+), 371 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 41e98a77ce..00cbd187aa 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -51,8 +51,8 @@ jobs: path: openvino_package.tar.gz if-no-files-found: 'error' - openvino_build: - name: Build OpenVINO package + genai_nodejs_bindings: + name: Produce genai nodejs binaries archive needs: [openvino_download] if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 @@ -88,30 +88,11 @@ jobs: submodules: 'true' ref: ${{ env.OV_BRANCH}} - # - # Dependencies - # - - name: Install build dependencies run: | sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache - - name: Setup Python ${{ env.PYTHON_VERSION }} - uses: actions/setup-python@v5 - with: - python-version: ${{ env.PYTHON_VERSION }} - cache: 'pip' - - - name: Install python dependencies - run: | - # For Python API: build and wheel packaging - python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt - - # - # Build - # - - name: Setup ccache uses: actions/cache@v4 with: @@ -119,371 +100,36 @@ jobs: # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-ov + ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} - - name: CMake configure - OpenVINO + - name: Build openvino + genai run: | - cmake \ - -G "${{ env.CMAKE_GENERATOR }}" \ - -DENABLE_CPPLINT=OFF \ - -DENABLE_NCC_STYLE=OFF \ - -DENABLE_TESTS=OFF \ - -DENABLE_STRICT_DEPENDENCIES=OFF \ - -DENABLE_SYSTEM_OPENCL=ON \ - -DCMAKE_VERBOSE_MAKEFILE=ON \ - -DCPACK_GENERATOR=TGZ \ - -DENABLE_JS=ON \ - -DENABLE_SAMPLES=ON \ - -DENABLE_OV_ONNX_FRONTEND=OFF \ - -DENABLE_OV_PADDLE_FRONTEND=OFF \ - -DENABLE_OV_PYTORCH_FRONTEND=ON \ - -DENABLE_OV_TF_FRONTEND=ON \ - -DENABLE_OV_TF_LITE_FRONTEND=OFF \ - -DENABLE_INTEL_GPU=OFF \ - -DENABLE_INTEL_NPU=ON \ - -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ - -DCMAKE_C_COMPILER_LAUNCHER=ccache \ - -DENABLE_PYTHON=ON \ - -DENABLE_WHEEL=ON \ - -S ${OPENVINO_REPO} \ - -B ${BUILD_DIR} - - - name: Clean ccache stats - run: ccache --zero-stats --show-config - - - name: Cmake build - OpenVINO - run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} - - - name: Show ccache stats - run: ccache --show-stats - - - name: Cmake install - OpenVINO - run: | - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake - cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake - - - name: Pack Artifacts - run: | - pushd ${INSTALL_DIR} - tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * - popd + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_WHEEL=OFF \ + -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ${OPENVINO_REPO} -B ./build + cmake --build ./build/ --target package -j # # Upload build artifacts and logs # - - name: Upload openvino package + - name: Upload genai nodejs bindings archive if: ${{ always() }} uses: actions/upload-artifact@v4 with: - name: openvino_package - path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' - # genai_python_lib: - # name: OpenVINO genai extension (cmake + wheel) - # needs: [ openvino_download, openvino_build ] - # if: | - # always() && - # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - # timeout-minutes: 120 - # defaults: - # run: - # shell: bash - # runs-on: ubuntu-20.04-16-cores - # env: - # CMAKE_GENERATOR: Unix Makefiles - # CMAKE_BUILD_PARALLEL_LEVEL: null - # OV_INSTALL_DIR: ${{ github.workspace }}/ov - # CCACHE_DIR: ${{ github.workspace }}/ccache - # CCACHE_MAXSIZE: 500Mi - # CMAKE_CXX_COMPILER_LAUNCHER: ccache - # CMAKE_C_COMPILER_LAUNCHER: ccache - - # steps: - # - name: Clone openvino.genai - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - # - name: Download OpenVINO package - # uses: actions/download-artifact@v4 - # with: - # name: openvino_package - # path: ${{ env.OV_INSTALL_DIR }} - - # - name: Extract OpenVINO packages - # run: | - # pushd ${OV_INSTALL_DIR} - # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - # popd - - # - name: Set apt - # run: | - # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - # - name: Install build dependencies - # run: | - # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - # sudo apt-get install ccache - - # - name: Setup ccache - # uses: actions/cache@v4 - # with: - # # Should save cache only if run in the master branch of the base repo - # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - # path: ${{ env.CCACHE_DIR }} - # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - # - name: Build genai - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - # cmake --build ./build/ --config Release -j - - # - name: Test bindings - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - # python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template - # env: - # PYTHONPATH: "./build/:$PYTHONPATH" - - # - name: Test bindings (wheel) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - # python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" - - # - run: > - # source ${OV_INSTALL_DIR}/setupvars.sh - # && python -m pytest -v ./tests/python_tests/test_vlm_api.py - - # genai_python_lib_whisper: - # name: OpenVINO genai extension whisper tests (cmake + wheel) - # needs: [ openvino_download, openvino_build ] - # if: | - # always() && - # (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - # timeout-minutes: 90 - # defaults: - # run: - # shell: bash - # runs-on: ubuntu-20.04-16-cores - # env: - # CMAKE_GENERATOR: Unix Makefiles - # CMAKE_BUILD_PARALLEL_LEVEL: null - # OV_INSTALL_DIR: ${{ github.workspace }}/ov - # CCACHE_DIR: ${{ github.workspace }}/ccache - # CCACHE_MAXSIZE: 500Mi - # CMAKE_CXX_COMPILER_LAUNCHER: ccache - # CMAKE_C_COMPILER_LAUNCHER: ccache - - # steps: - # - name: Clone openvino.genai - # uses: actions/checkout@v4 - # with: - # submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - # - name: Download OpenVINO package - # uses: actions/download-artifact@v4 - # with: - # name: openvino_package - # path: ${{ env.OV_INSTALL_DIR }} - - # - name: Extract OpenVINO packages - # run: | - # pushd ${OV_INSTALL_DIR} - # tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - # popd - - # - name: Set apt - # run: | - # echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - # echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - # echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - # echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - # - name: Install build dependencies - # run: | - # sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - # sudo apt-get install ccache - - # - name: Setup ccache - # uses: actions/cache@v4 - # with: - # # Should save cache only if run in the master branch of the base repo - # # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - # save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - # path: ${{ env.CCACHE_DIR }} - # key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} - # restore-keys: | - # ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release - - # - name: Build genai - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ - # cmake --build ./build/ --config Release --target py_openvino_genai -j - - # - name: Test bindings - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager - # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke - # env: - # PYTHONPATH: "./build/:$PYTHONPATH" - - # - name: Test bindings (wheel) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels - # python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" - - genai_package: - name: OpenVINO genai extension (install to OpenVINO package) - strategy: - matrix: - build-type: [Release, Debug] - needs: [ openvino_download, openvino_build ] - if: | - always() && - (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') - timeout-minutes: 60 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - env: - CMAKE_BUILD_PARALLEL_LEVEL: null - OV_INSTALL_DIR: ${{ github.workspace }}/ov - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 500Mi - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - - steps: - - name: Clone openvino.genai - uses: actions/checkout@v4 - with: - submodules: recursive - - # - name: Setup Python ${{ env.PYTHON_VERSION }} - # uses: actions/setup-python@v5 - # with: - # python-version: ${{ env.PYTHON_VERSION }} - # cache: 'pip' - - - name: Download OpenVINO package - uses: actions/download-artifact@v4 - with: - name: openvino_package - path: ${{ env.OV_INSTALL_DIR }} - - - name: Extract OpenVINO packages - run: | - pushd ${OV_INSTALL_DIR} - tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 - popd - - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null - - - name: Install build dependencies - run: | - sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh - sudo apt-get install ccache - - - name: Setup ccache - uses: actions/cache@v4 - with: - save-always: true - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }} - - - name: Build genai - run: | - source ${OV_INSTALL_DIR}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ./ -B ./build/ - cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j - - # - name: Build and Install dependencies - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - # python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels - # optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - # optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny - - # - name: Install samples - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} - - # - name: Build samples (Release) - # if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build - # run: | - # ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace - - # - name: Build samples (Debug) - # if: ${{ 'Release' != matrix.build-type }} - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j - # cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace - - # - name: Test C++ samples (greedy_causal_lm) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" - - # - name: Test C++ samples (whisper_speech_recognition) - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav - # timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav - - # - name: Test python samples (multinomial_causal_lm) - # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 - - # - name: Test python samples (whisper_speech_recognition) - # if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only - # run: | - # source ${OV_INSTALL_DIR}/setupvars.sh - # timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav - Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_package] + needs: [openvino_download, genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 785bf8d90973acaba18a94614c323dcef605ecf2 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 14:09:45 +0100 Subject: [PATCH 15/74] Remove setupvars.sh run --- .github/workflows/linux.yml | 1 - 1 file changed, 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 00cbd187aa..4fc5caf3cf 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -106,7 +106,6 @@ jobs: - name: Build openvino + genai run: | - source ${OV_INSTALL_DIR}/setupvars.sh cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ From e8c53f286c8a083a08c806bb4cbca61c49ada1b9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 15:37:00 +0100 Subject: [PATCH 16/74] Fix ov + genai workflow --- .github/workflows/linux.yml | 16 +++++++--------- 1 file changed, 7 insertions(+), 9 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 4fc5caf3cf..b334defe9b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -66,10 +66,9 @@ jobs: CMAKE_GENERATOR: 'Ninja Multi-Config' CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache - OPENVINO_REPO: ${{ github.workspace }}/openvino - INSTALL_DIR: ${{ github.workspace }}/openvino/install - BUILD_DIR: ${{ github.workspace }}/openvino/build - CCACHE_DIR: ${{ github.workspace }}/ccache + OPENVINO_REPO: ${{ github.workspace }}/../openvino + BUILD_DIR: ${{ github.workspace }}/../build + CCACHE_DIR: ${{ github.workspace }}/../ccache CCACHE_MAXSIZE: 2000Mi steps: @@ -106,13 +105,12 @@ jobs: - name: Build openvino + genai run: | - cmake -DOPENVINO_EXTRA_MODULES=./ -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cd .. && cmake -DOPENVINO_EXTRA_MODULES=openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ - -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ${OPENVINO_REPO} -B ./build - cmake --build ./build/ --target package -j + -S openvino -B ./build + cmake --build ./build --target package -j # # Upload build artifacts and logs @@ -123,7 +121,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: genai_nodejs_bindings - path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + path: ../build/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' Overall_Status: From 82ae75f4f68850676ae9e5dd1d9db773a26f7cc0 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 15:47:22 +0100 Subject: [PATCH 17/74] Fix ov + genai workflow --- .github/workflows/linux.yml | 19 +++++++++++++------ 1 file changed, 13 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index b334defe9b..ec6dab48da 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -66,9 +66,10 @@ jobs: CMAKE_GENERATOR: 'Ninja Multi-Config' CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache - OPENVINO_REPO: ${{ github.workspace }}/../openvino - BUILD_DIR: ${{ github.workspace }}/../build - CCACHE_DIR: ${{ github.workspace }}/../ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + GENAI_REPO: ${{ github.workspace }}/openvino.genai + BUILD_DIR: ${{ github.workspace }}/build + CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 2000Mi steps: @@ -92,6 +93,12 @@ jobs: sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache + - name: Clone GenAI + uses: actions/checkout@v4 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + - name: Setup ccache uses: actions/cache@v4 with: @@ -105,11 +112,11 @@ jobs: - name: Build openvino + genai run: | - cd .. && cmake -DOPENVINO_EXTRA_MODULES=openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=./openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S openvino -B ./build + -S ./openvino -B ./build cmake --build ./build --target package -j # @@ -121,7 +128,7 @@ jobs: uses: actions/upload-artifact@v4 with: name: genai_nodejs_bindings - path: ../build/genai_nodejs_bindings.tar.gz + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' Overall_Status: From 963fc1d3446fe0ccd206cbe7474e615c83fe7024 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 27 Nov 2024 17:43:01 +0100 Subject: [PATCH 18/74] Fix path to genai folder --- .github/workflows/linux.yml | 44 +++++++++---------------------------- 1 file changed, 10 insertions(+), 34 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index ec6dab48da..9ccd38ed9f 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,44 +17,13 @@ concurrency: env: PYTHON_VERSION: '3.9' - OV_BRANCH: 0080d90974ca84f9a6d359da3388a2a18a93b753 + OV_BRANCH: 1c9b23ccca1cca72e3f11f8cdad2e88fc7f7e633 OV_TARBALL: '' jobs: - openvino_download: - name: Download OpenVINO package - outputs: - status: ${{ steps.openvino_download.outcome }} - timeout-minutes: 10 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04 - - steps: - - name: Download OpenVINO build - id: openvino_download - run: | - wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz - tar -tvf openvino_package.tar.gz - continue-on-error: true - - # - # Upload to artifacts - # - - - name: Upload openvino package - if: steps.openvino_download.outcome == 'success' - uses: actions/upload-artifact@v4 - with: - name: openvino_package - path: openvino_package.tar.gz - if-no-files-found: 'error' genai_nodejs_bindings: name: Produce genai nodejs binaries archive - needs: [openvino_download] - if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -99,6 +68,12 @@ jobs: path: ${{ env.GENAI_REPO }} submodules: recursive + - name: Check GenAI repo contents + run: | + ls -la ${{ env.GENAI_REPO }}/src/js + cd ${{ env.GENAI_REPO }} + git log + - name: Setup ccache uses: actions/cache@v4 with: @@ -112,8 +87,9 @@ jobs: - name: Build openvino + genai run: | - cmake -DOPENVINO_EXTRA_MODULES=./openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ -S ./openvino -B ./build @@ -133,7 +109,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, genai_nodejs_bindings] + needs: [genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 1d952b6d7d8d8c34a831e786627c1d84db752476 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 28 Nov 2024 10:29:32 +0100 Subject: [PATCH 19/74] Change OV branch hash --- .github/workflows/linux.yml | 8 +------- 1 file changed, 1 insertion(+), 7 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9ccd38ed9f..3d7b1fd751 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -17,7 +17,7 @@ concurrency: env: PYTHON_VERSION: '3.9' - OV_BRANCH: 1c9b23ccca1cca72e3f11f8cdad2e88fc7f7e633 + OV_BRANCH: 0080d90974ca84f9a6d359da3388a2a18a93b753 OV_TARBALL: '' jobs: @@ -68,12 +68,6 @@ jobs: path: ${{ env.GENAI_REPO }} submodules: recursive - - name: Check GenAI repo contents - run: | - ls -la ${{ env.GENAI_REPO }}/src/js - cd ${{ env.GENAI_REPO }} - git log - - name: Setup ccache uses: actions/cache@v4 with: From e909f03ecfa371a10e74b07e3b81246f583a5029 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 14:48:20 +0100 Subject: [PATCH 20/74] Align openvino_tokenizers version with master --- thirdparty/openvino_tokenizers | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/thirdparty/openvino_tokenizers b/thirdparty/openvino_tokenizers index aeb1bc2cb1..904046825b 160000 --- a/thirdparty/openvino_tokenizers +++ b/thirdparty/openvino_tokenizers @@ -1 +1 @@ -Subproject commit aeb1bc2cb1646650d6ff0fa258ca775bd20796d6 +Subproject commit 904046825b6378bae74f16f302b40599aa88d5b3 From c5f5f85d7035c409dfe2478a522f7eb38996c33e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 14:49:06 +0100 Subject: [PATCH 21/74] Move rpath setting to libopenvino_genai part --- src/cpp/CMakeLists.txt | 13 +++++++++++++ src/js/CMakeLists.txt | 13 ------------- 2 files changed, 13 insertions(+), 13 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 308ac119f4..ef719ff1be 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -138,6 +138,19 @@ if(CPACK_GENERATOR STREQUAL "NPM") set(LIBRARY_DESTINATION .) set(ARCHIVE_DESTINATION .) set(RUNTIME_DESTINATION .) + + # setting RPATH / LC_RPATH depending on platform + if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") + elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") + endif() + + if(rpaths) + set_target_properties(${TARGET_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") + endif() else() set(LIBRARY_DESTINATION runtime/lib/${ARCH_DIR}) set(ARCHIVE_DESTINATION runtime/lib/${ARCH_DIR}) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index d7ea811618..275803a804 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -73,19 +73,6 @@ set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".node" ) -# setting RPATH / LC_RPATH depending on platform -if(LINUX) - # to find libopenvino_genai.so in the same folder - set(rpaths "$ORIGIN") -elseif(APPLE) - # to find libopenvino_genai.dylib in the same folder - set(rpaths "@loader_path") -endif() - -if(rpaths) - set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") -endif() - install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} From 6d7cbcd8722c3ce1df3b3f3c0bc84bd84ebb429e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:22:00 +0100 Subject: [PATCH 22/74] Add ENABLE_JS option to features.cmake --- .github/workflows/linux.yml | 2 +- cmake/features.cmake | 1 + 2 files changed, 2 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3d7b1fd751..7e87a0e992 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -82,7 +82,7 @@ jobs: - name: Build openvino + genai run: | cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ diff --git a/cmake/features.cmake b/cmake/features.cmake index 0434b21ee9..999ee1d81e 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -3,3 +3,4 @@ # option(ENABLE_PYTHON "Enable Python API build" ON) +option(ENABLE_JS "Enable JS API build" OFF) From d99956977f206660b9bf9f6ee9632f663a5a6b8a Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:29:30 +0100 Subject: [PATCH 23/74] Rollback worflow files --- .github/workflows/assign_issue.yml | 10 +- .github/workflows/bandit.yml | 10 +- .github/workflows/causal_lm_cpp.yml | 117 ++--- .github/workflows/job_vlm_sample_llava.yml | 20 +- .github/workflows/labeler.yml | 4 +- .github/workflows/lcm_dreamshaper_cpp.yml | 86 ++-- .github/workflows/linux.yml | 426 +++++++++++++++++- .github/workflows/llm_bench-python.yml | 51 +-- .github/workflows/mac.yml | 24 +- .../workflows/stable_diffusion_1_5_cpp.yml | 60 +-- .github/workflows/windows.yml | 26 +- 11 files changed, 629 insertions(+), 205 deletions(-) diff --git a/.github/workflows/assign_issue.yml b/.github/workflows/assign_issue.yml index 68bfb6cf2d..4a4579e2c7 100644 --- a/.github/workflows/assign_issue.yml +++ b/.github/workflows/assign_issue.yml @@ -1,10 +1,10 @@ name: Take Issue -# on: -# issue_comment: -# types: -# - created -# - edited +on: + issue_comment: + types: + - created + - edited permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/bandit.yml b/.github/workflows/bandit.yml index 356e66e6f7..fce770d101 100644 --- a/.github/workflows/bandit.yml +++ b/.github/workflows/bandit.yml @@ -1,9 +1,9 @@ name: python -m bandit --recursive --configfile bandit.yml . -# on: -# pull_request: -# paths-ignore: -# - 'thirdparty' -# - '**.md' +on: + pull_request: + paths-ignore: + - 'thirdparty' + - '**.md' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions jobs: bandit: diff --git a/.github/workflows/causal_lm_cpp.yml b/.github/workflows/causal_lm_cpp.yml index 9d1b5564b5..504e303fb5 100644 --- a/.github/workflows/causal_lm_cpp.yml +++ b/.github/workflows/causal_lm_cpp.yml @@ -1,12 +1,12 @@ name: causal_lm_cpp -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -16,10 +16,10 @@ concurrency: cancel-in-progress: true env: - l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241105_x86_64.tgz - l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz - m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/m_openvino_toolkit_macos_12_6_2025.0.0.dev20241105_x86_64.tgz - w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + l_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241205_x86_64.tgz + l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz + m_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/m_openvino_toolkit_macos_12_6_2025.0.0.dev20241205_x86_64.tgz + w_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip jobs: cpp-multinomial-greedy_causal_lm-ubuntu: runs-on: ubuntu-20.04-8-cores @@ -46,8 +46,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model openlm-research/open_llama_3b_v2 open_llama_3b_v2 optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T wget https://huggingface.co/smangrul/tinyllama_lora_sql/resolve/main/adapter_model.safetensors?download=true -O adapter_model.safetensors @@ -105,8 +105,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Compare env: @@ -118,7 +118,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('Why is the Sun yellow?', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -134,7 +134,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('69', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -150,7 +150,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('Hi', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -166,7 +166,7 @@ jobs: import transformers with open('pred.txt', 'r') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('return 0', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -182,7 +182,7 @@ jobs: import transformers with open('pred.txt', 'r', errors='ignore') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') tokenized = tokenizer('你好！ 你好嗎？', return_tensors='pt') for beam in transformers.LlamaForCausalLM.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0').generate(**tokenized, num_beam_groups=3, num_beams=15, num_return_sequences=15, diversity_penalty=1.0, max_new_tokens=20, early_stopping=False, length_penalty=1.0, no_repeat_ngram_size=9**9, do_sample=False): ref = ': ' + tokenizer.decode(beam[tokenized['input_ids'].numel():], skip_special_tokens=True) @@ -198,7 +198,7 @@ jobs: import transformers with open('pred.txt', 'r', errors='ignore') as file: predictions = file.read() - tokenizer = transformers.LlamaTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') + tokenizer = transformers.AutoTokenizer.from_pretrained('TinyLlama/TinyLlama-1.1B-Chat-v1.0') prompts = [ 'Alan Turing was a', 'return 0', @@ -241,8 +241,8 @@ jobs: - name: Download and convert model run: | call .\ov\setupvars.bat - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino -m TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T TinyLlama/TinyLlama-1.1B-intermediate-step-1431k-3T curl -o adapter_model.safetensors -s -L https://huggingface.co/smangrul/tinyllama_lora_sql/resolve/main/adapter_model.safetensors?download=true @@ -299,8 +299,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen-7B-Chat Qwen-7B-Chat - run: > . ./ov/setupvars.sh @@ -333,8 +333,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen1.5-7B-Chat Qwen1.5-7B-Chat - run: > . ./ov/setupvars.sh @@ -368,8 +368,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-2 phi-2 - run: > . ./ov/setupvars.sh @@ -403,8 +403,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model argilla/notus-7b-v1 notus-7b-v1 - run: > . ./ov/setupvars.sh @@ -438,8 +438,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-3b dolly-v2-3b optimum-cli export openvino --trust-remote-code --weight-format fp16 --model databricks/dolly-v2-7b dolly-v2-7b - name: run and compare @@ -488,8 +488,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --weight-format fp16 --model Qwen/Qwen-7B-Chat Qwen-7B-Chat --task text-generation-with-past - name: run and compare @@ -560,8 +560,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model microsoft/phi-1_5 phi-1_5 - name: Run Generation run: | @@ -591,7 +591,7 @@ jobs: PYTHONPATH: "./build" cpp-greedy_causal_lm-redpajama-3b-chat: - runs-on: ubuntu-20.04-4-cores + runs-on: ubuntu-20.04-8-cores defaults: run: shell: bash @@ -615,8 +615,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model ikala/redpajama-3b-chat redpajama-3b-chat - name: Run Generation run: | @@ -670,8 +670,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Compare env: @@ -681,9 +681,9 @@ jobs: printf 'What is 2 + 2?\nWhat is the previous answer?\nAdd 1 to it.\nSubtract 5 from it.\nWhy is the sun yellow?\nWhat was my first question?\n' > ./input.txt timeout 30s ./build/samples/cpp/chat_sample/chat_sample ./TinyLlama-1.1B-Chat-v1.0/ < input.txt > ./pred.txt python -c " - from transformers import LlamaTokenizer, AutoModelForCausalLM + from transformers import AutoTokenizer, AutoModelForCausalLM model_id = 'TinyLlama/TinyLlama-1.1B-Chat-v1.0' - tokenizer = LlamaTokenizer.from_pretrained(model_id) + tokenizer = AutoTokenizer.from_pretrained(model_id) model = AutoModelForCausalLM.from_pretrained(model_id) prompts = ['What is 2 + 2?', 'What is the previous answer?', 'Add 1 to it.', 'Subtract 5 from it.', 'Why is the sun yellow?', 'What was my first question?'] def gen_prompt(prompt): @@ -727,7 +727,7 @@ jobs: ov_link: ${{ env.l_u22_ov_link }} - uses: ./.github/actions/build_app with: - build_target: 'visual_language_chat py_openvino_genai' + build_target: 'visual_language_chat benchmark_vlm py_openvino_genai' - uses: ./.github/actions/install_python_deps - name: Download and convert tiny-random-minicpmv-2_6 model and an image run: | @@ -754,6 +754,12 @@ jobs: && ./build/samples/cpp/visual_language_chat/visual_language_chat ./tiny-random-minicpmv-2_6/ ./images/ <<< $'Describe the images?' | tee cpp.txt timeout-minutes: 2 + - name: Run benchmark_vlm C++ sample - tiny-random-minicpmv-2_6 + run: > + set -o pipefail + && source ./ov/setupvars.sh + && ./build/samples/cpp/visual_language_chat/benchmark_vlm -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3 + timeout-minutes: 2 - name: Run visual_language_chat Python sample - tiny-random-minicpmv-2_6 run: > set -o pipefail @@ -762,6 +768,13 @@ jobs: <<< $'Describe the images?' | tee py.txt env: PYTHONPATH: "./build/" + - name: Run benchmark_vlm Python sample - tiny-random-minicpmv-2_6 + run: > + set -o pipefail + && source ./ov/setupvars.sh + && ./samples/python/visual_language_chat/benchmark_vlm.py -m ./tiny-random-minicpmv-2_6/ -i ./images/cat.png -n 3 + env: + PYTHONPATH: "./build/" - name: Encode cpp.txt with Python encoding instead of terminal one shell: python run: | @@ -863,8 +876,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | @@ -909,8 +922,8 @@ jobs: - name: Download and convert and model run: | call .\ov\setupvars.bat - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | @@ -954,8 +967,8 @@ jobs: - name: Download and convert and model run: | source ./ov/setupvars.sh - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install -r ./samples/requirements.txt optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 - name: Run gtests run: | diff --git a/.github/workflows/job_vlm_sample_llava.yml b/.github/workflows/job_vlm_sample_llava.yml index 5d1ad4f59b..166284bd4b 100644 --- a/.github/workflows/job_vlm_sample_llava.yml +++ b/.github/workflows/job_vlm_sample_llava.yml @@ -1,17 +1,17 @@ name: visual_language_chat sample - LLaVA -# on: -# workflow_call: -# inputs: -# model_id: -# required: true -# type: string -# model_dir: -# required: true -# type: string +on: + workflow_call: + inputs: + model_id: + required: true + type: string + model_dir: + required: true + type: string env: - l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz + l_u22_ov_link: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz jobs: visual_language_chat_sample-ubuntu-llava: diff --git a/.github/workflows/labeler.yml b/.github/workflows/labeler.yml index 1e17202ec0..102dee120b 100644 --- a/.github/workflows/labeler.yml +++ b/.github/workflows/labeler.yml @@ -1,6 +1,6 @@ name: "Pull Request Labeler" -# on: -# - pull_request_target +on: +- pull_request_target permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions diff --git a/.github/workflows/lcm_dreamshaper_cpp.yml b/.github/workflows/lcm_dreamshaper_cpp.yml index f6c903187a..b3a36761e1 100644 --- a/.github/workflows/lcm_dreamshaper_cpp.yml +++ b/.github/workflows/lcm_dreamshaper_cpp.yml @@ -1,13 +1,13 @@ name: lcm_dreamshaper -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -18,8 +18,8 @@ concurrency: env: PYTHON_VERSION: '3.9' - LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241105_x86_64.tgz - WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu22_2025.0.0.dev20241205_x86_64.tgz + WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip OV_INSTALL_DIR: ${{ github.workspace }}/ov jobs: @@ -46,12 +46,12 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - - - name: Build app + + - name: Build apps run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python3 -m venv openvino_lcm_cpp @@ -59,27 +59,39 @@ jobs: - name: Install python dependencies run: | source openvino_lcm_cpp/bin/activate - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer run: | source openvino_lcm_cpp/bin/activate optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16 + wget -O ./image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png + wget -O ./mask_image.png https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png - - name: Run app + - name: Run heterogeneous_stable_diffusion run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + ${{ env.build_dir }}/samples/cpp/image_generation/heterogeneous_stable_diffusion ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" - - name: Run Python app + - name: Run heterogeneous_stable_diffusion.py run: | source openvino_lcm_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python ./samples/python/image_generation/heterogeneous_stable_diffusion.py ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} + - name: Run image2image + run: | + source ./ov/setupvars.sh + ${{ env.build_dir }}/samples/cpp/image_generation/image2image ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png + + - name: Run inpainting + run: | + source ./ov/setupvars.sh + ${{ env.build_dir }}/samples/cpp/image_generation/inpainting ./models/lcm_dreamshaper_v7/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" ./image.png ./mask_image.png + lcm_dreamshaper_v7_cpp-windows: runs-on: windows-2019 defaults: @@ -106,38 +118,58 @@ jobs: with: python-version: ${{ env.PYTHON_VERSION }} cache: 'pip' - + - name: Create virtual environment run: python -m venv openvino_lcm_cpp - - - name: Build app + + - name: Build apps run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion heterogeneous_stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Install python dependencies run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" optimum-cli export openvino --model SimianLuo/LCM_Dreamshaper_v7 --task stable-diffusion --weight-format fp16 models/lcm_dreamshaper_v7/FP16 + Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo.png' -OutFile 'image.png' + Invoke-WebRequest -Uri 'https://raw.githubusercontent.com/CompVis/latent-diffusion/main/data/inpainting_examples/overture-creations-5sI6fQgYIuo_mask.png' -OutFile 'mask_image.png' - - name: Run app + - name: Run heterogeneous_stable_diffusion run: > . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - & "${{ env.build_dir }}/samples/cpp/text2image/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + & "${{ env.build_dir }}/samples/cpp/image_generation/Release/heterogeneous_stable_diffusion.exe ./models/lcm_dreamshaper_v7/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + + - name: Run heterogeneous_stable_diffusion.py + run: | + . "./openvino_lcm_cpp/Scripts/Activate.ps1" + . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" + $env:Path += "${{ env.build_dir }}\openvino_genai" + python .\samples\python\image_generation\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + env: + PYTHONPATH: ${{ env.build_dir }} + + - name: Run image2image.py + run: | + . "./openvino_lcm_cpp/Scripts/Activate.ps1" + . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" + $env:Path += "${{ env.build_dir }}\openvino_genai" + python .\samples\python\image_generation\image2image.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png + env: + PYTHONPATH: ${{ env.build_dir }} - - name: Run Python app + - name: Run inpainting.py run: | . "./openvino_lcm_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\heterogeneous_stable_diffusion.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python .\samples\python\image_generation\inpainting.py .\models\lcm_dreamshaper_v7\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" .\image.png .\mask_image.png env: PYTHONPATH: ${{ env.build_dir }} diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 7e87a0e992..2916c340bf 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -21,9 +21,40 @@ env: OV_TARBALL: '' jobs: + openvino_download: + name: Download OpenVINO package + outputs: + status: ${{ steps.openvino_download.outcome }} + timeout-minutes: 10 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + + steps: + - name: Download OpenVINO build + id: openvino_download + run: | + wget ${{ env.OV_TARBALL}} --progress=bar:force:noscroll -O openvino_package.tar.gz + tar -tvf openvino_package.tar.gz + continue-on-error: true - genai_nodejs_bindings: - name: Produce genai nodejs binaries archive + # + # Upload to artifacts + # + + - name: Upload openvino package + if: steps.openvino_download.outcome == 'success' + uses: actions/upload-artifact@v4 + with: + name: openvino_package + path: openvino_package.tar.gz + if-no-files-found: 'error' + + openvino_build: + name: Build OpenVINO package + needs: [openvino_download] + if: needs.openvino_download.outputs.status != 'success' timeout-minutes: 150 defaults: run: @@ -36,8 +67,8 @@ jobs: CMAKE_CXX_COMPILER_LAUNCHER: ccache CMAKE_C_COMPILER_LAUNCHER: ccache OPENVINO_REPO: ${{ github.workspace }}/openvino - GENAI_REPO: ${{ github.workspace }}/openvino.genai - BUILD_DIR: ${{ github.workspace }}/build + INSTALL_DIR: ${{ github.workspace }}/openvino/install + BUILD_DIR: ${{ github.workspace }}/openvino/build CCACHE_DIR: ${{ github.workspace }}/ccache CCACHE_MAXSIZE: 2000Mi @@ -57,16 +88,29 @@ jobs: submodules: 'true' ref: ${{ env.OV_BRANCH}} + # + # Dependencies + # + - name: Install build dependencies run: | sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh sudo apt-get install ccache - - name: Clone GenAI - uses: actions/checkout@v4 + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 with: - path: ${{ env.GENAI_REPO }} - submodules: recursive + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Install python dependencies + run: | + # For Python API: build and wheel packaging + python3 -m pip install -r ${OPENVINO_REPO}/src/bindings/python/wheel/requirements-dev.txt + + # + # Build + # - name: Setup ccache uses: actions/cache@v4 @@ -75,35 +119,373 @@ jobs: # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-${{ github.sha }} restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} + ${{ runner.os }}-${{ runner.arch }}-ccache-ov - - name: Build openvino + genai + - name: CMake configure - OpenVINO run: | - cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ - -DENABLE_PYTHON=OFF \ - -DENABLE_WHEEL=OFF \ - -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ./openvino -B ./build - cmake --build ./build --target package -j + cmake \ + -G "${{ env.CMAKE_GENERATOR }}" \ + -DENABLE_CPPLINT=OFF \ + -DENABLE_NCC_STYLE=OFF \ + -DENABLE_TESTS=OFF \ + -DENABLE_STRICT_DEPENDENCIES=OFF \ + -DENABLE_SYSTEM_OPENCL=ON \ + -DCMAKE_VERBOSE_MAKEFILE=ON \ + -DCPACK_GENERATOR=TGZ \ + -DENABLE_JS=OFF \ + -DENABLE_SAMPLES=ON \ + -DENABLE_OV_ONNX_FRONTEND=OFF \ + -DENABLE_OV_PADDLE_FRONTEND=OFF \ + -DENABLE_OV_PYTORCH_FRONTEND=ON \ + -DENABLE_OV_TF_FRONTEND=ON \ + -DENABLE_OV_TF_LITE_FRONTEND=OFF \ + -DENABLE_INTEL_GPU=OFF \ + -DENABLE_INTEL_NPU=ON \ + -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \ + -DCMAKE_C_COMPILER_LAUNCHER=ccache \ + -DENABLE_PYTHON=ON \ + -DENABLE_WHEEL=ON \ + -S ${OPENVINO_REPO} \ + -B ${BUILD_DIR} + + - name: Clean ccache stats + run: ccache --zero-stats --show-config + + - name: Cmake build - OpenVINO + run: cmake --build ${BUILD_DIR} --parallel --config ${{ env.CMAKE_BUILD_TYPE }} + + - name: Show ccache stats + run: ccache --show-stats + + - name: Cmake install - OpenVINO + run: | + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -P ${BUILD_DIR}/cmake_install.cmake + cmake -DCMAKE_INSTALL_PREFIX=${INSTALL_DIR}/openvino_package -DCOMPONENT=python_wheels -P ${BUILD_DIR}/cmake_install.cmake + + - name: Pack Artifacts + run: | + pushd ${INSTALL_DIR} + tar -czvf ${BUILD_DIR}/openvino_package.tar.gz * + popd # # Upload build artifacts and logs # - - name: Upload genai nodejs bindings archive + - name: Upload openvino package if: ${{ always() }} uses: actions/upload-artifact@v4 with: - name: genai_nodejs_bindings - path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + name: openvino_package + path: ${{ env.BUILD_DIR }}/openvino_package.tar.gz if-no-files-found: 'error' + genai_python_lib: + name: OpenVINO genai extension (cmake + wheel) + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 120 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + CMAKE_GENERATOR: Unix Makefiles + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + cmake --build ./build/ --config Release -j + + - name: Test bindings + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template + env: + PYTHONPATH: "./build/:$PYTHONPATH" + + - name: Test bindings (wheel) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" + + - run: > + source ${OV_INSTALL_DIR}/setupvars.sh + && python -m pytest -v ./tests/python_tests/test_vlm_api.py + + genai_python_lib_whisper: + name: OpenVINO genai extension whisper tests (cmake + wheel) + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 90 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + CMAKE_GENERATOR: Unix Makefiles + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-release + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ./build/ + cmake --build ./build/ --config Release --target py_openvino_genai -j + + - name: Test bindings + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke + env: + PYTHONPATH: "./build/:$PYTHONPATH" + + - name: Test bindings (wheel) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels + python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" + + genai_package: + name: OpenVINO genai extension (install to OpenVINO package) + strategy: + matrix: + build-type: [Release, Debug] + needs: [ openvino_download, openvino_build ] + if: | + always() && + (needs.openvino_download.outputs.status == 'success' || needs.openvino_build.result == 'success') + timeout-minutes: 60 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04 + env: + CMAKE_BUILD_PARALLEL_LEVEL: null + OV_INSTALL_DIR: ${{ github.workspace }}/ov + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 500Mi + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + + steps: + - name: Clone openvino.genai + uses: actions/checkout@v4 + with: + submodules: recursive + + - name: Setup Python ${{ env.PYTHON_VERSION }} + uses: actions/setup-python@v5 + with: + python-version: ${{ env.PYTHON_VERSION }} + cache: 'pip' + + - name: Download OpenVINO package + uses: actions/download-artifact@v4 + with: + name: openvino_package + path: ${{ env.OV_INSTALL_DIR }} + + - name: Extract OpenVINO packages + run: | + pushd ${OV_INSTALL_DIR} + tar -xzf openvino_package.tar.gz -C ${OV_INSTALL_DIR} --strip-components=1 + popd + + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Install build dependencies + run: | + sudo ${OV_INSTALL_DIR}/install_dependencies/install_openvino_dependencies.sh + sudo apt-get install ccache + + - name: Setup ccache + uses: actions/cache@v4 + with: + save-always: true + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-genai-${{ matrix.build-type }} + + - name: Build genai + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ./ -B ./build/ + cmake --build ./build/ --config ${{ matrix.build-type }} --target package -j + + - name: Build and Install dependencies + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 + optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny + + - name: Install samples + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake --install ./build/ --config ${{ matrix.build-type }} --prefix ${OV_INSTALL_DIR} + + - name: Build samples (Release) + if: ${{ 'Release' == matrix.build-type }} # build_samples enforces Release build + run: | + ${OV_INSTALL_DIR}/samples/cpp/build_samples.sh -i ${{ github.workspace }}/s\ pace + + - name: Build samples (Debug) + if: ${{ 'Release' != matrix.build-type }} + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -S ${OV_INSTALL_DIR}/samples/cpp/ -B ./samples\ build/ && cmake --build ./samples\ build/ --config ${{ matrix.build-type }} -j + cmake --install ./samples\ build/ --config ${{ matrix.build-type }} --component samples_bin --prefix s\ pace + + - name: Test C++ samples (greedy_causal_lm) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/greedy_causal_lm ./TinyLlama-1.1B-Chat-v1.0/ "" + + - name: Test C++ samples (whisper_speech_recognition) + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + wget https://storage.openvinotoolkit.org/models_contrib/speech/2021.2/librispeech_s5/how_are_you_doing_today.wav + timeout 25s ${{ github.workspace }}/s\ pace/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav + + - name: Test python samples (multinomial_causal_lm) + if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${OV_INSTALL_DIR}/samples/python/multinomial_causal_lm/multinomial_causal_lm.py ./TinyLlama-1.1B-Chat-v1.0/ 0 + + - name: Test python samples (whisper_speech_recognition) + if: ${{ 'Release' == matrix.build-type }} # Python bindings can be built in Release only + run: | + source ${OV_INSTALL_DIR}/setupvars.sh + timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + Overall_Status: name: ci/gha_overall_status_linux - needs: [genai_nodejs_bindings] + needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] if: ${{ always() }} runs-on: ubuntu-latest steps: diff --git a/.github/workflows/llm_bench-python.yml b/.github/workflows/llm_bench-python.yml index 02e836349f..2c8f6a358a 100644 --- a/.github/workflows/llm_bench-python.yml +++ b/.github/workflows/llm_bench-python.yml @@ -3,17 +3,14 @@ name: llm_bench Python Test -# on: -# push: -# branches: [ "master" ] -# paths: -# - tools/llm_bench/** -# - tools/who_what_benchmark/** -# pull_request: -# paths: -# - tools/llm_bench/** -# - tools/who_what_benchmark/** -# - .github/workflows/llm_bench-python.yml +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -66,28 +63,28 @@ jobs: python ./tools/llm_bench/benchmark.py -m tiny-random-qwen -d cpu -n 1 -f pt env: GIT_LFS_SKIP_SMUDGE: 0 - - name: Test tiny-random-baichuan2 on Linux + - name: Test tiny-random-baichuan2 on Linux Optimum Intel run: | optimum-cli export openvino --model katuni4ka/tiny-random-baichuan2 --trust-remote-code --weight-format fp16 ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16 - python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16/ -d cpu -n 1 - - name: Test tiny-stable-diffusion on Linux + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-random-baichuan2/pytorch/dldt/FP16/ -d cpu -n 1 --optimum + - name: Test tiny-stable-diffusion on Linux Optimum Intel run: | optimum-cli export openvino --model segmind/tiny-sd --trust-remote-code --weight-format fp16 ./ov_models/tiny-sd/pytorch/dldt/FP16/ - python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-sd/pytorch/dldt/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 + python ./tools/llm_bench/benchmark.py -m ./ov_models/tiny-sd/pytorch/dldt/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --optimum - name: Test dreamlike-anime on Linux with GenAI run: | optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 ov_models/dreamlike-art-dreamlike-anime-1.0/FP16 - python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --genai + python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 - name: Test dreamlike-anime on Linux with GenAI and LoRA run: | wget -O ./ov_models/soulcard.safetensors https://civitai.com/api/download/models/72591 - python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --genai --lora ./ov_models/soulcard.safetensors --lora_alphas 0.7 + python ./tools/llm_bench/benchmark.py -m ./ov_models/dreamlike-art-dreamlike-anime-1.0/FP16/ -pf ./tools/llm_bench/prompts/stable-diffusion.jsonl -d cpu -n 1 --lora ./ov_models/soulcard.safetensors --lora_alphas 0.7 - name: Test TinyLlama-1.1B-Chat-v1.0 in Speculative Deconding mode on Linux run: | optimum-cli export openvino --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --trust-remote-code --weight-format fp16 ov_models/TinyLlama-1.1B-Chat-v1.0/FP16 optimum-cli export openvino --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 --trust-remote-code --weight-format int8 ov_models/TinyLlama-1.1B-Chat-v1.0/INT8 - python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --genai --assistant_confidence_threshold 0.4 - python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --genai --num_assistant_tokens 5 + python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --assistant_confidence_threshold 0.4 + python ./tools/llm_bench/benchmark.py -m ./ov_models/TinyLlama-1.1B-Chat-v1.0/FP16/ --draft_model ./ov_models/TinyLlama-1.1B-Chat-v1.0/INT8/ -p "Why is the Sun yellow?" -d cpu --draft_device cpu -n 1 --num_assistant_tokens 5 - name: Test whisper-tiny on Linux run: | GIT_LFS_SKIP_SMUDGE=1 git clone --depth 1 --branch main --single-branch https://huggingface.co/datasets/facebook/multilingual_librispeech @@ -97,15 +94,13 @@ jobs: tar zxvf data/mls_polish/train/audio/3283_1447_000.tar.gz -C data/mls_polish/train/audio/3283_1447_000/ cd .. optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny ./ov_models/whisper-tiny + python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 --optimum python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 - python ./tools/llm_bench/benchmark.py -m ./ov_models/whisper-tiny --media multilingual_librispeech/data/mls_polish/train/audio/3283_1447_000/3283_1447_000000.flac -d cpu -n 1 --genai - name: WWB Tests run: | - GIT_CLONE_PROTECTION_ACTIVE=false pip install -r ${{ env.WWB_PATH }}/requirements.txt pip install git+https://github.com/huggingface/optimum-intel.git - GIT_CLONE_PROTECTION_ACTIVE=false pip install ${{ env.WWB_PATH }} - python -m pip install -U --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --force-reinstall - python -m pytest -v tools/who_what_benchmark/tests + GIT_CLONE_PROTECTION_ACTIVE=false PIP_PRE=1 PIP_EXTRA_INDEX_URL=https://storage.openvinotoolkit.org/simple/wheels/nightly pip install ${{ env.WWB_PATH }} + python -m pytest -v ${{ env.WWB_PATH }}/tests stateful: runs-on: ubuntu-20.04 steps: @@ -122,9 +117,7 @@ jobs: grep beam_idx pytorch/dldt/FP32/openvino_model.xml - name: WWB Tests run: | - GIT_CLONE_PROTECTION_ACTIVE=false pip install -r tools/who_what_benchmark/requirements.txt - pip install git+https://github.com/huggingface/optimum-intel.git - GIT_CLONE_PROTECTION_ACTIVE=false pip install tools/who_what_benchmark/ pip install pytest - python -m pip install -U --pre openvino openvino-tokenizers openvino-genai --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly --force-reinstall - python -m pytest -v tools/who_what_benchmark/tests + pip install git+https://github.com/huggingface/optimum-intel.git + GIT_CLONE_PROTECTION_ACTIVE=false PIP_PRE=1 PIP_EXTRA_INDEX_URL=https://storage.openvinotoolkit.org/simple/wheels/nightly pip install ${{ env.WWB_PATH }} + python -m pytest -v ${{ env.WWB_PATH }}/tests diff --git a/.github/workflows/mac.yml b/.github/workflows/mac.yml index 09e14f2b20..7a4ee31beb 100644 --- a/.github/workflows/mac.yml +++ b/.github/workflows/mac.yml @@ -1,12 +1,12 @@ name: macOS (12, Python 3.9) -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -224,7 +224,7 @@ jobs: - name: Test bindings run: | source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template env: PYTHONPATH: "./build/:$PYTHONPATH" @@ -234,6 +234,7 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels python -c "from openvino_genai import LLMPipeline" + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/ --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" genai_python_lib_whisper: @@ -288,7 +289,7 @@ jobs: - name: Test bindings run: | source ${OV_INSTALL_DIR}/setupvars.sh - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke env: PYTHONPATH: "./build/:$PYTHONPATH" @@ -298,6 +299,7 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install . --verbose --find-links ${OV_INSTALL_DIR}/wheels python -c "from openvino_genai import LLMPipeline" + python -m pip install ./tools/who_what_benchmark --find-links ${OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_package: @@ -354,7 +356,7 @@ jobs: run: | source ${OV_INSTALL_DIR}/setupvars.sh python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${OV_INSTALL_DIR}/wheels optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny diff --git a/.github/workflows/stable_diffusion_1_5_cpp.yml b/.github/workflows/stable_diffusion_1_5_cpp.yml index f9357902dc..b355cd4f09 100644 --- a/.github/workflows/stable_diffusion_1_5_cpp.yml +++ b/.github/workflows/stable_diffusion_1_5_cpp.yml @@ -1,13 +1,13 @@ name: stable_diffusion_1_5_cpp -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -18,8 +18,8 @@ concurrency: env: PYTHON_VERSION: '3.10' - LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241105_x86_64.tgz - WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17289-7cf2bbb8391/w_openvino_toolkit_windows_2025.0.0.dev20241105_x86_64.zip + LINUX_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/l_openvino_toolkit_ubuntu20_2025.0.0.dev20241205_x86_64.tgz + WINDOWS_OV_ARCHIVE_URL: https://storage.openvinotoolkit.org/repositories/openvino/packages/nightly/2025.0.0-17539-6abe2e39391/w_openvino_toolkit_windows_2025.0.0.dev20241205_x86_64.zip OV_INSTALL_DIR: ${{ github.workspace }}/ov jobs: @@ -51,7 +51,7 @@ jobs: run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python3 -m venv openvino_sd_cpp @@ -59,7 +59,7 @@ jobs: - name: Install python dependencies run: | source openvino_sd_cpp/bin/activate - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer @@ -68,29 +68,29 @@ jobs: optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --weight-format fp16 --task stable-diffusion models/dreamlike-art-dreamlike-anime-1.0/FP16 wget -O ./models/soulcard.safetensors https://civitai.com/api/download/models/72591 - - name: Run main app + - name: Run text2image app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + ${{ env.build_dir }}/samples/cpp/image_generation/text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" - - name: Run LoRA app + - name: Run lora_text2image app run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - ${{ env.build_dir }}/samples/cpp/text2image/lora_stable_diffusion ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 + ${{ env.build_dir }}/samples/cpp/image_generation/lora_text2image ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 - - name: Run Python main app + - name: Run text2image.py app run: | source openvino_sd_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/main.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python ./samples/python/image_generation/text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} - - name: Run Python LoRA app + - name: Run lora_text2image.py app run: | source openvino_sd_cpp/bin/activate source ./ov/setupvars.sh - python ./samples/python/text2image/lora.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 + python ./samples/python/image_generation/lora_text2image.py ./models/dreamlike-art-dreamlike-anime-1.0/FP16 "curly-haired unicorn in the forest, anime, line" ./models/soulcard.safetensors 0.7 env: PYTHONPATH: ${{ env.build_dir }} @@ -125,7 +125,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" cmake -DCMAKE_BUILD_TYPE=Release -S ./ -B ${{ env.build_dir }} - cmake --build ${{ env.build_dir }} --config Release --target stable_diffusion lora_stable_diffusion py_openvino_genai --parallel + cmake --build ${{ env.build_dir }} --config Release --target text2image image2image inpainting heterogeneous_stable_diffusion lora_text2image py_openvino_genai --parallel - name: Create virtual environment run: python -m venv openvino_sd_cpp @@ -133,7 +133,7 @@ jobs: - name: Install python dependencies run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --pre --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --extra-index-url https://storage.openvinotoolkit.org/simple/wheels/nightly python -m pip install -r ./samples/requirements.txt - name: Download and convert models and tokenizer @@ -142,35 +142,35 @@ jobs: optimum-cli export openvino --model dreamlike-art/dreamlike-anime-1.0 --task stable-diffusion --weight-format fp16 models/dreamlike-art-dreamlike-anime-1.0/FP16 Invoke-WebRequest -Uri 'https://civitai.com/api/download/models/72591' -OutFile 'models/soulcard.safetensors' - - name: Run main app + - name: Run text2image app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - "${{ env.build_dir }}/samples/cpp/text2image/Release/stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" + "${{ env.build_dir }}/samples/cpp/image_generation/Release/text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting'" env: PATH: ${{ env.build_dir }}\openvino_genai - - name: Run LoRA app + - name: Run lora_text2image app run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - "${{ env.build_dir }}/samples/cpp/text2image/Release/lora_stable_diffusion.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7" + "${{ env.build_dir }}/samples/cpp/image_generation/Release/lora_text2image.exe ./models/dreamlike-art-dreamlike-anime-1.0/FP16 'curly-haired unicorn in the forest, anime, line' ./models/soulcard.safetensors 0.7" env: PATH: ${{ env.build_dir }}\openvino_genai - - name: Run Python main app + - name: Run text2image.py app run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\main.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" + python .\samples\python\image_generation\text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "cyberpunk cityscape like Tokyo New York with tall buildings at dusk golden hour cinematic lighting" env: PYTHONPATH: ${{ env.build_dir }} - - name: Run Python LoRA app + - name: Run lora_text2image.py app run: | . "./openvino_sd_cpp/Scripts/Activate.ps1" . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" $env:Path += "${{ env.build_dir }}\openvino_genai" - python .\samples\python\text2image\lora.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7 + python .\samples\python\image_generation\lora_text2image.py .\models\dreamlike-art-dreamlike-anime-1.0\FP16 "curly-haired unicorn in the forest, anime, line" .\models\soulcard.safetensors 0.7 env: PYTHONPATH: ${{ env.build_dir }} diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index f48e80413e..649d678c02 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -1,12 +1,12 @@ name: Windows (VS 2019, Python 3.11) -# on: -# workflow_dispatch: -# pull_request: -# merge_group: -# push: -# branches: -# - master -# - 'releases/**' +on: + workflow_dispatch: + pull_request: + merge_group: + push: + branches: + - master + - 'releases/**' permissions: read-all # Required by https://github.com/ossf/scorecard/blob/e23b8ad91fd6a64a0a971ca4fc0a4d1650725615/docs/checks.md#token-permissions @@ -235,7 +235,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_chat_generate_api.py::test_set_chat_template env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -244,6 +244,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install . --verbose --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/ --ignore ./tests/python_tests/test_whisper_generate_api.py --ignore ./tests/python_tests/test_vlm_api.py -k "not test_set_chat_template" genai_python_lib_whisper: @@ -299,7 +300,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k test_smoke env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -308,6 +309,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install . --verbose --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install ./tools/who_what_benchmark --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_whisper_generate_api.py -k "not test_smoke" genai_python_lib_vlm: @@ -363,7 +365,7 @@ jobs: - name: Test bindings run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" - python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels --upgrade-strategy eager + python -m pip install ./thirdparty/openvino_tokenizers/[transformers] -r ./tests/python_tests/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels python -m pytest -v ./tests/python_tests/test_vlm_api.py env: PYTHONPATH: "./build/" # cmd evaluates variables in a different way. Setting PYTHONPATH before setupvars.bat instead of doing that after solves that. @@ -425,7 +427,7 @@ jobs: run: | . "${{ env.OV_INSTALL_DIR }}/setupvars.ps1" python -m pip install ./thirdparty/openvino_tokenizers/[transformers] --find-links ${env:OV_INSTALL_DIR}/wheels - python -m pip install --upgrade-strategy eager -r ./samples/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels + python -m pip install -r ./samples/requirements.txt --find-links ${env:OV_INSTALL_DIR}/wheels optimum-cli export openvino --trust-remote-code --weight-format fp16 --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 optimum-cli export openvino --trust-remote-code --model openai/whisper-tiny whisper-tiny From b722d39f906ba46cf5e8ed612dcf26b59bd41d1a Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 11 Dec 2024 15:30:59 +0100 Subject: [PATCH 24/74] Add genai_nodejs_bindings linux workflow job --- .github/workflows/linux.yml | 81 ++++++++++++++++++++++++++++++++++++- 1 file changed, 80 insertions(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 2916c340bf..09554c2c1b 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -483,9 +483,88 @@ jobs: source ${OV_INSTALL_DIR}/setupvars.sh timeout 25s ${OV_INSTALL_DIR}/samples/python/whisper_speech_recognition/whisper_speech_recognition.py ./whisper-tiny/ how_are_you_doing_today.wav + genai_nodejs_bindings: + name: Produce genai nodejs binaries archive + timeout-minutes: 150 + defaults: + run: + shell: bash + runs-on: ubuntu-20.04-16-cores + env: + DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input + CMAKE_BUILD_TYPE: 'Release' + CMAKE_GENERATOR: 'Ninja Multi-Config' + CMAKE_CXX_COMPILER_LAUNCHER: ccache + CMAKE_C_COMPILER_LAUNCHER: ccache + OPENVINO_REPO: ${{ github.workspace }}/openvino + GENAI_REPO: ${{ github.workspace }}/openvino.genai + BUILD_DIR: ${{ github.workspace }}/build + CCACHE_DIR: ${{ github.workspace }}/ccache + CCACHE_MAXSIZE: 2000Mi + + steps: + - name: Set apt + run: | + echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null + echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null + echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null + echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + + - name: Clone OpenVINO + uses: actions/checkout@v4 + with: + repository: 'openvinotoolkit/openvino' + path: ${{ env.OPENVINO_REPO }} + submodules: 'true' + ref: ${{ env.OV_BRANCH}} + + - name: Install build dependencies + run: | + sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh + sudo apt-get install ccache + + - name: Clone GenAI + uses: actions/checkout@v4 + with: + path: ${{ env.GENAI_REPO }} + submodules: recursive + + - name: Setup ccache + uses: actions/cache@v4 + with: + # Should save cache only if run in the master branch of the base repo + # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push + save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} + path: ${{ env.CCACHE_DIR }} + key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} + restore-keys: | + ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} + + - name: Build openvino + genai + run: | + cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + cmake --build ./build --target package -j + + # + # Upload build artifacts and logs + # + + - name: Upload genai nodejs bindings archive + if: ${{ always() }} + uses: actions/upload-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + if-no-files-found: 'error' + Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper] + needs: [openvino_download, openvino_build, genai_python_lib, genai_package, genai_python_lib_whisper, genai_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From 3b4b3c0deee8036503ff1f673fdc925af4108a33 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:09:49 +0100 Subject: [PATCH 25/74] Specify rpath for genai_node_addon.node --- src/cpp/CMakeLists.txt | 4 ++-- src/js/CMakeLists.txt | 13 +++++++++++++ 2 files changed, 15 insertions(+), 2 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index ef719ff1be..2fcc37996a 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -141,10 +141,10 @@ if(CPACK_GENERATOR STREQUAL "NPM") # setting RPATH / LC_RPATH depending on platform if(LINUX) - # to find libopenvino_genai.so in the same folder + # to find libopenvino.so in the same folder set(rpaths "$ORIGIN") elseif(APPLE) - # to find libopenvino_genai.dylib in the same folder + # to find libopenvino.dylib in the same folder set(rpaths "@loader_path") endif() diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index 275803a804..d7ea811618 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -73,6 +73,19 @@ set_target_properties(${PROJECT_NAME} PROPERTIES SUFFIX ".node" ) +# setting RPATH / LC_RPATH depending on platform +if(LINUX) + # to find libopenvino_genai.so in the same folder + set(rpaths "$ORIGIN") +elseif(APPLE) + # to find libopenvino_genai.dylib in the same folder + set(rpaths "@loader_path") +endif() + +if(rpaths) + set_target_properties(${PROJECT_NAME} PROPERTIES INSTALL_RPATH "${rpaths}") +endif() + install(TARGETS ${PROJECT_NAME} LIBRARY DESTINATION . COMPONENT ${PROJECT_NAME} RUNTIME DESTINATION . COMPONENT ${PROJECT_NAME} From 17f2069c711ce1b748b5eaf9802cb36f44612657 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:13:43 +0100 Subject: [PATCH 26/74] Fix offsets --- src/cpp/CMakeLists.txt | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 2fcc37996a..7864158e06 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -177,9 +177,9 @@ install(EXPORT OpenVINOGenAITargets FILE OpenVINOGenAITargets.cmake include(CMakePackageConfigHelpers) configure_package_config_file("${OpenVINOGenAI_SOURCE_DIR}/cmake/templates/OpenVINOGenAIConfig.cmake.in" - "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) + "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" INSTALL_DESTINATION runtime/cmake) write_basic_package_version_file("${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" - VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) + VERSION ${OpenVINOGenAI_VERSION} COMPATIBILITY AnyNewerVersion) install(FILES "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfig.cmake" "${CMAKE_BINARY_DIR}/OpenVINOGenAIConfigVersion.cmake" DESTINATION runtime/cmake COMPONENT core_genai_dev) export(EXPORT OpenVINOGenAITargets FILE "${CMAKE_BINARY_DIR}/OpenVINOGenAITargets.cmake" NAMESPACE openvino::) From 27a97d81de0a96ce99d5f6cc4fb2b9ecd1a8f0b6 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 10:16:50 +0100 Subject: [PATCH 27/74] Fix path to js samples in readme Co-authored-by: Vladimir Zlobin --- src/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/README.md b/src/js/README.md index 77ebc0792c..19023eb3e9 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -19,7 +19,7 @@ TODO: Add instructions - To run sample you should have prepared model. Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples](../../samples/js/) +- Go to [samples/js](../../samples/js/) - Run `node app.js`, you should see: `User Prompt: ...` ### Using as npm Dependency From 6f0e0b45039627f202efbdb22617053404eb2d89 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 13:47:07 +0100 Subject: [PATCH 28/74] Align API --- samples/js/interactive.js | 56 ------------------------------------- src/js/lib/module.js | 20 ++++++------- src/js/tests/module.test.js | 24 ++++++++-------- 3 files changed, 22 insertions(+), 78 deletions(-) delete mode 100644 samples/js/interactive.js diff --git a/samples/js/interactive.js b/samples/js/interactive.js deleted file mode 100644 index 466326ced0..0000000000 --- a/samples/js/interactive.js +++ /dev/null @@ -1,56 +0,0 @@ -import path from 'node:path'; -import readline from 'readline'; - -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -main(); - -async function main() { - // Create interface for reading user input from stdin - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` - + 'Type something and press enter. Type "finish" to exit.'); - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - await pipeline.startChat(); - promptUser(); - - // Function to prompt the user for input - function promptUser() { - rl.question('> ', handleInput); - } - - // Function to handle user input - async function handleInput(input) { - input = input.trim(); - - // Check for exit command - if (input === 'finish') { - console.log('Goodbye!'); - await pipeline.finishChat(); - rl.close(); - process.exit(0); - } - - const result = await pipeline.generate(input, generationCallback); - console.log('\n'); - - // Wait for new input - promptUser(); - } - - function generationCallback(chunk) { - process.stdout.write(chunk); - } -} diff --git a/src/js/lib/module.js b/src/js/lib/module.js index 2f3d557711..b390712fa4 100644 --- a/src/js/lib/module.js +++ b/src/js/lib/module.js @@ -55,7 +55,14 @@ class LLMPipeline { return result; } - async generate(prompt, generationCallback, options = {}) { + async generate(prompt, generationCallbackOrOptions, generationCallback) { + let options = {}; + + if (!generationCallback) + generationCallback = generationCallbackOrOptions; + else + options = generationCallbackOrOptions; + if (!this.isInitialized) throw new Error('Pipeline is not initialized'); @@ -95,24 +102,17 @@ class LLMPipeline { } } -const availablePipelines = { LLMPipeline: LLMPipeline }; - class Pipeline { - static async create(pipelineType, modelPath, device = 'CPU') { - if (!Object.keys(availablePipelines).includes(pipelineType)) - throw new Error(`Pipeline type: '${pipelineType}' doesn't support`); - - const pipeline = new availablePipelines[pipelineType](modelPath, device); + static async LLMPipeline(modelPath, device = 'CPU') { + const pipeline = new LLMPipeline(modelPath, device); await pipeline.init(); return pipeline; } } -const availablePipelinesKeys = Object.keys(availablePipelines); export { addon, Pipeline, - availablePipelinesKeys as availablePipelines, }; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 90b2f5266d..672fa71638 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -11,7 +11,7 @@ describe('module', async () => { let pipeline = null; await before(async () => { - pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); }); @@ -23,8 +23,8 @@ describe('module', async () => { await it('should generate "Hello world"', async () => { const result = await pipeline.generate( 'Type "Hello world!" in English', + { temperature: '0', max_new_tokens: '4' }, () => {}, - { temperature: '0', max_new_tokens: '4' } ); assert.strictEqual(result, 'Hello world!'); @@ -33,7 +33,7 @@ describe('module', async () => { describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await assert.rejects( async () => await pipeline.init(), @@ -45,7 +45,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is already started', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); @@ -59,7 +59,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is not started', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await assert.rejects( () => pipeline.finishChat(), @@ -75,7 +75,7 @@ describe('generation parameters validation', () => { let pipeline = null; before(async () => { - pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); }); @@ -95,7 +95,7 @@ describe('generation parameters validation', () => { }); it('should throw an error if generationCallback is not a function', async () => { - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); await pipeline.startChat(); @@ -110,7 +110,7 @@ describe('generation parameters validation', () => { it('should throw an error if options specified but not an object', async () => { await assert.rejects( - async () => await pipeline.generate('prompt', () => {}, 'options'), + async () => await pipeline.generate('prompt', 'options', () => {}), { name: 'Error', message: 'Options must be an object', @@ -120,7 +120,7 @@ describe('generation parameters validation', () => { it('should perform generation with default options', async () => { try { - await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); } catch (error) { assert.fail(error); } @@ -129,14 +129,14 @@ describe('generation parameters validation', () => { }); it('should return a string as generation result', async () => { - const reply = await pipeline.generate('prompt', () => {}, { max_new_tokens: 1 }); + const reply = await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); assert.strictEqual(typeof reply, 'string'); }); it('should call generationCallback with string chunk', async () => { - await pipeline.generate('prompt', (chunk) => { + await pipeline.generate('prompt', { max_new_tokens: 1 }, (chunk) => { assert.strictEqual(typeof chunk, 'string'); - }, { max_new_tokens: 1 }); + }); }); }); From 855c893f5e84a93e6c2c1bf47dcddecccbb31a37 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 13:47:59 +0100 Subject: [PATCH 29/74] Align chat sample --- samples/js/app.js | 28 ---------- samples/js/{ => chat_sample}/.gitignore | 0 samples/js/chat_sample/README.md | 0 samples/js/chat_sample/chat_sample.js | 54 +++++++++++++++++++ .../js/{ => chat_sample}/package-lock.json | 11 ++-- samples/js/{ => chat_sample}/package.json | 2 +- 6 files changed, 63 insertions(+), 32 deletions(-) delete mode 100644 samples/js/app.js rename samples/js/{ => chat_sample}/.gitignore (100%) create mode 100644 samples/js/chat_sample/README.md create mode 100644 samples/js/chat_sample/chat_sample.js rename samples/js/{ => chat_sample}/package-lock.json (71%) rename samples/js/{ => chat_sample}/package.json (82%) diff --git a/samples/js/app.js b/samples/js/app.js deleted file mode 100644 index f57e2fad1d..0000000000 --- a/samples/js/app.js +++ /dev/null @@ -1,28 +0,0 @@ -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -const generationCallback = (chunk) => { - process.stdout.write(chunk); -}; - -const prompt = 'Who are you?'; -console.log(`User Prompt: "${prompt}"\n`); - -const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - -await pipeline.startChat(); -const result = await pipeline.generate( - prompt, - generationCallback, - { temperature: 0 }, -); -await pipeline.finishChat(); - -console.log(`\n\nGeneration result:\n"${result}"`) diff --git a/samples/js/.gitignore b/samples/js/chat_sample/.gitignore similarity index 100% rename from samples/js/.gitignore rename to samples/js/chat_sample/.gitignore diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md new file mode 100644 index 0000000000..e69de29bb2 diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js new file mode 100644 index 0000000000..fad1b507a8 --- /dev/null +++ b/samples/js/chat_sample/chat_sample.js @@ -0,0 +1,54 @@ +import readline from 'readline'; +import { Pipeline } from 'genai-node'; + +main(); + +function streamer(subword) { + process.stdout.write(subword); +} + +async function main() { + const MODEL_PATH = process.argv[2]; + + if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); + } + + const device = 'CPU'; // GPU can be used as well + + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + const pipe = await Pipeline.LLMPipeline(MODEL_PATH, device); + const config = { 'max_new_tokens': 100 }; + + await pipe.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('question:\n', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (!input) { + await pipe.finishChat(); + rl.close(); + process.exit(0); + } + + await pipe.generate(input, config, streamer); + console.log('\n----------'); + + promptUser(); + } +} diff --git a/samples/js/package-lock.json b/samples/js/chat_sample/package-lock.json similarity index 71% rename from samples/js/package-lock.json rename to samples/js/chat_sample/package-lock.json index 4b2d14913f..fbee0db012 100644 --- a/samples/js/package-lock.json +++ b/samples/js/chat_sample/package-lock.json @@ -9,13 +9,13 @@ "version": "1.0.0", "license": "Apache-2.0", "devDependencies": { - "genai-node": "../../src/js/" + "genai-node": "../../../src/js/" }, "engines": { "node": ">=21.0.0" } }, - "../../src/js": { + "../../../src/js": { "name": "genai-node", "version": "2024.5.0-preview", "dev": true, @@ -25,12 +25,17 @@ "darwin", "win32" ], + "devDependencies": { + "@huggingface/hub": "^0.21.0", + "global-agent": "^3.0.0", + "node-fetch": "^3.3.2" + }, "engines": { "node": ">=21.0.0" } }, "node_modules/genai-node": { - "resolved": "../../src/js", + "resolved": "../../../src/js", "link": true } } diff --git a/samples/js/package.json b/samples/js/chat_sample/package.json similarity index 82% rename from samples/js/package.json rename to samples/js/chat_sample/package.json index 0251bdfa14..8de12582e9 100644 --- a/samples/js/package.json +++ b/samples/js/chat_sample/package.json @@ -4,7 +4,7 @@ "license": "Apache-2.0", "type": "module", "devDependencies": { - "genai-node": "../../src/js/" + "genai-node": "../../../src/js/" }, "engines": { "node": ">=21.0.0" From df51af9f9a189c3d413bf253db03e9c7271fc9e1 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 14:23:55 +0100 Subject: [PATCH 30/74] Add README.md into js chat sample --- samples/js/chat_sample/README.md | 49 ++++++++++++++++++++++++++++++++ 1 file changed, 49 insertions(+) diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md index e69de29bb2..8381c98aa8 100644 --- a/samples/js/chat_sample/README.md +++ b/samples/js/chat_sample/README.md @@ -0,0 +1,49 @@ +# JavaScript chat_sample that supports most popular models like LLaMA 3 + +This example showcases inference of text-generation Large Language Models (LLMs): `chatglm`, `LLaMA`, `Qwen` and other models with the same signature. The application doesn't have many configuration options to encourage the reader to explore and modify the source code. For example, change the device for inference to GPU. The sample fearures `Pipeline.LLMPipeline` and configures it for the chat scenario. + +## Download and convert the model and tokenizers + +To convert model you have to use python package `optimum-intel`. +The `--upgrade-strategy eager` option is needed to ensure `optimum-intel` is upgraded to the latest version. + +Install [../../export-requirements.txt](../../export-requirements.txt) to convert a model. + +```sh +pip install --upgrade-strategy eager -r ../../export-requirements.txt +optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B-Chat-v1.0 TinyLlama-1.1B-Chat-v1.0 +``` + +## Run: + +Create `bin` folder in [../../../src/js](../../../src/js). +Compile GenAI JavaScript bindings archive first. Put its content into `bin` directory. + +Run `npm install` in current folder and then run a sample: + +`node chat_sample.js TinyLlama-1.1B-Chat-v1.0` + +Discrete GPUs (dGPUs) usually provide better performance compared to CPUs. It is recommended to run larger models on a dGPU with 32GB+ RAM. For example, the model meta-llama/Llama-2-13b-chat-hf can benefit from being run on a dGPU. Modify the source code to change the device for inference to the GPU. + +See https://github.com/openvinotoolkit/openvino.genai/blob/master/src/README.md#supported-models for the list of supported models. + +### Troubleshooting + +#### Unicode characters encoding error on Windows + +Example error: +``` +UnicodeEncodeError: 'charmap' codec can't encode character '\u25aa' in position 0: character maps to +``` + +If you encounter the error described in the example when sample is printing output to the Windows console, it is likely due to the default Windows encoding not supporting certain Unicode characters. To resolve this: +1. Enable Unicode characters for Windows cmd - open `Region` settings from `Control panel`. `Administrative`->`Change system locale`->`Beta: Use Unicode UTF-8 for worldwide language support`->`OK`. Reboot. +2. Enable UTF-8 mode by setting environment variable `PYTHONIOENCODING="utf8"`. + +#### Missing chat template + +If you encounter an exception indicating a missing "chat template" when launching the `ov::genai::LLMPipeline` in chat mode, it likely means the model was not tuned for chat functionality. To work this around, manually add the chat template to tokenizer_config.json of your model. +The following template can be used as a default, but it may not work properly with every model: +``` +"chat_template": "{% for message in messages %}{% if (message['role'] == 'user') %}{{'<|im_start|>user\n' + message['content'] + '<|im_end|>\n<|im_start|>assistant\n'}}{% elif (message['role'] == 'assistant') %}{{message['content'] + '<|im_end|>\n'}}{% endif %}{% endfor %}", +``` From a365ef8c9f90a981b38624f875c6cd1e2ebc1858 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 12 Dec 2024 15:53:50 +0100 Subject: [PATCH 31/74] Update build instructions --- src/js/README.md | 34 ++++++++++++++++++++++++++++++---- 1 file changed, 30 insertions(+), 4 deletions(-) diff --git a/src/js/README.md b/src/js/README.md index 19023eb3e9..2302ce1082 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -13,14 +13,40 @@ This is preview version, do not use it in production! ### Build Bindings -TODO: Add instructions +#### Build OpenVINO GenAI as OpenVINO Extra Module + +OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenVINO build process. This method simplifies the build process by integrating OpenVINO GenAI directly into the OpenVINO build. + +1. Clone OpenVINO repository: + ```sh + git clone --recursive https://github.com/openvinotoolkit/openvino.git + ``` +1. Configure CMake with OpenVINO extra modules: + ```sh + cmake -DOPENVINO_EXTRA_MODULES=*relative (from openvino folder) path to genai repository* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build + ``` +1. Build OpenVINO archive with GenAI: + ```sh + cmake --build ./build --target package -j + ``` + +1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. + Create `bin` directory by path `src/js/` and unarchive archive content to it. +1. Run tests to be sure that everything works: + `npm test` ### Perform Test Run - To run sample you should have prepared model. - Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/cpp/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples/js](../../samples/js/) -- Run `node app.js`, you should see: `User Prompt: ...` + Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/js/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) +- Go to [samples/js/chat_sample/](../../samples/js/chat_sample/) +- Read [README.md](../../samples/js/chat_sample/README.md) and follow steps there + to run **chat sample**. ### Using as npm Dependency From 6d0b501064f19e2e72e4c413d1073cc73c0ab9e9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 15:16:00 +0100 Subject: [PATCH 32/74] Fix review comments --- src/js/CMakeLists.txt | 3 ++- src/js/README.md | 7 +++++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/src/js/CMakeLists.txt b/src/js/CMakeLists.txt index d7ea811618..7e4ff0bea4 100644 --- a/src/js/CMakeLists.txt +++ b/src/js/CMakeLists.txt @@ -17,7 +17,8 @@ endif() project(genai_node_addon) -# Add definitions +# Specify NAPI version 8 +# supports v12.22.0+, v14.17.0+, v15.12.0+, 16.0.0 and all later Node.js versions add_definitions(-DNAPI_VERSION=8) include(FetchContent) diff --git a/src/js/README.md b/src/js/README.md index 2302ce1082..9092340d83 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -23,7 +23,7 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV ``` 1. Configure CMake with OpenVINO extra modules: ```sh - cmake -DOPENVINO_EXTRA_MODULES=*relative (from openvino folder) path to genai repository* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=*absolute path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ @@ -36,7 +36,10 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV ``` 1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. - Create `bin` directory by path `src/js/` and unarchive archive content to it. + Create `bin` directory by path `src/js/` and unpack archive content to it. + ```sh + tar -xzf genai_nodejs_bindings.tar.gz + ``` 1. Run tests to be sure that everything works: `npm test` From f241cda3ffaeeb03be638249809affb94adea9a2 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 16:13:10 +0100 Subject: [PATCH 33/74] Add run js api test in linux workflow --- .github/workflows/linux.yml | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1d8cdb0385..7a7e9fc130 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -469,6 +469,15 @@ jobs: -S ./openvino -B ./build cmake --build ./build --target package -j + - name: Run javascript tests + working-directory: ${{ env.GENAI_REPO }}/src/js + run: | + mkdir bin + cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin + cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz cd .. + npm install + npm test + # # Upload build artifacts and logs # From 1f2324b23e01222fb952218c8eb99605813ab4f9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 16 Dec 2024 16:52:54 +0100 Subject: [PATCH 34/74] Fix workflow --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 8487c7cf4e..bb448a0e49 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -470,7 +470,7 @@ jobs: run: | mkdir bin cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin - cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz cd .. + cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz && cd .. npm install npm test From dda494de1bd0f1351dd55882576cf25f2a92e90e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 19:29:14 +0100 Subject: [PATCH 35/74] Add test for js sample --- samples/js/chat_sample/chat_sample.js | 2 +- samples/js/chat_sample/package.json | 3 ++ samples/js/chat_sample/tests/usage.test.js | 57 ++++++++++++++++++++++ samples/js/interactive.js | 56 +++++++++++++++++++++ 4 files changed, 117 insertions(+), 1 deletion(-) create mode 100644 samples/js/chat_sample/tests/usage.test.js create mode 100644 samples/js/interactive.js diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js index fad1b507a8..9bca37d4b7 100644 --- a/samples/js/chat_sample/chat_sample.js +++ b/samples/js/chat_sample/chat_sample.js @@ -49,6 +49,6 @@ async function main() { await pipe.generate(input, config, streamer); console.log('\n----------'); - promptUser(); + if (!rl.closed) promptUser(); } } diff --git a/samples/js/chat_sample/package.json b/samples/js/chat_sample/package.json index 8de12582e9..24e66a120d 100644 --- a/samples/js/chat_sample/package.json +++ b/samples/js/chat_sample/package.json @@ -8,5 +8,8 @@ }, "engines": { "node": ">=21.0.0" + }, + "scripts": { + "test": "node tests/usage.test.js" } } diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js new file mode 100644 index 0000000000..639ed0a487 --- /dev/null +++ b/samples/js/chat_sample/tests/usage.test.js @@ -0,0 +1,57 @@ +import { env } from 'process'; +import { spawn } from 'child_process'; + +const MODEL_PATH = env.MODEL_PATH; + +if (!MODEL_PATH) + throw new Error( + 'Please environment variable MODEL_PATH to the path of the model directory' + ); + +const runTest = async () => { + return new Promise((resolve, reject) => { + const script = spawn('node', ['chat_sample.js', MODEL_PATH]); + let output = ''; + + // Collect output from stdout + script.stdout.on('data', (data) => { + output += data.toString(); + }); + + // Capture errors + script.stderr.on('data', (data) => { + reject(data.toString()); + }); + + // Send input after detecting the question prompt + script.stdout.once('data', (data) => { + if (data.toString().startsWith('question:')) { + script.stdin.write('Say exactly, without any changes, print it as is: "Hello world"\n'); // Provide input + script.stdin.end(); // Close stdin to signal EOF + } + }); + + // Check results when the process exits + script.on('close', (code) => { + if (code !== 0) { + return reject(`Process exited with code ${code}`); + } + // Validate the output + if (output.includes('"Hello world"')) { + resolve('Test passed!'); + } else { + reject('Test failed: Output did not match expected result.'); + } + }); + }); +}; + +runTest() + .then((message) => { + console.log(message); + process.exit(0); + }) + .catch((err) => { + console.error(err); + process.exit(1); + }); diff --git a/samples/js/interactive.js b/samples/js/interactive.js new file mode 100644 index 0000000000..466326ced0 --- /dev/null +++ b/samples/js/interactive.js @@ -0,0 +1,56 @@ +import path from 'node:path'; +import readline from 'readline'; + +import { Pipeline } from 'genai-node'; + +const MODEL_PATH = process.argv[2]; + +if (!MODEL_PATH) { + console.error('Please specify path to model directory\n' + + 'Run command must be: `node app.js *path_to_model_dir*`'); + process.exit(1); +} + +main(); + +async function main() { + // Create interface for reading user input from stdin + const rl = readline.createInterface({ + input: process.stdin, + output: process.stdout, + }); + + console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` + + 'Type something and press enter. Type "finish" to exit.'); + const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); + await pipeline.startChat(); + promptUser(); + + // Function to prompt the user for input + function promptUser() { + rl.question('> ', handleInput); + } + + // Function to handle user input + async function handleInput(input) { + input = input.trim(); + + // Check for exit command + if (input === 'finish') { + console.log('Goodbye!'); + await pipeline.finishChat(); + rl.close(); + process.exit(0); + } + + const result = await pipeline.generate(input, generationCallback); + console.log('\n'); + + // Wait for new input + promptUser(); + } + + function generationCallback(chunk) { + process.stdout.write(chunk); + } +} From 394d4250cf423eba3a4e5fdf2c0f1fd5869dd48f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 19:29:46 +0100 Subject: [PATCH 36/74] Extend linux workflow to run js samples test --- .github/workflows/linux.yml | 54 +++++++++++++++++++++++++++++++++++++ 1 file changed, 54 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index bb448a0e49..3e5952b62c 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -486,6 +486,60 @@ jobs: path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' + genai_nodejs_samples_tests: + name: NodeJS Samples Tests + needs: [ genai_nodejs_bindings ] + timeout-minutes: 30 + defaults: + run: + shell: bash + runs-on: aks-linux-2-cores-8gb + env: + INSTALL_DIR: ${{ github.workspace }}/ov + SRC_DIR: ${{ github.workspace }}/src + MODELS_DIR: ${{ github.workspace }}/models + JS_SRC_DIR: ${{ github.workspace }}/src/openvino.genai/src/js + JS_SAMPLES_DIR: ${{ github.workspace }}/src/openvino.genai/samples/js + BIN_DIR: ${{ github.workspace }}/src/openvino.genai/src/js/bin + steps: + - name: Clone openvino.genai + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.SRC_DIR }} + submodules: recursive + + - name: Download genai nodejs bindings archive + uses: actions/download-artifact@v4 + with: + name: genai_nodejs_bindings + path: ${{ env.BIN_DIR }} + + - name: Unpack genai nodejs bindings archive + run: tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz + working-directory: ${{ env.BIN_DIR }} + + - name: Install nodejs dependencies + run: npm install + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Download model for tests + run: npm run test_setup + working-directory: ${{ env.JS_SAMPLES_DIR }} + + - name: Link genai-node package + run: npm link + working-directory: ${{ env.JS_SRC_DIR }} + + - name: Install genai-node package to samples + run: npm link genai-node + working-directory: ${{ env.JS_SAMPLES_DIR }} + + - name: Run tests + run: npm test + env: + MODEL_PATH: ${{ env.JS_SRC_DIR }}/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + working-directory: ${{ env.JS_SAMPLES_DIR }} + Overall_Status: name: ci/gha_overall_status_linux needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings] From 31861422aba147a6fc47617353ecafa8ae860e09 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 20:17:56 +0100 Subject: [PATCH 37/74] Change runner for js samples tests --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3e5952b62c..9c89377bec 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -493,7 +493,7 @@ jobs: defaults: run: shell: bash - runs-on: aks-linux-2-cores-8gb + runs-on: ubuntu-20.04-16-cores env: INSTALL_DIR: ${{ github.workspace }}/ov SRC_DIR: ${{ github.workspace }}/src From 2fbd888f7940ebdb2592f2a6426fef387d08a896 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 20:37:54 +0100 Subject: [PATCH 38/74] Fix repo path --- .github/workflows/linux.yml | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 9c89377bec..df5eb4ca05 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -496,16 +496,16 @@ jobs: runs-on: ubuntu-20.04-16-cores env: INSTALL_DIR: ${{ github.workspace }}/ov - SRC_DIR: ${{ github.workspace }}/src + GENAI_REPO: ${{ github.workspace }}/openvino.genai MODELS_DIR: ${{ github.workspace }}/models - JS_SRC_DIR: ${{ github.workspace }}/src/openvino.genai/src/js - JS_SAMPLES_DIR: ${{ github.workspace }}/src/openvino.genai/samples/js - BIN_DIR: ${{ github.workspace }}/src/openvino.genai/src/js/bin + JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js + JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js + BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin steps: - name: Clone openvino.genai uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 with: - path: ${{ env.SRC_DIR }} + path: ${{ env.GENAI_REPO }} submodules: recursive - name: Download genai nodejs bindings archive From 0a0c39995fdee5e5fa7d2eee2d69a60164f46e14 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 21:30:23 +0100 Subject: [PATCH 39/74] Fix path --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index fd3cd08547..44a659f132 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -499,7 +499,7 @@ jobs: GENAI_REPO: ${{ github.workspace }}/openvino.genai MODELS_DIR: ${{ github.workspace }}/models JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js - JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js + JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js/chat_sample BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin steps: - name: Clone openvino.genai @@ -524,7 +524,7 @@ jobs: - name: Download model for tests run: npm run test_setup - working-directory: ${{ env.JS_SAMPLES_DIR }} + working-directory: ${{ env.JS_SRC_DIR }} - name: Link genai-node package run: npm link From 35c1c61f7fae0aceaa384aa5077a01e673996fc3 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 17 Dec 2024 22:02:10 +0100 Subject: [PATCH 40/74] Add debug output --- samples/js/chat_sample/tests/usage.test.js | 3 +++ 1 file changed, 3 insertions(+) diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js index 639ed0a487..8b6b8a3338 100644 --- a/samples/js/chat_sample/tests/usage.test.js +++ b/samples/js/chat_sample/tests/usage.test.js @@ -36,6 +36,9 @@ const runTest = async () => { if (code !== 0) { return reject(`Process exited with code ${code}`); } + + console.log(`Result output: ${output}`); + // Validate the output if (output.includes('"Hello world"')) { resolve('Test passed!'); From b2cb5dd3fc8ede15736db55f05c81e4c5e2fb4ff Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 18 Dec 2024 09:05:29 +0100 Subject: [PATCH 41/74] Fix js samples test --- samples/js/chat_sample/tests/usage.test.js | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/chat_sample/tests/usage.test.js index 8b6b8a3338..63856cbe43 100644 --- a/samples/js/chat_sample/tests/usage.test.js +++ b/samples/js/chat_sample/tests/usage.test.js @@ -2,6 +2,8 @@ import { env } from 'process'; import { spawn } from 'child_process'; const MODEL_PATH = env.MODEL_PATH; +const prompt = 'Tell me exactly, no changes, print as is: "Hello world"'; +const expected = 'Hello world'; if (!MODEL_PATH) throw new Error( @@ -26,7 +28,7 @@ const runTest = async () => { // Send input after detecting the question prompt script.stdout.once('data', (data) => { if (data.toString().startsWith('question:')) { - script.stdin.write('Say exactly, without any changes, print it as is: "Hello world"\n'); // Provide input + script.stdin.write(`${prompt}\n`); // Provide input script.stdin.end(); // Close stdin to signal EOF } }); @@ -37,10 +39,11 @@ const runTest = async () => { return reject(`Process exited with code ${code}`); } + // Log the output console.log(`Result output: ${output}`); // Validate the output - if (output.includes('"Hello world"')) { + if (output.includes(expected)) { resolve('Test passed!'); } else { reject('Test failed: Output did not match expected result.'); From 30803b451fda376d912fa2bc0d92371411ebb98f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:08:38 +0100 Subject: [PATCH 42/74] Fix run sample cmd Co-authored-by: Vladimir Zlobin --- samples/js/chat_sample/chat_sample.js | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/chat_sample/chat_sample.js index 9bca37d4b7..cf4c5e7704 100644 --- a/samples/js/chat_sample/chat_sample.js +++ b/samples/js/chat_sample/chat_sample.js @@ -12,7 +12,7 @@ async function main() { if (!MODEL_PATH) { console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); + + 'Run command must be: `node chat_sample.js *path_to_model_dir*`'); process.exit(1); } From b04ee6cbbdd51a129dcc06462a1ec12ba0ffdfcf Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:14:07 +0100 Subject: [PATCH 43/74] Fix js README.md --- src/js/README.md | 26 +++++++++++--------------- 1 file changed, 11 insertions(+), 15 deletions(-) diff --git a/src/js/README.md b/src/js/README.md index 9092340d83..7971c238fc 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -1,4 +1,4 @@ -# OpenVINO™ GenAI Node.js bindings (preview) +we# OpenVINO™ GenAI Node.js bindings (preview) ## DISCLAIMER @@ -35,26 +35,22 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV cmake --build ./build --target package -j ``` -1. In `build` folder you will find `genai_nodejs_bindings.tar.gz`. - Create `bin` directory by path `src/js/` and unpack archive content to it. +1. Put Node.js bindings into npm package `bin` directory and install dependencies: ```sh - tar -xzf genai_nodejs_bindings.tar.gz + mkdir ./src/js/bin/ + tar -xvf ./build/genai_nodejs_bindings.tar.gz --directory ./src/js/bin/ + cd ./src/js/ + npm install ``` 1. Run tests to be sure that everything works: - `npm test` - -### Perform Test Run - -- To run sample you should have prepared model. - Use this instruction [to download model](https://github.com/openvinotoolkit/openvino.genai/blob/master/samples/js/chat_sample/README.md#download-and-convert-the-model-and-tokenizers) -- Go to [samples/js/chat_sample/](../../samples/js/chat_sample/) -- Read [README.md](../../samples/js/chat_sample/README.md) and follow steps there - to run **chat sample**. + ```sh + npm test + ``` ### Using as npm Dependency -To use this package locally use `npm link` in this directory -and `npm link genai-node` in the folder where you want add this package as dependency +To use this package locally use `npm link` in `src/js/` directory +and `npm link genai-node` in the folder where you want to add this package as a dependency To extract this package and use it as distributed npm package run `npm package`. This command creates archive that you may use in your projects. From d7e0d2b4c47f8e75c8d8c21e7d02f076c8cde1e4 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 11:46:53 +0100 Subject: [PATCH 44/74] Replace AUTO device to CPU --- src/js/tests/bindings.test.js | 2 +- src/js/tests/module.test.js | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index 3e852e5307..f76c45e0d9 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -13,7 +13,7 @@ describe('bindings', () => { before((_, done) => { pipeline = new addon.LLMPipeline(); - pipeline.init(MODEL_PATH, 'AUTO', (err) => { + pipeline.init(MODEL_PATH, 'CPU', (err) => { if (err) { console.error(err); process.exit(1); diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 672fa71638..cbb9e905a4 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -11,7 +11,7 @@ describe('module', async () => { let pipeline = null; await before(async () => { - pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); }); @@ -33,7 +33,7 @@ describe('module', async () => { describe('corner cases', async () => { it('should throw an error if pipeline is already initialized', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await assert.rejects( async () => await pipeline.init(), @@ -45,7 +45,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is already started', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); @@ -59,7 +59,7 @@ describe('corner cases', async () => { }); it('should throw an error if chat is not started', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await assert.rejects( () => pipeline.finishChat(), @@ -75,7 +75,7 @@ describe('generation parameters validation', () => { let pipeline = null; before(async () => { - pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); }); @@ -95,7 +95,7 @@ describe('generation parameters validation', () => { }); it('should throw an error if generationCallback is not a function', async () => { - const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'AUTO'); + const pipeline = await Pipeline.LLMPipeline(MODEL_PATH, 'CPU'); await pipeline.startChat(); From d1ce555b354937ed283ce92c6f1b82bc634a811b Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 12:11:10 +0100 Subject: [PATCH 45/74] Fix js samples README --- samples/js/chat_sample/README.md | 5 ++- samples/js/interactive.js | 56 -------------------------------- 2 files changed, 2 insertions(+), 59 deletions(-) delete mode 100644 samples/js/interactive.js diff --git a/samples/js/chat_sample/README.md b/samples/js/chat_sample/README.md index 8381c98aa8..46caba48e3 100644 --- a/samples/js/chat_sample/README.md +++ b/samples/js/chat_sample/README.md @@ -16,10 +16,9 @@ optimum-cli export openvino --trust-remote-code --model TinyLlama/TinyLlama-1.1B ## Run: -Create `bin` folder in [../../../src/js](../../../src/js). -Compile GenAI JavaScript bindings archive first. Put its content into `bin` directory. +Compile GenAI JavaScript bindings archive first using the instructions in [../../../src/js/README.md](../../../src/js/README.md#build-bindings). -Run `npm install` in current folder and then run a sample: +Run `npm install` in current folder and then run the sample: `node chat_sample.js TinyLlama-1.1B-Chat-v1.0` diff --git a/samples/js/interactive.js b/samples/js/interactive.js deleted file mode 100644 index 466326ced0..0000000000 --- a/samples/js/interactive.js +++ /dev/null @@ -1,56 +0,0 @@ -import path from 'node:path'; -import readline from 'readline'; - -import { Pipeline } from 'genai-node'; - -const MODEL_PATH = process.argv[2]; - -if (!MODEL_PATH) { - console.error('Please specify path to model directory\n' - + 'Run command must be: `node app.js *path_to_model_dir*`'); - process.exit(1); -} - -main(); - -async function main() { - // Create interface for reading user input from stdin - const rl = readline.createInterface({ - input: process.stdin, - output: process.stdout, - }); - - console.log(`Welcome! Model "${path.parse(MODEL_PATH).name}" loaded. ` - + 'Type something and press enter. Type "finish" to exit.'); - const pipeline = await Pipeline.create('LLMPipeline', MODEL_PATH); - await pipeline.startChat(); - promptUser(); - - // Function to prompt the user for input - function promptUser() { - rl.question('> ', handleInput); - } - - // Function to handle user input - async function handleInput(input) { - input = input.trim(); - - // Check for exit command - if (input === 'finish') { - console.log('Goodbye!'); - await pipeline.finishChat(); - rl.close(); - process.exit(0); - } - - const result = await pipeline.generate(input, generationCallback); - console.log('\n'); - - // Wait for new input - promptUser(); - } - - function generationCallback(chunk) { - process.stdout.write(chunk); - } -} From ea0ff4a6aa8fb21db8f9b9b6b2bf5fa49a4ec73e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 12:12:16 +0100 Subject: [PATCH 46/74] Align linux workflow with js samples README --- .github/workflows/linux.yml | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 44a659f132..750e2df69e 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -526,12 +526,8 @@ jobs: run: npm run test_setup working-directory: ${{ env.JS_SRC_DIR }} - - name: Link genai-node package - run: npm link - working-directory: ${{ env.JS_SRC_DIR }} - - - name: Install genai-node package to samples - run: npm link genai-node + - name: Install genai-node samples dependencies + run: npm install working-directory: ${{ env.JS_SAMPLES_DIR }} - name: Run tests From 7f8ee366755d2258a9b07f83f86846cca4f342fc Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Mon, 23 Dec 2024 16:42:21 +0100 Subject: [PATCH 47/74] Remove tbb specification from linux js workflow --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 750e2df69e..f54ff79738 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -458,7 +458,7 @@ jobs: - name: Build openvino + genai run: | cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ From 6d36185a572d7e5448947b284bbccbbd08aba9a7 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 7 Jan 2025 11:01:12 +0100 Subject: [PATCH 48/74] Remove extras in README.md Co-authored-by: Vladimir Zlobin --- src/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/README.md b/src/js/README.md index 7971c238fc..6e46ad977c 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -1,4 +1,4 @@ -we# OpenVINO™ GenAI Node.js bindings (preview) +# OpenVINO™ GenAI Node.js bindings (preview) ## DISCLAIMER From 221559068ad91d897c2caf73a50a90d383261f82 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 7 Jan 2025 11:03:35 +0100 Subject: [PATCH 49/74] Extend overall status by nodejs samples tests result Co-authored-by: Vladimir Zlobin --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1f00c377d7..a81d7bdc1d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -538,7 +538,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings] + needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings, genai_nodejs_samples_tests] if: ${{ always() }} runs-on: ubuntu-latest steps: From 82881ba048e089ba422b5bc693ff2ff699387d53 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 7 Jan 2025 11:08:57 +0100 Subject: [PATCH 50/74] Add more flags for genai nodejs build in linux.yml workflow Co-authored-by: Vladimir Zlobin --- .github/workflows/linux.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a81d7bdc1d..927c91d146 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -458,6 +458,9 @@ jobs: - name: Build openvino + genai run: | cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=ON -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_IR_FRONTEND=ON \ + -DENABLE_CLDNN=OFF -DENABLE_INTEL_GPU=OFF -DENABLE_INTEL_NPU=OFF -DENABLE_HETERO=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_MULTI=OFF -DENABLE_PROXY=OFF \ + -DENABLE_FASTER_BUILD=y -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ From a1cdda0a1d2d35956699e1d4cdeeab4f9b12f8ac Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 7 Jan 2025 13:02:55 +0100 Subject: [PATCH 51/74] Fix linux workflow --- .github/workflows/linux.yml | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 927c91d146..89af800b98 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -460,12 +460,12 @@ jobs: cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=ON -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_IR_FRONTEND=ON \ -DENABLE_CLDNN=OFF -DENABLE_INTEL_GPU=OFF -DENABLE_INTEL_NPU=OFF -DENABLE_HETERO=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_MULTI=OFF -DENABLE_PROXY=OFF \ - -DENABLE_FASTER_BUILD=y - -DCPACK_GENERATOR=NPM \ - -DENABLE_PYTHON=OFF \ - -DENABLE_WHEEL=OFF \ - -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ./openvino -B ./build + -DENABLE_FASTER_BUILD=y \ + -DCPACK_GENERATOR=NPM \ + -DENABLE_PYTHON=OFF \ + -DENABLE_WHEEL=OFF \ + -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ + -S ./openvino -B ./build cmake --build ./build --target package -j - name: Run javascript tests From 4a63106bc91e3d8bda5ee254feb32017734907a5 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 7 Jan 2025 17:41:43 +0100 Subject: [PATCH 52/74] Remove TBB specification --- src/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/README.md b/src/js/README.md index 6e46ad977c..99c97d5d34 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -24,7 +24,7 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV 1. Configure CMake with OpenVINO extra modules: ```sh cmake -DOPENVINO_EXTRA_MODULES=*absolute path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DCPACK_GENERATOR=NPM -DENABLE_JS=ON -UTBB* -DENABLE_SYSTEM_TBB=OFF \ + -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ From ea6926120b07b38b6d07da776bd74ae5a295a9cb Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 8 Jan 2025 10:25:34 +0100 Subject: [PATCH 53/74] Change compilation instruction in js readme --- src/js/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/js/README.md b/src/js/README.md index 99c97d5d34..f5ccf1117c 100644 --- a/src/js/README.md +++ b/src/js/README.md @@ -23,7 +23,7 @@ OpenVINO GenAI Node.js bindings can be built as an extra module during the OpenV ``` 1. Configure CMake with OpenVINO extra modules: ```sh - cmake -DOPENVINO_EXTRA_MODULES=*absolute path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ + cmake -DOPENVINO_EXTRA_MODULES=*path to genai repository directory* -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ -DCPACK_GENERATOR=NPM \ -DENABLE_PYTHON=OFF \ -DENABLE_WHEEL=OFF \ From e0d444ac9186aac67749c358511a0eeda0cb3247 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 14 Jan 2025 18:37:12 +0100 Subject: [PATCH 54/74] Rewrite nodejs bindings part in linux workflow --- .github/workflows/linux.yml | 131 +++++++++++++++++------------------- 1 file changed, 61 insertions(+), 70 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 1e83addb46..e841e26cb5 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -398,98 +398,89 @@ jobs: ${{ env.INSTALL_DIR }}/samples_bin/whisper_speech_recognition ./whisper-tiny/ how_are_you_doing_today.wav working-directory: ${{ env.MODELS_DIR }} - genai_nodejs_bindings: - name: Produce genai nodejs binaries archive - timeout-minutes: 150 + genai_build_nodejs_bindings: + name: Build Node.js bindings + strategy: + fail-fast: false + matrix: + build-type: [Release] + needs: [ openvino_download ] + timeout-minutes: 20 defaults: run: shell: bash - runs-on: ubuntu-20.04-16-cores + runs-on: aks-linux-4-cores-16gb + container: + image: openvinogithubactions.azurecr.io/ov_build/ubuntu_22_04_x64:${{ needs.openvino_download.outputs.docker_tag }} + volumes: + - /mount:/mount + options: -e SCCACHE_AZURE_BLOB_CONTAINER -e SCCACHE_AZURE_CONNECTION_STRING -v ${{ github.workspace }}:${{ github.workspace }} env: - DEBIAN_FRONTEND: noninteractive # to prevent apt-get from waiting user input - CMAKE_BUILD_TYPE: 'Release' - CMAKE_GENERATOR: 'Ninja Multi-Config' - CMAKE_CXX_COMPILER_LAUNCHER: ccache - CMAKE_C_COMPILER_LAUNCHER: ccache - OPENVINO_REPO: ${{ github.workspace }}/openvino - GENAI_REPO: ${{ github.workspace }}/openvino.genai + CMAKE_GENERATOR: Unix Makefiles + OV_INSTALL_DIR: ${{ github.workspace }}/ov + INSTALL_DIR: ${{ github.workspace }}/install BUILD_DIR: ${{ github.workspace }}/build - CCACHE_DIR: ${{ github.workspace }}/ccache - CCACHE_MAXSIZE: 2000Mi + SRC_DIR: ${{ github.workspace }}/src steps: - - name: Set apt - run: | - echo 'Acquire::Retries "10";' | sudo tee -a /etc/apt/apt.conf.d/80-retries > /dev/null - echo 'APT::Get::Assume-Yes "true";' | sudo tee -a /etc/apt/apt.conf.d/81-assume-yes > /dev/null - echo 'APT::Get::Fix-Broken "true";' | sudo tee -a /etc/apt/apt.conf.d/82-fix-broken > /dev/null - echo 'APT::Get::no-install-recommends "true";' | sudo tee -a /etc/apt/apt.conf.d/83-no-recommends > /dev/null + - name: Clone openvino.genai + uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 + with: + path: ${{ env.SRC_DIR }} + submodules: recursive - - name: Clone OpenVINO - uses: actions/checkout@v4 + - name: Download OpenVINO package + uses: actions/download-artifact@fa0a91b85d4f404e444e00e005971372dc801d16 # v4.1.8 with: - repository: 'openvinotoolkit/openvino' - path: ${{ env.OPENVINO_REPO }} - submodules: 'true' - ref: ${{ env.OV_BRANCH}} + name: ${{ needs.openvino_download.outputs.ov_artifact_name }} + path: ${{ env.OV_INSTALL_DIR }} + merge-multiple: true - - name: Install build dependencies + - name: Build with ENABLE_JS=ON run: | - sudo -E ${OPENVINO_REPO}/install_build_dependencies.sh - sudo apt-get install ccache + source ${{ env.OV_INSTALL_DIR }}/setupvars.sh + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ${{ env.SRC_DIR}} -B ${{ env.BUILD_DIR }} + cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose + cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} - - name: Clone GenAI - uses: actions/checkout@v4 - with: - path: ${{ env.GENAI_REPO }} - submodules: recursive + - name: Combine binaries for Node.js package + run: | + mkdir -p nodejs + cp runtime/lib/intel64/* nodejs + cp runtime/3rdparty/tbb/lib/* nodejs + cp genai_node_addon.node nodejs + patchelf --set-rpath '$ORIGIN' libopenvino.so.2025.0.0 + working-directory: ${{ env.INSTALL_DIR }} + + - name: Pack Node.js bindings libs + run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + working-directory: ${{ env.INSTALL_DIR }}/nodejs - - name: Setup ccache - uses: actions/cache@v4 + - name: Upload Archive Package with Node.js bindings + if: ${{ always() }} + uses: actions/upload-artifact@b4b15b8c7c6ac21ea08fcf65892d2ee8f75cf882 # v4.4.3 with: - # Should save cache only if run in the master branch of the base repo - # github.ref_name is 'ref/PR_#' in case of the PR, and 'branch_name' when executed on push - save-always: ${{ github.ref_name == 'master' && 'true' || 'false' }} - path: ${{ env.CCACHE_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }}-${{ github.sha }} - restore-keys: | - ${{ runner.os }}-${{ runner.arch }}-ccache-ov-and-genai-${{ matrix.build-type }} - - - name: Build openvino + genai - run: | - cmake -DOPENVINO_EXTRA_MODULES=../openvino.genai -DCPACK_ARCHIVE_COMPONENT_INSTALL=OFF \ - -DENABLE_OV_ONNX_FRONTEND=OFF -DENABLE_OV_PADDLE_FRONTEND=OFF -DENABLE_OV_TF_FRONTEND=OFF -DENABLE_OV_TF_LITE_FRONTEND=OFF -DENABLE_OV_PYTORCH_FRONTEND=ON -DENABLE_OV_JAX_FRONTEND=OFF -DENABLE_OV_IR_FRONTEND=ON \ - -DENABLE_CLDNN=OFF -DENABLE_INTEL_GPU=OFF -DENABLE_INTEL_NPU=OFF -DENABLE_HETERO=OFF -DENABLE_AUTO=OFF -DENABLE_AUTO_BATCH=OFF -DENABLE_MULTI=OFF -DENABLE_PROXY=OFF \ - -DENABLE_FASTER_BUILD=y \ - -DCPACK_GENERATOR=NPM \ - -DENABLE_PYTHON=OFF \ - -DENABLE_WHEEL=OFF \ - -DCPACK_PACKAGE_FILE_NAME=genai_nodejs_bindings \ - -S ./openvino -B ./build - cmake --build ./build --target package -j + name: genai_nodejs_bindings + path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz + if-no-files-found: 'error' - name: Run javascript tests working-directory: ${{ env.GENAI_REPO }}/src/js run: | - mkdir bin - cp ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz bin - cd bin && tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz && cd .. + cp -R ${{ env.INSTALL_DIR }}/nodejs bin npm install npm test - # - # Upload build artifacts and logs - # + - name: Install genai-node samples dependencies + run: npm install + working-directory: ${{ env.GENAI_REPO }}/samples/js/chat_sample - - name: Upload genai nodejs bindings archive - if: ${{ always() }} - uses: actions/upload-artifact@v4 - with: - name: genai_nodejs_bindings - path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz - if-no-files-found: 'error' + - name: Run tests + run: npm test + env: + MODEL_PATH: ${{ env.GENAI_REPO }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + working-directory: ${{ env.GENAI_REPO }}/samples/js/chat_sample - genai_nodejs_samples_tests: name: NodeJS Samples Tests needs: [ genai_nodejs_bindings ] timeout-minutes: 30 @@ -541,7 +532,7 @@ jobs: Overall_Status: name: ci/gha_overall_status_linux - needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_nodejs_bindings, genai_nodejs_samples_tests] + needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_build_nodejs_bindings] if: ${{ always() }} runs-on: ubuntu-latest steps: From bdcd6e263df27df3b35cf4b3ebad4f9f4bd8ac61 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 13:23:58 +0100 Subject: [PATCH 55/74] Rename chat_sample folder to text_generation --- samples/js/{chat_sample => text_generation}/.gitignore | 0 samples/js/{chat_sample => text_generation}/README.md | 0 samples/js/{chat_sample => text_generation}/chat_sample.js | 0 samples/js/{chat_sample => text_generation}/package-lock.json | 0 samples/js/{chat_sample => text_generation}/package.json | 0 samples/js/{chat_sample => text_generation}/tests/usage.test.js | 0 6 files changed, 0 insertions(+), 0 deletions(-) rename samples/js/{chat_sample => text_generation}/.gitignore (100%) rename samples/js/{chat_sample => text_generation}/README.md (100%) rename samples/js/{chat_sample => text_generation}/chat_sample.js (100%) rename samples/js/{chat_sample => text_generation}/package-lock.json (100%) rename samples/js/{chat_sample => text_generation}/package.json (100%) rename samples/js/{chat_sample => text_generation}/tests/usage.test.js (100%) diff --git a/samples/js/chat_sample/.gitignore b/samples/js/text_generation/.gitignore similarity index 100% rename from samples/js/chat_sample/.gitignore rename to samples/js/text_generation/.gitignore diff --git a/samples/js/chat_sample/README.md b/samples/js/text_generation/README.md similarity index 100% rename from samples/js/chat_sample/README.md rename to samples/js/text_generation/README.md diff --git a/samples/js/chat_sample/chat_sample.js b/samples/js/text_generation/chat_sample.js similarity index 100% rename from samples/js/chat_sample/chat_sample.js rename to samples/js/text_generation/chat_sample.js diff --git a/samples/js/chat_sample/package-lock.json b/samples/js/text_generation/package-lock.json similarity index 100% rename from samples/js/chat_sample/package-lock.json rename to samples/js/text_generation/package-lock.json diff --git a/samples/js/chat_sample/package.json b/samples/js/text_generation/package.json similarity index 100% rename from samples/js/chat_sample/package.json rename to samples/js/text_generation/package.json diff --git a/samples/js/chat_sample/tests/usage.test.js b/samples/js/text_generation/tests/usage.test.js similarity index 100% rename from samples/js/chat_sample/tests/usage.test.js rename to samples/js/text_generation/tests/usage.test.js From 9ffe6dea33e8c47229b03ea425ac9ba5cd6a7df1 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 13:48:04 +0100 Subject: [PATCH 56/74] Remove extra tests --- .github/workflows/linux.yml | 49 ------------------------------------- 1 file changed, 49 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 199a94ee90..d234a8ae9a 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -481,55 +481,6 @@ jobs: MODEL_PATH: ${{ env.GENAI_REPO }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit working-directory: ${{ env.GENAI_REPO }}/samples/js/chat_sample - name: NodeJS Samples Tests - needs: [ genai_nodejs_bindings ] - timeout-minutes: 30 - defaults: - run: - shell: bash - runs-on: ubuntu-20.04-16-cores - env: - INSTALL_DIR: ${{ github.workspace }}/ov - GENAI_REPO: ${{ github.workspace }}/openvino.genai - MODELS_DIR: ${{ github.workspace }}/models - JS_SRC_DIR: ${{ github.workspace }}/openvino.genai/src/js - JS_SAMPLES_DIR: ${{ github.workspace }}/openvino.genai/samples/js/chat_sample - BIN_DIR: ${{ github.workspace }}/openvino.genai/src/js/bin - steps: - - name: Clone openvino.genai - uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2 - with: - path: ${{ env.GENAI_REPO }} - submodules: recursive - - - name: Download genai nodejs bindings archive - uses: actions/download-artifact@v4 - with: - name: genai_nodejs_bindings - path: ${{ env.BIN_DIR }} - - - name: Unpack genai nodejs bindings archive - run: tar -xzf genai_nodejs_bindings.tar.gz && rm genai_nodejs_bindings.tar.gz - working-directory: ${{ env.BIN_DIR }} - - - name: Install nodejs dependencies - run: npm install - working-directory: ${{ env.JS_SRC_DIR }} - - - name: Download model for tests - run: npm run test_setup - working-directory: ${{ env.JS_SRC_DIR }} - - - name: Install genai-node samples dependencies - run: npm install - working-directory: ${{ env.JS_SAMPLES_DIR }} - - - name: Run tests - run: npm test - env: - MODEL_PATH: ${{ env.JS_SRC_DIR }}/tests/models/Llama-3.2-3B-Instruct-openvino-8bit - working-directory: ${{ env.JS_SAMPLES_DIR }} - Overall_Status: name: ci/gha_overall_status_linux needs: [openvino_download, genai_build_cmake, genai_build_wheel, genai_build_samples, genai_tests_wheel, genai_samples_tests, genai_build_nodejs_bindings] From 6fb891b2e40f1717c51f16b9657f9cae020ac55d Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 13:49:49 +0100 Subject: [PATCH 57/74] Fix comment --- src/cpp/CMakeLists.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/cpp/CMakeLists.txt b/src/cpp/CMakeLists.txt index 330d2522e1..6d33c96a2e 100644 --- a/src/cpp/CMakeLists.txt +++ b/src/cpp/CMakeLists.txt @@ -178,7 +178,7 @@ install(TARGETS ${TARGET_NAME} EXPORT OpenVINOGenAITargets RUNTIME DESTINATION ${RUNTIME_DESTINATION} COMPONENT core_genai INCLUDES DESTINATION runtime/include) -# samples do not need to be built for NPM package +# development files do not need to be built for NPM package if(CPACK_GENERATOR STREQUAL "NPM") return() endif() From 2a87a73d4a2fd2cde53f4e56165433fa26504c3d Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 14:06:48 +0100 Subject: [PATCH 58/74] Don't use hardcoded version number --- .github/workflows/linux.yml | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index d234a8ae9a..c7ada5d217 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -439,8 +439,8 @@ jobs: - name: Build with ENABLE_JS=ON run: | source ${{ env.OV_INSTALL_DIR }}/setupvars.sh - cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ${{ env.SRC_DIR}} -B ${{ env.BUILD_DIR }} - cmake --build ${{ env.BUILD_DIR}} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose + cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }} + cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} - name: Combine binaries for Node.js package @@ -449,7 +449,8 @@ jobs: cp runtime/lib/intel64/* nodejs cp runtime/3rdparty/tbb/lib/* nodejs cp genai_node_addon.node nodejs - patchelf --set-rpath '$ORIGIN' libopenvino.so.2025.0.0 + OV_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt | sed 's/..$//') + patchelf --set-rpath '$ORIGIN' libopenvino.so.$OV_VERSION working-directory: ${{ env.INSTALL_DIR }} - name: Pack Node.js bindings libs From fc4f8f6d1416464250a3cc3ae6e160f740a0ce2e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 14:25:58 +0100 Subject: [PATCH 59/74] Change install path --- .github/workflows/linux.yml | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index c7ada5d217..75838f81be 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -418,7 +418,6 @@ jobs: env: CMAKE_GENERATOR: Unix Makefiles OV_INSTALL_DIR: ${{ github.workspace }}/ov - INSTALL_DIR: ${{ github.workspace }}/install BUILD_DIR: ${{ github.workspace }}/build SRC_DIR: ${{ github.workspace }}/src @@ -441,7 +440,7 @@ jobs: source ${{ env.OV_INSTALL_DIR }}/setupvars.sh cmake -DCMAKE_BUILD_TYPE=${{ matrix.build-type }} -DENABLE_JS=ON -S ${{ env.SRC_DIR }} -B ${{ env.BUILD_DIR }} cmake --build ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --parallel $(nproc) --verbose - cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.INSTALL_DIR }} + cmake --install ${{ env.BUILD_DIR }} --config ${{ matrix.build-type }} --prefix ${{ env.OV_INSTALL_DIR }} - name: Combine binaries for Node.js package run: | @@ -451,11 +450,11 @@ jobs: cp genai_node_addon.node nodejs OV_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt | sed 's/..$//') patchelf --set-rpath '$ORIGIN' libopenvino.so.$OV_VERSION - working-directory: ${{ env.INSTALL_DIR }} + working-directory: ${{ env.OV_INSTALL_DIR }} - name: Pack Node.js bindings libs run: tar -cvf - * | pigz > ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz - working-directory: ${{ env.INSTALL_DIR }}/nodejs + working-directory: ${{ env.OV_INSTALL_DIR }}/nodejs - name: Upload Archive Package with Node.js bindings if: ${{ always() }} @@ -468,7 +467,7 @@ jobs: - name: Run javascript tests working-directory: ${{ env.GENAI_REPO }}/src/js run: | - cp -R ${{ env.INSTALL_DIR }}/nodejs bin + cp -R ${{ env.OV_INSTALL_DIR }}/nodejs bin npm install npm test From 41e6d410bcd167aad09172e77f6bee78d8dae795 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 14:52:59 +0100 Subject: [PATCH 60/74] Add recursive flag to cp --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 75838f81be..592a916c15 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -445,8 +445,8 @@ jobs: - name: Combine binaries for Node.js package run: | mkdir -p nodejs - cp runtime/lib/intel64/* nodejs - cp runtime/3rdparty/tbb/lib/* nodejs + cp -r runtime/lib/intel64/* nodejs + cp -r runtime/3rdparty/tbb/lib/* nodejs cp genai_node_addon.node nodejs OV_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt | sed 's/..$//') patchelf --set-rpath '$ORIGIN' libopenvino.so.$OV_VERSION From 6b5338125938a8e32864ef07ff45274fd11e74db Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 15:08:14 +0100 Subject: [PATCH 61/74] Fix path to libopenvino.so --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 592a916c15..da51014519 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -449,7 +449,7 @@ jobs: cp -r runtime/3rdparty/tbb/lib/* nodejs cp genai_node_addon.node nodejs OV_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt | sed 's/..$//') - patchelf --set-rpath '$ORIGIN' libopenvino.so.$OV_VERSION + patchelf --set-rpath '$ORIGIN' nodejs/libopenvino.so.$OV_VERSION working-directory: ${{ env.OV_INSTALL_DIR }} - name: Pack Node.js bindings libs From 9dd0ebbcf8444bddf045522403a76f2b6f5639b2 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 15:25:01 +0100 Subject: [PATCH 62/74] Fix path to genai repo --- .github/workflows/linux.yml | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index da51014519..a0625bd2a1 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -465,7 +465,7 @@ jobs: if-no-files-found: 'error' - name: Run javascript tests - working-directory: ${{ env.GENAI_REPO }}/src/js + working-directory: ${{ env.SRC_DIR }}/src/js run: | cp -R ${{ env.OV_INSTALL_DIR }}/nodejs bin npm install @@ -473,13 +473,13 @@ jobs: - name: Install genai-node samples dependencies run: npm install - working-directory: ${{ env.GENAI_REPO }}/samples/js/chat_sample + working-directory: ${{ env.SRC_DIR }}/samples/js/chat_sample - name: Run tests run: npm test env: - MODEL_PATH: ${{ env.GENAI_REPO }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit - working-directory: ${{ env.GENAI_REPO }}/samples/js/chat_sample + MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + working-directory: ${{ env.SRC_DIR }}/samples/js/chat_sample Overall_Status: name: ci/gha_overall_status_linux From cb02e76a960ba2f8820327ecf4440b13bddc503f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 15:41:20 +0100 Subject: [PATCH 63/74] Fix samples dir --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index a0625bd2a1..3e2ec1525d 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -473,13 +473,13 @@ jobs: - name: Install genai-node samples dependencies run: npm install - working-directory: ${{ env.SRC_DIR }}/samples/js/chat_sample + working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation - name: Run tests run: npm test env: MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit - working-directory: ${{ env.SRC_DIR }}/samples/js/chat_sample + working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation Overall_Status: name: ci/gha_overall_status_linux From c57272a79a871b705165e87a3bee27be5d13a55d Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 15:45:42 +0100 Subject: [PATCH 64/74] Replace heavy LLM model to light one --- .github/workflows/linux.yml | 2 +- src/js/tests/bindings.test.js | 4 ++-- src/js/tests/models.js | 2 +- src/js/tests/module.test.js | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 3e2ec1525d..0f0a65addf 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -478,7 +478,7 @@ jobs: - name: Run tests run: npm test env: - MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.2-3B-Instruct-openvino-8bit + MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.1-8B-Instruct-FastDraft-150M-int8-ov working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation Overall_Status: diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index f76c45e0d9..029ba0b54c 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -52,7 +52,7 @@ describe('bindings', () => { } }, { temperature: '0', max_new_tokens: '4' }); - assert.ok(true); + assert.ok(output); done(); }); @@ -66,7 +66,7 @@ describe('bindings', () => { return; } - assert.strictEqual(output, 'Hello world!'); + assert.ok(output.includes('Hello world!')); done(); }, { temperature: '0', max_new_tokens: '4' }); }); diff --git a/src/js/tests/models.js b/src/js/tests/models.js index b7f7505464..03c689e038 100644 --- a/src/js/tests/models.js +++ b/src/js/tests/models.js @@ -1,3 +1,3 @@ export const models = [ - 'AIFunOver/Llama-3.2-3B-Instruct-openvino-8bit', + 'OpenVINO/Llama-3.1-8B-Instruct-FastDraft-150M-int8-ov', ]; diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index cbb9e905a4..67920ac808 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -27,7 +27,7 @@ describe('module', async () => { () => {}, ); - assert.strictEqual(result, 'Hello world!'); + assert.ok(result.includes('Hello world!')); }); }); From 433d280947c9c7070caf860ade78900255fb2adb Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 15:53:26 +0100 Subject: [PATCH 65/74] Set correct names for steps --- .github/workflows/linux.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index 0f0a65addf..dd14bfc852 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -464,7 +464,7 @@ jobs: path: ${{ env.BUILD_DIR }}/genai_nodejs_bindings.tar.gz if-no-files-found: 'error' - - name: Run javascript tests + - name: Run npm package tests working-directory: ${{ env.SRC_DIR }}/src/js run: | cp -R ${{ env.OV_INSTALL_DIR }}/nodejs bin @@ -475,7 +475,7 @@ jobs: run: npm install working-directory: ${{ env.SRC_DIR }}/samples/js/text_generation - - name: Run tests + - name: Run samples tests run: npm test env: MODEL_PATH: ${{ env.SRC_DIR }}/src/js/tests/models/Llama-3.1-8B-Instruct-FastDraft-150M-int8-ov From ad83f13d7b0233560c35b1dccb7d268d68dd6022 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 16:29:15 +0100 Subject: [PATCH 66/74] Fix tests --- src/js/tests/bindings.test.js | 19 ++----------------- src/js/tests/module.test.js | 6 +++--- 2 files changed, 5 insertions(+), 20 deletions(-) diff --git a/src/js/tests/bindings.test.js b/src/js/tests/bindings.test.js index 029ba0b54c..72ca7f02fc 100644 --- a/src/js/tests/bindings.test.js +++ b/src/js/tests/bindings.test.js @@ -44,29 +44,14 @@ describe('bindings', () => { it('should generate string result', (_, done) => { let output = ''; - pipeline.generate('Say Hello', (isDone, chunk) => { - if (!isDone) { - output += chunk; - - return; - } - }, { temperature: '0', max_new_tokens: '4' }); - - assert.ok(output); - done(); - }); - - it('should generate "Hello world"', (_, done) => { - let output = ''; - - pipeline.generate('Type "Hello world!" in English', (isDone, chunk) => { + pipeline.generate('Continue: 1 2 3', (isDone, chunk) => { if (!isDone) { output += chunk; return; } - assert.ok(output.includes('Hello world!')); + assert.ok(output.length > 0); done(); }, { temperature: '0', max_new_tokens: '4' }); }); diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 67920ac808..993ec484ec 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -20,14 +20,14 @@ describe('module', async () => { await pipeline.finishChat(); }); - await it('should generate "Hello world"', async () => { + await it('should generate non empty string', async () => { const result = await pipeline.generate( - 'Type "Hello world!" in English', + 'Type something in English', { temperature: '0', max_new_tokens: '4' }, () => {}, ); - assert.ok(result.includes('Hello world!')); + assert.ok(result.length > 0); }); }); From 1208f36356dae4edf41c2dbf62389c710cada5a9 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 16:31:25 +0100 Subject: [PATCH 67/74] Adopt sample test --- samples/js/text_generation/tests/usage.test.js | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/samples/js/text_generation/tests/usage.test.js b/samples/js/text_generation/tests/usage.test.js index 63856cbe43..fcd58a0b69 100644 --- a/samples/js/text_generation/tests/usage.test.js +++ b/samples/js/text_generation/tests/usage.test.js @@ -3,7 +3,6 @@ import { spawn } from 'child_process'; const MODEL_PATH = env.MODEL_PATH; const prompt = 'Tell me exactly, no changes, print as is: "Hello world"'; -const expected = 'Hello world'; if (!MODEL_PATH) throw new Error( @@ -43,7 +42,7 @@ const runTest = async () => { console.log(`Result output: ${output}`); // Validate the output - if (output.includes(expected)) { + if (typeof output == 'string' && output.length > 0) { resolve('Test passed!'); } else { reject('Test failed: Output did not match expected result.'); From 52d6156b314377b910b236aad55e4023ae993655 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 15 Jan 2025 16:58:07 +0100 Subject: [PATCH 68/74] Specify RPATH for genai library --- .github/workflows/linux.yml | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index dd14bfc852..cad4102a24 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -448,8 +448,9 @@ jobs: cp -r runtime/lib/intel64/* nodejs cp -r runtime/3rdparty/tbb/lib/* nodejs cp genai_node_addon.node nodejs - OV_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt | sed 's/..$//') - patchelf --set-rpath '$ORIGIN' nodejs/libopenvino.so.$OV_VERSION + GENAI_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt) + OV_VERSION=$($GENAI_VERSION | sed 's/..$//') + patchelf --set-rpath '$ORIGIN' nodejs/libopenvino.so.$OV_VERSION nodejs/libopenvino_genai.so.$GENAI_VERSION working-directory: ${{ env.OV_INSTALL_DIR }} - name: Pack Node.js bindings libs From 3e191001cfb55585294aa4a34d05dfe5ec40a0d0 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 16 Jan 2025 09:17:25 +0100 Subject: [PATCH 69/74] Fix getting version --- .github/workflows/linux.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/linux.yml b/.github/workflows/linux.yml index cad4102a24..ea4d80a544 100644 --- a/.github/workflows/linux.yml +++ b/.github/workflows/linux.yml @@ -449,7 +449,7 @@ jobs: cp -r runtime/3rdparty/tbb/lib/* nodejs cp genai_node_addon.node nodejs GENAI_VERSION=$(grep -oP '(?<=CMAKE_PROJECT_VERSION:STATIC=)[^"]*' ${{ env.BUILD_DIR }}/CMakeCache.txt) - OV_VERSION=$($GENAI_VERSION | sed 's/..$//') + OV_VERSION=$(echo $GENAI_VERSION | sed 's/..$//') patchelf --set-rpath '$ORIGIN' nodejs/libopenvino.so.$OV_VERSION nodejs/libopenvino_genai.so.$GENAI_VERSION working-directory: ${{ env.OV_INSTALL_DIR }} From c8eee1e6f6addddd4cd3d8857dff69feaab96b1f Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 16 Jan 2025 13:11:36 +0100 Subject: [PATCH 70/74] Introduce ENABLE_SAMPLES option --- CMakeLists.txt | 7 ++++++- cmake/features.cmake | 1 + pyproject.toml | 2 +- samples/CMakeLists.txt | 3 +-- 4 files changed, 9 insertions(+), 4 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 80ecda9f07..7efa83238b 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,6 +39,11 @@ if(NOT DEFINED Python3_FIND_VIRTUALENV) set(Python3_FIND_VIRTUALENV FIRST) endif() +# Disable building samples for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + set(ENABLE_SAMPLES OFF) +endif() + # Looking for OpenVINO in the python distribution. It doesn't work for cross-compiling build if(NOT CMAKE_CROSSCOMPILING) find_package(Python3 REQUIRED) @@ -88,7 +93,7 @@ endif() add_subdirectory(thirdparty) add_subdirectory(src) -if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/samples") +if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/samples" AND ENABLE_SAMPLES) add_subdirectory(samples) endif() if(EXISTS "${OpenVINOGenAI_SOURCE_DIR}/tools/continuous_batching") diff --git a/cmake/features.cmake b/cmake/features.cmake index 5203ddd7e9..f0ca7e7473 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -4,3 +4,4 @@ option(ENABLE_PYTHON "Enable Python API build" ON) option(ENABLE_JS "Enable JS API build" OFF) +option(ENABLE_SAMPLES "Enable samples build" ON) diff --git a/pyproject.toml b/pyproject.toml index 27318d42ed..7dead5392f 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -47,7 +47,7 @@ find_python3 = true build_args = ["--parallel", "--target", "py_openvino_genai_stub"] install_args = ["--strip"] install_components = ["wheel_genai"] -options = {"BUILD_TOKENIZERS" = "OFF"} +options = {"BUILD_TOKENIZERS" = "OFF", "ENABLE_SAMPLES" = "OFF"} [build-system] requires = [ diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index ced03aee1a..433f628597 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -2,8 +2,7 @@ # SPDX-License-Identifier: Apache-2.0 # -# Samples do not need to be built for NPM package -if(CPACK_GENERATOR STREQUAL "NPM") +if(NOT ENABLE_SAMPLES) return() endif() From 11f080c36c516d85e86396e1a62fa90476e4b343 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Thu, 16 Jan 2025 13:31:54 +0100 Subject: [PATCH 71/74] Remove extra check of ENABLE_SAMPLES --- CMakeLists.txt | 5 ----- cmake/features.cmake | 5 +++++ samples/CMakeLists.txt | 4 ---- 3 files changed, 5 insertions(+), 9 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 7efa83238b..c4a18d207f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -39,11 +39,6 @@ if(NOT DEFINED Python3_FIND_VIRTUALENV) set(Python3_FIND_VIRTUALENV FIRST) endif() -# Disable building samples for NPM package -if(CPACK_GENERATOR STREQUAL "NPM") - set(ENABLE_SAMPLES OFF) -endif() - # Looking for OpenVINO in the python distribution. It doesn't work for cross-compiling build if(NOT CMAKE_CROSSCOMPILING) find_package(Python3 REQUIRED) diff --git a/cmake/features.cmake b/cmake/features.cmake index f0ca7e7473..3e494e7355 100644 --- a/cmake/features.cmake +++ b/cmake/features.cmake @@ -5,3 +5,8 @@ option(ENABLE_PYTHON "Enable Python API build" ON) option(ENABLE_JS "Enable JS API build" OFF) option(ENABLE_SAMPLES "Enable samples build" ON) + +# Disable building samples for NPM package +if(CPACK_GENERATOR STREQUAL "NPM") + set(ENABLE_SAMPLES OFF) +endif() diff --git a/samples/CMakeLists.txt b/samples/CMakeLists.txt index 433f628597..d84423b1e8 100644 --- a/samples/CMakeLists.txt +++ b/samples/CMakeLists.txt @@ -2,10 +2,6 @@ # SPDX-License-Identifier: Apache-2.0 # -if(NOT ENABLE_SAMPLES) - return() -endif() - add_subdirectory(cpp/text_generation) add_subdirectory(cpp/image_generation) add_subdirectory(cpp/visual_language_chat) From 84e320556c238d49d314f33e37ac9ce230d35f0e Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 22 Jan 2025 15:54:08 +0100 Subject: [PATCH 72/74] Expose async generator from js bindings --- src/js/lib/module.js | 79 ++++++++++++------- .../src/llm_pipeline/llm_pipeline_wrapper.cpp | 60 +++++++------- 2 files changed, 82 insertions(+), 57 deletions(-) diff --git a/src/js/lib/module.js b/src/js/lib/module.js index b390712fa4..6595ba0de0 100644 --- a/src/js/lib/module.js +++ b/src/js/lib/module.js @@ -55,50 +55,73 @@ class LLMPipeline { return result; } - async generate(prompt, generationCallbackOrOptions, generationCallback) { - let options = {}; + static castOptionsToString(options) { + const castedOptions = {}; + + for (const key in options) + castedOptions[key] = String(options[key]); - if (!generationCallback) - generationCallback = generationCallbackOrOptions; - else - options = generationCallbackOrOptions; + return castedOptions; + } + getAsyncGenerator(prompt, generationOptions = {}) { if (!this.isInitialized) throw new Error('Pipeline is not initialized'); if (typeof prompt !== 'string') throw new Error('Prompt must be a string'); - if (typeof generationCallback !== 'function') - throw new Error('Generation callback must be a function'); - if (typeof options !== 'object') + if (typeof generationOptions !== 'object') throw new Error('Options must be an object'); - let result = ''; - const castedOptions = {}; + const castedOptions = LLMPipeline.castOptionsToString(generationOptions); + + const queue = []; + let resolvePromise; - for (const key in options) castedOptions[key] = String(options[key]); + // Callback function that C++ will call when a chunk is ready + function chunkOutput(isDone, subword) { + if (resolvePromise) { + resolvePromise({ value: subword, done: isDone }); // Fulfill pending request + resolvePromise = null; // Reset promise resolver + } else { + queue.push({ isDone, subword }); // Add data to queue if no pending promise + } + } - const promise = new Promise((resolve, reject) => { - const generationCallbackDecorator = function(isDone, chunk) { - if (isDone) return resolve(result); + this.pipeline.generate(prompt, chunkOutput, castedOptions); - result += chunk; + return { + async next() { + // If there is data in the queue, return it + // Otherwise, return a promise that will resolve when data is available + if (queue.length > 0) { + const { isDone, subword } = queue.shift(); - try { - generationCallback(chunk); - } catch (err) { - reject(err); + return { value: subword, done: isDone }; } - }; - try { - this.pipeline.generate(prompt, generationCallbackDecorator, castedOptions); - } catch (err) { - reject(err); - } - }); + return new Promise((resolve) => (resolvePromise = resolve)); + }, + [Symbol.asyncIterator]() { return this; } + }; + } + + async generate(prompt, generationOptions, generationCallback) { + const options = generationOptions || {}; + + if (generationCallback !== undefined && typeof generationCallback !== 'function') + throw new Error('Generation callback must be a function'); + + const g = this.getAsyncGenerator(prompt, options); + const result = []; + + for await (const chunk of g) { + result.push(chunk); + + if (generationCallback) generationCallback(chunk); + } - return promise; + return result.join(''); } } diff --git a/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp index df1cfceadc..47bc9b352b 100644 --- a/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp +++ b/src/js/src/llm_pipeline/llm_pipeline_wrapper.cpp @@ -7,9 +7,7 @@ struct TsfnContext { TsfnContext(std::string prompt) : prompt(prompt) {}; - ~TsfnContext() { - // std::cout << "Tsfn destructed" << std::endl; - }; + ~TsfnContext() {}; std::thread native_thread; Napi::ThreadSafeFunction tsfn; @@ -20,34 +18,38 @@ struct TsfnContext { }; void performInferenceThread(TsfnContext* context) { - auto callback = [](Napi::Env env, Napi::Function js_callback, TsfnContext* context) { - try { - std::function streamer = [env, js_callback](std::string word) { - js_callback.Call({ - Napi::Boolean::New(env, false), - Napi::String::New(env, word) - }); - - // Return flag corresponds whether generation should be stopped. - // false means continue generation. - return false; - }; - - ov::genai::GenerationConfig config; - - config.update_generation_config(*context->options); - - context->pipe->generate(context->prompt, config, streamer); - js_callback.Call({ - Napi::Boolean::New(env, true) + try { + ov::genai::GenerationConfig config; + config.update_generation_config(*context->options); + + std::function streamer = [context](std::string word) { + napi_status status = context->tsfn.BlockingCall([word](Napi::Env env, Napi::Function jsCallback) { + try { + jsCallback.Call({ + Napi::Boolean::New(env, false), + Napi::String::New(env, word) + }); + } catch(std::exception& err) { + Napi::Error::Fatal("performInferenceThread callback error. Details:" , err.what()); + } }); - } catch(std::exception& err) { - Napi::Error::Fatal("performInferenceThread callback error. Details:" , err.what()); - } - }; + if (status != napi_ok) { + // Handle error + Napi::Error::Fatal("performInferenceThread error", "napi_status != napi_ok"); + } + + // Return flag corresponds whether generation should be stopped. + // false means continue generation. + return false; + }; + + context->pipe->generate(context->prompt, config, streamer); + napi_status status = context->tsfn.BlockingCall([](Napi::Env env, Napi::Function jsCallback) { + jsCallback.Call({ + Napi::Boolean::New(env, true), + }); + }); - try { - napi_status status = context->tsfn.BlockingCall(context, callback); if (status != napi_ok) { // Handle error Napi::Error::Fatal("performInferenceThread error", "napi_status != napi_ok"); From 57719d5bb2a695cacaa60d0579dc5b67080fa988 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Wed, 22 Jan 2025 15:54:18 +0100 Subject: [PATCH 73/74] Adopt tests --- src/js/tests/module.test.js | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/js/tests/module.test.js b/src/js/tests/module.test.js index 993ec484ec..0825625d3f 100644 --- a/src/js/tests/module.test.js +++ b/src/js/tests/module.test.js @@ -100,7 +100,7 @@ describe('generation parameters validation', () => { await pipeline.startChat(); await assert.rejects( - async () => await pipeline.generate('prompt'), + async () => await pipeline.generate('prompt', {}, false), { name: 'Error', message: 'Generation callback must be a function', @@ -120,7 +120,7 @@ describe('generation parameters validation', () => { it('should perform generation with default options', async () => { try { - await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); + await pipeline.generate('prompt', { max_new_tokens: 1 }); } catch (error) { assert.fail(error); } @@ -129,7 +129,7 @@ describe('generation parameters validation', () => { }); it('should return a string as generation result', async () => { - const reply = await pipeline.generate('prompt', { max_new_tokens: 1 }, () => {}); + const reply = await pipeline.generate('prompt', { max_new_tokens: 1 }); assert.strictEqual(typeof reply, 'string'); }); From f9b4044cc01b11b0dd73a10a56636f4436166021 Mon Sep 17 00:00:00 2001 From: Vishniakov Nikolai Date: Tue, 28 Jan 2025 13:48:45 +0100 Subject: [PATCH 74/74] Add missed symbols definition --- src/js/.gitignore | 1 - src/js/thirdparty/node-lib.def | 147 +++++++++++++++++++++++ src/js/thirdparty/win_delay_load_hook.cc | 52 ++++++++ 3 files changed, 199 insertions(+), 1 deletion(-) create mode 100644 src/js/thirdparty/node-lib.def create mode 100644 src/js/thirdparty/win_delay_load_hook.cc diff --git a/src/js/.gitignore b/src/js/.gitignore index 638d8a5bb7..8990d8c418 100644 --- a/src/js/.gitignore +++ b/src/js/.gitignore @@ -2,6 +2,5 @@ bin bin.* build -thirdparty node_modules tests/models diff --git a/src/js/thirdparty/node-lib.def b/src/js/thirdparty/node-lib.def new file mode 100644 index 0000000000..8d46bbec84 --- /dev/null +++ b/src/js/thirdparty/node-lib.def @@ -0,0 +1,147 @@ +NAME NODE.EXE +EXPORTS +napi_async_destroy +napi_async_init +napi_cancel_async_work +napi_create_async_work +napi_create_buffer +napi_create_buffer_copy +napi_create_external_buffer +napi_delete_async_work +napi_fatal_error +napi_get_buffer_info +napi_get_node_version +napi_is_buffer +napi_make_callback +napi_module_register +napi_queue_async_work +napi_adjust_external_memory +napi_call_function +napi_close_escapable_handle_scope +napi_close_handle_scope +napi_coerce_to_bool +napi_coerce_to_number +napi_coerce_to_object +napi_coerce_to_string +napi_create_array +napi_create_array_with_length +napi_create_arraybuffer +napi_create_dataview +napi_create_double +napi_create_error +napi_create_external +napi_create_external_arraybuffer +napi_create_function +napi_create_int32 +napi_create_int64 +napi_create_object +napi_create_promise +napi_create_range_error +napi_create_reference +napi_create_string_latin1 +napi_create_string_utf16 +napi_create_string_utf8 +napi_create_symbol +napi_create_type_error +napi_create_typedarray +napi_create_uint32 +napi_define_class +napi_define_properties +napi_delete_element +napi_delete_property +napi_delete_reference +napi_escape_handle +napi_get_and_clear_last_exception +napi_get_array_length +napi_get_arraybuffer_info +napi_get_boolean +napi_get_cb_info +napi_get_dataview_info +napi_get_element +napi_get_global +napi_get_last_error_info +napi_get_named_property +napi_get_new_target +napi_get_null +napi_get_property +napi_get_property_names +napi_get_prototype +napi_get_reference_value +napi_get_typedarray_info +napi_get_undefined +napi_get_value_bool +napi_get_value_double +napi_get_value_external +napi_get_value_int32 +napi_get_value_int64 +napi_get_value_string_latin1 +napi_get_value_string_utf16 +napi_get_value_string_utf8 +napi_get_value_uint32 +napi_get_version +napi_has_element +napi_has_named_property +napi_has_own_property +napi_has_property +napi_instanceof +napi_is_array +napi_is_arraybuffer +napi_is_dataview +napi_is_error +napi_is_exception_pending +napi_is_promise +napi_is_typedarray +napi_new_instance +napi_open_escapable_handle_scope +napi_open_handle_scope +napi_reference_ref +napi_reference_unref +napi_reject_deferred +napi_remove_wrap +napi_resolve_deferred +napi_run_script +napi_set_element +napi_set_named_property +napi_set_property +napi_strict_equals +napi_throw +napi_throw_error +napi_throw_range_error +napi_throw_type_error +napi_typeof +napi_unwrap +napi_wrap +napi_get_uv_event_loop +napi_add_env_cleanup_hook +napi_close_callback_scope +napi_fatal_exception +napi_open_callback_scope +napi_remove_env_cleanup_hook +napi_acquire_threadsafe_function +napi_call_threadsafe_function +napi_create_threadsafe_function +napi_get_threadsafe_function_context +napi_ref_threadsafe_function +napi_release_threadsafe_function +napi_unref_threadsafe_function +napi_add_finalizer +napi_create_date +napi_get_date_value +napi_is_date +napi_create_bigint_int64 +napi_create_bigint_uint64 +napi_create_bigint_words +napi_get_all_property_names +napi_get_instance_data +napi_get_value_bigint_int64 +napi_get_value_bigint_uint64 +napi_get_value_bigint_words +napi_set_instance_data +napi_detach_arraybuffer +napi_is_detached_arraybuffer +napi_add_async_cleanup_hook +napi_remove_async_cleanup_hook +napi_check_object_type_tag +napi_object_freeze +napi_object_seal +napi_type_tag_object diff --git a/src/js/thirdparty/win_delay_load_hook.cc b/src/js/thirdparty/win_delay_load_hook.cc new file mode 100644 index 0000000000..9e652fa4df --- /dev/null +++ b/src/js/thirdparty/win_delay_load_hook.cc @@ -0,0 +1,52 @@ +/* + * When this file is linked to a DLL, it sets up a delay-load hook that + * intervenes when the DLL is trying to load 'node.exe' or 'iojs.exe' + * dynamically. Instead of trying to locate the .exe file it'll just return + * a handle to the process image. + * + * This allows compiled addons to work when node.exe or iojs.exe is renamed. + */ + +#ifdef _MSC_VER + +#ifndef WIN32_LEAN_AND_MEAN +#define WIN32_LEAN_AND_MEAN +#endif + +#include + +#include +#include + +static HMODULE node_dll = NULL; +static HMODULE nw_dll = NULL; + +static FARPROC WINAPI load_exe_hook(unsigned int event, DelayLoadInfo* info) { + if (event == dliNotePreGetProcAddress) { + FARPROC ret = NULL; + ret = GetProcAddress(node_dll, info->dlp.szProcName); + if (ret) + return ret; + ret = GetProcAddress(nw_dll, info->dlp.szProcName); + return ret; + } + if (event == dliStartProcessing) { + node_dll = GetModuleHandleA("node.dll"); + nw_dll = GetModuleHandleA("nw.dll"); + return NULL; + } + if (event != dliNotePreLoadLibrary) + return NULL; + + if (_stricmp(info->szDll, "node.exe") != 0) + return NULL; + + // Fall back to the current process + if(!node_dll) node_dll = GetModuleHandleA(NULL); + + return (FARPROC) node_dll; +} + +decltype(__pfnDliNotifyHook2) __pfnDliNotifyHook2 = load_exe_hook; + +#endif