From 863be3dd4cdf33bcd1a784066ead186a9bea3b38 Mon Sep 17 00:00:00 2001 From: Eric Mehl Date: Wed, 6 Nov 2024 18:01:24 -0500 Subject: [PATCH 01/26] PathFilter : Prevent path drops on read only nodes --- Changes.md | 4 ++++ python/GafferSceneUI/PathFilterUI.py | 13 +++++++++++-- python/GafferUI/VectorDataWidget.py | 3 +++ 3 files changed, 18 insertions(+), 2 deletions(-) diff --git a/Changes.md b/Changes.md index 9b969194603..9825fcdcf67 100644 --- a/Changes.md +++ b/Changes.md @@ -1,7 +1,11 @@ 1.4.15.x (relative to 1.4.15.1) ======== +Fixes +----- +- PathFilter : Fixed bug allowing dropping paths onto read-only `PathFilter` nodes in the graph. +- VectorDataWidget : Fixed bug allowing dropping paths onto read-only widgets. 1.4.15.1 (relative to 1.4.15.0) ======== diff --git a/python/GafferSceneUI/PathFilterUI.py b/python/GafferSceneUI/PathFilterUI.py index 7ade786a71e..ca430c1d891 100644 --- a/python/GafferSceneUI/PathFilterUI.py +++ b/python/GafferSceneUI/PathFilterUI.py @@ -255,9 +255,14 @@ def __filterPlug( node ) : return filterPlugs[0] return None +def __editable( plug ) : + + return not Gaffer.MetadataAlgo.readOnly( plug ) and plug.settable() + def __dropMode( nodeGadget, event ) : - if __pathsPlug( nodeGadget.node() ) is None : + pathsPlug = __pathsPlug( nodeGadget.node() ) + if pathsPlug is None : filter = None filterPlug = __filterPlug( nodeGadget.node() ) @@ -267,9 +272,13 @@ def __dropMode( nodeGadget, event ) : if filterPlug.getInput() is not None : filter = filterPlug.source().node() if filter is None : - return __DropMode.Replace + return __DropMode.Replace if __editable( filterPlug ) else __DropMode.None_ elif not isinstance( filter, GafferScene.PathFilter ) : return __DropMode.None_ + pathsPlug = __pathsPlug( filter ) + + if not __editable( pathsPlug ) : + return __DropMode.None_ if event.modifiers & event.Modifiers.Shift : return __DropMode.Add diff --git a/python/GafferUI/VectorDataWidget.py b/python/GafferUI/VectorDataWidget.py index da097cc5e85..483eed8f55e 100644 --- a/python/GafferUI/VectorDataWidget.py +++ b/python/GafferUI/VectorDataWidget.py @@ -621,6 +621,9 @@ def __addRows( self, button ) : def __dragEnter( self, widget, event ) : + if not self.getEditable() : + return False + if event.sourceWidget is self.__tableViewHolder and widget is not self.__buttonRow[1]: # we don't accept drags from ourself unless the target is the remove button return False From 42f7ab3f3bdee74692254eaf7ceee99e7fdadcbf Mon Sep 17 00:00:00 2001 From: Eric Mehl Date: Thu, 14 Nov 2024 15:29:26 -0500 Subject: [PATCH 02/26] Path / SetFilterUI : Add `notEditable` pointers --- python/GafferSceneUI/PathFilterUI.py | 14 ++++++++++---- python/GafferSceneUI/SetFilterUI.py | 14 ++++++++++---- 2 files changed, 20 insertions(+), 8 deletions(-) diff --git a/python/GafferSceneUI/PathFilterUI.py b/python/GafferSceneUI/PathFilterUI.py index ca430c1d891..5811eddde44 100644 --- a/python/GafferSceneUI/PathFilterUI.py +++ b/python/GafferSceneUI/PathFilterUI.py @@ -236,7 +236,7 @@ def __popupMenu( menuDefinition, plugValueWidget ) : GafferUI.Pointer.registerPointer( "removeObjects", GafferUI.Pointer( "removeObjects.png", imath.V2i( 53, 14 ) ) ) GafferUI.Pointer.registerPointer( "replaceObjects", GafferUI.Pointer( "replaceObjects.png", imath.V2i( 53, 14 ) ) ) -__DropMode = enum.Enum( "__DropMode", [ "None_", "Add", "Remove", "Replace" ] ) +__DropMode = enum.Enum( "__DropMode", [ "None_", "Add", "Remove", "Replace", "NotEditable" ] ) __originalDragPointer = None @@ -272,13 +272,13 @@ def __dropMode( nodeGadget, event ) : if filterPlug.getInput() is not None : filter = filterPlug.source().node() if filter is None : - return __DropMode.Replace if __editable( filterPlug ) else __DropMode.None_ + return __DropMode.Replace if __editable( filterPlug ) else __DropMode.NotEditable elif not isinstance( filter, GafferScene.PathFilter ) : return __DropMode.None_ pathsPlug = __pathsPlug( filter ) if not __editable( pathsPlug ) : - return __DropMode.None_ + return __DropMode.NotEditable if event.modifiers & event.Modifiers.Shift : return __DropMode.Add @@ -351,7 +351,10 @@ def __dragMove( nodeGadget, event ) : if __originalDragPointer is None : return False - GafferUI.Pointer.setCurrent( __dropMode( nodeGadget, event ).name.lower() + "Objects" ) + dropMode = __dropMode( nodeGadget, event ) + GafferUI.Pointer.setCurrent( + dropMode.name.lower() + "Objects" if dropMode != __DropMode.NotEditable else "notEditable" + ) return True @@ -361,6 +364,9 @@ def __drop( nodeGadget, event ) : if __originalDragPointer is None : return False + if __dropMode( nodeGadget, event ) == __DropMode.NotEditable : + return True + pathsPlug = __pathsPlug( nodeGadget.node() ) if pathsPlug is None : pathsPlug = __pathsPlug( __filterPlug( nodeGadget.node() ).source().node() ) diff --git a/python/GafferSceneUI/SetFilterUI.py b/python/GafferSceneUI/SetFilterUI.py index 742f5363ae6..95d57d12818 100644 --- a/python/GafferSceneUI/SetFilterUI.py +++ b/python/GafferSceneUI/SetFilterUI.py @@ -111,7 +111,7 @@ GafferUI.Pointer.registerPointer( "removeSets", GafferUI.Pointer( "pointerRemoveSets.png", imath.V2i( 53, 14 ) ) ) GafferUI.Pointer.registerPointer( "replaceSets", GafferUI.Pointer( "pointerReplaceSets.png", imath.V2i( 53, 14 ) ) ) -__DropMode = enum.Enum( "__DropMode", [ "None_", "Add", "Remove", "Replace" ] ) +__DropMode = enum.Enum( "__DropMode", [ "None_", "Add", "Remove", "Replace", "NotEditable" ] ) __originalDragPointer = None @@ -146,13 +146,13 @@ def __dropMode( nodeGadget, event ) : if nodeGadget.node()["filter"].getInput() is not None : filter = nodeGadget.node()["filter"].source().node() if filter is None : - return __DropMode.Replace if __editable( nodeGadget.node()["filter"] ) else __DropMode.None_ + return __DropMode.Replace if __editable( nodeGadget.node()["filter"] ) else __DropMode.NotEditable elif not isinstance( filter, GafferScene.SetFilter ) : return __DropMode.None_ setsPlug = filter["setExpression"] if not __editable( setsPlug ) : - return __DropMode.None_ + return __DropMode.NotEditable if event.modifiers & event.Modifiers.Shift : return __DropMode.Add @@ -201,7 +201,10 @@ def __dragMove( nodeGadget, event ) : if __originalDragPointer is None : return False - GafferUI.Pointer.setCurrent( __dropMode( nodeGadget, event ).name.lower() + "Sets" ) + dropMode = __dropMode( nodeGadget, event ) + GafferUI.Pointer.setCurrent( + dropMode.name.lower() + "Sets" if dropMode != __DropMode.NotEditable else "notEditable" + ) return True @@ -211,6 +214,9 @@ def __drop( nodeGadget, event ) : if __originalDragPointer is None : return False + if __dropMode( nodeGadget, event ) == __DropMode.NotEditable : + return True + setsPlug = __setsPlug( nodeGadget.node() ) if setsPlug is None : setsPlug = __setsPlug( nodeGadget.node()["filter"].source().node() ) From c1be2dabb6c85b6ea093e4744e1e9a4eb6667bc0 Mon Sep 17 00:00:00 2001 From: Murray Stevenson <50844517+murraystevenson@users.noreply.github.com> Date: Sat, 16 Nov 2024 11:24:46 +1100 Subject: [PATCH 03/26] CI : Drop GCC9 builds --- .github/workflows/main.yml | 24 +++++------------------- 1 file changed, 5 insertions(+), 19 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 1079d396d95..53579a6c382 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -28,19 +28,18 @@ jobs: # and then use `include` to define their settings. name: [ - linux-gcc9, - linux-debug-gcc9, linux-gcc11, + linux-debug-gcc11, windows, ] include: - - name: linux-gcc9 + - name: linux-gcc11 os: ubuntu-20.04 buildType: RELEASE publish: true - containerImage: ghcr.io/gafferhq/build/build:2.1.2 + containerImage: ghcr.io/gafferhq/build/build:3.0.0 # GitHub container builds run as root. This causes failures for tests that # assert that filesystem permissions are respected, because root doesn't # respect permissions. So we run the final test suite as a dedicated @@ -49,11 +48,11 @@ jobs: sconsCacheMegabytes: 400 jobs: 4 - - name: linux-debug-gcc9 + - name: linux-debug-gcc11 os: ubuntu-20.04 buildType: DEBUG publish: false - containerImage: ghcr.io/gafferhq/build/build:2.1.2 + containerImage: ghcr.io/gafferhq/build/build:3.0.0 testRunner: su testUser -c testArguments: -excludedCategories performance # Debug builds are ludicrously big, so we must use a larger cache @@ -61,19 +60,6 @@ jobs: sconsCacheMegabytes: 2500 jobs: 4 - - name: linux-gcc11 - os: ubuntu-20.04 - buildType: RELEASE - publish: true - containerImage: ghcr.io/gafferhq/build/build:3.0.0 - # GitHub container builds run as root. This causes failures for tests that - # assert that filesystem permissions are respected, because root doesn't - # respect permissions. So we run the final test suite as a dedicated - # test user rather than as root. - testRunner: su testUser -c - sconsCacheMegabytes: 400 - jobs: 4 - - name: windows os: windows-2019 buildType: RELEASE From 7ddf9b1cef4479027993847ca1f29bfeae235db8 Mon Sep 17 00:00:00 2001 From: Murray Stevenson <50844517+murraystevenson@users.noreply.github.com> Date: Sat, 16 Nov 2024 11:26:00 +1100 Subject: [PATCH 04/26] CI : Update to node20 actions --- .github/workflows/main.yml | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 53579a6c382..7930d2f7f63 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -79,16 +79,12 @@ jobs: ARNOLD_FORCE_ABORT_ON_LICENSE_FAIL: 0 # And don't abort because the license isn't found GAFFER_BUILD_DIR: "./build" GAFFER_CACHE_DIR: "./sconsCache" - # GitHub have moved to running actions on Node20, which prevents them from - # running on CentOS 7. The below allows actions to continue running on Node16 - # until October. - ACTIONS_ALLOW_USE_UNSECURE_NODE_VERSION: true steps: - - uses: actions/checkout@v3 + - uses: actions/checkout@v4 - - uses: ilammy/msvc-dev-cmd@v1.12.1 + - uses: ilammy/msvc-dev-cmd@v1.13.0 with: sdk: 10.0.17763.0 @@ -158,7 +154,7 @@ jobs: if: runner.os == 'Windows' - name: Cache - uses: actions/cache@v3 + uses: actions/cache@v4 with: path: ${{ env.GAFFER_CACHE_DIR }} key: ${{ runner.os }}-${{ matrix.containerImage }}-${{env.GAFFER_DEPENDENCIES_HASH}}-${{ matrix.buildType }}-${{ github.sha }} @@ -245,10 +241,13 @@ jobs: echo "::remove-matcher owner=validateRelease::" if: matrix.publish - - uses: actions/upload-artifact@v3 + - uses: actions/upload-artifact@v4 with: name: ${{ env.GAFFER_BUILD_NAME }} path: ${{ env.GAFFER_BUILD_NAME }}.${{ env.PACKAGE_EXTENSION }} + # Using compression-level 0 avoids compressing our already compressed + # package and results in a significantly faster upload. + compression-level: 0 if: matrix.publish - name: Publish Release From 4a8b9fed9548a79fea1b8623803bd64a2f8f33da Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 22 Nov 2024 11:00:41 +0000 Subject: [PATCH 05/26] Bump version to 1.4.15.2 --- Changes.md | 7 ++++++- SConstruct | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/Changes.md b/Changes.md index 9825fcdcf67..838942fd268 100644 --- a/Changes.md +++ b/Changes.md @@ -1,4 +1,9 @@ -1.4.15.x (relative to 1.4.15.1) +1.4.15.x (relative to 1.4.15.2) +======== + + + +1.4.15.2 (relative to 1.4.15.1) ======== Fixes diff --git a/SConstruct b/SConstruct index 67fd47c3129..7b6ec201560 100644 --- a/SConstruct +++ b/SConstruct @@ -64,7 +64,7 @@ if codecs.lookup( locale.getpreferredencoding() ).name != "utf-8" : gafferMilestoneVersion = 1 # for announcing major milestones - may contain all of the below gafferMajorVersion = 4 # backwards-incompatible changes gafferMinorVersion = 15 # new backwards-compatible features -gafferPatchVersion = 1 # bug fixes +gafferPatchVersion = 2 # bug fixes gafferVersionSuffix = "" # used for alpha/beta releases : "a1", "b2", etc. # All of the following must be considered when determining From 74d9bfc592ecbf6a3194e8327859c51b728af9b5 Mon Sep 17 00:00:00 2001 From: Eric Mehl Date: Fri, 22 Nov 2024 17:08:46 -0500 Subject: [PATCH 06/26] Color*PlugValueWidget : Removed `scoped = False` --- python/GafferUI/ColorChooserPlugValueWidget.py | 3 +-- python/GafferUI/ColorSwatchPlugValueWidget.py | 3 +-- 2 files changed, 2 insertions(+), 4 deletions(-) diff --git a/python/GafferUI/ColorChooserPlugValueWidget.py b/python/GafferUI/ColorChooserPlugValueWidget.py index 9640e4b3ecd..a98b9e6f3b2 100644 --- a/python/GafferUI/ColorChooserPlugValueWidget.py +++ b/python/GafferUI/ColorChooserPlugValueWidget.py @@ -86,8 +86,7 @@ def __init__( self, plugs, **kw ) : functools.partial( Gaffer.WeakMethod( self.__dynamicSliderBackgroundsChanged ) ) ) self.__colorChooser.optionsMenuSignal().connect( - functools.partial( Gaffer.WeakMethod( self.__colorChooserOptionsMenu ) ), - scoped = False + functools.partial( Gaffer.WeakMethod( self.__colorChooserOptionsMenu ) ) ) self.__lastChangedReason = None diff --git a/python/GafferUI/ColorSwatchPlugValueWidget.py b/python/GafferUI/ColorSwatchPlugValueWidget.py index 0ed0604bf55..35b7b6f29d2 100644 --- a/python/GafferUI/ColorSwatchPlugValueWidget.py +++ b/python/GafferUI/ColorSwatchPlugValueWidget.py @@ -151,8 +151,7 @@ def __init__( self, plugs, parentWindow ) : functools.partial( Gaffer.WeakMethod( self.__dynamicSliderBackgroundsChanged ) ) ) self.colorChooser().optionsMenuSignal().connect( - functools.partial( Gaffer.WeakMethod( self.__colorChooserOptionsMenu ) ), - scoped = False + functools.partial( Gaffer.WeakMethod( self.__colorChooserOptionsMenu ) ) ) self.confirmButton.clickedSignal().connect( Gaffer.WeakMethod( self.__buttonClicked ) ) From 724d1828f570183c3c0a860e1d364f1389ee2a09 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Sun, 27 Oct 2024 18:03:24 +0000 Subject: [PATCH 07/26] PlugLayout : Update activations when children added/removed --- Changes.md | 3 +++ python/GafferUI/PlugLayout.py | 1 + 2 files changed, 4 insertions(+) diff --git a/Changes.md b/Changes.md index df8ced87413..adbdeec461c 100644 --- a/Changes.md +++ b/Changes.md @@ -1,7 +1,10 @@ 1.5.x.x (relative to 1.5.1.0) ======= +API +--- +- PlugLayout : Activations may now depend on the presence of certain plugs, as they are now reevaluated when child plugs are added and removed. 1.5.1.0 (relative to 1.5.0.1) ======= diff --git a/python/GafferUI/PlugLayout.py b/python/GafferUI/PlugLayout.py index 03686619eb1..ecabda5c7e5 100644 --- a/python/GafferUI/PlugLayout.py +++ b/python/GafferUI/PlugLayout.py @@ -483,6 +483,7 @@ def __childAddedOrRemoved( self, *unusedArgs ) : # we do a lazy update so we can batch up several changes into one. # upheaval is over. self.__layoutDirty = True + self.__activationsDirty = True self.__updateLazily() def __plugMetadataChanged( self, plug, key, reason ) : From 597afa5f0778af6cbf72def1cf725d8735c50835 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:04:24 +0000 Subject: [PATCH 08/26] GafferML : Add library and module boilerplate --- SConstruct | 35 +++++++++++++- bin/gaffer | 13 +++++ bin/gaffer.cmd | 6 +++ include/GafferML/Export.h | 43 +++++++++++++++++ include/GafferML/TypeIds.h | 55 ++++++++++++++++++++++ python/GafferML/__init__.py | 49 +++++++++++++++++++ python/GafferMLTest/__init__.py | 39 +++++++++++++++ python/GafferMLUI/__init__.py | 37 +++++++++++++++ python/GafferMLUITest/DocumentationTest.py | 54 +++++++++++++++++++++ python/GafferMLUITest/NodeUITest.py | 52 ++++++++++++++++++++ python/GafferMLUITest/__init__.py | 41 ++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 42 +++++++++++++++++ 12 files changed, 465 insertions(+), 1 deletion(-) create mode 100644 include/GafferML/Export.h create mode 100644 include/GafferML/TypeIds.h create mode 100644 python/GafferML/__init__.py create mode 100644 python/GafferMLTest/__init__.py create mode 100644 python/GafferMLUI/__init__.py create mode 100644 python/GafferMLUITest/DocumentationTest.py create mode 100644 python/GafferMLUITest/NodeUITest.py create mode 100644 python/GafferMLUITest/__init__.py create mode 100644 src/GafferMLModule/GafferMLModule.cpp diff --git a/SConstruct b/SConstruct index 246d37c3575..786e65cfc99 100644 --- a/SConstruct +++ b/SConstruct @@ -307,6 +307,12 @@ options.Add( ) ) +options.Add( + "ONNX_ROOT", + "The directory in which the ONNX runtime is installed. Used to build GafferML", + "", +) + # general variables options.Add( @@ -771,7 +777,7 @@ commandEnv["ENV"]["PYTHONPATH"] = commandEnv.subst( os.path.pathsep.join( [ "$BU # SIP on MacOS prevents DYLD_LIBRARY_PATH being passed down so we make sure # we also pass through to gaffer the other base vars it uses to populate paths # for third-party support. -for v in ( 'ARNOLD_ROOT', 'DELIGHT_ROOT' ) : +for v in ( 'ARNOLD_ROOT', 'DELIGHT_ROOT', 'ONNX_ROOT' ) : commandEnv["ENV"][ v ] = commandEnv[ v ] def runCommand( command ) : @@ -1109,6 +1115,33 @@ libraries = { }, }, + "GafferML" : { + "envAppends" : { + "CPPPATH" : [ "$ONNX_ROOT/include" ], + "LIBPATH" : [ "$ONNX_ROOT/lib" ], + "LIBS" : [ "Gaffer", "GafferImage", "onnxruntime" ], + }, + "pythonEnvAppends" : { + "CPPPATH" : [ "$ONNX_ROOT/include" ], + "LIBPATH" : [ "$ONNX_ROOT/lib" ], + "LIBS" : [ "GafferBindings", "GafferImage", "GafferML", "onnxruntime" ], + }, + "requiredOptions" : [ "ONNX_ROOT" ], + }, + + "GafferMLTest" : { + "requiredOptions" : [ "ONNX_ROOT" ], + "additionalFiles" : glob.glob( "python/GafferMLTest/models/*" ) + }, + + "GafferMLUI" : { + "requiredOptions" : [ "ONNX_ROOT" ], + }, + + "GafferMLUITest" : { + "requiredOptions" : [ "ONNX_ROOT" ], + }, + "IECoreArnold" : { "envAppends" : { "LIBPATH" : [ "$ARNOLD_ROOT/bin" ] if env["PLATFORM"] != "win32" else [ "$ARNOLD_ROOT/bin", "$ARNOLD_ROOT/lib" ], diff --git a/bin/gaffer b/bin/gaffer index 6eb2e2608a0..5bc5a6726db 100755 --- a/bin/gaffer +++ b/bin/gaffer @@ -305,6 +305,19 @@ if [[ -n $DELIGHT ]] ; then fi +# Set up ONNX +########################################################################## + +if [[ -n $ONNX_ROOT ]] ; then + + if [[ `uname` = "Linux" ]] ; then + appendToPath "$ONNX_ROOT/lib" LD_LIBRARY_PATH + else + appendToPath "$ONNX_ROOT/lib" DYLD_LIBRARY_PATH + fi + +fi + # Set up 3rd Party extensions ########################################################################## diff --git a/bin/gaffer.cmd b/bin/gaffer.cmd index 7de2be18fe2..635eb611ba3 100644 --- a/bin/gaffer.cmd +++ b/bin/gaffer.cmd @@ -140,6 +140,12 @@ if "%CYCLES_ROOT%" NEQ "" ( call :prependToPath "%CYCLES_ROOT%\bin" PATH ) +rem ONNX +rem ==== + +if "%ONNX_ROOT%" NEQ "" ( + call :appendToPath "%ONNX_ROOT%\lib" PATH +) rem Set up 3rd party extensions rem Batch files are awkward at `for` loops. The default `for`, without `/f` diff --git a/include/GafferML/Export.h b/include/GafferML/Export.h new file mode 100644 index 00000000000..4c1bfdd418e --- /dev/null +++ b/include/GafferML/Export.h @@ -0,0 +1,43 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above copyright +// notice, this list of conditions and the following disclaimer. +// +// * Redistributions in binary form must reproduce the above copyright +// notice, this list of conditions and the following disclaimer in the +// documentation and/or other materials provided with the distribution. +// +// * Neither the name of Image Engine Design nor the names of any +// other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "IECore/Export.h" + +#ifdef GafferML_EXPORTS + #define GAFFERML_API IECORE_EXPORT +#else + #define GAFFERML_API IECORE_IMPORT +#endif diff --git a/include/GafferML/TypeIds.h b/include/GafferML/TypeIds.h new file mode 100644 index 00000000000..95125b0d2a0 --- /dev/null +++ b/include/GafferML/TypeIds.h @@ -0,0 +1,55 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +namespace GafferML +{ + +enum TypeId +{ + TensorTypeId = 110451, + TensorPlugTypeId = 110452, + ImageToTensorTypeId = 110453, + TensorToImageTypeId = 110454, + InferenceTypeId = 110455, + TensorReaderTypeId = 110456, + DataToTensorTypeId = 110457, + + LastTypeId = 110500 +}; + +} // namespace GafferML diff --git a/python/GafferML/__init__.py b/python/GafferML/__init__.py new file mode 100644 index 00000000000..707fcbc0d8f --- /dev/null +++ b/python/GafferML/__init__.py @@ -0,0 +1,49 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import os +import pathlib + +__import__( "Gaffer" ) +__import__( "GafferImage" ) + +if hasattr( os, "add_dll_directory" ) : + os.add_dll_directory( ( pathlib.Path( os.environ["ONNX_ROOT"] ) / "lib" ).resolve() ) +del os, pathlib # Don't pollute the namespace + +from ._GafferML import * + +__import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferML" ) diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py new file mode 100644 index 00000000000..00238c96ef5 --- /dev/null +++ b/python/GafferMLTest/__init__.py @@ -0,0 +1,39 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +if __name__ == "__main__": + import unittest + unittest.main() diff --git a/python/GafferMLUI/__init__.py b/python/GafferMLUI/__init__.py new file mode 100644 index 00000000000..573d1b5fbe2 --- /dev/null +++ b/python/GafferMLUI/__init__.py @@ -0,0 +1,37 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +__import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferMLUI" ) diff --git a/python/GafferMLUITest/DocumentationTest.py b/python/GafferMLUITest/DocumentationTest.py new file mode 100644 index 00000000000..45b5fdff387 --- /dev/null +++ b/python/GafferMLUITest/DocumentationTest.py @@ -0,0 +1,54 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import GafferUITest + +import GafferImage +import GafferML +import GafferMLUI + +class DocumentationTest( GafferUITest.TestCase ) : + + def test( self ) : + + self.maxDiff = None + self.assertNodesAreDocumented( + GafferML, + additionalTerminalPlugTypes = ( GafferImage.ImagePlug, ) + ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferMLUITest/NodeUITest.py b/python/GafferMLUITest/NodeUITest.py new file mode 100644 index 00000000000..a3ceecfd67f --- /dev/null +++ b/python/GafferMLUITest/NodeUITest.py @@ -0,0 +1,52 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import Gaffer +import GafferUI +import GafferUITest +import GafferML +import GafferMLUI + +class NodeUITest( GafferUITest.TestCase ) : + + def testLifetimes( self ) : + + self.assertNodeUIsHaveExpectedLifetime( GafferML ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferMLUITest/__init__.py b/python/GafferMLUITest/__init__.py new file mode 100644 index 00000000000..ffd46580d91 --- /dev/null +++ b/python/GafferMLUITest/__init__.py @@ -0,0 +1,41 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +from .DocumentationTest import DocumentationTest +from .NodeUITest import NodeUITest + +if __name__ == "__main__": + unittest.main() diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp new file mode 100644 index 00000000000..b999c69b8f0 --- /dev/null +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -0,0 +1,42 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2012, John Haddon. All rights reserved. +// Copyright (c) 2013-2015, Image Engine Design Inc. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "boost/python.hpp" + +BOOST_PYTHON_MODULE( _GafferML ) +{ +} From 24ec3f10fecf850cb397dd40f232548f893ab7ba Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:23:50 +0000 Subject: [PATCH 09/26] JH config : Build GafferML --- config/jh/options | 2 ++ 1 file changed, 2 insertions(+) diff --git a/config/jh/options b/config/jh/options index 19bf995b04d..6aa96aef0b5 100644 --- a/config/jh/options +++ b/config/jh/options @@ -50,3 +50,5 @@ DELIGHT_ROOT = os.environ["DELIGHT"] ARNOLD_ROOT = os.environ["ARNOLD_ROOT"] VTUNE_ROOT = "/disk1/apps/intel/system_studio_2018/vtune_amplifier_2018.1.0.535340" GAFFERCORTEX=1 + +ONNX_ROOT = os.environ["ONNX_ROOT"] From 0e5eb746dffe6d8aa1481d33696676eaed84bd44 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:10:45 +0000 Subject: [PATCH 10/26] GafferML : Add Tensor class This is a wrapper class that will allow us to pass ONNX values through Gaffer's computation graph. --- include/GafferML/Tensor.h | 108 +++++++ python/GafferMLTest/TensorTest.py | 181 ++++++++++++ python/GafferMLTest/__init__.py | 2 + src/GafferML/Tensor.cpp | 405 ++++++++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 119 ++++++++ 5 files changed, 815 insertions(+) create mode 100644 include/GafferML/Tensor.h create mode 100644 python/GafferMLTest/TensorTest.py create mode 100644 src/GafferML/Tensor.cpp diff --git a/include/GafferML/Tensor.h b/include/GafferML/Tensor.h new file mode 100644 index 00000000000..225ba5a80c1 --- /dev/null +++ b/include/GafferML/Tensor.h @@ -0,0 +1,108 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Export.h" +#include "GafferML/TypeIds.h" + +#include "IECore/Data.h" + +#include "onnxruntime_cxx_api.h" + +#include + +namespace GafferML +{ + +/// Thin wrapper around an `Ort::Value`, allowing it to be passed +/// through a graph of ComputeNodes via TensorPlugs. +class GAFFERML_API Tensor : public IECore::Object +{ + + public : + + Tensor(); + Tensor( Ort::Value &&value ); + + /// Constructs from varieties of `IECore::TypedData`. The Tensor references `data` directly + /// without copying, so it must not be modified after being passed to the constructor. + /// If `shape` is not specified, then it will be inferred automatically from the data layout. + Tensor( const IECore::ConstDataPtr &data, std::vector shape = std::vector() ); + + IE_CORE_DECLAREEXTENSIONOBJECT( GafferML::Tensor, GafferML::TensorTypeId, IECore::Object ); + + /// Only const access to the `Ort::Value` is provided. This lets us + /// implement `Object::copy()` extremely cheaply, which is important + /// when accessing a Tensor value from a Python Expression. + const Ort::Value &value() const; + + /// Convenience accessors + /// ===================== + /// + /// These don't do anything that can't be achieved directly with + /// `value()` and the Ort API, but are provided for symmetry with + /// the Python bindings. + + std::vector shape() const; + + /// Conversion to `IECore::Data` + /// ============================ + + IECore::DataPtr asData(); + IECore::ConstDataPtr asData() const; + + private : + + struct State : public IECore::RefCounted + { + State( Ort::Value &&value, IECore::ConstDataPtr data = nullptr ); + Ort::Value value; + // If we were constructed from TypedData, then this keeps it alive for + // as long as `value` references it. If we constructed from + // `Ort::Value` directly, then this is null and `value` owns its own + // data. + IECore::ConstDataPtr data; + }; + IE_CORE_DECLAREPTR( State ); + + ConstStatePtr m_state; + +}; + +IE_CORE_DECLAREPTR( Tensor ); + +} // namespace GafferML diff --git a/python/GafferMLTest/TensorTest.py b/python/GafferMLTest/TensorTest.py new file mode 100644 index 00000000000..6b20ddd2aeb --- /dev/null +++ b/python/GafferMLTest/TensorTest.py @@ -0,0 +1,181 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import imath + +import IECore +import GafferTest +import GafferML + +class TensorTest( GafferTest.TestCase ) : + + def testAsData( self ) : + + for data in [ + IECore.BoolVectorData( [ True, False, True ] ), + IECore.FloatVectorData( [ 1, 2, 3 ] ), + IECore.DoubleVectorData( [ 1, 2, 3 ] ), + IECore.IntVectorData( [ 1, 2, 3 ] ), + IECore.UInt64VectorData( [ 1, 2, 3 ] ), + ] : + + tensor = GafferML.Tensor( data, [ 1, 3 ] ) + self.assertEqual( tensor.asData(), data ) + + self.assertIsNone( GafferML.Tensor().asData() ) + + def testInvalidShapeThrows( self ) : + + with self.assertRaisesRegex( RuntimeError, "not enough space: expected 16, got 12" ) : + GafferML.Tensor( IECore.FloatVectorData( [ 1, 2, 3 ] ), [ 4 ] ) + + def testExplicitShape( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3, 4, 5, 6 ] ), [ 1, 1, 2, 3 ] ) + self.assertEqual( tensor.shape(), [ 1, 1, 2, 3 ] ) + + tensor = GafferML.Tensor() + with self.assertRaisesRegex( RuntimeError, "Null tensor" ) : + tensor.shape() + + def testAutomaticShape( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3, 4, 5 ] ) ) + self.assertEqual( tensor.shape(), [ 5 ] ) + + tensor = GafferML.Tensor( IECore.V2iVectorData( [ imath.V2i( 1, 2 ), imath.V2i( 3, 4 ), imath.V2i( 5, 6 ) ] ) ) + self.assertEqual( tensor.shape(), [ 3, 2 ] ) + + tensor = GafferML.Tensor( IECore.V3fVectorData( [ imath.V3f( 1 ), imath.V3f( 2 ) ] ) ) + self.assertEqual( tensor.shape(), [ 2, 3 ] ) + + tensor = GafferML.Tensor( IECore.V3fData( imath.V3f( 1 ) ) ) + self.assertEqual( tensor.shape(), [ 3 ] ) + + tensor = GafferML.Tensor( IECore.Box3fData( imath.Box3f( imath.V3f( 1 ), imath.V3f( 2 ) ) ) ) + self.assertEqual( tensor.shape(), [ 2, 3 ] ) + + tensor = GafferML.Tensor( IECore.Color4fData( imath.Color4f( 0 ) ) ) + self.assertEqual( tensor.shape(), [ 4 ] ) + + def testMemoryUsage( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( 100 ), [ 100 ] ) + self.assertEqual( tensor.memoryUsage(), 440 ) + + def testHash( self ) : + + tensors = [ + GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3 ] ), [ 1, 3 ] ), + GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3 ] ), [ 3, 1 ] ), + GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3, 4 ] ), [ 4 ] ) + ] + + self.assertEqual( len( { t.hash() for t in tensors } ), len( tensors ) ) + + def testCopy( self ) : + + data = IECore.IntVectorData( [ 1, 2, 3 ] ) + tensor1 = GafferML.Tensor( data, [ 3 ] ) + tensor2 = tensor1.copy() + self.assertEqual( tensor2, tensor1 ) + self.assertEqual( tensor2.asData(), data ) + self.assertEqual( tensor2.shape(), tensor1.shape() ) + + def testIsEqual( self ) : + + data = IECore.IntVectorData( [ 1, 2, 3 ] ) + tensor1 = GafferML.Tensor( data, [ 3 ] ) + + tensor2 = GafferML.Tensor( data, [ 3 ] ) + self.assertEqual( tensor1, tensor2 ) + + tensor2 = GafferML.Tensor( data.copy(), [ 3 ] ) + self.assertEqual( tensor1, tensor2 ) + + tensor2 = GafferML.Tensor( IECore.IntVectorData( [ 1, 2, 3 ] ), [ 3 ] ) + self.assertEqual( tensor1, tensor2 ) + + tensor2 = GafferML.Tensor( data, [ 1, 3 ] ) + self.assertNotEqual( tensor1, tensor2 ) # Different shape + + tensor2 = GafferML.Tensor( IECore.IntVectorData( [ 3, 2, 1 ] ), [ 3 ] ) + self.assertNotEqual( tensor1, tensor2 ) # Different data + + def testDefaultRepr( self ) : + + self.assertEqual( repr( GafferML.Tensor() ), "GafferML.Tensor()" ) + + def testConstructFromUnsupportedDataType( self ) : + + with self.assertRaisesRegex( RuntimeError, "Unsupported data type `PathMatcherData`" ) : + GafferML.Tensor( IECore.PathMatcherData() ) + + def testGetItem1D( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( range( 0, 100 ) ) ) + for i in range( 0, 100 ) : + self.assertEqual( tensor[i], i ) + + def testGetItem2D( self ) : + + data = IECore.V3fVectorData( [ imath.V3f( 1, 2, 3 ), imath.V3f( 4, 5, 6 ) ] ) + tensor = GafferML.Tensor( data ) + + for i in range( 0, 2 ) : + for j in range( 0, 3 ) : + v = tensor[i, j] + self.assertIsInstance( v, float ) + self.assertEqual( v, data[i][j] ) + + def testGetItemOutOfRange( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( [ 1, 2 ] ) ) + with self.assertRaisesRegex( RuntimeError, "invalid location range" ) : + tensor[-1] + with self.assertRaisesRegex( RuntimeError, "invalid location range" ) : + tensor[2] + + def testGetItemWrongDimensions( self ) : + + tensor = GafferML.Tensor( IECore.IntVectorData( [ 1, 2 ] ) ) + with self.assertRaisesRegex( RuntimeError, "location dimensions do not match shape size" ) : + tensor[0, 1] + +if __name__ == "__main__" : + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 00238c96ef5..60f7088ec36 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -34,6 +34,8 @@ # ########################################################################## +from .TensorTest import TensorTest + if __name__ == "__main__": import unittest unittest.main() diff --git a/src/GafferML/Tensor.cpp b/src/GafferML/Tensor.cpp new file mode 100644 index 00000000000..05c7bc3e306 --- /dev/null +++ b/src/GafferML/Tensor.cpp @@ -0,0 +1,405 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/Tensor.h" + +#include "IECore/DataAlgo.h" +#include "IECore/TypeTraits.h" +#include "IECore/VectorTypedData.h" + +#include "fmt/format.h" + +using namespace std; +using namespace IECore; +using namespace GafferML; + +namespace +{ + +// The Ort C++ API defines `TypeToTensorType::type` for all C++ types +// (int, float etc) that are supported in tensors. But it leaves it +// completely undefined for other types. HasTensorType allows us to detect +// the supported types. + +template +struct HasTensorType : std::false_type {}; + +template +struct HasTensorType ) != 0>> : std::true_type {}; + +// ShapeTraits allows us to automatically determine the tensor layout for +// types like Imath::Vec3 etc. +template +struct ShapeTraits +{ + static constexpr std::array shape = {}; +}; + +template +struct ShapeTraits> +{ + static constexpr std::array shape = { 2 }; +}; + +template +struct ShapeTraits> +{ + static constexpr std::array shape = { 3 }; +}; + +template +struct ShapeTraits> +{ + static constexpr std::array shape = { 4 }; +}; + +template +struct ShapeTraits> +{ + static constexpr std::array shape = { 2, T::dimensions() }; +}; + +template +void dispatchTensorData( const Ort::Value &value, F &&functor ) +{ + const auto elementType = value.GetTensorTypeAndShapeInfo().GetElementType(); + switch( elementType ) + { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 : + functor( value.GetTensorData() ); + break; + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 : + functor( value.GetTensorData() ); + break; + default : + throw IECore::Exception( fmt::format( "Unsupported element type {}", elementType ) ); + } +} + +DataPtr dataFromValue( const Ort::Value &value ) +{ + DataPtr result; + dispatchTensorData( + value, + [&] ( const auto *data ) { + + using ElementType = remove_const_t>; + using DataType = TypedData>; + using PtrType = typename DataType::Ptr; + + PtrType d = new DataType; + const size_t count = value.GetTensorTypeAndShapeInfo().GetElementCount(); + d->writable().insert( d->writable().end(), data, data + count ); + result = d; + + } + ); + return result; +} + +} // namespace + +IE_CORE_DEFINEOBJECTTYPEDESCRIPTION( Tensor ); + +Tensor::State::State( Ort::Value &&value, IECore::ConstDataPtr data ) + : value( std::move( value ) ), data( data ) +{ + if( value && !value.IsTensor() ) + { + /// \todo Maybe we'll loosen this restriction at some point, + /// or maybe we'll create wrappers for the other Value types. + /// For the moment we just want to know if something unexpected + /// happens. + throw IECore::Exception( "Ort::Value is not a tensor" ); + } +} + +Tensor::Tensor() + : m_state( new State( Ort::Value( nullptr ) ) ) +{ +} + +Tensor::Tensor( Ort::Value &&value ) + : m_state( new State( std::move( value ) ) ) +{ +} + +Tensor::Tensor( const IECore::ConstDataPtr &data, std::vector shape ) +{ + IECore::dispatch( + data.get(), + [&] ( auto typedData ) -> void { + + using DataType = remove_const_t>; + using BaseType = typename DataType::BaseType; + + if( !shape.size() ) + { + // Automatically infer shape from type. + if constexpr( TypeTraits::IsVectorTypedData::value ) + { + shape.push_back( typedData->readable().size() ); + using ShapeT = ShapeTraits; + shape.insert( shape.end(), begin( ShapeT::shape ), end( ShapeT::shape ) ); + } + else + { + using ShapeT = ShapeTraits; + shape.insert( shape.end(), begin( ShapeT::shape ), end( ShapeT::shape ) ); + } + } + + if constexpr( std::is_same_v ) + { + // Special case for the vector of bool fiasco. + Ort::AllocatorWithDefaultOptions allocator; + Ort::Value value = Ort::Value::CreateTensor( + allocator, shape.data(), shape.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL + ); + std::copy( typedData->readable().begin(), typedData->readable().end(), value.GetTensorMutableData() ); + m_state = new State{ std::move( value ), nullptr }; + } + else if constexpr( HasTensorType::value ) + { + Ort::MemoryInfo memoryInfo = Ort::MemoryInfo::CreateCpu( OrtArenaAllocator, OrtMemTypeDefault ); + m_state = new State{ + Ort::Value::CreateTensor( + memoryInfo.GetConst(), + // `const_cast()` is OK because we only provide const access to the + // `Ort::Value` after construction. + const_cast( typedData )->baseWritable(), typedData->baseSize(), + shape.data(), shape.size() + ), + data + }; + } + else + { + throw IECore::Exception( fmt::format( "Unsupported data type `{}`", DataType::staticTypeName() ) ); + } + + } + ); +} + +bool Tensor::isEqualTo( const IECore::Object *other ) const +{ + if( !Object::isEqualTo( other ) ) + { + return false; + } + + auto otherTensor = static_cast( other ); + if( m_state == otherTensor->m_state ) + { + return true; + } + else if( !m_state->value && !otherTensor->m_state->value ) + { + return true; + } + else if( !m_state->value || !otherTensor->m_state->value ) + { + return false; + } + else if( shape() != otherTensor->shape() ) + { + return false; + } + + // Everything else is equal. Need to compare tensor data now. + + if( m_state->data && otherTensor->m_state->data ) + { + // If both tensors are backed by `IECore::Data`, then compare that. + // This has a fast path for when the underlying data is shared. + return m_state->data->isEqualTo( otherTensor->m_state->data.get() ); + } + + // Compare the buffers ourselves. + + if( + m_state->value.GetTensorTypeAndShapeInfo().GetElementType() != + otherTensor->m_state->value.GetTensorTypeAndShapeInfo().GetElementType() + ) + { + return false; + } + + bool equal; + dispatchTensorData( + m_state->value, + [&] ( const auto *data ) { + + using ElementType = remove_const_t>; + const auto *otherData = otherTensor->m_state->value.GetTensorData(); + const size_t count = m_state->value.GetTensorTypeAndShapeInfo().GetElementCount(); + equal = memcmp( data, otherData, count * sizeof( *data ) ) == 0; + } + ); + + return equal; +} + +void Tensor::hash( IECore::MurmurHash &h ) const +{ + Object::hash( h ); + + if( !m_state->value ) + { + return; + } + + dispatchTensorData( + m_state->value, + [&] ( const auto *data ) { + const size_t count = m_state->value.GetTensorTypeAndShapeInfo().GetElementCount(); + h.append( data, count ); + } + ); + + auto s = shape(); + h.append( s.data(), s.size() ); +} + +void Tensor::copyFrom( const IECore::Object *other, IECore::Object::CopyContext *context ) +{ + Object::copyFrom( other, context ); + // Because our public API only provides const access to the value, + // our copy can be extremely cheap, and just share ownership with + // the original. + m_state = static_cast( other )->m_state; +} + +void Tensor::save( IECore::Object::SaveContext *context ) const +{ + Object::save( context ); + throw IECore::NotImplementedException( "Tensor::save" ); +} + +void Tensor::load( IECore::Object::LoadContextPtr context ) +{ + Object::load( context ); + throw IECore::NotImplementedException( "Tensor::load" ); +} + +void Tensor::memoryUsage( IECore::Object::MemoryAccumulator &accumulator ) const +{ + Object::memoryUsage( accumulator ); + + if( m_state->data ) + { + // Register the memory used by data if we have it. This can + // be shared with other objects, in which case the MemoryAccumulator + // is smart enough not to double-count it. + accumulator.accumulate( m_state->data.get() ); + } + else if( m_state->value ) + { + // Ort::Value owns the data. Calculate memory usage. + dispatchTensorData( + m_state->value, + [&] ( const auto *data ) { + + const size_t count = m_state->value.GetTensorTypeAndShapeInfo().GetElementCount(); + accumulator.accumulate( m_state.get(), count * sizeof( *data ) ); + } + ); + } +} + +const Ort::Value &Tensor::value() const +{ + return m_state->value; +} + +std::vector Tensor::shape() const +{ + if( !m_state->value ) + { + throw IECore::Exception( "Null tensor" ); + } + return m_state->value.GetTensorTypeAndShapeInfo().GetShape(); +} + +IECore::DataPtr Tensor::asData() +{ + if( !m_state->value ) + { + return nullptr; + } + + if( m_state->data ) + { + return m_state->data->copy(); + } + return dataFromValue( m_state->value ); +} + +IECore::ConstDataPtr Tensor::asData() const +{ + if( !m_state->value ) + { + return nullptr; + } + + if( m_state->data ) + { + return m_state->data; + } + return dataFromValue( m_state->value ); +} diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index b999c69b8f0..48ba2f5ad20 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -37,6 +37,125 @@ #include "boost/python.hpp" +#include "GafferML/Tensor.h" + +#include "IECorePython/RunTimeTypedBinding.h" + +#include "boost/python/suite/indexing/container_utils.hpp" + +#include "fmt/format.h" + +using namespace boost::python; +using namespace IECore; +using namespace GafferML; + +namespace +{ + +TensorPtr tensorConstructorWrapper( const DataPtr &data, object pythonShape ) +{ + if( pythonShape != object() ) + { + std::vector shape; + boost::python::container_utils::extend_container( shape, pythonShape ); + return new Tensor( data, shape ); + } + else + { + return new Tensor( data ); + } +} + +list tensorShapeWrapper( const Tensor &tensor ) +{ + list result; + for( const auto &x : tensor.shape() ) + { + result.append( x ); + } + return result; +} + +std::string tensorRepr( const Tensor &tensor ) +{ + if( !tensor.value() ) + { + // The most common use of `repr()` is in serialising the + // empty default value for TensorPlug constructors. Make sure + // we have a nice clean serialisation for that. + return "GafferML.Tensor()"; + } + else + { + // We don't have a good `repr()` for this - just return a default one + // and the ValuePlugSerialiser will attempt a base 64 encoding instead. + return fmt::format( "", (void *)&tensor ); + } +} + +template +object tensorGetItemTyped( const Tensor &tensor, const std::vector &location ) +{ + return object( + const_cast( tensor.value() ).At( location ) + ); +} + +object tensorGetItem( const Tensor &tensor, const std::vector &location ) +{ + const auto elementType = tensor.value().GetTensorTypeAndShapeInfo().GetElementType(); + /// \todo Should we make Tensor.cpp's `dispatchTensorData()` public and use + /// it here? + switch( elementType ) + { + case ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_DOUBLE : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_BOOL : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT16 : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16 : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32 : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32 : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64 : + return tensorGetItemTyped( tensor, location ); + case ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64 : + return tensorGetItemTyped( tensor, location ); + default : + throw IECore::Exception( fmt::format( "Unsupported element type {}", elementType ) ); + } +} + +object tensorGetItem1D( const Tensor &tensor, int64_t index ) +{ + return tensorGetItem( tensor, { index } ); +} + +object tensorGetItemND( const Tensor &tensor, tuple index ) +{ + std::vector location; + boost::python::container_utils::extend_container( location, index ); + return tensorGetItem( tensor, location ); +} + +} // namespace + BOOST_PYTHON_MODULE( _GafferML ) { + + IECorePython::RunTimeTypedClass() + .def( init<>() ) + .def( "__init__", make_constructor( tensorConstructorWrapper, default_call_policies(), ( arg( "data" ), arg( "shape" ) = object() ) ) ) + .def( "asData", (IECore::DataPtr (Tensor::*)())&Tensor::asData ) + .def( "shape", &tensorShapeWrapper ) + .def( "__repr__", &tensorRepr ) + .def( "__getitem__", &tensorGetItem1D ) + .def( "__getitem__", &tensorGetItemND ) + ; + } From 7728139de8263728fb2c6c6dbc19ba6a89a624d6 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:12:54 +0000 Subject: [PATCH 11/26] GafferML : Add TensorPlug This will be used for passing Tensor values between nodes. --- include/GafferML/TensorPlug.h | 51 ++++++++++++++++++++ python/GafferMLTest/TensorPlugTest.py | 68 +++++++++++++++++++++++++++ python/GafferMLTest/__init__.py | 1 + src/GafferML/TensorPlug.cpp | 50 ++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 5 ++ 5 files changed, 175 insertions(+) create mode 100644 include/GafferML/TensorPlug.h create mode 100644 python/GafferMLTest/TensorPlugTest.py create mode 100644 src/GafferML/TensorPlug.cpp diff --git a/include/GafferML/TensorPlug.h b/include/GafferML/TensorPlug.h new file mode 100644 index 00000000000..daf0589088b --- /dev/null +++ b/include/GafferML/TensorPlug.h @@ -0,0 +1,51 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Tensor.h" +#include "GafferML/TypeIds.h" + +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferML +{ + +using TensorPlug = Gaffer::TypedObjectPlug; + +IE_CORE_DECLAREPTR( TensorPlug ); + +} // namespace GafferML diff --git a/python/GafferMLTest/TensorPlugTest.py b/python/GafferMLTest/TensorPlugTest.py new file mode 100644 index 00000000000..19ef5e11932 --- /dev/null +++ b/python/GafferMLTest/TensorPlugTest.py @@ -0,0 +1,68 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import IECore + +import Gaffer +import GafferTest +import GafferML + +class TensorPlugTest( GafferTest.TestCase ) : + + def testDefaultValue( self ) : + + plug = GafferML.TensorPlug() + self.assertEqual( plug.defaultValue(), GafferML.Tensor() ) + self.assertEqual( plug.getValue(), GafferML.Tensor() ) + + plug = GafferML.TensorPlug( defaultValue = GafferML.Tensor( IECore.IntVectorData( [ 1, 2, ] ), [ 2 ] ) ) + self.assertEqual( plug.defaultValue(), GafferML.Tensor( IECore.IntVectorData( [ 1, 2, ] ), [ 2 ] ) ) + + def testSerialisationOfDynamicPlugs( self ) : + + script = Gaffer.ScriptNode() + script["node"] = Gaffer.Node() + script["node"]["user"]["p"] = GafferML.TensorPlug( flags = Gaffer.Plug.Flags.Default | Gaffer.Plug.Flags.Dynamic ) + + script2 = Gaffer.ScriptNode() + script2.execute( script.serialise() ) + self.assertIsInstance( script2["node"]["user"]["p"], GafferML.TensorPlug ) + self.assertEqual( script2["node"]["user"]["p"].getValue(), GafferML.Tensor() ) + +if __name__ == "__main__" : + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 60f7088ec36..0e696891b1e 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -35,6 +35,7 @@ ########################################################################## from .TensorTest import TensorTest +from .TensorPlugTest import TensorPlugTest if __name__ == "__main__": import unittest diff --git a/src/GafferML/TensorPlug.cpp b/src/GafferML/TensorPlug.cpp new file mode 100644 index 00000000000..7812279d488 --- /dev/null +++ b/src/GafferML/TensorPlug.cpp @@ -0,0 +1,50 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/TensorPlug.h" + +#include "Gaffer/TypedObjectPlugImplementation.h" + +using namespace GafferML; + +namespace Gaffer +{ + +GAFFER_PLUG_DEFINE_TEMPLATE_TYPE( TensorPlug, TensorPlugTypeId ) + +template class Gaffer::TypedObjectPlug; + +} // namespace Gaffer diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index 48ba2f5ad20..5643ac0f060 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -37,7 +37,10 @@ #include "boost/python.hpp" +#include "GafferBindings/TypedObjectPlugBinding.h" + #include "GafferML/Tensor.h" +#include "GafferML/TensorPlug.h" #include "IECorePython/RunTimeTypedBinding.h" @@ -158,4 +161,6 @@ BOOST_PYTHON_MODULE( _GafferML ) .def( "__getitem__", &tensorGetItemND ) ; + GafferBindings::TypedObjectPlugClass(); + } From 5dcee4ce26b5de7962674fce4808139b39f29c6b Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:14:58 +0000 Subject: [PATCH 12/26] GafferML : Add DataToTensor node This allows data from elsewhere in Gaffer to be converted for use in GafferML. --- include/GafferML/DataToTensor.h | 102 +++++++++++ include/GafferML/DataToTensor.inl | 54 ++++++ python/GafferMLTest/DataToTensorTest.py | 109 ++++++++++++ python/GafferMLTest/__init__.py | 1 + python/GafferMLUI/DataToTensorUI.py | 227 ++++++++++++++++++++++++ python/GafferMLUI/__init__.py | 2 + src/GafferML/DataToTensor.cpp | 189 ++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 62 +++++++ 8 files changed, 746 insertions(+) create mode 100644 include/GafferML/DataToTensor.h create mode 100644 include/GafferML/DataToTensor.inl create mode 100644 python/GafferMLTest/DataToTensorTest.py create mode 100644 python/GafferMLUI/DataToTensorUI.py create mode 100644 src/GafferML/DataToTensor.cpp diff --git a/include/GafferML/DataToTensor.h b/include/GafferML/DataToTensor.h new file mode 100644 index 00000000000..529d87e3765 --- /dev/null +++ b/include/GafferML/DataToTensor.h @@ -0,0 +1,102 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Export.h" +#include "GafferML/TensorPlug.h" + +#include "Gaffer/ComputeNode.h" +#include "Gaffer/NumericPlug.h" +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferML +{ + +class GAFFERML_API DataToTensor : public Gaffer::ComputeNode +{ + + public : + + explicit DataToTensor( const std::string &name=defaultName() ); + ~DataToTensor() override; + + GAFFER_NODE_DECLARE_TYPE( GafferML::DataToTensor, DataToTensorTypeId, Gaffer::ComputeNode ); + + enum class ShapeMode + { + Automatic, + Custom + }; + + bool canSetup( const Gaffer::ValuePlug *prototypeDataPlug ); + void setup( const Gaffer::ValuePlug *prototypeDataPlug ); + + template + T *dataPlug(); + template + const T *dataPlug() const; + + Gaffer::IntPlug *shapeModePlug(); + const Gaffer::IntPlug *shapeModePlug() const; + + Gaffer::Int64VectorDataPlug *shapePlug(); + const Gaffer::Int64VectorDataPlug *shapePlug() const; + + TensorPlug *tensorPlug(); + const TensorPlug *tensorPlug() const; + + void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override; + + protected : + + void hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + void compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const override; + + Gaffer::ValuePlug::CachePolicy computeCachePolicy( const Gaffer::ValuePlug *output ) const override; + + private : + + static size_t g_firstPlugIndex; + static const IECore::InternedString g_dataPlugName; + +}; + +IE_CORE_DECLAREPTR( DataToTensor ) + +} // namespace GafferML + +#include "GafferML/DataToTensor.inl" \ No newline at end of file diff --git a/include/GafferML/DataToTensor.inl b/include/GafferML/DataToTensor.inl new file mode 100644 index 00000000000..2c0f70310f1 --- /dev/null +++ b/include/GafferML/DataToTensor.inl @@ -0,0 +1,54 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +namespace GafferML +{ + +template +T *DataToTensor::dataPlug() +{ + return getChild( g_dataPlugName ); +} + +template +const T *DataToTensor::dataPlug() const +{ + return getChild( g_dataPlugName ); +} + +} // namespace GafferML diff --git a/python/GafferMLTest/DataToTensorTest.py b/python/GafferMLTest/DataToTensorTest.py new file mode 100644 index 00000000000..c22ab972c9d --- /dev/null +++ b/python/GafferMLTest/DataToTensorTest.py @@ -0,0 +1,109 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import imath + +import IECore +import Gaffer +import GafferTest +import GafferML + +class DataToTensorTest( GafferTest.TestCase ) : + + def testBeforeSetup( self ) : + + script = Gaffer.ScriptNode() + script["node"] = GafferML.DataToTensor() + self.assertIsNone( script["node"].getChild( "data" ) ) + self.assertTrue( script["node"].canSetup( Gaffer.FloatVectorDataPlug() ) ) + self.assertEqual( script["node"]["tensor"].getValue(), GafferML.Tensor() ) + + serialisation = script.serialise() + self.assertNotIn( "setup", serialisation ) + + script2 = Gaffer.ScriptNode() + script2.execute( serialisation ) + self.assertIsNone( script2["node"].getChild( "data" ) ) + self.assertTrue( script2["node"].canSetup( Gaffer.FloatVectorDataPlug() ) ) + self.assertEqual( script2["node"]["tensor"].getValue(), GafferML.Tensor() ) + + def testSetup( self ) : + + script = Gaffer.ScriptNode() + script["node"] = GafferML.DataToTensor() + + prototypeDataPlug = Gaffer.FloatVectorDataPlug() + self.assertTrue( script["node"].canSetup( Gaffer.FloatVectorDataPlug() ) ) + script["node"].setup( prototypeDataPlug ) + self.assertIsInstance( script["node"]["data"], Gaffer.FloatVectorDataPlug ) + self.assertFalse( script["node"]["data"].isSame( prototypeDataPlug ) ) + self.assertFalse( script["node"].canSetup( prototypeDataPlug ) ) + + serialisation = script.serialise() + self.assertIn( "setup", serialisation ) + + script2 = Gaffer.ScriptNode() + script2.execute( serialisation ) + self.assertEqual( script2["node"].keys(), script["node"].keys() ) + self.assertIsInstance( script2["node"]["data"], Gaffer.FloatVectorDataPlug ) + self.assertFalse( script2["node"].canSetup( prototypeDataPlug ) ) + + def testTensor( self ) : + + node = GafferML.DataToTensor() + node.setup( Gaffer.FloatVectorDataPlug( defaultValue = IECore.FloatVectorData( [ 1, 2, 3 ] ) ) ) + + tensor = node["tensor"].getValue() + self.assertEqual( tensor.shape(), [ 3 ] ) + self.assertEqual( tensor.asData(), IECore.FloatVectorData( [ 1, 2, 3 ] ) ) + + def testShapeModes( self ) : + + node = GafferML.DataToTensor() + node.setup( Gaffer.V2iVectorDataPlug( defaultValue = IECore.V2iVectorData( [ imath.V2i( i ) for i in range( 0, 3 ) ] ) ) ) + + tensor = node["tensor"].getValue() + self.assertEqual( tensor.shape(), [ 3, 2 ] ) + + node["shapeMode"].setValue( node.ShapeMode.Custom ) + node["shape"].setValue( IECore.Int64VectorData( [ 1, 1, 1, 6 ] ) ) + tensor = node["tensor"].getValue() + self.assertEqual( tensor.shape(), [ 1, 1, 1, 6 ] ) + +if __name__ == "__main__" : + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 0e696891b1e..7d461fb0ccf 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -36,6 +36,7 @@ from .TensorTest import TensorTest from .TensorPlugTest import TensorPlugTest +from .DataToTensorTest import DataToTensorTest if __name__ == "__main__": import unittest diff --git a/python/GafferMLUI/DataToTensorUI.py b/python/GafferMLUI/DataToTensorUI.py new file mode 100644 index 00000000000..a7cdfb2a9b2 --- /dev/null +++ b/python/GafferMLUI/DataToTensorUI.py @@ -0,0 +1,227 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import functools + +import imath + +import IECore + +import Gaffer +import GafferUI +import GafferML + +Gaffer.Metadata.registerNode( + + GafferML.DataToTensor, + + "description", + """ + Converts Gaffer data to tensors for use with the Inference node. + Potential data sources include PrimitiveVariableQuery nodes to fetch data + from 3D scenes, or expressions to generate arbitrary input data. + """, + + "layout:customWidget:setupButton:widgetType", "GafferMLUI.DataToTensorUI._SetupWidget", + "layout:customWidget:setupButton:section", "Settings", + "layout:customWidget:setupButton:visibilityActivator", lambda node : "data" not in node, + + "noduleLayout:customGadget:setupButton:gadgetType", "GafferMLUI.DataToTensorUI._SetupGadget", + "noduleLayout:customGadget:setupButton:index", 0, + + "layout:activator:isSetup", lambda node : "data" in node, + + plugs = { + + "data" : [ + + "description", + """ + The data to be converted. + """, + + "layout:index", 0, + "noduleLayout:index", 0, + + ], + + "shapeMode" : [ + + "description", + """ + Method used to determine the shape of the tensor. + + - Automatic : Derives the shape from the data automatically. For example, a V3fVectorData of size 10 + would give a shape of `[ 10, 3 ]`. + - Custom : The shape is specified manually using the `shape` plug. + """, + + "plugValueWidget:type", "GafferUI.PresetsPlugValueWidget", + "preset:Automatic", GafferML.DataToTensor.ShapeMode.Automatic, + "preset:Custom", GafferML.DataToTensor.ShapeMode.Custom, + + "layout:index", 1, + "layout:visibilityActivator", "isSetup", + + "nodule:type", "", + + ], + + "shape" : [ + + "description", + """ + Defines the shape of the tensor. The product of the shape + must equal the number of elements provided by the `data` + plug. + + Only used when ShapeMode is Custom. + """, + + "layout:index", 2, + "noduleLayout:index", 2, + + "layout:activator", lambda plug : plug.node()["shapeMode"].getValue() == GafferML.DataToTensor.ShapeMode.Custom, + "layout:visibilityActivator", "isSetup", + + ], + + "tensor" : [ + + "description", + """ + The output tensor. + """, + + "layout:visibilityActivator", False, + + ], + + } +) + +class _SetupGadget( GafferUI.PlugAdder ) : + + def __init__( self, node ) : + + GafferUI.PlugAdder.__init__( self ) + + self.__node = node + self.__node.childAddedSignal().connect( Gaffer.WeakMethod( self.__childAddedOrRemoved ) ) + self.__node.childRemovedSignal().connect( Gaffer.WeakMethod( self.__childAddedOrRemoved ) ) + + self.__updateVisibility() + + def canCreateConnection( self, endpoint ) : + + if not GafferUI.PlugAdder.canCreateConnection( self, endpoint ) : + return False + + return ( + self.__node.canSetup( endpoint ) and + endpoint.direction() == Gaffer.Plug.Direction.Out + ) + + def createConnection( self, endpoint ) : + + with Gaffer.UndoScope( self.__node.scriptNode() ) : + self.__node.setup( endpoint ) + self.__node["data"].setInput( endpoint ) + + def __childAddedOrRemoved( self, node, child ) : + + self.__updateVisibility() + + def __updateVisibility( self ) : + + self.setVisible( "data" not in self.__node ) + +GafferUI.NoduleLayout.registerCustomGadget( "GafferMLUI.DataToTensorUI._SetupGadget", _SetupGadget ) + +class _SetupWidget( GafferUI.Widget ) : + + def __init__( self, node ) : + + self.__node = node + self.__row = GafferUI.ListContainer( GafferUI.ListContainer.Orientation.Horizontal ) + + GafferUI.Widget.__init__( self, self.__row ) + + with self.__row : + + GafferUI.Spacer( imath.V2i( GafferUI.PlugWidget.labelWidth(), 1 ) ) + + GafferUI.MenuButton( + menu = GafferUI.Menu( Gaffer.WeakMethod( self.__menuDefinition ), title = "Choose Data Type" ), + image = "plus.png", hasFrame = False + ) + + GafferUI.Spacer( imath.V2i( 1 ), imath.V2i( 999999, 1 ), parenting = { "expand" : True } ) + + def __menuDefinition( self, menu ) : + + result = IECore.MenuDefinition() + + def setup( node, plugType ) : + + with Gaffer.UndoScope( node.scriptNode() ) : + node.setup( plugType() ) + + for plugType in ( + Gaffer.BoolVectorDataPlug, + Gaffer.IntVectorDataPlug, + Gaffer.FloatVectorDataPlug, + None, + Gaffer.V2iVectorDataPlug, + Gaffer.V3iVectorDataPlug, + Gaffer.V2fVectorDataPlug, + Gaffer.V3fVectorDataPlug, + None, + Gaffer.Color3fVectorDataPlug, + Gaffer.Color4fVectorDataPlug, + ) : + if plugType is None : + result.append( "/Divider{}".format( result.size() ), { "divider" : True } ) + else : + result.append( + "/" + plugType.__name__.replace( "VectorDataPlug", "" ), + { + "command" : functools.partial( setup, self.__node, plugType ), + "active" : not Gaffer.MetadataAlgo.readOnly( self.__node ), + } + ) + + return result diff --git a/python/GafferMLUI/__init__.py b/python/GafferMLUI/__init__.py index 573d1b5fbe2..37e153c13ed 100644 --- a/python/GafferMLUI/__init__.py +++ b/python/GafferMLUI/__init__.py @@ -34,4 +34,6 @@ # ########################################################################## +from . import DataToTensorUI + __import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferMLUI" ) diff --git a/src/GafferML/DataToTensor.cpp b/src/GafferML/DataToTensor.cpp new file mode 100644 index 00000000000..36ccc992331 --- /dev/null +++ b/src/GafferML/DataToTensor.cpp @@ -0,0 +1,189 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/DataToTensor.h" + +#include "Gaffer/Context.h" +#include "Gaffer/PlugAlgo.h" + +using namespace std; +using namespace Imath; +using namespace IECore; +using namespace Gaffer; +using namespace GafferML; + +GAFFER_NODE_DEFINE_TYPE( DataToTensor ); + +size_t DataToTensor::g_firstPlugIndex = 0; +const IECore::InternedString DataToTensor::g_dataPlugName( "data" ); + +DataToTensor::DataToTensor( const std::string &name ) + : ComputeNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new IntPlug( "shapeMode", Plug::In, (int)ShapeMode::Automatic, (int)ShapeMode::Automatic, (int)ShapeMode::Custom ) ); + addChild( new Int64VectorDataPlug( "shape" ) ); + addChild( new TensorPlug( "tensor", Plug::Out ) ); +} + +DataToTensor::~DataToTensor() +{ +} + +bool DataToTensor::canSetup( const Gaffer::ValuePlug *prototypeDataPlug ) +{ + if( dataPlug() ) + { + return false; + } + + /// \todo Check type + return true; +} + +void DataToTensor::setup( const Gaffer::ValuePlug *prototypeDataPlug ) +{ + if( dataPlug() ) + { + throw IECore::Exception( "DataToTensor already has a \"data\" plug." ); + } + + PlugPtr dataPlug = prototypeDataPlug->createCounterpart( g_dataPlugName, Plug::In ); + dataPlug->setFlags( Plug::Serialisable, true ); + addChild( dataPlug ); +} + +Gaffer::IntPlug *DataToTensor::shapeModePlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::IntPlug *DataToTensor::shapeModePlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::Int64VectorDataPlug *DataToTensor::shapePlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::Int64VectorDataPlug *DataToTensor::shapePlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +TensorPlug *DataToTensor::tensorPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const TensorPlug *DataToTensor::tensorPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +void DataToTensor::affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const +{ + ComputeNode::affects( input, outputs ); + + if( + input == dataPlug() || + input == shapeModePlug() || + input == shapePlug() + ) + { + outputs.push_back( tensorPlug() ); + } +} + +void DataToTensor::hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + if( output == tensorPlug() ) + { + ComputeNode::hash( output, context, h ); + if( auto d = dataPlug() ) + { + d->hash( h ); + shapeModePlug()->hash( h ); + shapePlug()->hash( h ); + } + } + else + { + ComputeNode::hash( output, context, h ); + } +} + +void DataToTensor::compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const +{ + if( output == tensorPlug() ) + { + if( auto d = dataPlug() ) + { + ConstInt64VectorDataPtr shapeData; + if( shapeModePlug()->getValue() == (int)ShapeMode::Custom ) + { + shapeData = shapePlug()->getValue(); + } + static const vector g_automaticShape; + ConstDataPtr bufferData = PlugAlgo::getValueAsData( d ); + ConstTensorPtr tensorData = new Tensor( bufferData, shapeData ? shapeData->readable() : g_automaticShape ); + static_cast( output )->setValue( tensorData ); + } + else + { + output->setToDefault(); + } + } + else + { + ComputeNode::compute( output, context ); + } +} + +Gaffer::ValuePlug::CachePolicy DataToTensor::computeCachePolicy( const Gaffer::ValuePlug *output ) const +{ + if( output == tensorPlug() ) + { + // Tensors can be really big. Prevent concurrent creation of identical + // tensors that could cause a memory spike before the cache deduplicates + // them. + return ValuePlug::CachePolicy::TaskCollaboration; + } + return ComputeNode::computeCachePolicy( output ); +} + diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index 5643ac0f060..ed51a719411 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -37,8 +37,10 @@ #include "boost/python.hpp" +#include "GafferBindings/DependencyNodeBinding.h" #include "GafferBindings/TypedObjectPlugBinding.h" +#include "GafferML/DataToTensor.h" #include "GafferML/Tensor.h" #include "GafferML/TensorPlug.h" @@ -50,7 +52,9 @@ using namespace boost::python; using namespace IECore; +using namespace Gaffer; using namespace GafferML; +using namespace GafferBindings; namespace { @@ -146,6 +150,51 @@ object tensorGetItemND( const Tensor &tensor, tuple index ) return tensorGetItem( tensor, location ); } +void dataToTensorSetupWrapper( DataToTensor &dataToTensor, ValuePlug &prototypeDataPlug ) +{ + IECorePython::ScopedGILRelease gilRelease; + dataToTensor.setup( &prototypeDataPlug ); +} + +class DataToTensorSerialiser : public NodeSerialiser +{ + + bool childNeedsConstruction( const Gaffer::GraphComponent *child, const Serialisation &serialisation ) const override + { + auto dataToTensor = child->parent(); + if( child == dataToTensor->dataPlug() ) + { + // We'll serialise a `setup()` call to construct this. + return false; + } + return NodeSerialiser::childNeedsConstruction( child, serialisation ); + } + + std::string postConstructor( const Gaffer::GraphComponent *graphComponent, const std::string &identifier, Serialisation &serialisation ) const override + { + std::string result = NodeSerialiser::postConstructor( graphComponent, identifier, serialisation ); + + auto dataPlug = static_cast( graphComponent )->dataPlug(); + if( !dataPlug ) + { + return result; + } + + if( result.size() ) + { + result += "\n"; + } + + // Add a call to `setup()` to recreate the plug. + + const Serialiser *plugSerialiser = Serialisation::acquireSerialiser( dataPlug ); + result += identifier + ".setup( " + plugSerialiser->constructor( dataPlug, serialisation ) + " )\n"; + + return result; + } + +}; + } // namespace BOOST_PYTHON_MODULE( _GafferML ) @@ -163,4 +212,17 @@ BOOST_PYTHON_MODULE( _GafferML ) GafferBindings::TypedObjectPlugClass(); + { + scope s = GafferBindings::DependencyNodeClass() + .def( "canSetup", &DataToTensor::canSetup, ( arg( "prototypeDataPlug" ) ) ) + .def( "setup", &dataToTensorSetupWrapper, ( arg( "prototypeDataPlug" ) ) ) + ; + + enum_( "ShapeMode" ) + .value( "Automatic", DataToTensor::ShapeMode::Automatic ) + .value( "Custom", DataToTensor::ShapeMode::Custom ) + ; + Serialisation::registerSerialiser( DataToTensor::staticTypeId(), new DataToTensorSerialiser ); + } + } From df99cff95b1f4b4bf2e8ccb9e6f2d882897c895b Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:19:55 +0000 Subject: [PATCH 13/26] GafferML : Add Inference node This forms the meat of GafferML, loading ONNX models and performing inference using data from an array of input TensorPlugs. --- include/GafferML/Inference.h | 95 ++++++ python/GafferMLTest/InferenceTest.py | 169 +++++++++++ python/GafferMLTest/__init__.py | 1 + python/GafferMLTest/models/add.onnx | Bin 0 -> 129 bytes python/GafferMLUI/InferenceUI.py | 132 +++++++++ python/GafferMLUI/__init__.py | 1 + src/GafferML/Inference.cpp | 401 ++++++++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 11 + 8 files changed, 810 insertions(+) create mode 100644 include/GafferML/Inference.h create mode 100644 python/GafferMLTest/InferenceTest.py create mode 100644 python/GafferMLTest/models/add.onnx create mode 100644 python/GafferMLUI/InferenceUI.py create mode 100644 src/GafferML/Inference.cpp diff --git a/include/GafferML/Inference.h b/include/GafferML/Inference.h new file mode 100644 index 00000000000..537f0abd7d6 --- /dev/null +++ b/include/GafferML/Inference.h @@ -0,0 +1,95 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Export.h" +#include "GafferML/TensorPlug.h" + +#include "Gaffer/ArrayPlug.h" +#include "Gaffer/ComputeNode.h" +#include "Gaffer/StringPlug.h" +#include "Gaffer/TypedObjectPlug.h" + +namespace GafferML +{ + +class GAFFERML_API Inference : public Gaffer::ComputeNode +{ + + public : + + explicit Inference( const std::string &name=defaultName() ); + ~Inference() override; + + GAFFER_NODE_DECLARE_TYPE( GafferML::Inference, InferenceTypeId, Gaffer::ComputeNode ); + + void loadModel(); + + Gaffer::StringPlug *modelPlug(); + const Gaffer::StringPlug *modelPlug() const; + + Gaffer::ArrayPlug *inPlug(); + const Gaffer::ArrayPlug *inPlug() const; + + Gaffer::ArrayPlug *outPlug(); + const Gaffer::ArrayPlug *outPlug() const; + + void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override; + + protected : + + void hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + void compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const override; + Gaffer::ValuePlug::CachePolicy computeCachePolicy( const Gaffer::ValuePlug *output ) const override; + + private : + + // We assume that if a model has multiple outputs, then it is more + // efficient to compute them all at once. We do that and cache it + // on this plug, then dole out individual results from the children + // of `outPlug()`. + /// \todo Verify the assumption. + Gaffer::CompoundObjectPlug *inferencePlug(); + const Gaffer::CompoundObjectPlug *inferencePlug() const; + + static size_t g_firstPlugIndex; + +}; + +IE_CORE_DECLAREPTR( Inference ) + +} // namespace GafferML diff --git a/python/GafferMLTest/InferenceTest.py b/python/GafferMLTest/InferenceTest.py new file mode 100644 index 00000000000..b9f0615f5d0 --- /dev/null +++ b/python/GafferMLTest/InferenceTest.py @@ -0,0 +1,169 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import os +import subprocess +import pathlib +import unittest + +import IECore + +import Gaffer +import GafferTest +import GafferML + +## \todo Test cancellation. For this, we need a model that takes long enough to compute +# but is small enough to package with the tests. +class InferenceTest( GafferTest.TestCase ) : + + def testLoadModel( self ) : + + script = Gaffer.ScriptNode() + + script["inference"] = GafferML.Inference() + script["inference"]["model"].setValue( pathlib.Path( __file__ ).parent / "models" / "add.onnx" ) + script["inference"].loadModel() + + def assertLoaded( inference ) : + + self.assertEqual( inference["in"].keys(), [ "in0", "in1" ] ) + self.assertIsInstance( inference["in"]["in0"], GafferML.TensorPlug ) + self.assertIsInstance( inference["in"]["in1"], GafferML.TensorPlug ) + self.assertEqual( Gaffer.Metadata.value( inference["in"]["in0"], "label" ), "x" ) + self.assertEqual( Gaffer.Metadata.value( inference["in"]["in1"], "label" ), "y" ) + + self.assertEqual( inference["out"].keys(), [ "out0" ] ) + self.assertIsInstance( inference["out"]["out0"], GafferML.TensorPlug ) + self.assertEqual( Gaffer.Metadata.value( inference["out"]["out0"], "label" ), "sum" ) + + assertLoaded( script["inference"] ) + + script2 = Gaffer.ScriptNode() + script2.execute( script.serialise() ) + assertLoaded( script2["inference"] ) + + def testCompute( self ) : + + inference = GafferML.Inference() + inference["model"].setValue( pathlib.Path( __file__ ).parent / "models" / "add.onnx" ) + inference.loadModel() + + inference["in"][0].setValue( + GafferML.Tensor( IECore.FloatVectorData( [ 1 ] * 60 ), [ 3, 4, 5 ] ) + ) + + inference["in"][1].setValue( + GafferML.Tensor( IECore.FloatVectorData( [ 2 ] * 60 ), [ 3, 4, 5 ] ) + ) + + self.assertEqual( + inference["out"][0].getValue().asData(), + IECore.FloatVectorData( [ 3 ] * 60 ) + ) + + inference["in"][1].setValue( + GafferML.Tensor( IECore.FloatVectorData( [ 3 ] * 60 ), [ 3, 4, 5 ] ) + ) + + self.assertEqual( + inference["out"][0].getValue().asData(), + IECore.FloatVectorData( [ 4 ] * 60 ) + ) + + def testComputeError( self ) : + + inference = GafferML.Inference() + inference["model"].setValue( pathlib.Path( __file__ ).parent / "models" / "add.onnx" ) + inference.loadModel() + + inference["in"][0].setValue( + GafferML.Tensor( IECore.FloatVectorData( [ 1 ] * 60 ), [ 60 ] ) + ) + + inference["in"][1].setValue( + GafferML.Tensor( IECore.FloatVectorData( [ 2 ] * 60 ), [ 3, 4, 5 ] ) + ) + + with self.assertRaisesRegex( Gaffer.ProcessException, "Invalid rank for input" ) : + inference["out"][0].getValue() + + def testModelSearchPaths( self ) : + + node = GafferML.Inference() + node["model"].setValue( "add.onnx" ) + + testPath = str( pathlib.Path( __file__ ).parent / "models" ) + if os.environ.get( "GAFFERML_MODEL_PATHS", "" ) != testPath : + + self.assertRaises( RuntimeError, node.loadModel ) + env = os.environ.copy() + env["GAFFERML_MODEL_PATHS"] = testPath + try : + subprocess.check_output( + [ str( Gaffer.executablePath() ), "test", "GafferMLTest.InferenceTest.testModelSearchPaths" ], + env = env, stderr = subprocess.STDOUT + ) + except subprocess.CalledProcessError as e : + self.fail( e.output ) + + else : + + node.loadModel() + self.assertEqual( len( node["in"] ), 2 ) + self.assertEqual( len( node["out"] ), 1 ) + + def testLoadModelKeepsConnections( self ) : + + dataToTensor1 = GafferML.DataToTensor() + dataToTensor2 = GafferML.DataToTensor() + destinationPlug = GafferML.TensorPlug() + + inference = GafferML.Inference() + inference["model"].setValue( pathlib.Path( __file__ ).parent / "models" / "add.onnx" ) + inference.loadModel() + + inference["in"][0].setInput( dataToTensor1["tensor"] ) + inference["in"][1].setInput( dataToTensor2["tensor"] ) + destinationPlug.setInput( inference["out"][0] ) + + inference.loadModel() + + self.assertTrue( inference["in"][0].getInput().isSame( dataToTensor1["tensor"] ) ) + self.assertTrue( inference["in"][1].getInput().isSame( dataToTensor2["tensor"] ) ) + self.assertTrue( destinationPlug.getInput().isSame( inference["out"][0] ) ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 7d461fb0ccf..069dc4c2888 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -37,6 +37,7 @@ from .TensorTest import TensorTest from .TensorPlugTest import TensorPlugTest from .DataToTensorTest import DataToTensorTest +from .InferenceTest import InferenceTest if __name__ == "__main__": import unittest diff --git a/python/GafferMLTest/models/add.onnx b/python/GafferMLTest/models/add.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8aa4ab2bc385ac70420f1d31868757a543650cb9 GIT binary patch literal 129 zcmd;J7vf1uOwLZtOVKS!EiSRj Tip : If a relative path is used, it will be searched for + > in all the filesystem locations specified by the `GAFFERML_MODEL_PATHS` + > environment variable. + """, + + "nodule:type", "", + "plugValueWidget:type", "GafferUI.FileSystemPathPlugValueWidget", + "path:leaf", True, + "path:valid", True, + "path:bookmarks", "onnx", + "fileSystemPath:extensions", "onnx", + + ], + + "in" : [ + + "description", + """ + The inputs to the model. + """, + + "nodule:type", "GafferUI::CompoundNodule", + "noduleLayout:spacing", 1.0, + # Disable ArrayPlug "+" button. + "noduleLayout:customGadget:addButton:gadgetType", "", + ## \todo Add a widget which displays type/shape requirements etc + # for each input. + "plugValueWidget:type", "", + + ], + + "out" : [ + + "description", + """ + The outputs from the model. + """, + + "nodule:type", "GafferUI::CompoundNodule", + # Disable ArrayPlug "+" button. + "noduleLayout:customGadget:addButton:gadgetType", "", + "noduleLayout:spacing", 1.0, + "plugValueWidget:type", "", + + ], + + } +) + +class _LoadButton( GafferUI.PlugValueWidget ) : + + def __init__( self, node, **kw ) : + + button = GafferUI.Button( image = "refresh.png", hasFrame = False ) + GafferUI.PlugValueWidget.__init__( self, button, node["model"], **kw ) + + button.clickedSignal().connect( Gaffer.WeakMethod( self.__clicked ) ) + + def __clicked( self, button ) : + + with self.context() : + if self.getPlug().getValue() : + with GafferUI.ErrorDialogue.ErrorHandler( + title = "Error loading model", + parentWindow = self.ancestor( GafferUI.Window ) + ) : + with Gaffer.UndoScope( self.scriptNode() ) : + self.getPlug().node().loadModel() diff --git a/python/GafferMLUI/__init__.py b/python/GafferMLUI/__init__.py index 37e153c13ed..de754cc80af 100644 --- a/python/GafferMLUI/__init__.py +++ b/python/GafferMLUI/__init__.py @@ -35,5 +35,6 @@ ########################################################################## from . import DataToTensorUI +from . import InferenceUI __import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferMLUI" ) diff --git a/src/GafferML/Inference.cpp b/src/GafferML/Inference.cpp new file mode 100644 index 00000000000..3fa6d98f8b4 --- /dev/null +++ b/src/GafferML/Inference.cpp @@ -0,0 +1,401 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/Inference.h" + +#include "Gaffer/Context.h" +#include "Gaffer/Metadata.h" + +#include "IECore/SearchPath.h" +#include "IECore/StringAlgo.h" + +#include "onnxruntime_cxx_api.h" + +#include +#include + +using namespace std; +using namespace Imath; +using namespace IECore; +using namespace Gaffer; +using namespace GafferML; + +////////////////////////////////////////////////////////////////////////// +// Internal utilities +////////////////////////////////////////////////////////////////////////// + +namespace +{ + +Ort::Env &acquireEnv() +{ + static Ort::Env g_env( ORT_LOGGING_LEVEL_WARNING, "Gaffer" ); + return g_env; +} + +// Constructing a session (loading a model) is relatively expensive, +// so we only ever create a single session per model. I can't find +// a reference for this in the docs, but `Session::Run()` is thread-safe +// and can be called concurrently by multiple clients : +// +// https://github.com/microsoft/onnxruntime/issues/114 +Ort::Session &acquireSession( const std::string &fileName ) +{ + static std::mutex g_mutex; + static std::unordered_map g_map; + lock_guard lock( g_mutex ); + + auto it = g_map.find( fileName ); + if( it != g_map.end() ) + { + return it->second; + } + + const char *sp = getenv( "GAFFERML_MODEL_PATHS" ); + IECore::SearchPath searchPath( sp ? sp : "" ); + + /// \todo Convert SearchPath to deal in `std::filesystem` rather than `boost::filesystem`. + std::filesystem::path path = searchPath.find( fileName ).string(); + if( path.empty() ) + { + throw Exception( fmt::format( "Could not find file \"{}\" on GAFFERML_MODEL_PATHS", fileName ) ); + } + + it = g_map.try_emplace( fileName, acquireEnv(), path.c_str(), Ort::SessionOptions() ).first; + return it->second; +} + +struct AsyncWaiter +{ + + AsyncWaiter( Ort::RunOptions &runOptions ) + : m_runOptions( runOptions ) + { + } + + void wait( const IECore::Canceller *canceller ) + { + while( true ) + { + std::unique_lock lock( m_mutex ); + m_conditionVariable.wait_for( lock, std::chrono::milliseconds( 100 ) ); + + if( m_resultStatus ) + { + // Run has completed. Throw if it errored or was cancelled, + // otherwise return. + Ort::ThrowOnError( *m_resultStatus ); + IECore::Canceller::check( canceller ); + return; + } + else if( canceller && canceller->cancelled() ) + { + m_runOptions.SetTerminate(); + } + } + } + + static void callback( void *userData, OrtValue **outputs, size_t numOutputs, OrtStatusPtr status ) + { + // Run has completed. Set status so we can pick it up in `wait()`. + auto that = (AsyncWaiter *)userData; + { + std::unique_lock lock( that->m_mutex ); + that->m_resultStatus = status; + } + that->m_conditionVariable.notify_all(); + } + + private : + + Ort::RunOptions &m_runOptions; + std::mutex m_mutex; + std::condition_variable m_conditionVariable; + std::optional m_resultStatus; + +}; + +} // namespace + +////////////////////////////////////////////////////////////////////////// +// Inference +////////////////////////////////////////////////////////////////////////// + +GAFFER_NODE_DEFINE_TYPE( Inference ); + +size_t Inference::g_firstPlugIndex = 0; + +Inference::Inference( const std::string &name ) + : ComputeNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new StringPlug( "model" ) ); + addChild( new ArrayPlug( "in", Plug::In, new TensorPlug( "in0" ), 0, std::numeric_limits::max(), Plug::Default, false ) ); + addChild( new ArrayPlug( "out", Plug::Out, new TensorPlug( "out0" ), 0, std::numeric_limits::max(), Plug::Default, false ) ); + addChild( new CompoundObjectPlug( "__inference", Plug::Out ) ); +} + +Inference::~Inference() +{ +} + +void Inference::loadModel() +{ + Ort::Session &session = acquireSession( modelPlug()->getValue() ); + + // Input and output names can contain characters like `.` that cannot be + // used in plug names. Furthermore, many models have inputs and outputs + // which are interchangeable other than trivial differences in naming. So + // instead of using the names as plug names, we store inputs and outputs as + // ArrayPlugs, where only index matters. Then we add label metadata using + // the true name, to make the UI a little more helpful. + + size_t numInputs = 0; + for( size_t i = 0; i < session.GetInputCount(); ++i ) + { + if( session.GetInputTypeInfo( i ).GetONNXType() != ONNXType::ONNX_TYPE_TENSOR ) + { + continue; + } + + numInputs++; + inPlug()->resize( std::max( inPlug()->children().size(), numInputs ) ); // Add new plug if needed. + + Ort::AllocatedStringPtr ortName = session.GetInputNameAllocated( i, Ort::AllocatorWithDefaultOptions() ); + IECore::ConstStringDataPtr label = new StringData( ortName.get() ); + Metadata::registerValue( inPlug()->getChild( i ), "label", label ); + Metadata::registerValue( inPlug()->getChild( i ), "noduleLayout:label", label ); + } + inPlug()->resize( numInputs ); // Remove old plugs we don't need. + + size_t numOutputs = 0; + for( size_t i = 0; i < session.GetOutputCount(); ++i ) + { + if( session.GetOutputTypeInfo( i ).GetONNXType() != ONNXType::ONNX_TYPE_TENSOR ) + { + continue; + } + + numOutputs++; + outPlug()->resize( std::max( outPlug()->children().size(), numOutputs ) ); + + Ort::AllocatedStringPtr ortName = session.GetOutputNameAllocated( i, Ort::AllocatorWithDefaultOptions() ); + IECore::ConstStringDataPtr label = new StringData( ortName.get() ); + Metadata::registerValue( outPlug()->getChild( i ), "label", label ); + Metadata::registerValue( outPlug()->getChild( i ), "noduleLayout:label", label ); + } + outPlug()->resize( numOutputs ); +} + +Gaffer::StringPlug *Inference::modelPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const Gaffer::StringPlug *Inference::modelPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::ArrayPlug *Inference::inPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::ArrayPlug *Inference::inPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::ArrayPlug *Inference::outPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::ArrayPlug *Inference::outPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +Gaffer::CompoundObjectPlug *Inference::inferencePlug() +{ + return getChild( g_firstPlugIndex + 3); +} + +const Gaffer::CompoundObjectPlug *Inference::inferencePlug() const +{ + return getChild( g_firstPlugIndex + 3 ); +} + +void Inference::affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const +{ + ComputeNode::affects( input, outputs ); + + if( + input == modelPlug() || + input->parent() == inPlug() + ) + { + outputs.push_back( inferencePlug() ); + } + + if( input == inferencePlug() ) + { + for( auto p : Gaffer::ValuePlug::Range( *outPlug() ) ) + { + outputs.push_back( p.get() ); + } + } +} + +void Inference::hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + if( output == inferencePlug() ) + { + ComputeNode::hash( output, context, h ); + modelPlug()->hash( h ); + for( auto &p : TensorPlug::InputRange( *inPlug() ) ) + { + p->hash( h ); + } + } + else if( output->parent() == outPlug() ) + { + ComputeNode::hash( output, context, h ); + inferencePlug()->hash( h ); + } + else + { + ComputeNode::hash( output, context, h ); + } +} + +void Inference::compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const +{ + if( output == inferencePlug() ) + { + // Set up input and output tensor arrays. + + const string model = modelPlug()->getValue(); + Ort::Session &session = acquireSession( model ); + + vector inputNameOwners; + vector inputNames; + vector inputOwners; + vector inputs; + + for( auto &p : TensorPlug::InputRange( *inPlug() ) ) + { + int inputIndex = StringAlgo::numericSuffix( p->getName().string() ); + inputNameOwners.push_back( session.GetInputNameAllocated( inputIndex, Ort::AllocatorWithDefaultOptions() ) ); + inputNames.push_back( inputNameOwners.back().get() ); + inputOwners.push_back( p->getValue() ); + inputs.push_back( inputOwners.back()->value() ); + } + + vector outputNameOwners; + vector outputNames; + vector outputs; + for( auto &p : TensorPlug::OutputRange( *outPlug() ) ) + { + int outputIndex = StringAlgo::numericSuffix( p->getName().string() ); + outputNameOwners.push_back( session.GetOutputNameAllocated( outputIndex, Ort::AllocatorWithDefaultOptions() ) ); + outputNames.push_back( outputNameOwners.back().get() ); + outputs.push_back( Ort::Value( nullptr ) ); + } + + // Run inference asynchronously on an ONNX thread. This allows us + // to check for cancellation via our AsyncWaiter. + + Ort::RunOptions runOptions; + AsyncWaiter waiter( runOptions ); + + session.RunAsync( + runOptions, inputNames.data(), + // The Ort C++ API wants us to pass `Ort::Value *`, but `Ort::Value` + // is non-copyable and the original `Ort::Value` instances are in + // separate TensorDatas and can't be moved. But `Ort::Value` has the + // same layout as `OrtValue *` (the underlying C type) so we can + // just reinterpret cast from the latter. Indeed, `Run()` is going + // to cast straight back to `OrtValue *` to call the C API! + reinterpret_cast( inputs.data() ), + inputs.size(), + outputNames.data(), + outputs.data(), + outputNames.size(), + waiter.callback, + &waiter + ); + + waiter.wait( context->canceller() ); + + CompoundObjectPtr result = new CompoundObject; + for( size_t i = 0; i < outputs.size(); ++i ) + { + result->members()[outPlug()->children()[i]->getName()] = new Tensor( std::move( outputs[i] ) ); + } + + static_cast( output )->setValue( result ); + } + else if( output->parent() == outPlug() ) + { + ConstCompoundObjectPtr inferenceData = inferencePlug()->getValue(); + static_cast( output )->setValue( inferenceData->member( output->getName() ) ); + } + else + { + ComputeNode::compute( output, context ); + } +} + +Gaffer::ValuePlug::CachePolicy Inference::computeCachePolicy( const Gaffer::ValuePlug *output ) const +{ + if( output == inferencePlug() ) + { + // We're not actually capable of task collaboration, because all the work is done by ONNX + // on its own threads. But we use the TaskCollaboration policy to avoid concurrent computes + // of the same thing, which would be incredibly wasteful. + return ValuePlug::CachePolicy::TaskCollaboration; + } + else if( output->parent() == outPlug() ) + { + // We're just going to reference data that is already cached in + // `inferencePlug()`. Avoid double-counting of cache memory by not + // caching again (the compute is fast enough that we don't care anyway). + return ValuePlug::CachePolicy::Uncached; + } + return ComputeNode::computeCachePolicy( output ); +} diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index ed51a719411..e28e46b419f 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -41,6 +41,7 @@ #include "GafferBindings/TypedObjectPlugBinding.h" #include "GafferML/DataToTensor.h" +#include "GafferML/Inference.h" #include "GafferML/Tensor.h" #include "GafferML/TensorPlug.h" @@ -195,6 +196,12 @@ class DataToTensorSerialiser : public NodeSerialiser }; +void loadModelWrapper( Inference &inference ) +{ + IECorePython::ScopedGILRelease gilRelease; + inference.loadModel(); +} + } // namespace BOOST_PYTHON_MODULE( _GafferML ) @@ -225,4 +232,8 @@ BOOST_PYTHON_MODULE( _GafferML ) Serialisation::registerSerialiser( DataToTensor::staticTypeId(), new DataToTensorSerialiser ); } + GafferBindings::DependencyNodeClass() + .def( "loadModel", &loadModelWrapper ) + ; + } From 6d95369782e8013882d16fd23f9d82aa86d04394 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:21:26 +0000 Subject: [PATCH 14/26] GafferML : Add ImageToTensor This is a node which converts images from GafferImage into tensors for use by the Inference node. --- include/GafferML/ImageToTensor.h | 93 +++++++ python/GafferMLTest/ImageToTensorTest.py | 128 ++++++++++ python/GafferMLTest/__init__.py | 1 + python/GafferMLUI/ImageToTensorUI.py | 118 +++++++++ python/GafferMLUI/__init__.py | 1 + src/GafferML/ImageToTensor.cpp | 303 +++++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 3 + 7 files changed, 647 insertions(+) create mode 100644 include/GafferML/ImageToTensor.h create mode 100644 python/GafferMLTest/ImageToTensorTest.py create mode 100644 python/GafferMLUI/ImageToTensorUI.py create mode 100644 src/GafferML/ImageToTensor.cpp diff --git a/include/GafferML/ImageToTensor.h b/include/GafferML/ImageToTensor.h new file mode 100644 index 00000000000..51b75108f83 --- /dev/null +++ b/include/GafferML/ImageToTensor.h @@ -0,0 +1,93 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Export.h" +#include "GafferML/TensorPlug.h" + +#include "GafferImage/ImagePlug.h" + +#include "Gaffer/ComputeNode.h" +#include "Gaffer/StringPlug.h" + +namespace GafferML +{ + +class GAFFERML_API ImageToTensor : public Gaffer::ComputeNode +{ + + public : + + explicit ImageToTensor( const std::string &name=defaultName() ); + ~ImageToTensor() override; + + GAFFER_NODE_DECLARE_TYPE( GafferML::ImageToTensor, ImageToTensorTypeId, Gaffer::ComputeNode ); + + GafferImage::ImagePlug *imagePlug(); + const GafferImage::ImagePlug *imagePlug() const; + + Gaffer::StringPlug *viewPlug(); + const Gaffer::StringPlug *viewPlug() const; + + Gaffer::StringVectorDataPlug *channelsPlug(); + const Gaffer::StringVectorDataPlug *channelsPlug() const; + + Gaffer::BoolPlug *interleaveChannelsPlug(); + const Gaffer::BoolPlug *interleaveChannelsPlug() const; + + TensorPlug *tensorPlug(); + const TensorPlug *tensorPlug() const; + + void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override; + + protected : + + void hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + void compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const override; + + Gaffer::ValuePlug::CachePolicy hashCachePolicy( const Gaffer::ValuePlug *output ) const override; + Gaffer::ValuePlug::CachePolicy computeCachePolicy( const Gaffer::ValuePlug *output ) const override; + + private : + + static size_t g_firstPlugIndex; + +}; + +IE_CORE_DECLAREPTR( ImageToTensor ) + +} // namespace GafferML diff --git a/python/GafferMLTest/ImageToTensorTest.py b/python/GafferMLTest/ImageToTensorTest.py new file mode 100644 index 00000000000..4b7eca608aa --- /dev/null +++ b/python/GafferMLTest/ImageToTensorTest.py @@ -0,0 +1,128 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import imath + +import IECore + +import Gaffer +import GafferTest +import GafferImage +import GafferML + +class ImageToTensorTest( GafferTest.TestCase ) : + + def testMissingChannels( self ) : + + checker = GafferImage.Checkerboard() + tensor = GafferML.ImageToTensor() + tensor["image"].setInput( checker["out"] ) + tensor["channels"].setValue( IECore.StringVectorData( [ "Y" ] ) ) + + with self.assertRaisesRegex( Gaffer.ProcessException, 'Channel "Y" does not exist' ) : + tensor["tensor"].getValue() + + def testShufflingChannelsChangesHash( self ) : + + checker = GafferImage.Checkerboard() + tensor = GafferML.ImageToTensor() + tensor["image"].setInput( checker["out"] ) + + self.assertEqual( tensor["channels"].getValue(), IECore.StringVectorData( [ "R", "G", "B" ] ) ) + h1 = tensor["tensor"].hash() + + tensor["channels"].setValue( IECore.StringVectorData( [ "B", "G", "R" ] ) ) + self.assertNotEqual( tensor["tensor"].hash(), h1 ) + + def testView( self ) : + + left = GafferImage.Constant() + left["color"].setValue( imath.Color4f( 1, 0, 0, 1 ) ) + left["format"].setValue( GafferImage.Format( 1, 1 ) ) + + right = GafferImage.Constant() + right["color"].setValue( imath.Color4f( 0, 1, 0, 1 ) ) + right["format"].setValue( GafferImage.Format( 1, 1 ) ) + + createViews = GafferImage.CreateViews() + createViews["views"].resize( 2 ) + createViews["views"][0]["name"].setValue( "left" ) + createViews["views"][0]["value"].setInput( left["out" ]) + createViews["views"][1]["name"].setValue( "right" ) + createViews["views"][1]["value"].setInput( right["out" ]) + + imageToTensor = GafferML.ImageToTensor() + imageToTensor["image"].setInput( createViews["out"] ) + + with self.assertRaisesRegex( Gaffer.ProcessException, "View does not exist" ) : + imageToTensor["tensor"].getValue() + + imageToTensor["view"].setValue( "left" ) + self.assertEqual( + imageToTensor["tensor"].getValue().asData(), + IECore.FloatVectorData( [ 1, 0, 0 ] ) + ) + + imageToTensor["view"].setValue( "right" ) + self.assertEqual( + imageToTensor["tensor"].getValue().asData(), + IECore.FloatVectorData( [ 0, 1, 0 ] ) + ) + + def testDataWindowAffectsHash( self ) : + + checker = GafferImage.Checkerboard() + checker["format"].setValue( + GafferImage.Format( GafferImage.ImagePlug.tileSize(), GafferImage.ImagePlug.tileSize() ) + ) + tileHash = checker["out"].channelDataHash( "R", imath.V2i( 0 ) ) + + imageToTensor = GafferML.ImageToTensor() + imageToTensor["image"].setInput( checker["out"] ) + h = imageToTensor["tensor"].hash() + tensor = imageToTensor["tensor"].getValue() + + checker["format"].setValue( + GafferImage.Format( GafferImage.ImagePlug.tileSize() - 1, GafferImage.ImagePlug.tileSize() - 1 ) + ) + self.assertEqual( checker["out"].channelDataHash( "R", imath.V2i( 0 ) ), tileHash ) + self.assertNotEqual( imageToTensor["tensor"].hash(), h ) + self.assertNotEqual( imageToTensor["tensor"].getValue(), tensor ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 069dc4c2888..6c6c8d2202e 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -38,6 +38,7 @@ from .TensorPlugTest import TensorPlugTest from .DataToTensorTest import DataToTensorTest from .InferenceTest import InferenceTest +from .ImageToTensorTest import ImageToTensorTest if __name__ == "__main__": import unittest diff --git a/python/GafferMLUI/ImageToTensorUI.py b/python/GafferMLUI/ImageToTensorUI.py new file mode 100644 index 00000000000..396d03ccfac --- /dev/null +++ b/python/GafferMLUI/ImageToTensorUI.py @@ -0,0 +1,118 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import Gaffer +import GafferML + +Gaffer.Metadata.registerNode( + + GafferML.ImageToTensor, + + "description", + """ + Converts images to tensors for use with the Inference node. + + > Note : Only the data window is converted, as it would typically be + > wasteful to convert and process the empty pixels outside the data window. + > If this is necessary, merge the image over a Constant image before + > conversion. + """, + + plugs = { + + "image" : [ + + "description", + """ + The image to be converted. + """, + + ], + + "view" : [ + + "description", + """ + The image view to take the tensor data from. + """, + + "plugValueWidget:type", "GafferImageUI.ViewPlugValueWidget", + + "noduleLayout:visible", False, + + ], + + "channels" : [ + + "description", + """ + The list of channels to convert. Channels are added to the + tensor in the order specified, so can be shuffled by changing + the order. For example, an order of `[ "B", "G", "R" ]` might + be needed for use with models trained on images using OpenCV + conventions. + """, + + "noduleLayout:visible", False, + + ], + + "interleaveChannels" : [ + + "description", + """ + Interleaves the channel data, so that all channels for a single + pixel are adjacent in memory. Whether or not this is needed depends + on the input requirements of the model the tensor is used with. + """, + + "noduleLayout:visible", False, + + ], + + "tensor" : [ + + "description", + """ + The output tensor. + """, + + "layout:visibilityActivator", lambda plug : False, + + ], + + } +) diff --git a/python/GafferMLUI/__init__.py b/python/GafferMLUI/__init__.py index de754cc80af..466a103c9e9 100644 --- a/python/GafferMLUI/__init__.py +++ b/python/GafferMLUI/__init__.py @@ -36,5 +36,6 @@ from . import DataToTensorUI from . import InferenceUI +from . import ImageToTensorUI __import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferMLUI" ) diff --git a/src/GafferML/ImageToTensor.cpp b/src/GafferML/ImageToTensor.cpp new file mode 100644 index 00000000000..0380edb4738 --- /dev/null +++ b/src/GafferML/ImageToTensor.cpp @@ -0,0 +1,303 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/ImageToTensor.h" + +#include "GafferImage/BufferAlgo.h" +#include "GafferImage/ImageAlgo.h" +#include "GafferImage/Sampler.h" + +#include "Gaffer/Context.h" + +#include "boost/container/flat_map.hpp" + +#include "onnxruntime_cxx_api.h" + +using namespace std; +using namespace Imath; +using namespace IECore; +using namespace Gaffer; +using namespace GafferImage; +using namespace GafferML; + +GAFFER_NODE_DEFINE_TYPE( ImageToTensor ); + +size_t ImageToTensor::g_firstPlugIndex = 0; + +ImageToTensor::ImageToTensor( const std::string &name ) + : ComputeNode( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new ImagePlug( "image", Plug::In ) ); + addChild( new StringPlug( "view", Plug::In, "default" ) ); + addChild( new StringVectorDataPlug( "channels", Plug::In, new StringVectorData( { "R", "G", "B" } ) ) ); + addChild( new BoolPlug( "interleaveChannels" ) ); + addChild( new TensorPlug( "tensor", Plug::Out ) ); +} + +ImageToTensor::~ImageToTensor() +{ +} + +GafferImage::ImagePlug *ImageToTensor::imagePlug() +{ + return getChild( g_firstPlugIndex ); +} + +const GafferImage::ImagePlug *ImageToTensor::imagePlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::StringPlug *ImageToTensor::viewPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::StringPlug *ImageToTensor::viewPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::StringVectorDataPlug *ImageToTensor::channelsPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::StringVectorDataPlug *ImageToTensor::channelsPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +Gaffer::BoolPlug *ImageToTensor::interleaveChannelsPlug() +{ + return getChild( g_firstPlugIndex + 3 ); +} + +const Gaffer::BoolPlug *ImageToTensor::interleaveChannelsPlug() const +{ + return getChild( g_firstPlugIndex + 3 ); +} + +TensorPlug *ImageToTensor::tensorPlug() +{ + return getChild( g_firstPlugIndex + 4 ); +} + +const TensorPlug *ImageToTensor::tensorPlug() const +{ + return getChild( g_firstPlugIndex + 4 ); +} + +void ImageToTensor::affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const +{ + ComputeNode::affects( input, outputs ); + + if( + input == imagePlug()->viewNamesPlug() || + input == imagePlug()->dataWindowPlug() || + input == imagePlug()->channelNamesPlug() || + input == imagePlug()->channelDataPlug() || + input == viewPlug() || + input == channelsPlug() || + input == interleaveChannelsPlug() + ) + { + outputs.push_back( tensorPlug() ); + } +} + +void ImageToTensor::hash( const Gaffer::ValuePlug *output, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + if( output == tensorPlug() ) + { + ComputeNode::hash( output, context, h ); + + ConstStringVectorDataPtr channels = channelsPlug()->getValue(); + interleaveChannelsPlug()->hash( h ); + + ImagePlug::ViewScope viewScope( context ); + const std::string view = viewPlug()->getValue(); + viewScope.setViewNameChecked( &view, imagePlug()->viewNames().get() ); + + ConstStringVectorDataPtr inChannels = imagePlug()->channelNamesPlug()->getValue(); + for( const auto &channelName : channels->readable() ) + { + if( !ImageAlgo::channelExists( inChannels->readable(), channelName ) ) + { + throw IECore::Exception( fmt::format( "Channel \"{}\" does not exist", channelName ) ); + } + } + + const Box2i dataWindow = imagePlug()->dataWindow(); + + ImageAlgo::parallelGatherTiles( + imagePlug(), + channels->readable(), + // Tile + [&] ( const ImagePlug *image, const string &channelName, const Imath::V2i &tileOrigin ) + { + IECore::Canceller::check( context->canceller() ); + return image->channelDataPlug()->hash(); + }, + // Gather + [&] ( const ImagePlug *image, const string &channelName, const Imath::V2i &tileOrigin, const IECore::MurmurHash &tileHash ) + { + h.append( tileHash ); + }, + dataWindow, + ImageAlgo::TopToBottom + ); + + h.append( dataWindow ); + } + else + { + ComputeNode::hash( output, context, h ); + } +} + +void ImageToTensor::compute( Gaffer::ValuePlug *output, const Gaffer::Context *context ) const +{ + if( output == tensorPlug() ) + { + ConstStringVectorDataPtr channelsData = channelsPlug()->getValue(); + const auto &channels = channelsData->readable(); + const bool interleaveChannels = interleaveChannelsPlug()->getValue(); + + ImagePlug::ViewScope viewScope( context ); + const std::string view = viewPlug()->getValue(); + viewScope.setViewNameChecked( &view, imagePlug()->viewNames().get() ); + + ConstStringVectorDataPtr inChannels = imagePlug()->channelNamesPlug()->getValue(); + for( const auto &channelName : channels ) + { + if( !ImageAlgo::channelExists( inChannels->readable(), channelName ) ) + { + throw IECore::Exception( fmt::format( "Channel \"{}\" does not exist", channelName ) ); + } + } + + const Box2i dataWindow = imagePlug()->dataWindow(); + const size_t numPixels = dataWindow.size().x * dataWindow.size().y; + + FloatVectorDataPtr bufferData = new FloatVectorData; + vector &buffer = bufferData->writable(); + buffer.resize( numPixels * channels.size() ); + + boost::container::flat_map channelIndices; + for( size_t i = 0; i < channels.size(); ++i ) + { + channelIndices[channels[i]] = i; + } + + ImageAlgo::parallelProcessTiles( + imagePlug(), + channels, + [&] ( const ImagePlug *image, const string &channelName, const Imath::V2i &tileOrigin ) + { + IECore::Canceller::check( context->canceller() ); + + ConstFloatVectorDataPtr channelData = image->channelDataPlug()->getValue(); + const Box2i tileBound( tileOrigin, tileOrigin + V2i( ImagePlug::tileSize() ) ); + const Box2i validTileBound = BufferAlgo::intersection( tileBound, dataWindow ); + + const size_t channelIndex = channelIndices[channelName]; + float *dstData = buffer.data(); + size_t dstStride; + if( interleaveChannels ) + { + dstData += channelIndex; + dstStride = channels.size(); + } + else + { + dstData += numPixels * channelIndex; + dstStride = 1; + } + + const float *sourceData = channelData->readable().data(); + + for( V2i p = validTileBound.min; p.y < validTileBound.max.y; ++p.y ) + { + size_t dstIndex = BufferAlgo::index( V2i( p.x, dataWindow.max.y - p.y - 1 ), dataWindow ) * dstStride; + size_t srcIndex = BufferAlgo::index( p, tileBound ); + for( int x = validTileBound.min.x; x < validTileBound.max.x; ++x ) + { + dstData[dstIndex] = sourceData[srcIndex++]; + dstIndex += dstStride; + } + } + } + ); + + vector shape; + if( interleaveChannels ) + { + shape = { 1, dataWindow.size().y, dataWindow.size().x, (int64_t)channels.size() }; + } + else + { + shape = { 1, (int64_t)channels.size(), dataWindow.size().y, dataWindow.size().x }; + } + + ConstTensorPtr tensor = new Tensor( bufferData, shape ); + static_cast( output )->setValue( tensor ); + } + else + { + ComputeNode::compute( output, context ); + } +} + +Gaffer::ValuePlug::CachePolicy ImageToTensor::hashCachePolicy( const Gaffer::ValuePlug *output ) const +{ + if( output == tensorPlug() ) + { + return ValuePlug::CachePolicy::TaskCollaboration; + } + return ComputeNode::hashCachePolicy( output ); +} + +Gaffer::ValuePlug::CachePolicy ImageToTensor::computeCachePolicy( const Gaffer::ValuePlug *output ) const +{ + if( output == tensorPlug() ) + { + return ValuePlug::CachePolicy::TaskCollaboration; + } + return ComputeNode::computeCachePolicy( output ); +} + diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index e28e46b419f..8f1418ad918 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -41,6 +41,7 @@ #include "GafferBindings/TypedObjectPlugBinding.h" #include "GafferML/DataToTensor.h" +#include "GafferML/ImageToTensor.h" #include "GafferML/Inference.h" #include "GafferML/Tensor.h" #include "GafferML/TensorPlug.h" @@ -236,4 +237,6 @@ BOOST_PYTHON_MODULE( _GafferML ) .def( "loadModel", &loadModelWrapper ) ; + GafferBindings::DependencyNodeClass(); + } From 5e886b922984cae8aacf90f51e2921eab193c950 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:22:15 +0000 Subject: [PATCH 15/26] GafferML : Add TensorToImage This allows tensors to be converted back to GafferImage images, after they have been processed by the Inference node. --- include/GafferML/TensorToImage.h | 91 +++++++ python/GafferMLTest/TensorToImageTest.py | 130 +++++++++ python/GafferMLTest/__init__.py | 1 + python/GafferMLUI/TensorToImageUI.py | 95 +++++++ python/GafferMLUI/__init__.py | 1 + src/GafferML/TensorToImage.cpp | 329 +++++++++++++++++++++++ src/GafferMLModule/GafferMLModule.cpp | 2 + 7 files changed, 649 insertions(+) create mode 100644 include/GafferML/TensorToImage.h create mode 100644 python/GafferMLTest/TensorToImageTest.py create mode 100644 python/GafferMLUI/TensorToImageUI.py create mode 100644 src/GafferML/TensorToImage.cpp diff --git a/include/GafferML/TensorToImage.h b/include/GafferML/TensorToImage.h new file mode 100644 index 00000000000..de0eafcc35e --- /dev/null +++ b/include/GafferML/TensorToImage.h @@ -0,0 +1,91 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#pragma once + +#include "GafferML/Export.h" +#include "GafferML/TensorPlug.h" + +#include "GafferImage/FlatImageSource.h" + +namespace GafferML +{ + +class GAFFERML_API TensorToImage : public GafferImage::FlatImageSource +{ + + public : + + explicit TensorToImage( const std::string &name=defaultName() ); + ~TensorToImage() override; + + GAFFER_NODE_DECLARE_TYPE( GafferML::TensorToImage, TensorToImageTypeId, GafferImage::FlatImageSource ); + + TensorPlug *tensorPlug(); + const TensorPlug *tensorPlug() const; + + Gaffer::StringVectorDataPlug *channelsPlug(); + const Gaffer::StringVectorDataPlug *channelsPlug() const; + + Gaffer::BoolPlug *interleavedChannelsPlug(); + const Gaffer::BoolPlug *interleavedChannelsPlug() const; + + void affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const override; + + protected : + + void hashMetadata( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + IECore::ConstCompoundDataPtr computeMetadata( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const override; + + void hashFormat( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + GafferImage::Format computeFormat( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const override; + + void hashDataWindow( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + Imath::Box2i computeDataWindow( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const override; + + void hashChannelNames( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + IECore::ConstStringVectorDataPtr computeChannelNames( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const override; + + void hashChannelData( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const override; + IECore::ConstFloatVectorDataPtr computeChannelData( const std::string &channelName, const Imath::V2i &tileOrigin, const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const override; + + static size_t g_firstPlugIndex; + +}; + +IE_CORE_DECLAREPTR( TensorToImage ) + +} // namespace GafferML diff --git a/python/GafferMLTest/TensorToImageTest.py b/python/GafferMLTest/TensorToImageTest.py new file mode 100644 index 00000000000..b7a2b05d7a5 --- /dev/null +++ b/python/GafferMLTest/TensorToImageTest.py @@ -0,0 +1,130 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import unittest + +import imath + +import IECore + +import Gaffer +import GafferTest +import GafferImage +import GafferImageTest +import GafferML + +class TensorToImageTest( GafferImageTest.ImageTestCase ) : + + def testNoInput( self ) : + + node = GafferML.TensorToImage() + with self.assertRaisesRegex( Gaffer.ProcessException, "Empty tensor" ) : + node["out"].dataWindow() + + def testNonMatchingChannels( self ) : + + tensor = GafferML.Tensor( + IECore.Color3fVectorData( [ imath.Color3f( 1, 2, 3 ) ] ), + [ 1, 1, 3 ] + ) + + tensorToImage = GafferML.TensorToImage() + tensorToImage["tensor"].setValue( tensor ) + tensorToImage["interleavedChannels"].setValue( True ) + self.assertEqual( tensorToImage["out"].dataWindow(), imath.Box2i( imath.V2i( 0 ), imath.V2i( 1 ) ) ) + + # Only two channels specified. + + tensorToImage["channels"].setValue( IECore.StringVectorData( [ "R", "G" ] ) ) + self.assertEqual( tensorToImage["out"].channelNames(), IECore.StringVectorData( [ "R", "G" ] ) ) + self.assertEqual( tensorToImage["out"].channelData( "R", imath.V2i( 0 ) )[0], 1 ) + self.assertEqual( tensorToImage["out"].channelData( "G", imath.V2i( 0 ) )[0], 2 ) + + with self.assertRaisesRegex( RuntimeError, 'Invalid channel "B"' ) : + tensorToImage["out"].channelData( "B", imath.V2i( 0 ) ) + + # Duplicate channels specified. We just take the first. + + tensorToImage["channels"].setValue( IECore.StringVectorData( [ "R", "R", "B" ] ) ) + self.assertEqual( tensorToImage["out"].channelNames(), IECore.StringVectorData( [ "R", "B" ] ) ) + self.assertEqual( tensorToImage["out"].channelData( "R", imath.V2i( 0 ) )[0], 1 ) + self.assertEqual( tensorToImage["out"].channelData( "B", imath.V2i( 0 ) )[0], 3 ) + + with self.assertRaisesRegex( RuntimeError, 'Invalid channel "G' ) : + tensorToImage["out"].channelData( "G", imath.V2i( 0 ) ) + + # Too many channels specified. We error if the extra channel is accessed. + + tensorToImage["channels"].setValue( IECore.StringVectorData( [ "R", "G", "B", "A" ] ) ) + self.assertEqual( tensorToImage["out"].channelNames(), IECore.StringVectorData( [ "R", "G", "B", "A" ] ) ) + self.assertEqual( tensorToImage["out"].channelData( "R", imath.V2i( 0 ) )[0], 1 ) + self.assertEqual( tensorToImage["out"].channelData( "G", imath.V2i( 0 ) )[0], 2 ) + self.assertEqual( tensorToImage["out"].channelData( "B", imath.V2i( 0 ) )[0], 3 ) + + with self.assertRaisesRegex( RuntimeError, 'Channel "A" out of range' ) : + tensorToImage["out"].channelData( "A", imath.V2i( 0 ) ) + + # Channels skipped by entering empty strings. + + tensorToImage["channels"].setValue( IECore.StringVectorData( [ "R", "", "B" ] ) ) + self.assertEqual( tensorToImage["out"].channelNames(), IECore.StringVectorData( [ "R", "B" ] ) ) + self.assertEqual( tensorToImage["out"].channelData( "R", imath.V2i( 0 ) )[0], 1 ) + self.assertEqual( tensorToImage["out"].channelData( "B", imath.V2i( 0 ) )[0], 3 ) + + with self.assertRaisesRegex( RuntimeError, 'Invalid channel "G' ) : + tensorToImage["out"].channelData( "G", imath.V2i( 0 ) ) + + def testRoundTripWithImageToTensor( self ) : + + image = GafferImage.Checkerboard() + + imageToTensor = GafferML.ImageToTensor() + imageToTensor["image"].setInput( image["out"] ) + imageToTensor["channels"].setInput( image["out"]["channelNames"]) + + tensorToImage = GafferML.TensorToImage() + tensorToImage["tensor"].setInput( imageToTensor["tensor"] ) + tensorToImage["channels"].setInput( image["out"]["channelNames"]) + + self.assertImagesEqual( tensorToImage["out"], image["out"] ) + + imageToTensor["interleaveChannels"].setValue( True ) + tensorToImage["interleavedChannels"].setValue( True ) + + self.assertImagesEqual( tensorToImage["out"], image["out"] ) + +if __name__ == "__main__": + unittest.main() diff --git a/python/GafferMLTest/__init__.py b/python/GafferMLTest/__init__.py index 6c6c8d2202e..b926385c6b5 100644 --- a/python/GafferMLTest/__init__.py +++ b/python/GafferMLTest/__init__.py @@ -39,6 +39,7 @@ from .DataToTensorTest import DataToTensorTest from .InferenceTest import InferenceTest from .ImageToTensorTest import ImageToTensorTest +from .TensorToImageTest import TensorToImageTest if __name__ == "__main__": import unittest diff --git a/python/GafferMLUI/TensorToImageUI.py b/python/GafferMLUI/TensorToImageUI.py new file mode 100644 index 00000000000..d9e04ef5c77 --- /dev/null +++ b/python/GafferMLUI/TensorToImageUI.py @@ -0,0 +1,95 @@ +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above +# copyright notice, this list of conditions and the following +# disclaimer. +# +# * Redistributions in binary form must reproduce the above +# copyright notice, this list of conditions and the following +# disclaimer in the documentation and/or other materials provided with +# the distribution. +# +# * Neither the name of John Haddon nor the names of +# any other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import Gaffer +import GafferML + +Gaffer.Metadata.registerNode( + + GafferML.TensorToImage, + + plugs = { + + "tensor" : [ + + "description", + """ + The input tensor to be turned into an image. Typically this would be connected + to the output of an Inference node that is doing image processing. + """, + + "plugValueWidget:type", "", + "nodule:type", "GafferUI::StandardNodule", + + ], + + "channels" : [ + + "description", + """ + The names to give to the channels in the output image. These + channels are unpacked from the tensor in the order in which they are + specified. For example, an order of `[ "B", "G", "R" ]` might be + needed for use with models trained on images using OpenCV + conventions. An empty channel name may be used to skip a channel + when unpacking. + """, + + ], + + "interleavedChannels" : [ + + "description", + """ + Indicates that the channels are interleaved in the input tensor, in + which case they will be deinterleaved when converting to the output + image. Whether or not channels are interleaved will depend on the + model from which the tensor is obtained. + """, + + ], + + "out" : [ + + "description", + """ + The output image. + """, + + ], + + } +) diff --git a/python/GafferMLUI/__init__.py b/python/GafferMLUI/__init__.py index 466a103c9e9..446a2d81fac 100644 --- a/python/GafferMLUI/__init__.py +++ b/python/GafferMLUI/__init__.py @@ -37,5 +37,6 @@ from . import DataToTensorUI from . import InferenceUI from . import ImageToTensorUI +from . import TensorToImageUI __import__( "IECore" ).loadConfig( "GAFFER_STARTUP_PATHS", subdirectory = "GafferMLUI" ) diff --git a/src/GafferML/TensorToImage.cpp b/src/GafferML/TensorToImage.cpp new file mode 100644 index 00000000000..3f2ec34716a --- /dev/null +++ b/src/GafferML/TensorToImage.cpp @@ -0,0 +1,329 @@ +////////////////////////////////////////////////////////////////////////// +// +// Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +// +// Redistribution and use in source and binary forms, with or without +// modification, are permitted provided that the following conditions are +// met: +// +// * Redistributions of source code must retain the above +// copyright notice, this list of conditions and the following +// disclaimer. +// +// * Redistributions in binary form must reproduce the above +// copyright notice, this list of conditions and the following +// disclaimer in the documentation and/or other materials provided with +// the distribution. +// +// * Neither the name of John Haddon nor the names of +// any other contributors to this software may be used to endorse or +// promote products derived from this software without specific prior +// written permission. +// +// THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +// IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +// THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +// PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +// CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +// EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +// PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +// PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +// LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +// NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +// SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +// +////////////////////////////////////////////////////////////////////////// + +#include "GafferML/TensorToImage.h" + +#include "GafferImage/BufferAlgo.h" +#include "GafferImage/ImageAlgo.h" +#include "GafferImage/Sampler.h" + +#include "Gaffer/Context.h" + +using namespace std; +using namespace Imath; +using namespace IECore; +using namespace Gaffer; +using namespace GafferImage; +using namespace GafferML; + +////////////////////////////////////////////////////////////////////////// +// Internal utilities +////////////////////////////////////////////////////////////////////////// + +namespace +{ + +struct ImageShape +{ + Box2i dataWindow; + size_t numChannels; +}; + +ImageShape imageShape( const Tensor *tensor, bool interleavedChannels ) +{ + if( !tensor->value() ) + { + throw IECore::Exception( "Empty tensor" ); + } + + const auto shape = tensor->value().GetTensorTypeAndShapeInfo().GetShape(); + if( shape.size() < 3 ) + { + throw IECore::Exception( "Expected tensor with at least 3 dimensions" ); + } + + size_t i = shape.size() - 3; + for( size_t d = 0; d < i; ++d ) + { + if( shape[d] != 1 ) + { + throw IECore::Exception( + fmt::format( + "Expected {} dimensional tensor to have size 1 in dimension {}", + shape.size(), d + ) + ); + } + } + + if( interleavedChannels ) + { + return { + Box2i( V2i( 0 ), V2i( (int)shape[i+1], (int)shape[i] ) ), + (size_t)shape[i+2] + }; + } + else + { + return { + Box2i( V2i( 0 ), V2i( (int)shape[i+2], (int)shape[i+1] ) ), + (size_t)shape[i] + }; + } +} + +} // namespace + +////////////////////////////////////////////////////////////////////////// +// TensorToImage +////////////////////////////////////////////////////////////////////////// + +GAFFER_NODE_DEFINE_TYPE( TensorToImage ); + +size_t TensorToImage::g_firstPlugIndex = 0; + +TensorToImage::TensorToImage( const std::string &name ) + : FlatImageSource( name ) +{ + storeIndexOfNextChild( g_firstPlugIndex ); + addChild( new TensorPlug( "tensor" ) ); + addChild( new StringVectorDataPlug( "channels", Plug::In, new StringVectorData( { "R", "G", "B" } ) ) ); + addChild( new BoolPlug( "interleavedChannels" ) ); +} + +TensorToImage::~TensorToImage() +{ +} + +TensorPlug *TensorToImage::tensorPlug() +{ + return getChild( g_firstPlugIndex ); +} + +const TensorPlug *TensorToImage::tensorPlug() const +{ + return getChild( g_firstPlugIndex ); +} + +Gaffer::StringVectorDataPlug *TensorToImage::channelsPlug() +{ + return getChild( g_firstPlugIndex + 1 ); +} + +const Gaffer::StringVectorDataPlug *TensorToImage::channelsPlug() const +{ + return getChild( g_firstPlugIndex + 1 ); +} + +Gaffer::BoolPlug *TensorToImage::interleavedChannelsPlug() +{ + return getChild( g_firstPlugIndex + 2 ); +} + +const Gaffer::BoolPlug *TensorToImage::interleavedChannelsPlug() const +{ + return getChild( g_firstPlugIndex + 2 ); +} + +void TensorToImage::affects( const Gaffer::Plug *input, AffectedPlugsContainer &outputs ) const +{ + FlatImageSource::affects( input, outputs ); + + if( input == channelsPlug() ) + { + outputs.push_back( outPlug()->channelNamesPlug() ); + } + + if( input == tensorPlug() || input == interleavedChannelsPlug() ) + { + outputs.push_back( outPlug()->dataWindowPlug() ); + } + + if( input == tensorPlug() || input == channelsPlug() || input == interleavedChannelsPlug() ) + { + outputs.push_back( outPlug()->channelDataPlug() ); + } + + if( input == outPlug()->dataWindowPlug() ) + { + outputs.push_back( outPlug()->formatPlug() ); + } +} + +void TensorToImage::hashMetadata( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + h = outPlug()->metadataPlug()->defaultHash(); +} + +IECore::ConstCompoundDataPtr TensorToImage::computeMetadata( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const +{ + return outPlug()->metadataPlug()->defaultValue(); +} + +void TensorToImage::hashFormat( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + FlatImageSource::hashFormat( parent, context, h ); + outPlug()->dataWindowPlug()->hash( h ); +} + +GafferImage::Format TensorToImage::computeFormat( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const +{ + return Format( outPlug()->dataWindowPlug()->getValue() ); +} + +void TensorToImage::hashDataWindow( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + FlatImageSource::hashDataWindow( parent, context, h ); + tensorPlug()->hash( h ); + interleavedChannelsPlug()->hash( h ); +} + +Imath::Box2i TensorToImage::computeDataWindow( const Gaffer::Context *context, const ImagePlug *parent ) const +{ + ConstTensorPtr tensor = tensorPlug()->getValue(); + return imageShape( tensor.get(), interleavedChannelsPlug()->getValue() ).dataWindow; +} + +void TensorToImage::hashChannelNames( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + FlatImageSource::hashChannelNames( parent, context, h ); + channelsPlug()->hash( h ); +} + +IECore::ConstStringVectorDataPtr TensorToImage::computeChannelNames( const Gaffer::Context *context, const GafferImage::ImagePlug *parent ) const +{ + ConstStringVectorDataPtr channels = channelsPlug()->getValue()->copy(); + // `channels` might be in a non-standard order, to facilitate unpacking from + // a shuffled buffer, and could contain duplicates since it's easy to create + // them while shuffling the list in the UI. Sort into a more natural order + // and remove duplicates. + StringVectorDataPtr result = new StringVectorData( ImageAlgo::sortedChannelNames( channels->readable() ) ); + result->writable().erase( + std::unique( + result->writable().begin(), + result->writable().end() + ), + result->writable().end() + ); + // Remove empty channel names since they wouldn't be valid in the output. + result->writable().erase( + std::remove_if( + result->writable().begin(), + result->writable().end(), + [] ( const string &channelName ) { + return channelName.empty(); + } + ), + result->writable().end() + ); + return result; +} + +void TensorToImage::hashChannelData( const GafferImage::ImagePlug *parent, const Gaffer::Context *context, IECore::MurmurHash &h ) const +{ + FlatImageSource::hashChannelData( parent, context, h ); + { + ImagePlug::GlobalScope globalScope( context ); + tensorPlug()->hash( h ); + channelsPlug()->hash( h ); + interleavedChannelsPlug()->hash( h ); + } + + h.append( context->get( ImagePlug::channelNameContextName ) ); + h.append( context->get( ImagePlug::tileOriginContextName ) ); +} + +IECore::ConstFloatVectorDataPtr TensorToImage::computeChannelData( const std::string &channelName, const Imath::V2i &tileOrigin, const Gaffer::Context *context, const ImagePlug *parent ) const +{ + ConstTensorPtr tensorData; + ConstStringVectorDataPtr channelsData; + bool interleavedChannels; + { + ImagePlug::GlobalScope globalScope( context ); + tensorData = tensorPlug()->getValue(); + channelsData = channelsPlug()->getValue(); + interleavedChannels = interleavedChannelsPlug()->getValue(); + } + + const auto channelIt = std::find( channelsData->readable().begin(), channelsData->readable().end(), channelName ); + if( channelIt == channelsData->readable().end() ) + { + throw IECore::Exception( fmt::format( "Invalid channel \"{}\"", channelName ) ); + } + const size_t channelIndex = channelIt - channelsData->readable().begin(); + + const ImageShape imageShape = ::imageShape( tensorData.get(), interleavedChannels ); + if( channelIndex >= imageShape.numChannels ) + { + throw IECore::Exception( fmt::format( "Channel \"{}\" out of range", channelName ) ); + } + + FloatVectorDataPtr outData = new FloatVectorData; + vector &out = outData->writable(); + + const Box2i dataWindow = imageShape.dataWindow; + const Box2i tileBound( tileOrigin, tileOrigin + V2i( ImagePlug::tileSize() ) ); + const Box2i validTileBound = BufferAlgo::intersection( dataWindow, tileBound ); + out.resize( ImagePlug::tileSize() * ImagePlug::tileSize() ); + + const float *sourceData = tensorData->value().GetTensorData(); + size_t sourceStride; + if( interleavedChannels ) + { + sourceData += channelIndex; + sourceStride = imageShape.numChannels; + } + else + { + sourceData += dataWindow.size().x * dataWindow.size().y * channelIndex; + sourceStride = 1; + } + float *dstData = out.data(); + + for( V2i p = validTileBound.min; p.y < validTileBound.max.y ; ++p.y ) + { + size_t srcIndex = BufferAlgo::index( V2i( p.x, dataWindow.max.y - p.y - 1 ), dataWindow ) * sourceStride; + size_t dstIndex = BufferAlgo::index( p, tileBound ); + + for( int x = validTileBound.min.x; x < validTileBound.max.x; ++x ) + { + dstData[dstIndex++] = sourceData[srcIndex]; + srcIndex += sourceStride; + } + } + + return outData; +} diff --git a/src/GafferMLModule/GafferMLModule.cpp b/src/GafferMLModule/GafferMLModule.cpp index 8f1418ad918..dc0e32b16cf 100644 --- a/src/GafferMLModule/GafferMLModule.cpp +++ b/src/GafferMLModule/GafferMLModule.cpp @@ -45,6 +45,7 @@ #include "GafferML/Inference.h" #include "GafferML/Tensor.h" #include "GafferML/TensorPlug.h" +#include "GafferML/TensorToImage.h" #include "IECorePython/RunTimeTypedBinding.h" @@ -238,5 +239,6 @@ BOOST_PYTHON_MODULE( _GafferML ) ; GafferBindings::DependencyNodeClass(); + GafferBindings::DependencyNodeClass(); } From a4cf447b4ae4a92f8837bf63a01d78b7be354991 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:27:59 +0000 Subject: [PATCH 16/26] GUI startup : Add GafferML nodes to NodeMenu And advertise them in Changes.md. --- Changes.md | 11 +++++++++++ startup/gui/menus.py | 12 ++++++++++++ 2 files changed, 23 insertions(+) diff --git a/Changes.md b/Changes.md index adbdeec461c..c057cc17a63 100644 --- a/Changes.md +++ b/Changes.md @@ -1,6 +1,17 @@ 1.5.x.x (relative to 1.5.1.0) ======= +> Caution : The GafferML features introduced in this release are considered experimental, and are not subject to the usual backwards compatibility guarantees that apply to the rest of Gaffer. + +Features +-------- + +- GafferML : Added a new module with the following nodes for running maching learning models via ONNX Runtime : + - DataToTensor : Converts Gaffer data to tensors. + - Inference : Loads ONNX models and performance inference using an array of input tensors. + - ImageToTensor : Converts images to tensors for use with the Inference node. + - TensorToImage : Converts tensors back to images following inference. + API --- diff --git a/startup/gui/menus.py b/startup/gui/menus.py index 3d11f018454..260e976bb2c 100644 --- a/startup/gui/menus.py +++ b/startup/gui/menus.py @@ -555,6 +555,18 @@ def __usdLightCreator( lightType ) : nodeMenu.append( "/Dispatch/Frame Mask", GafferDispatch.FrameMask, searchText = "FrameMask" ) nodeMenu.append( "/Dispatch/Local Dispatcher", GafferDispatch.LocalDispatcher, searchText = "LocalDispatcher" ) +# ML nodes + +if os.environ.get( "ONNX_ROOT" ) and moduleSearchPath.find( "GafferML" ) : + + import GafferML + import GafferMLUI + + nodeMenu.append( "/ML/Data To Tensor", GafferML.DataToTensor, searchText = "DataToTensor" ) + nodeMenu.append( "/ML/Image To Tensor", GafferML.ImageToTensor, searchText = "ImageToTensor" ) + nodeMenu.append( "/ML/Tensor To Image", GafferML.TensorToImage, searchText = "TensorToImage" ) + nodeMenu.append( "/ML/Inference", GafferML.Inference, searchText = "Inference" ) + # Utility nodes nodeMenu.append( "/Utility/Expression", Gaffer.Expression ) From 1e151d04697e8e297da39f8c2c7f8d1553427583 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 15 Nov 2024 13:08:04 +0000 Subject: [PATCH 17/26] CI : Build and test GafferML --- .github/workflows/main.yml | 4 +- .github/workflows/main/installONNX.py | 63 +++++++++++++++++++++++++++ .github/workflows/main/sconsOptions | 1 + 3 files changed, 67 insertions(+), 1 deletion(-) create mode 100755 .github/workflows/main/installONNX.py diff --git a/.github/workflows/main.yml b/.github/workflows/main.yml index 768795a5dec..9c401080c34 100644 --- a/.github/workflows/main.yml +++ b/.github/workflows/main.yml @@ -66,7 +66,7 @@ jobs: publish: true containerImage: testRunner: Invoke-Expression - testArguments: -excludedCategories performance GafferTest GafferVDBTest GafferUSDTest GafferSceneTest GafferDispatchTest GafferOSLTest GafferImageTest GafferUITest GafferImageUITest GafferSceneUITest GafferDispatchUITest GafferOSLUITest GafferUSDUITest GafferVDBUITest GafferDelightUITest GafferTractorTest GafferTractorUITest + testArguments: -excludedCategories performance GafferTest GafferVDBTest GafferUSDTest GafferSceneTest GafferDispatchTest GafferOSLTest GafferImageTest GafferUITest GafferImageUITest GafferSceneUITest GafferDispatchUITest GafferOSLUITest GafferUSDUITest GafferVDBUITest GafferDelightUITest GafferTractorTest GafferTractorUITest GafferMLTest GafferMLUITest sconsCacheMegabytes: 800 jobs: 4 @@ -139,6 +139,8 @@ jobs: echo GAFFER_DEPENDENCIES_HASH=`python .github/workflows/main/installDependencies.py ${{ matrix.dependenciesURL != '' && format( '--archiveURL {0}', matrix.dependenciesURL ) || '' }} --dependenciesDir ${{ env.GAFFER_BUILD_DIR }} --outputFormat "{archiveDigest}"` >> $GITHUB_ENV ./.github/workflows/main/installDelight.py echo DELIGHT=$GITHUB_WORKSPACE/3delight >> $GITHUB_ENV + ./.github/workflows/main/installONNX.py + echo ONNX_ROOT=$GITHUB_WORKSPACE/onnxruntime >> $GITHUB_ENV shell: bash - name: Install Mesa (Windows) diff --git a/.github/workflows/main/installONNX.py b/.github/workflows/main/installONNX.py new file mode 100755 index 00000000000..56ad0df1871 --- /dev/null +++ b/.github/workflows/main/installONNX.py @@ -0,0 +1,63 @@ +#!/usr/bin/env python3 +########################################################################## +# +# Copyright (c) 2024, Cinesite VFX Ltd. All rights reserved. +# +# Redistribution and use in source and binary forms, with or without +# modification, are permitted provided that the following conditions are +# met: +# +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# +# * Neither the name of Image Engine Design nor the names of any +# other contributors to this software may be used to endorse or +# promote products derived from this software without specific prior +# written permission. +# +# THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS +# IS" AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, +# THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +# PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +# CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +# EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +# PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +# PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF +# LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING +# NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS +# SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. +# +########################################################################## + +import pathlib +import sys +import shutil +import subprocess +from urllib.request import urlretrieve + +version = "1.19.2" + +if sys.platform == "linux" : + url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-linux-x64-{version}.tgz" +elif sys.platform == "darwin" : + url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-osx-arm64-{version}.tgz" +elif sys.platform == "win32" : + url = f"https://github.com/microsoft/onnxruntime/releases/download/v{version}/onnxruntime-win-x64-{version}.zip" + +print( "Downloading ONNX \"{}\"".format( url ) ) +archiveFileName, headers = urlretrieve( url ) + +if sys.platform in ( "linux", "darwin" ) : + subprocess.check_call( + [ "tar", "-xf", archiveFileName ] + ) +else : + subprocess.check_call( + [ "7z", "x", archiveFileName, "-o./", "-y" ] + ) + +shutil.move( pathlib.Path( url ).stem, "onnxruntime" ) \ No newline at end of file diff --git a/.github/workflows/main/sconsOptions b/.github/workflows/main/sconsOptions index 61fbf0ff529..04969ac3e19 100644 --- a/.github/workflows/main/sconsOptions +++ b/.github/workflows/main/sconsOptions @@ -45,6 +45,7 @@ BUILD_CACHEDIR = os.environ["GAFFER_CACHE_DIR"] ARNOLD_ROOT = os.environ.get( "ARNOLD_ROOT", "" ) DELIGHT_ROOT = os.environ["DELIGHT"] +ONNX_ROOT = os.environ["ONNX_ROOT"] BUILD_DIR = os.environ["GAFFER_BUILD_DIR"] INSTALL_DIR = os.path.join( "install", os.environ["GAFFER_BUILD_NAME"] ) From 674f3a8b1ab5f2eea707baa9e9d589ccda0d4555 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Tue, 26 Nov 2024 11:38:00 +0000 Subject: [PATCH 18/26] TensorToImage : Check tensor data type --- python/GafferMLTest/TensorToImageTest.py | 14 ++++++++++++++ src/GafferML/TensorToImage.cpp | 7 +++++++ 2 files changed, 21 insertions(+) diff --git a/python/GafferMLTest/TensorToImageTest.py b/python/GafferMLTest/TensorToImageTest.py index b7a2b05d7a5..29a05993133 100644 --- a/python/GafferMLTest/TensorToImageTest.py +++ b/python/GafferMLTest/TensorToImageTest.py @@ -126,5 +126,19 @@ def testRoundTripWithImageToTensor( self ) : self.assertImagesEqual( tensorToImage["out"], image["out"] ) + def testNonFloatTensor( self ) : + + tensor = GafferML.Tensor( + IECore.IntVectorData( [ 1, 2, 3 ] ), + [ 1, 1, 3 ] + ) + + tensorToImage = GafferML.TensorToImage() + tensorToImage["tensor"].setValue( tensor ) + tensorToImage["interleavedChannels"].setValue( True ) + + with self.assertRaisesRegex( RuntimeError, "Unsupported tensor data type" ) : + tensorToImage["out"].channelData( "R", imath.V2i( 0 ) ) + if __name__ == "__main__": unittest.main() diff --git a/src/GafferML/TensorToImage.cpp b/src/GafferML/TensorToImage.cpp index 3f2ec34716a..89821b1398e 100644 --- a/src/GafferML/TensorToImage.cpp +++ b/src/GafferML/TensorToImage.cpp @@ -291,6 +291,13 @@ IECore::ConstFloatVectorDataPtr TensorToImage::computeChannelData( const std::st throw IECore::Exception( fmt::format( "Channel \"{}\" out of range", channelName ) ); } + const ONNXTensorElementDataType elementType = tensorData->value().GetTensorTypeAndShapeInfo().GetElementType(); + if( elementType != ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT ) + { + /// \todo Support other types by converting to float. + throw IECore::Exception( fmt::format( "Unsupported tensor data type \"{}\"", elementType ) ); + } + FloatVectorDataPtr outData = new FloatVectorData; vector &out = outData->writable(); From a69879153e0c39230988a28c8bda52f77474f635 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Wed, 27 Nov 2024 09:27:36 +0000 Subject: [PATCH 19/26] GraphEditor config : Fix error in location drop handler When the drag originates outside Gaffer, `event.sourceWidget` is `None`, in which case we were getting the following error : ``` ERROR : File "/home/john/dev/build/gaffer-1.4/startup/gui/graphEditor.py", line 253, in __dropLocationData ERROR : sourceEditor = event.sourceWidget.ancestor( GafferUI.Editor ) ERROR : AttributeError: 'NoneType' object has no attribute 'ancestor' ``` This had gone unnoticed before, because typically such events were being accepted by `__fileDragEnter()` first and never reached the location drop handler. But when the file is of an unhandled type, the file handler returns False and the location handler comes into play. --- Changes.md | 3 +++ startup/gui/graphEditor.py | 3 ++- 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 838942fd268..367e5d7ddc9 100644 --- a/Changes.md +++ b/Changes.md @@ -1,7 +1,10 @@ 1.4.15.x (relative to 1.4.15.2) ======== +Fixes +----- +- GraphEditor : Fixed errors when dragging an unknown file type into the GraphEditor. 1.4.15.2 (relative to 1.4.15.1) ======== diff --git a/startup/gui/graphEditor.py b/startup/gui/graphEditor.py index c7b870fd65e..8f90b731242 100644 --- a/startup/gui/graphEditor.py +++ b/startup/gui/graphEditor.py @@ -245,7 +245,8 @@ def __dropLocationData( event ) : if ( not isinstance( event.data, IECore.StringVectorData ) or len( event.data ) != 1 or - not event.data[0].startswith( "/" ) + not event.data[0].startswith( "/" ) or + event.sourceWidget is None ) : return None From 5d19325c222087a2135136cb883d31e3b81ce405 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Wed, 27 Nov 2024 09:48:26 +0000 Subject: [PATCH 20/26] Widget : Set `DragDropEvent.sourceWidget` from foreign drags Our assumption was that such drags would always comes from outside of Gaffer - the foreign drag handling was originally added so that we could drag files from the system browser into the GraphEditor. In that case there is no source of information for `sourceWidget`. But some folks prefer to code their Gaffer extensions in pure Qt, with a tiny shim to host them in GafferUI. In this case, we _can_ find a suitable `sourceWidget`, which we now do. --- Changes.md | 1 + python/GafferUI/Widget.py | 6 +++++- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 367e5d7ddc9..92ac4650230 100644 --- a/Changes.md +++ b/Changes.md @@ -5,6 +5,7 @@ Fixes ----- - GraphEditor : Fixed errors when dragging an unknown file type into the GraphEditor. +- Widget : Fixed `event.sourceWidget` for DragDropEvents generated from a Qt native drag within the same Gaffer process. This will now reference the `GafferUI.Widget` that the Qt source widget belongs to, if any. 1.4.15.2 (relative to 1.4.15.1) ======== diff --git a/python/GafferUI/Widget.py b/python/GafferUI/Widget.py index 25cf66e2bae..238ed7ca10f 100644 --- a/python/GafferUI/Widget.py +++ b/python/GafferUI/Widget.py @@ -1578,9 +1578,13 @@ def __foreignDragEnter( self, qObject, qEvent ) : Widget._modifiers( qEvent.keyboardModifiers() ), ) dragDropEvent.data = data - dragDropEvent.sourceWidget = None dragDropEvent.destinationWidget = None + if isinstance( qEvent.source(), QtWidgets.QWidget ) : + dragDropEvent.sourceWidget = GafferUI.Widget._owner( qEvent.source() ) + else : + dragDropEvent.sourceWidget = None + if widget._dragEnterSignal( widget, dragDropEvent ) : qEvent.acceptProposedAction() self.__foreignDragDropEvent = dragDropEvent From ed8273195de13c5aa27196cb03d2b384351d824e Mon Sep 17 00:00:00 2001 From: Eric Mehl Date: Wed, 4 Dec 2024 14:59:08 -0500 Subject: [PATCH 21/26] FPSGadget : Draw on `Front` layer --- src/GafferUI/FPSGadget.cpp | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/GafferUI/FPSGadget.cpp b/src/GafferUI/FPSGadget.cpp index 30339d630b6..3c316152707 100644 --- a/src/GafferUI/FPSGadget.cpp +++ b/src/GafferUI/FPSGadget.cpp @@ -59,7 +59,7 @@ FPSGadget::~FPSGadget() void FPSGadget::renderLayer( Gadget::Layer layer, const Style *style, Gadget::RenderReason reason ) const { - if( layer != Layer::Main ) + if( layer != Layer::Front ) { return; } @@ -109,7 +109,7 @@ void FPSGadget::renderLayer( Gadget::Layer layer, const Style *style, Gadget::Re unsigned FPSGadget::layerMask() const { - return (unsigned)Layer::Main; + return (unsigned)Layer::Front; } From f5ca501954f9aacc1cdefc9ce3fc1c03b0354218 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Thu, 5 Dec 2024 11:08:40 +0000 Subject: [PATCH 22/26] MergeScenes : Reduce code duplication --- src/GafferScene/MergeScenes.cpp | 44 +++++++-------------------------- 1 file changed, 9 insertions(+), 35 deletions(-) diff --git a/src/GafferScene/MergeScenes.cpp b/src/GafferScene/MergeScenes.cpp index f5545af591a..b169fe4030a 100644 --- a/src/GafferScene/MergeScenes.cpp +++ b/src/GafferScene/MergeScenes.cpp @@ -302,41 +302,15 @@ void MergeScenes::compute( Gaffer::ValuePlug *output, const Gaffer::Context *con void MergeScenes::hashActiveInputs( const Gaffer::Context *context, IECore::MurmurHash &h ) const { - const ScenePath &scenePath = context->get( ScenePlug::scenePathContextName ); - - if( scenePath.empty() ) - { - h.append( (uint64_t)connectedInputs().to_ulong() ); - } - else - { - InputMask parentActiveInputs; - { - ScenePath parentPath = scenePath; parentPath.pop_back(); - ScenePlug::PathScope parentScope( context, &parentPath ); - parentActiveInputs = activeInputsPlug()->getValue(); - } - - if( parentActiveInputs.count() == 1 ) - { - h.append( (uint64_t)parentActiveInputs.to_ulong() ); - } - else - { - InputMask activeInputs; - visit( - parentActiveInputs, - [&scenePath, &activeInputs] ( InputType type, size_t index, const ScenePlug *scene ) { - if( scene->exists( scenePath ) ) - { - activeInputs[index] = true; - } - return true; - } - ); - h.append( (uint64_t)activeInputs.to_ulong() ); - } - } + // We anticipate very few unique values for the active inputs, as in most + // cases they are inherited directly down the hierarchy. So we want to use a + // perfect hash to avoid making lots of duplicate cache entries containing + // those repeated values. This means our hash needs to use + // `scene->existsPlug()->getValue()` rather than + // `scene->existsPlug()->hash()`. Which means our hash function is actually + // _identical_ to our compute, so we might as well just call it rather than + // duplicate the code. + h.append( computeActiveInputs( context ) ); } int MergeScenes::computeActiveInputs( const Gaffer::Context *context ) const From dd1e11a09f29f95ed265f0233a00d08d7f595866 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Thu, 5 Dec 2024 11:06:27 +0000 Subject: [PATCH 23/26] MergeScenes : Remove unnecessary temporary contexts We're already in the right context to call `existsPlug()->getValue()` directly. Instead we were calling `exists()` which was constructing a new context identical to the one we already had. --- Changes.md | 5 +++++ src/GafferScene/MergeScenes.cpp | 2 +- 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index c057cc17a63..9791764e226 100644 --- a/Changes.md +++ b/Changes.md @@ -12,6 +12,11 @@ Features - ImageToTensor : Converts images to tensors for use with the Inference node. - TensorToImage : Converts tensors back to images following inference. +Improvements +------------ + +- MergeScenes : Removed unnecessary temporary contexts. + API --- diff --git a/src/GafferScene/MergeScenes.cpp b/src/GafferScene/MergeScenes.cpp index b169fe4030a..ceaae3503f1 100644 --- a/src/GafferScene/MergeScenes.cpp +++ b/src/GafferScene/MergeScenes.cpp @@ -351,7 +351,7 @@ int MergeScenes::computeActiveInputs( const Gaffer::Context *context ) const visit( parentActiveInputs, [&result, &scenePath] ( InputType type, size_t index, const ScenePlug *scene ) { - if( scene->exists( scenePath ) ) + if( scene->existsPlug()->getValue() ) { result[index] = true; } From 673d38488ee86f9a75ff0012ccf08e44ba85cdcb Mon Sep 17 00:00:00 2001 From: John Haddon Date: Thu, 5 Dec 2024 09:42:32 +0000 Subject: [PATCH 24/26] MergeScenes : Fix handling of inputs without a computed source Examples of such an input might be a promoted plug that hasn't yet been connected to anything. In this case we were treating that input as active for every single location declared by the other inputs, and if it was the first input it would take precedence over the other inputs when in Keep mode. This would mean all attributes being lost from the location. I did look into an alternative fix : defaulting `ScenePlug.exists` to false. But that caused problems for the Parent node when omitting the primary input and parenting children to `/`. There just is no good default value for `exists`; it should be true for the root and false for all other locations. --- Changes.md | 5 +++++ python/GafferSceneTest/MergeScenesTest.py | 21 +++++++++++++++++- src/GafferScene/MergeScenes.cpp | 26 +++++++++++++++++++++-- 3 files changed, 49 insertions(+), 3 deletions(-) diff --git a/Changes.md b/Changes.md index 9791764e226..4c123db8dca 100644 --- a/Changes.md +++ b/Changes.md @@ -17,6 +17,11 @@ Improvements - MergeScenes : Removed unnecessary temporary contexts. +Fixes +----- + +- MergeScenes : Fixed bug handling input connections not originating from the output of another node. These could cause locations provided by other inputs to lose all their properties. + API --- diff --git a/python/GafferSceneTest/MergeScenesTest.py b/python/GafferSceneTest/MergeScenesTest.py index b711c140a3d..30d4add15ff 100644 --- a/python/GafferSceneTest/MergeScenesTest.py +++ b/python/GafferSceneTest/MergeScenesTest.py @@ -234,7 +234,7 @@ def testGlobals( self ) : def testSingleInputPassThrough( self ) : - sphere = GafferScene.Sphere() + sphere = GafferSceneTest.TestLight() sphere["transform"]["translate"].setValue( imath.V3f( 1, 2, 3 ) ) sphere["sets"].setValue( "A" ) cube = GafferScene.Cube() @@ -249,6 +249,25 @@ def testSingleInputPassThrough( self ) : self.assertScenesEqual( merge["out"], group["out"] ) self.assertSceneHashesEqual( merge["out"], group["out"] ) + merge["in"][1].setInput( group["out"] ) + merge["in"][0].setInput( None ) + + self.assertScenesEqual( merge["out"], group["out"] ) + self.assertSceneHashesEqual( merge["out"], group["out"] ) + + def testFirstInputEmptyPassThrough( self ) : + + merge = GafferScene.MergeScenes() + + emptyScene = GafferScene.ScenePlug() + merge["in"][0].setInput( emptyScene ) + + light = GafferSceneTest.TestLight() + merge["in"][1].setInput( light["out"] ) + + self.assertScenesEqual( merge["out"], light["out"] ) + self.assertSceneHashesEqual( merge["out"], light["out"], checks = self.allSceneChecks - { "sets" } ) + def testNoInputsPassThrough( self ) : merge = GafferScene.MergeScenes() diff --git a/src/GafferScene/MergeScenes.cpp b/src/GafferScene/MergeScenes.cpp index ceaae3503f1..2d7b2da14a9 100644 --- a/src/GafferScene/MergeScenes.cpp +++ b/src/GafferScene/MergeScenes.cpp @@ -320,8 +320,28 @@ int MergeScenes::computeActiveInputs( const Gaffer::Context *context ) const InputMask result; if( scenePath.empty() ) { - // Root - result = connectedInputs(); + // Root. Every input is active here, but there's a wrinkle : the default + // value for `ScenePlug.exists` is `true`, and this is the value we'll + // get if the input is not from a computed output. This would mean that + // the input would claim to be active for _any_ scene location. We deal + // with this once here at the root rather than repeat the workaround at + // each descendant location; + visit( + connectedInputs(), + [&result, &scenePath] ( InputType type, size_t index, const ScenePlug *scene ) { + if( scene->childNamesPlug()->getValue()->readable().size() ) + { + result[index] = true; + } + return true; + } + ); + if( result.none() ) + { + // Make sure that at least one input is active, so we have + // something to use as a pass-through. + result[0] = true; + } } else { @@ -849,6 +869,8 @@ IECore::ConstInternedStringVectorDataPtr MergeScenes::computeSetNames( const Gaf void MergeScenes::hashSet( const IECore::InternedString &setName, const Gaffer::Context *context, const ScenePlug *parent, IECore::MurmurHash &h ) const { + /// \todo It might be a good idea to implement a pass-through for + /// the cases where the set only exists in one of the inputs. visit( connectedInputs(), [&] ( InputType type, size_t index, const ScenePlug *scene ) { From 300dda0b3ded7d82a9e31f47a3fd0280e8d3ace2 Mon Sep 17 00:00:00 2001 From: John Haddon Date: Fri, 29 Nov 2024 16:58:38 +0000 Subject: [PATCH 25/26] CatalogueUI : Don't "steal" irrelevant drags We were accepting any drag which provided StringVectorData, when we should only have been accepting those which originated in the image listing (because we are using the drag to reorder images). This meant we were accepting drags of paths from the HierarchyView, and then clobbering the custom pointer in `__pathListingDragLeave`. This gave people the impression that the drag was broken, when in fact you could still continue and drop elsewhere (despite the pointer indicating otherwise). Also tweaked the drag move logic so we use exactly the same checks in both enter and move. --- Changes.md | 1 + python/GafferImageUI/CatalogueUI.py | 4 ++-- 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/Changes.md b/Changes.md index 92ac4650230..97b4b27a0e7 100644 --- a/Changes.md +++ b/Changes.md @@ -6,6 +6,7 @@ Fixes - GraphEditor : Fixed errors when dragging an unknown file type into the GraphEditor. - Widget : Fixed `event.sourceWidget` for DragDropEvents generated from a Qt native drag within the same Gaffer process. This will now reference the `GafferUI.Widget` that the Qt source widget belongs to, if any. +- Catalogue : Fixed bug which "stole" drags that crossed the image listing but which were destined elsewhere, for instance a drag from the HierarchyView to a PathFilter in the GraphEditor. 1.4.15.2 (relative to 1.4.15.1) ======== diff --git a/python/GafferImageUI/CatalogueUI.py b/python/GafferImageUI/CatalogueUI.py index 1c0945ecd92..b0e1b0e2b82 100644 --- a/python/GafferImageUI/CatalogueUI.py +++ b/python/GafferImageUI/CatalogueUI.py @@ -1079,7 +1079,7 @@ def __dropImage( self, eventData ) : def __pathListingDragEnter( self, widget, event ) : - if isinstance( event.data, IECore.StringVectorData ) : + if event.sourceWidget is widget and isinstance( event.data, IECore.StringVectorData ) and event.data : # Allow reordering of images self.__moveToPath = None self.__mergeGroupId += 1 @@ -1101,7 +1101,7 @@ def __pathListingDragLeave( self, widget, event ) : def __pathListingDragMove( self, listing, event ) : - if not event.data or not isinstance( event.data, IECore.StringVectorData ) : + if not ( event.sourceWidget is listing and isinstance( event.data, IECore.StringVectorData ) and event.data ) : return targetPath = self.__pathListing.pathAt( event.line.p0 ) From 6d0ae46543bcbbe833b110d76a0702ab1528759b Mon Sep 17 00:00:00 2001 From: John Haddon Date: Tue, 3 Dec 2024 16:06:58 +0000 Subject: [PATCH 26/26] GadgetWidget : Fix signal handling bug This was broken in 9e052380bc2c272067827d6d0144b00cb3d41fbb, which switched from using a scoped to an unscoped connection. We need a scoped connection so that we automatically remove the connection to the previous viewport when making the connection to the new one. --- Changes.md | 1 + python/GafferUI/GadgetWidget.py | 2 +- python/GafferUITest/GadgetWidgetTest.py | 13 +++++++++++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/Changes.md b/Changes.md index 97b4b27a0e7..005da9cfb71 100644 --- a/Changes.md +++ b/Changes.md @@ -7,6 +7,7 @@ Fixes - GraphEditor : Fixed errors when dragging an unknown file type into the GraphEditor. - Widget : Fixed `event.sourceWidget` for DragDropEvents generated from a Qt native drag within the same Gaffer process. This will now reference the `GafferUI.Widget` that the Qt source widget belongs to, if any. - Catalogue : Fixed bug which "stole" drags that crossed the image listing but which were destined elsewhere, for instance a drag from the HierarchyView to a PathFilter in the GraphEditor. +- GadgetWidget : Fixed signal handling bug in `setViewportGadget()`. This could cause the widget to attempt to redraw unnecessarily when the _old_ viewport requested a redraw. 1.4.15.2 (relative to 1.4.15.1) ======== diff --git a/python/GafferUI/GadgetWidget.py b/python/GafferUI/GadgetWidget.py index 49fe3e19cbf..0327ec7a804 100644 --- a/python/GafferUI/GadgetWidget.py +++ b/python/GafferUI/GadgetWidget.py @@ -106,7 +106,7 @@ def setViewportGadget( self, viewportGadget ) : self.__viewportGadget.setVisible( False ) self.__viewportGadget = viewportGadget - self.__viewportGadget.renderRequestSignal().connect( Gaffer.WeakMethod( self.__renderRequest ), scoped = False ) + self.__renderRequestConnection = self.__viewportGadget.renderRequestSignal().connect( Gaffer.WeakMethod( self.__renderRequest ), scoped = True ) size = self.size() if size.x and size.y : self.__viewportGadget.setViewport( size ) diff --git a/python/GafferUITest/GadgetWidgetTest.py b/python/GafferUITest/GadgetWidgetTest.py index aaadcc88556..3f8c8bbfb77 100644 --- a/python/GafferUITest/GadgetWidgetTest.py +++ b/python/GafferUITest/GadgetWidgetTest.py @@ -66,5 +66,18 @@ def testViewportVisibility( self ) : self.assertFalse( vg1.visible() ) self.assertFalse( vg2.visible() ) + def testConnectionLifetime( self ) : + + gadgetWidget = GafferUI.GadgetWidget() + viewportGadget1 = gadgetWidget.getViewportGadget() + self.assertEqual( viewportGadget1.renderRequestSignal().numSlots(), 1 ) + + viewportGadget2 = GafferUI.ViewportGadget() + self.assertEqual( viewportGadget2.renderRequestSignal().numSlots(), 0 ) + + gadgetWidget.setViewportGadget( viewportGadget2 ) + self.assertEqual( viewportGadget1.renderRequestSignal().numSlots(), 0 ) + self.assertEqual( viewportGadget2.renderRequestSignal().numSlots(), 1 ) + if __name__ == "__main__": unittest.main()