diff --git a/build.gradle b/build.gradle index b98e663c5..854e18700 100644 --- a/build.gradle +++ b/build.gradle @@ -18,12 +18,13 @@ def openrndrUseSnapshot = false apply plugin: 'org.jetbrains.dokka' project.ext { - openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.44-rc.14" + openrndrVersion = openrndrUseSnapshot? "0.4.0-SNAPSHOT" : "0.3.44" kotlinVersion = "1.4.0" spekVersion = "2.0.12" libfreenectVersion = "0.5.7-1.5.4" gsonVersion = "2.8.6" antlrVersion = "4.8-1" + tensorflowVersion = "0.2.0" } switch (org.gradle.internal.os.OperatingSystem.current()) { @@ -61,12 +62,12 @@ allprojects { group 'org.openrndr.extra' - repositories { if (openrndrUseSnapshot) { mavenLocal() } + mavenCentral() jcenter() maven { url = "https://dl.bintray.com/openrndr/openrndr" diff --git a/orx-tensorflow-natives-linux-x64/build.gradle b/orx-tensorflow-natives-linux-x64/build.gradle new file mode 100644 index 000000000..b09e6511e --- /dev/null +++ b/orx-tensorflow-natives-linux-x64/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtimeOnly "org.tensorflow:tensorflow-core-api:$tensorflowVersion:linux-x86_64" +} \ No newline at end of file diff --git a/orx-tensorflow-natives-macos/build.gradle b/orx-tensorflow-natives-macos/build.gradle new file mode 100644 index 000000000..9a6b5dc0d --- /dev/null +++ b/orx-tensorflow-natives-macos/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtimeOnly "org.tensorflow:tensorflow-core-api:$tensorflowVersion:macosx-x86_64" +} \ No newline at end of file diff --git a/orx-tensorflow-natives-windows/build.gradle b/orx-tensorflow-natives-windows/build.gradle new file mode 100644 index 000000000..095544b04 --- /dev/null +++ b/orx-tensorflow-natives-windows/build.gradle @@ -0,0 +1,3 @@ +dependencies { + runtimeOnly "org.tensorflow:tensorflow-core-api:$tensorflowVersion:windows-x86_64" +} \ No newline at end of file diff --git a/orx-tensorflow/build.gradle b/orx-tensorflow/build.gradle new file mode 100644 index 000000000..ac645ed9e --- /dev/null +++ b/orx-tensorflow/build.gradle @@ -0,0 +1,53 @@ +java { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 +} +sourceSets { + demo { + java { + srcDirs = ["src/demo/kotlin"] + compileClasspath += main.getCompileClasspath() + runtimeClasspath += main.getRuntimeClasspath() + } + } + wrapgen { + java { + srcDirs = ["src/wrapgen/kotlin"] + compileClasspath += main.getCompileClasspath() + runtimeClasspath += main.getRuntimeClasspath() + } + } +} + +compileWrapgenKotlin { + sourceCompatibility = JavaVersion.VERSION_1_8 + targetCompatibility = JavaVersion.VERSION_1_8 + + kotlinOptions { + jvmTarget = "1.8" + apiVersion = "1.4" + languageVersion = "1.4" + } +} + + +dependencies { + implementation "com.google.code.gson:gson:$gsonVersion" + demoImplementation("org.openrndr:openrndr-core:$openrndrVersion") + + + demoRuntimeOnly(project(":orx-tensorflow-natives-$openrndrOS")) + demoRuntimeOnly("org.openrndr:openrndr-gl3:$openrndrVersion") + demoRuntimeOnly("org.openrndr:openrndr-gl3-natives-$openrndrOS:$openrndrVersion") + demoRuntimeOnly("org.openrndr:openrndr-extensions:$openrndrVersion") + demoImplementation("org.openrndr:openrndr-ffmpeg:$openrndrVersion") + demoRuntimeOnly("org.openrndr:openrndr-ffmpeg-natives-$openrndrOS:$openrndrVersion") + demoImplementation(project(":orx-fx")) + demoImplementation(sourceSets.getByName("main").output) + compile "org.tensorflow:tensorflow-core-api:$tensorflowVersion" + + // -- wrapgen + wrapgenImplementation 'com.github.javaparser:javaparser-core:3.15.21' +} + + diff --git a/orx-tensorflow/src/main/kotlin/Tensor.kt b/orx-tensorflow/src/main/kotlin/Tensor.kt new file mode 100644 index 000000000..bc55525b2 --- /dev/null +++ b/orx-tensorflow/src/main/kotlin/Tensor.kt @@ -0,0 +1,195 @@ +package org.openrndr.extra.tensorflow + + +import org.openrndr.draw.ColorBuffer +import org.openrndr.draw.ColorFormat +import org.openrndr.draw.ColorType +import org.openrndr.draw.colorBuffer +import org.openrndr.extra.tensorflow.arrays.* +import org.tensorflow.Tensor +import org.tensorflow.ndarray.StdArrays +import org.tensorflow.ndarray.buffer.DataBuffers +import org.tensorflow.types.* +import org.tensorflow.types.family.TType +import java.nio.ByteBuffer +import java.nio.ByteOrder + +fun ColorBuffer.copyTo(tensor: Tensor) { + val buffer = ByteBuffer.allocateDirect(effectiveWidth * effectiveHeight * format.componentCount * 4) + buffer.order(ByteOrder.nativeOrder()) + this.read(buffer, targetType = ColorType.FLOAT32) + buffer.rewind() + val dataBuffer = DataBuffers.of(buffer.asFloatBuffer()) + tensor.data().write(dataBuffer) +} + +@JvmName("copyToTUint8") +fun ColorBuffer.copyTo(tensor: Tensor) { + val buffer = ByteBuffer.allocateDirect(effectiveWidth * effectiveHeight * format.componentCount) + buffer.order(ByteOrder.nativeOrder()) + this.read(buffer, targetType = ColorType.UINT8) + buffer.rewind() + val dataBuffer = DataBuffers.of(buffer) + tensor.data().write(dataBuffer) +} + + +fun Tensor.copyTo(colorBuffer: ColorBuffer) { + val s = shape() + require(s.numDimensions() == 2 || s.numDimensions() == 3) + + val components = when { + s.numDimensions() == 3 -> s.size(2).toInt() + s.numDimensions() == 4 -> s.size(3).toInt() + else -> 1 + } + + val format = when (components) { + 4 -> ColorFormat.RGBa + 3 -> ColorFormat.RGB + 2 -> ColorFormat.RG + 1 -> ColorFormat.R + else -> error("only supports 1, 2, 3, or 4 components") + } + val buffer = ByteBuffer.allocateDirect(this.numBytes().toInt()) + buffer.order(ByteOrder.nativeOrder()) + val dataBuffer = DataBuffers.of(buffer.asFloatBuffer()) + data().read(dataBuffer) + buffer.rewind() + colorBuffer.write(buffer, sourceFormat = format, sourceType = ColorType.FLOAT32) +} + + +fun Tensor.summary() { + println("type: ${this.dataType().name()}") + println("shape: [${this.shape().asArray().joinToString(", ")}]") +} + +fun Tensor.toIntArray(): IntArray { + val elementCount = this.numBytes() / 4 + val tensorData = data() + val targetArray = IntArray(elementCount.toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toLongArray(): LongArray { + val elementCount = this.numBytes() / 8 + val tensorData = data() + val targetArray = LongArray(elementCount.toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toByteArray(): ByteArray { + val elementCount = this.numBytes() / 8 + val tensorData = data() + val targetArray = ByteArray(elementCount.toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + + +fun Tensor.toFloatArray(): FloatArray { + val elementCount = this.numBytes() / 4 + val tensorData = data() + val targetArray = FloatArray(elementCount.toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toFloatArray2D(): FloatArray2D { + val shape = this.shape() + require(shape.numDimensions() == 2) { + "tensor has ${shape.numDimensions()} dimensions, need 2" + } + val tensorData = data() + val targetArray = floatArray2D(shape.size(0).toInt(), shape.size(1).toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toFloatArray3D(): FloatArray3D { + val shape = this.shape() + require(shape.numDimensions() == 3) { + "tensor has ${shape.numDimensions()} dimensions, need 3" + } + val tensorData = data() + val targetArray = floatArray3D(shape.size(0).toInt(), shape.size(1).toInt(), shape.size(2).toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toFloatArray4D(): FloatArray4D { + val shape = this.shape() + require(shape.numDimensions() == 4) { + "tensor has ${shape.numDimensions()} dimensions, need 4" + } + val tensorData = data() + val targetArray = floatArray4D(shape.size(0).toInt(), shape.size(1).toInt(), shape.size(2).toInt(), shape.size(3).toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toDoubleArray(): DoubleArray { + val elementCount = this.numBytes() / 8 + val tensorData = data() + val targetArray = DoubleArray(elementCount.toInt()) + StdArrays.copyFrom(tensorData, targetArray) + return targetArray +} + +fun Tensor.toColorBuffer(target: ColorBuffer? = null): ColorBuffer { + val s = shape() + require(s.numDimensions() == 2 || s.numDimensions() == 3) + + val width = (if (s.numDimensions() == 3) s.size(1) else s.size(0)).toInt() + val height = (if (s.numDimensions() == 3) s.size(2) else s.size(1)).toInt() + val components = if (s.numDimensions() == 3) s.size(0).toInt() else 1 + + val format = when (components) { + 4 -> ColorFormat.RGBa + 3 -> ColorFormat.RGB + 2 -> ColorFormat.RG + 1 -> ColorFormat.R + else -> error("only supports 1, 2, 3, or 4 components") + } + + val targetColorBuffer = target?: colorBuffer(width, height, format = format, type = ColorType.FLOAT32) + val floatArray = toFloatArray() + val bb = ByteBuffer.allocateDirect(width * height * components * 4) + bb.order(ByteOrder.nativeOrder()) + val fb = bb.asFloatBuffer() + fb.put(floatArray) + bb.rewind() + targetColorBuffer.write(bb) + return targetColorBuffer +} + + +@JvmName("toColorBufferTInt8") +fun Tensor.toColorBuffer(target: ColorBuffer? = null): ColorBuffer { + val s = shape() + require(s.numDimensions() == 2 || s.numDimensions() == 3) + + val width = (if (s.numDimensions() == 3) s.size(1) else s.size(0)).toInt() + val height = (if (s.numDimensions() == 3) s.size(2) else s.size(1)).toInt() + val components = if (s.numDimensions() == 3) s.size(0).toInt() else 1 + + val format = when (components) { + 4 -> ColorFormat.RGBa + 3 -> ColorFormat.RGB + 2 -> ColorFormat.RG + 1 -> ColorFormat.R + else -> error("only supports 1, 2, 3, or 4 components") + } + + val byteArray = toByteArray() + val targetColorBuffer = target?: colorBuffer(width, height, format = format, type = ColorType.UINT8) + val bb = ByteBuffer.allocateDirect(width * height * components ) + bb.order(ByteOrder.nativeOrder()) + bb.put(byteArray) + bb.rewind() + targetColorBuffer.write(bb) + return targetColorBuffer +} diff --git a/orx-tensorflow/src/main/kotlin/arrays/Arrays.kt b/orx-tensorflow/src/main/kotlin/arrays/Arrays.kt new file mode 100644 index 000000000..db79826d9 --- /dev/null +++ b/orx-tensorflow/src/main/kotlin/arrays/Arrays.kt @@ -0,0 +1,83 @@ +package org.openrndr.extra.tensorflow.arrays + +typealias FloatArray2D = Array +typealias FloatArray3D = Array> +typealias FloatArray4D = Array>> +typealias FloatArray5D = Array>>> +typealias FloatArray6D = Array>>>> + +typealias IntArray2D = Array +typealias IntArray3D = Array> +typealias IntArray4D = Array>> +typealias IntArray5D = Array>>> +typealias IntArray6D = Array>>>> + +typealias BooleanArray2D = Array +typealias BooleanArray3D = Array> +typealias BooleanArray4D = Array>> +typealias BooleanArray5D = Array>>> +typealias BooleanArray6D = Array>>>> + +typealias LongArray2D = Array +typealias LongArray3D = Array> +typealias LongArray4D = Array>> +typealias LongArray5D = Array>>> +typealias LongArray6D = Array>>>> + +typealias ByteArray2D = Array +typealias ByteArray3D = Array> +typealias ByteArray4D = Array>> +typealias ByteArray5D = Array>>> +typealias ByteArray6D = Array>>>> + +typealias DoubleArray2D = Array +typealias DoubleArray3D = Array> +typealias DoubleArray4D = Array>> +typealias DoubleArray5D = Array>>> +typealias DoubleArray6D = Array>>>> + +fun floatArray2D(y: Int, x: Int): FloatArray2D = Array(y) { FloatArray(x) } +fun floatArray3D(z: Int, y: Int, x: Int): FloatArray3D = Array(z) { Array(y) { FloatArray(x) } } +fun floatArray4D(w: Int, z: Int, y: Int, x: Int): FloatArray4D = Array(w) { Array(z) { Array(y) { FloatArray(x) } } } + +fun doubleArray2D(y: Int, x: Int): DoubleArray2D = Array(y) { DoubleArray(x) } +fun doubleArray3D(z: Int, y: Int, x: Int): DoubleArray3D = Array(z) { Array(y) { DoubleArray(x) } } +fun doubleArray4D(w: Int, z: Int, y: Int, x: Int): DoubleArray4D = Array(w) { Array(z) { Array(y) { DoubleArray(x) } } } + +fun intArray2D(y: Int, x: Int): IntArray2D = Array(y) { IntArray(x) } +fun intArray3D(z: Int, y: Int, x: Int): IntArray3D = Array(z) { Array(y) { IntArray(x) } } +fun intArray4D(w: Int, z: Int, y: Int, x: Int): IntArray4D = Array(w) { Array(z) { Array(y) { IntArray(x) } } } + +fun longArray2D(y: Int, x: Int): LongArray2D = Array(y) { LongArray(x) } +fun longArray3D(z: Int, y: Int, x: Int): LongArray3D = Array(z) { Array(y) { LongArray(x) } } +fun longArray4D(w: Int, z: Int, y: Int, x: Int): LongArray4D = Array(w) { Array(z) { Array(y) { LongArray(x) } } } + +fun byteArray2D(y: Int, x: Int): ByteArray2D = Array(y) { ByteArray(x) } +fun byteArray3D(z: Int, y: Int, x: Int): ByteArray3D = Array(z) { Array(y) { ByteArray(x) } } +fun byteArray4D(w: Int, z: Int, y: Int, x: Int): ByteArray4D = Array(w) { Array(z) { Array(y) { ByteArray(x) } } } + +fun booleanArray2D(y: Int, x: Int): BooleanArray2D = Array(y) { BooleanArray(x) } +fun booleanArray3D(z: Int, y: Int, x: Int): BooleanArray3D = Array(z) { Array(y) { BooleanArray(x) } } +fun booleanArray4D(w: Int, z: Int, y: Int, x: Int): BooleanArray4D = Array(w) { Array(z) { Array(y) { BooleanArray(x) } } } + +operator fun FloatArray2D.get(y: Int, x: Int) = this[y][x] +operator fun FloatArray3D.get(z: Int, y: Int, x: Int) = this[z][y][x] +operator fun FloatArray4D.get(w: Int, z: Int, y: Int, x: Int) = this[w][z][y][x] + +operator fun DoubleArray2D.get(y: Int, x: Int) = this[y][x] +operator fun DoubleArray3D.get(z: Int, y: Int, x: Int) = this[z][y][x] +operator fun DoubleArray4D.get(w: Int, z: Int, y: Int, x: Int) = this[w][z][y][x] + +operator fun IntArray2D.get(y: Int, x: Int) = this[y][x] +operator fun IntArray3D.get(z: Int, y: Int, x: Int) = this[z][y][x] +operator fun IntArray4D.get(w: Int, z: Int, y: Int, x: Int) = this[w][z][y][x] + +operator fun LongArray2D.get(y: Int, x: Int) = this[y][x] +operator fun LongArray3D.get(z: Int, y: Int, x: Int) = this[z][y][x] +operator fun LongArray4D.get(w: Int, z: Int, y: Int, x: Int) = this[w][z][y][x] + +operator fun ByteArray2D.get(y: Int, x: Int) = this[y][x] +operator fun ByteArray3D.get(z: Int, y: Int, x: Int) = this[z][y][x] +operator fun ByteArray4D.get(w: Int, z: Int, y: Int, x: Int) = this[w][z][y][x] + + diff --git a/settings.gradle b/settings.gradle index c75d75914..163b1d77b 100644 --- a/settings.gradle +++ b/settings.gradle @@ -39,6 +39,10 @@ include 'openrndr-demos', 'orx-shapes', 'orx-syphon', 'orx-temporal-blur', + 'orx-tensorflow', + 'orx-tensorflow-natives-linux-x64', + 'orx-tensorflow-natives-windows', + 'orx-tensorflow-natives-macos', 'orx-timer', 'orx-time-operators', 'orx-kinect-common',