diff --git a/packages/flame_3d/assets/shaders/spatial_material.shaderbundle b/packages/flame_3d/assets/shaders/spatial_material.shaderbundle index 016d3a9ed33..841e7a8b93b 100644 Binary files a/packages/flame_3d/assets/shaders/spatial_material.shaderbundle and b/packages/flame_3d/assets/shaders/spatial_material.shaderbundle differ diff --git a/packages/flame_3d/lib/src/graphics/graphics_device.dart b/packages/flame_3d/lib/src/graphics/graphics_device.dart index a5e5117a5ee..83a24cb8f7e 100644 --- a/packages/flame_3d/lib/src/graphics/graphics_device.dart +++ b/packages/flame_3d/lib/src/graphics/graphics_device.dart @@ -3,6 +3,7 @@ import 'dart:ui'; import 'package:flame_3d/game.dart'; import 'package:flame_3d/resources.dart'; +import 'package:flame_3d/src/graphics/joints_info.dart'; import 'package:flutter_gpu/gpu.dart' as gpu; enum BlendState { @@ -48,6 +49,10 @@ class GraphicsDevice { Size _previousSize = Size.zero; + /// Must be set by the rendering pipeline before elements are bound. + /// Can be accessed by elements in their bind method. + final JointsInfo jointsInfo = JointsInfo(); + /// Must be set by the rendering pipeline before elements are bound. /// Can be accessed by elements in their bind method. final LightingInfo lightingInfo = LightingInfo(); diff --git a/packages/flame_3d/lib/src/graphics/joints_info.dart b/packages/flame_3d/lib/src/graphics/joints_info.dart new file mode 100644 index 00000000000..c21204b211c --- /dev/null +++ b/packages/flame_3d/lib/src/graphics/joints_info.dart @@ -0,0 +1,13 @@ +import 'package:flame_3d/core.dart'; + +class JointsInfo { + /// Joints per surface idx + Map> jointTransformsPerSurface = {}; + + /// Joints for the current surface + List jointTransforms = []; + + void setSurface(int surfaceIdx) { + jointTransforms = jointTransformsPerSurface[surfaceIdx] ?? []; + } +} diff --git a/packages/flame_3d/lib/src/resources/material/spatial_material.dart b/packages/flame_3d/lib/src/resources/material/spatial_material.dart index dd12426b6bb..4b9ed6ef28c 100644 --- a/packages/flame_3d/lib/src/resources/material/spatial_material.dart +++ b/packages/flame_3d/lib/src/resources/material/spatial_material.dart @@ -16,7 +16,15 @@ class SpatialMaterial extends Material { vertexShader: Shader( _library['TextureVertex']!, slots: [ - UniformSlot.value('VertexInfo', {'model', 'view', 'projection'}), + UniformSlot.value('VertexInfo', { + 'model', + 'view', + 'projection', + }), + UniformSlot.value( + 'JointMatrices', + List.generate(_maxJoints, (idx) => 'joint$idx').toSet(), + ), ], ), fragmentShader: Shader( @@ -56,6 +64,7 @@ class SpatialMaterial extends Material { @override void bind(GraphicsDevice device) { _bindVertexInfo(device); + _bindJointMatrices(device); _bindMaterial(device); _bindCamera(device); } @@ -67,6 +76,19 @@ class SpatialMaterial extends Material { ..setMatrix4('VertexInfo.projection', device.projection); } + void _bindJointMatrices(GraphicsDevice device) { + final jointTransforms = device.jointsInfo.jointTransforms; + if (jointTransforms.length > _maxJoints) { + throw Exception( + 'At most $_maxJoints joints per surface are supported;' + ' found ${jointTransforms.length}', + ); + } + for (final (idx, transform) in jointTransforms.indexed) { + vertexShader.setMatrix4('JointMatrices.joint$idx', transform); + } + } + void _bindMaterial(GraphicsDevice device) { _applyLights(device); fragmentShader @@ -89,4 +111,6 @@ class SpatialMaterial extends Material { static final _library = gpu.ShaderLibrary.fromAsset( 'packages/flame_3d/assets/shaders/spatial_material.shaderbundle', )!; + + static const _maxJoints = 16; } diff --git a/packages/flame_3d/lib/src/resources/mesh/mesh.dart b/packages/flame_3d/lib/src/resources/mesh/mesh.dart index 765b03f5acb..c2fa8b42b13 100644 --- a/packages/flame_3d/lib/src/resources/mesh/mesh.dart +++ b/packages/flame_3d/lib/src/resources/mesh/mesh.dart @@ -26,7 +26,8 @@ class Mesh extends Resource { int get vertexCount => _surfaces.fold(0, (p, e) => p + e.vertexCount); void bind(GraphicsDevice device) { - for (final surface in _surfaces) { + for (final (idx, surface) in _surfaces.indexed) { + device.jointsInfo.setSurface(idx); device.bindSurface(surface); } } diff --git a/packages/flame_3d/lib/src/resources/mesh/surface.dart b/packages/flame_3d/lib/src/resources/mesh/surface.dart index 3b4594300d9..b561768b47a 100644 --- a/packages/flame_3d/lib/src/resources/mesh/surface.dart +++ b/packages/flame_3d/lib/src/resources/mesh/surface.dart @@ -18,6 +18,7 @@ class Surface extends Resource { required List vertices, required List indices, this.material, + this.jointMap, /** * If `true`, the normals will be calculated if they are not provided. */ @@ -34,15 +35,16 @@ class Surface extends Resource { _vertices = Float32List.fromList( normalizedVertices.fold([], (p, v) => p..addAll(v.storage)), ).buffer; - _vertexCount = _vertices.lengthInBytes ~/ (normalizedVertices.length * 9); + _vertexCount = normalizedVertices.length; _indices = Uint16List.fromList(indices).buffer; - _indexCount = _indices.lengthInBytes ~/ 2; + _indexCount = indices.length; _calculateAabb(normalizedVertices); } Material? material; + Map? jointMap; Aabb3 get aabb => _aabb; late Aabb3 _aabb; diff --git a/packages/flame_3d/lib/src/resources/mesh/vertex.dart b/packages/flame_3d/lib/src/resources/mesh/vertex.dart index c1fd8fac805..a11e8801fda 100644 --- a/packages/flame_3d/lib/src/resources/mesh/vertex.dart +++ b/packages/flame_3d/lib/src/resources/mesh/vertex.dart @@ -18,14 +18,20 @@ class Vertex { required Vector2 texCoord, this.color = const Color(0xFFFFFFFF), Vector3? normal, + Vector4? joints, + Vector4? weights, }) : position = position.immutable, texCoord = texCoord.immutable, normal = normal?.immutable, + joints = joints?.immutable, + weights = weights?.immutable, _storage = Float32List.fromList([ ...position.storage, // 1, 2, 3 ...texCoord.storage, // 4, 5 ...color.storage, // 6, 7, 8, 9 ...(normal ?? Vector3.zero()).storage, // 10, 11, 12 + ...(joints ?? Vector4.zero()).storage, // 13, 14, 15, 16 + ...(weights ?? Vector4.zero()).storage, // 17, 18, 19, 20 ]); Float32List get storage => _storage; @@ -40,6 +46,12 @@ class Vertex { /// The normal vector of the vertex. final ImmutableVector3? normal; + /// The joints of the vertex. + final ImmutableVector4? joints; + + /// The weights of the vertex. + final ImmutableVector4? weights; + /// The color on the vertex. final Color color; @@ -49,16 +61,27 @@ class Vertex { position == other.position && texCoord == other.texCoord && normal == other.normal && - color == other.color; + color == other.color && + joints == other.joints && + weights == other.weights; @override - int get hashCode => Object.hashAll([position, texCoord, normal, color]); + int get hashCode => Object.hashAll([ + position, + texCoord, + normal, + color, + joints, + weights, + ]); Vertex copyWith({ Vector3? position, Vector2? texCoord, Vector3? normal, Color? color, + Vector4? joints, + Vector4? weights, }) { // TODO(wolfenrain): optimize this. return Vertex( @@ -66,6 +89,8 @@ class Vertex { texCoord: texCoord ?? this.texCoord.mutable, normal: normal ?? this.normal?.mutable, color: color ?? this.color, + joints: joints ?? this.joints?.mutable, + weights: weights ?? this.weights?.mutable, ); } diff --git a/packages/flame_3d/lib/src/resources/shader/uniform_value.dart b/packages/flame_3d/lib/src/resources/shader/uniform_value.dart index a138dfdf7f9..6b27f8d004c 100644 --- a/packages/flame_3d/lib/src/resources/shader/uniform_value.dart +++ b/packages/flame_3d/lib/src/resources/shader/uniform_value.dart @@ -3,6 +3,7 @@ import 'dart:typed_data'; import 'package:flame_3d/graphics.dart'; import 'package:flame_3d/resources.dart'; +import 'package:ordered_set/comparing.dart'; /// {@template uniform_value} /// Instance of a uniform value. Represented by a [ByteBuffer]. @@ -21,7 +22,9 @@ class UniformValue extends UniformInstance { if (super.resource == null) { var previousIndex = -1; - final data = _storage.entries.fold>([], (p, e) { + final entries = _storage.entries.toList() + ..sort(Comparing.on((c) => c.key)); + final data = entries.fold>([], (p, e) { if (previousIndex + 1 != e.key) { final field = slot.fields.indexed.firstWhere((e) => e.$1 == previousIndex + 1); diff --git a/packages/flame_3d/pubspec.yaml b/packages/flame_3d/pubspec.yaml index 425399a95cd..4649e69d328 100644 --- a/packages/flame_3d/pubspec.yaml +++ b/packages/flame_3d/pubspec.yaml @@ -18,6 +18,7 @@ dependencies: flutter_gpu: sdk: flutter meta: ^1.12.0 + ordered_set: ^6.0.1 vector_math: ^2.1.4 dev_dependencies: diff --git a/packages/flame_3d/shaders/spatial_material.vert b/packages/flame_3d/shaders/spatial_material.vert index 2aecb6f08e9..f7dfc0b4966 100644 --- a/packages/flame_3d/shaders/spatial_material.vert +++ b/packages/flame_3d/shaders/spatial_material.vert @@ -4,6 +4,8 @@ in vec3 vertexPosition; in vec2 vertexTexCoord; in vec4 vertexColor; in vec3 vertexNormal; +in vec4 vertexJoints; +in vec4 vertexWeights; out vec2 fragTexCoord; out vec4 fragColor; @@ -16,18 +18,91 @@ uniform VertexInfo { mat4 projection; } vertex_info; +uniform JointMatrices { + mat4 joint0; + mat4 joint1; + mat4 joint2; + mat4 joint3; + mat4 joint4; + mat4 joint5; + mat4 joint6; + mat4 joint7; + mat4 joint8; + mat4 joint9; + mat4 joint10; + mat4 joint11; + mat4 joint12; + mat4 joint13; + mat4 joint14; + mat4 joint15; +} joints; + +mat4 jointMat(float jointIndex) { + if (jointIndex == 0.0) { + return joints.joint0; + } else if (jointIndex == 1.0) { + return joints.joint1; + } else if (jointIndex == 2.0) { + return joints.joint2; + } else if (jointIndex == 3.0) { + return joints.joint3; + } else if (jointIndex == 4.0) { + return joints.joint4; + } else if (jointIndex == 5.0) { + return joints.joint5; + } else if (jointIndex == 6.0) { + return joints.joint6; + } else if (jointIndex == 7.0) { + return joints.joint7; + } else if (jointIndex == 8.0) { + return joints.joint8; + } else if (jointIndex == 9.0) { + return joints.joint9; + } else if (jointIndex == 10.0) { + return joints.joint10; + } else if (jointIndex == 11.0) { + return joints.joint11; + } else if (jointIndex == 12.0) { + return joints.joint12; + } else if (jointIndex == 13.0) { + return joints.joint13; + } else if (jointIndex == 14.0) { + return joints.joint14; + } else if (jointIndex == 15.0) { + return joints.joint15; + } else { + return mat4(0.0); + } +} + +mat4 computeSkinMatrix() { + if (vertexWeights.x == 0.0 && vertexWeights.y == 0.0 && vertexWeights.z == 0.0 && vertexWeights.w == 0.0) { + // no weights, skip skinning + return mat4(1.0); + } + + return vertexWeights.x * jointMat(vertexJoints.x) + + vertexWeights.y * jointMat(vertexJoints.y) + + vertexWeights.z * jointMat(vertexJoints.z) + + vertexWeights.w * jointMat(vertexJoints.w); +} + void main() { + mat4 skinMatrix = computeSkinMatrix(); + vec3 position = (skinMatrix * vec4(vertexPosition, 1.0)).xyz; + vec3 normal = normalize((skinMatrix * vec4(vertexNormal, 0.0)).xyz); + // Calculate the modelview projection matrix mat4 modelViewProjection = vertex_info.projection * vertex_info.view * vertex_info.model; // Transform the vertex position - gl_Position = modelViewProjection * vec4(vertexPosition, 1.0); + gl_Position = modelViewProjection * vec4(position, 1.0); // Pass the interpolated values to the fragment shader fragTexCoord = vertexTexCoord; fragColor = vertexColor; // Calculate the world-space position and normal - fragPosition = vec3(vertex_info.model * vec4(vertexPosition, 1.0)); - fragNormal = mat3(transpose(inverse(vertex_info.model))) * vertexNormal; + fragPosition = vec3(vertex_info.model * vec4(position, 1.0)); + fragNormal = mat3(transpose(inverse(vertex_info.model))) * normal; }