Assimp and OpenGL Skinning Trouble

[TwisterGE]

Lately I've been trying to create a skeletal animation system for my engine, but with no success...

After following several tutorials on the internet, I managed to get something working, however I get a very strange behaviour...
That is supposed to be a worm:


Here's the Mesh loader code:

Mesh _ret_mesh = null;
    try {
        byte[] _data = IOUtils.toByteArray(stream);
        ByteBuffer data = BufferUtils.createByteBuffer(_data.length);
        data.put(_data);
        data.flip();

        AIScene scene = Assimp.aiImportFileFromMemory(
                data, ...
        );

        ...

        PointerBuffer meshes = scene.mMeshes();

        _ret_mesh = new Mesh();
        AIMesh mesh = AIMesh.create(meshes.get(0));

        for (int f = 0; f < mesh.mNumFaces(); f++) {
            AIFace face = mesh.mFaces().get(f);
            for (int j = 0; j < 3; j++) {
                int index = face.mIndices().get(j);

                AIColor4D color = null;
                AIVector3D norm = null;
                AIVector3D tex = null;
                AIVector3D pos = mesh.mVertices().get(index);

                Color mcolor = new Color(1, 1, 1);
                Vec3 mnorm = new Vec3(0, 0, 1);
                Vec2 mtex = new Vec2(0, 0);

                if (mesh.mNormals() != null) {
                    norm = mesh.mNormals().get(index);
                    mnorm.set(norm.x(), norm.y(), norm.z());
                }
                if (mesh.mTextureCoords(0) != null) {
                    tex = mesh.mTextureCoords(0).get(index);
                    mtex.set(tex.x(), tex.y());
                }
                if (mesh.mColors(0) != null) {
                    color = mesh.mColors(0).get();
                    mcolor.set(color.r(), color.g(), color.b(), color.a());
                }

                Vertex vert = new Vertex();
                vert.position = new Vec3(pos.x(), pos.y(), pos.z());
                vert.normal = mnorm;
                vert.texCoord = mtex;
                vert.color = mcolor;

                _ret_mesh.pushVertex(vert);
                _ret_mesh.pushIndex(index);
            }
        }

        if (mesh.mNumBones() > 0) {
            // Loads bone ids and weights
            loadBones(mesh, _ret_mesh);
            loadSkeleton(scene, _ret_mesh);
        }
        _ret_mesh.finalize();
        _ret_mesh.setSkeleton(skeleton);
    } catch (IOException e) {
        e.printStackTrace();
    }

    return _ret_mesh;
}

private void loadSkeleton(AIScene scene, Mesh mesh) {
    skeleton = new Skeleton();
    skeleton.setInverseGlobalTransform(Mat4.assimp(scene.mRootNode().mTransformation()).invert());
    AINode root = findRootBone(scene.mRootNode());
    buildSkeleton(root, null);
}

private void buildSkeleton(AINode current, Bone parent) {
    String boneName = current.mName().dataString();
    if (bone_names.contains(boneName)) {
        int bone_id = bone_name_map.get(boneName);
        Mat4 bone_trans = bind_pose_map.get(bone_id);
        Bone bone = new Bone(bone_id, bone_trans);
        Transform bone_t = new Transform(Mat4.assimp(current.mTransformation()).transpose());
        bone.setTransform(bone_t);
        if (parent == null) {
            skeleton.root = bone;
        } else {
            parent.addChild(bone);
        }
        PointerBuffer children = current.mChildren();
        for (int i = 0; i < children.remaining(); i++) {
            buildSkeleton(AINode.create(children.get(i)), bone);
        }
    }
}

private AINode findRootBone(AINode node) {
    String boneName = node.mName().dataString();
    if (bone_names.contains(boneName)) {
        return node;
    }
    AINode result = null;
    for (int i = 0; i < node.mNumChildren(); i++) {
        result = findRootBone(new AINode(node.mChildren().getByteBuffer(i, 1288)));
        if (result != null) {
            return result;
        }
    }
    return null;
}

private void loadBones(AIMesh mesh, Mesh retmesh) {
    if (mesh.mNumBones() > 0) {
        int numBones = 0;
        PointerBuffer bones = mesh.mBones();
        for (int k = 0; k < bones.remaining(); k++) {
            int bone_id = 0;
            AIBone bone = AIBone.create(bones.get(k));
            String bone_name = bone.mName().dataString();
            if (!bone_name_map.containsKey(bone_name)) {
                bone_id = numBones;
                numBones++;

                bone_name_map.put(bone_name, bone_id);
                bone_names.add(bone_name);
                bind_pose_map.put(bone_id, Mat4.assimp(bone.mOffsetMatrix()).transpose());
            } else {
                bone_id = bone_name_map.get(bone_name);
                bone_name_map.replace(bone_name, bone_id);
                bind_pose_map.replace(bone_id, Mat4.assimp(bone.mOffsetMatrix()).transpose());
            }

            for (int j = 0; j < bone.mNumWeights(); j++) {
                int vid = bone.mWeights().get(j).mVertexId();
                float weight = bone.mWeights().get(j).mWeight();

                for (int i = 0; i < WEIGHTS_PER_VERTEX; i++) {
                    Vertex vert = retmesh.getVertices().get(vid);
                    if (vert.weights.get(i) == 0.0f) {
                        vert.weights.set(i, weight);
                        vert.ids.set(i, bone_id);
                        break;
                    }
                }
            }
        }
    }
}

This is where I update the skeleton:

private void readHierarchy(Bone bone, Mat4 parentTransform) {
    Mat4 boneTransform = bone.getTransform().getTransformation();
    Mat4 globalTransform = parentTransform.mul(boneTransform);
    bone.worldTransform = globalTransform;
    bone.skinnedTransform = inverseGlobalTransform.mul(globalTransform).mul(bone.bindPose);
    for (Bone c : bone.getChildren()) {
        readHierarchy(c, globalTransform);
    }
}

public void update() {
    ArrayList<Bone> bones = getBones();
    matrixMap.clear();

    int i = 0;
    while (matrixMap.size() < bones.size()) {
        matrixMap.put(bones.get(i).id, Mat4.identity());
        i++;
    }

    readHierarchy(root, Mat4.identity());
    for (Bone bone : bones) {
        matrixMap.replace(bone.id, bone.skinnedTransform);
    }
}

And the vertex shader:

#version 330

const int MAX_BONES = 64;
const int MAX_WEIGHTS = 4;

layout (location = 0) in vec3 v_position;
layout (location = 1) in vec3 v_normal;
layout (location = 2) in vec2 v_texco;
layout (location = 3) in vec4 v_color;
layout (location = 4) in vec4 v_bone_weights;
layout (location = 5) in vec4 v_bone_ids;

out DATA {
    vec3 normal;
    vec4 position;
    vec4 color;
    vec2 uv;
} vs_out;

uniform mat4 modelViewMatrix;
uniform mat4 projectionMatrix;
uniform mat4 normalMatrix;
uniform mat4 modelMatrix;

uniform bool hasBones;
uniform mat4 bones[MAX_BONES];

void main() {
    // Bones and Skinning
    vec4 skinned_vertex = vec4(0.0);
    vec4 skinned_normal = vec4(0.0);

    if (hasBones) {
        for (int i = 0; i < MAX_WEIGHTS; i++) {
            vec4 local_position = bones[int(v_bone_ids[i])] * vec4(v_position, 1.0);
            skinned_vertex += local_position * v_bone_weights[i];

            vec4 world_normal = bones[int(v_bone_ids[i])] * vec4(v_normal, 0.0);
            skinned_normal += world_normal * v_bone_weights[i];
        }
    } else {
        skinned_vertex = vec4(v_position, 1.0);
        skinned_normal = vec4(v_normal, 0.0);
    }
    //

    vec4 _position = modelViewMatrix * skinned_vertex;
    gl_Position = projectionMatrix * _position;

    vs_out.position = modelMatrix * skinned_vertex;
    vs_out.normal = (normalMatrix * skinned_normal).xyz;
    vs_out.color = v_color;
    vs_out.uv = v_texco;
}

I can't find the problem, It's been more than 1 week and my project is stuck because of this...

[Kai]

Your code looks correct and the animation also looks convincing/correct with respect to how a hierarchical skeleton animation would behave, had its bones been transformed individually, like you likely do.
The problem lies in your way of deciding what transformations to apply to each bone. When you model the anatomy of a worm/snake as a hierarchical set of bones where the transformation of a "parent"/upper part directly affects all descendants/children, you have to counter any rotation of a bone by a corresponding rotation of the child bone in the opposite direction so that for all descending bones the transformation will not apply.
All in all, procedurally animating a hierarhical skeleton to make it look like a worm is very complicated and should probably instead be done manually via keyframes in a digitial content creation tool.

[spasi]

This is too much code to review without running it. Any chance you could provide a simplified standalone version that we can run and debug? (including the worm data)

Anyway, what is the problem exactly? Is the animation wrong? Is the geometry/skinning wrong? (looks like the latter)

One thing that looks wrong to me is these two lines:

_ret_mesh.pushVertex(vert);
_ret_mesh.pushIndex(index);

You're pushing a vertex for each face index, but if you're doing indexed rendering, then you should have less vertices than indices. Shouldn't you be first loading the geometry data (vertex pos/normal/texcoord/weight) and then load the indices in a separate pass?

And a few code-style tips. This loop:

for ( int f = 0; f < mesh.mNumFaces(); f++ ) {
	AIFace face = mesh.mFaces().get(f);
	// ...
}

can be replaced with:

AIFace.Buffer faces = mesh.mFaces();
for ( int f = 0; f < faces.remaining(); f++ ) { // faces.remaining() will be equal to mesh.mNumFaces()
	AIFace face = faces.get(f);
	// ...
}

and this:

for ( int i = 0; i < node.mNumChildren(); i++ ) {
	result = findRootBone(new AINode(node.mChildren().getByteBuffer(i, 1288)));
	if ( result != null ) {
		return result;
	}
}

with:

PointerBuffer children = node.mChildren();
for ( int i = 0; i < children.remaining(); i++ ) { // children.remaining() will be equal to mesh.mNumChildren()
	result = findRootBone(AINode.create(children.get(i))); // no need to create a ByteBuffer, or manually specify the struct size
	if ( result != null ) {
		return result;
	}
}

and so on. There's indeed an extra line of code in both cases, but it reduces the API surface you're using and there's less room for error.

[TwisterGE]

Thanks, but the problem is not animating, the problem is as seen in the GIF, the mesh is not reacting to any bone except the root and the worm model is not being fully rendered for some reason...
This is what it should look like:

[spasi]

Have you tried rendering the mesh without skinning? Just to verify that the mesh data have been loaded correctly with Assimp.

[TwisterGE]

Yes, it looks right to me (without skinning)


And the code is much better now. I divided the vertex and index loading process into 2 passes.

Any chance you could provide a simplified standalone version that we can run and debug? (including the worm data)

Well I can send the whole project, it is not too big
(uses Eclipse IDE)
https://drive.google.com/file/d/0B4Q7HeW7vYDILUxpRUdiVE5UdlE/view?usp=sharing

[spasi]

The bug is in how you update the shader uniforms. Specifically, the loop at Renderer:242 is only updating the skeleton's first bone, because Shader::createProgramAndLoadUniforms() is only registering "bones[0]" in the uniforms HashMap. I implemented a hacky fix by replacing Shader:176 with:

if (a.get(0) != 1)
	addUniformArray(name.substring(0, name.indexOf('[')), a.get(0));
else
	addUniform (name);

You should now be able to see the worm animating, but in the opposite direction relative to the skeleton. To fix that problem, I removed ".invert()" from MeshResource:137.

Finally, a performance tip: The loop at Renderer:242 should be replaced with a single call to glUniformMatrix4fv. This function is able to upload multiple matrices to a matrix array uniform. You should refactor your code so that the entire skeleton pose is uploaded with one call, using the location of bones[0].

[TwisterGE]

Oh....!!!
That's it, it solved my problem, thank you very much!