GTX 1080 suspected memory leak

[jakethesnake]

Hi,

I've started distributing a lwjgl 3.0 game, with an engine built from scratch and it seems to be working fine on most platforms, except players with a Nvidia GTX 1080 card.

[LWJGL] OpenGL debug message
ID: 0x0
Source: API
Type: ERROR
Severity: HIGH
Message: Unknown internal debug message. The NVIDIA OpenGL driver has encountered
an out of memory error. This application might
behave inconsistently and fail.
(pid=25544 javaw.exe 32bit)
[LWJGL] OpenGL debug message
ID: 0x505
Source: API
Type: ERROR
Severity: HIGH
Message: GL_OUT_OF_MEMORY error generated. Failed to allocate memory for texture.
class snake2d.SoundCore sucessfully destroyed
class snake2d.GraphicContext was sucessfully destroyed
Core was sucessfully disposed

I get those two errors and then I terminate the game since geGetError returns an out of memory.

I upload two 4096^2 textures and a handful of <1mb VBO's. On my GPUz it's about 220 MB. The game is designed to run on low level GPUs. And there's definatelty not a memmory leak on my rig, nor any of the handful AMD/NVIDIA/Intel cards I've tried it out on, but it seems rater limited to 1080 .

And as I don't have access to a 1080 card my hands are kind of tied. So I was hoping one of you knew the problem / have a card and want to help me out.

The game is downloadable online: https://www.indiedb.com/games/songs-of-syx/downloads/songs-of-syx-demo-v1

The code is quite straight forward.

Texture creation:

                width = i.width;
		height = i.height;
		
		if (width > MAX_SIZE || height > MAX_SIZE)
			throw new RuntimeException();
		
		id = glGenTextures();
		
		glActiveTexture(GL_TEXTURE0);
		glBindTexture(GL_TEXTURE_RECTANGLE, id);
		
		//some strange filters. Experiment!
		glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MIN_FILTER, GL_NEAREST);
		glTexParameteri(GL_TEXTURE_RECTANGLE, GL_TEXTURE_MAG_FILTER, pixelated ? GL_NEAREST : GL_LINEAR);
		glTexImage2D(GL_TEXTURE_RECTANGLE, 0, GL_RGBA8, width, height, 0, GL_RGBA, GL_UNSIGNED_BYTE, i.data());
		
		GlHelper.checkErrors();

The checkErrors at the bottom with throw if there are any glerrors. It doesn't.
This is the VBO:

import static org.lwjgl.opengl.GL11.*;
import static org.lwjgl.opengl.GL15.*;
import static org.lwjgl.opengl.GL20.*;
import static org.lwjgl.opengl.GL30.*;

import java.nio.ByteBuffer;

import org.lwjgl.opengl.GL11;
import org.lwjgl.system.MemoryUtil;

import snake2d.util.color.COLOR;
import snake2d.util.color.OPACITY;

class VboParticles {

	private final int vertexArrayID;
	private final int attributeElementID;

	protected final ByteBuffer buffer;

	public final int MAX_ELEMENTS = 64000;
	private final int ELEMENT_SIZE;
	private final int BUFFER_SIZE;

	private final int NR_OF_ATTRIBUTES;

	protected int count = 0;

	private final byte byteZero = 0;
	private final byte byteFull = -1;

	private final int[] vFrom = new int[255];
	private final int[] vTo = new int[255];
	private final int[] size = new int[255];
	private int current = 0;
	private final VboShaderAbs shader;

	static VboParticles getDebug(SETTINGS sett) {
		ShaderDebug shader = new ShaderDebug(sett.getNativeWidth(), sett.getNativeHeight());
		return new VboParticles(shader, sett.getPointSize());
	}

	static VboParticles getForTexture(int width, int height) {
		ShaderTexture shader = new ShaderTexture(width, height);
		return new VboParticles(shader, 1);
	}

	static VboParticles getDeffered(SETTINGS sett) {
		ShaderDeffered shader = new ShaderDeffered(sett.getNativeWidth(), sett.getNativeHeight());
		return new VboParticles(shader, sett.getPointSize());
	}

	public VboParticles(VboShaderAbs shader, int pointSize) {

		VboAttribute[] attributes = new VboAttribute[] { new VboAttribute(2, GL_SHORT, false, 2), // position
				new VboAttribute(4, GL_UNSIGNED_BYTE, true, 1), // normal
				new VboAttribute(4, GL_UNSIGNED_BYTE, true, 1) // color
		};

		NR_OF_ATTRIBUTES = attributes.length;

		int byteStride = 0;
		for (VboAttribute v : attributes) {
			byteStride += v.getSizeInBytes();
		}

		ELEMENT_SIZE = byteStride;
		BUFFER_SIZE = ELEMENT_SIZE * MAX_ELEMENTS;
		buffer = MemoryUtil.memAlloc(BUFFER_SIZE);

		vertexArrayID = glGenVertexArrays();
		glBindVertexArray(vertexArrayID);

		attributeElementID = glGenBuffers();
		glBindBuffer(GL_ARRAY_BUFFER, attributeElementID);

		int index = 0;
		int pointerOffset = 0;
		for (VboAttribute v : attributes) {
			glVertexAttribPointer(index, v.getAmount(), v.getGlType(), v.isNormalized(), byteStride, pointerOffset);
			index++;
			pointerOffset += v.getSizeInBytes();
		}

		glBufferData(GL_ARRAY_BUFFER, buffer, GL_STREAM_DRAW);

		int[] indices = new int[MAX_ELEMENTS];
		for (int i = 0; i < indices.length; i++) {
			indices[i] = i;
		}

		ByteBuffer indicesBuffer = MemoryUtil.memAlloc(indices.length*4);
		indicesBuffer.asIntBuffer().put(indices).flip();

		int vertexFixID = glGenBuffers();
		glBindBuffer(GL_ELEMENT_ARRAY_BUFFER, vertexFixID);
		glBufferData(GL_ELEMENT_ARRAY_BUFFER, indicesBuffer, GL_STATIC_READ);

		glBindVertexArray(0);
		
		MemoryUtil.memFree(indicesBuffer);

		// glEnable(GL_POINT_SIZE);
		glPointSize(pointSize);
		this.shader = shader;

		size[0] = 1;

	}

	private static class ShaderDebug extends VboShaderAbs {

		ShaderDebug(float width, float height) {

			String VERTEX = "#version 330 core" + "\n"
					+ getScreenVec(width, height)
					+ "const vec2 trans = vec2(-1.0,1.0);" + "\n"

					+ "layout(location = 0) in vec2 in_position;" + "\n" + "layout(location = 2) in vec4 in_color;"
					+ "\n"

					+ "out vec4 vColor;" + "\n"

					+ "void main(){" + "\n" + "vColor = vec4(in_color.xyz*2.0, in_color.w);" + "\n"
					+ "gl_Position = vec4((in_position * screen)+trans, 0.0, 1.0);" + "\n" + "}";

			String FRAGMENT = "#version 330 core" + "\n"

					+ "in vec4 vColor;" + "\n"

					+ "out vec4 out_diffuse;" + "\n"

					+ "void main(){" + "\n" + "out_diffuse = vColor;" + "\n"

					+ "}" + "\n";

			super.compile(VERTEX, FRAGMENT);

		}

	}
	
	private static class ShaderTexture extends VboShaderAbs {

		ShaderTexture(float width, float height) {

			String VERTEX = "#version 330 core" + "\n"

					+ getScreenVec(width, height) 
					+ "const vec2 trans = vec2(-1.0,1.0);" + "\n"

					+ "layout(location = 0) in vec2 in_position;" + "\n" + "layout(location = 2) in vec4 in_color;"
					+ "\n"

					+ "out vec4 vColor;" + "\n"

					+ "void main(){" + "\n" + "vColor = in_color;" + "\n"
					+ "gl_Position = vec4((in_position * screen)+trans, 0.0, 1.0);" + "\n" + "}";

			String FRAGMENT = "#version 330 core" + "\n"

					+ "in vec4 vColor;" + "\n"

					+ "out vec4 out_diffuse;" + "\n"

					+ "void main(){" + "\n" + "out_diffuse = vColor;" + "\n"

					+ "}" + "\n";

			super.compile(VERTEX, FRAGMENT);

		}

	}

	private static class ShaderDeffered extends VboShaderAbs {

		ShaderDeffered(float width, float height) {

			String VERTEX = "#version 330 core" + "\n"

					+ getScreenVec(width, height)  
					+ "const vec2 trans = vec2(-1.0,1.0);" + "\n"

					+ "layout(location = 0) in vec2 in_position;" + "\n" + "layout(location = 1) in vec4 in_normal;"
					+ "\n" + "layout(location = 2) in vec4 in_color;" + "\n"

					+ "out vec4 vColor;" + "\n" + "out vec4 vNormal;" + "\n"

					+ "void main(){" + "\n" + "vColor = vec4(in_color.xyz*2.0, in_color.w);" + "\n"
					+ "vNormal = in_normal;" + "\n" + "gl_Position = vec4((in_position * screen)+trans, 0.0, 1.0);"
					+ "\n" + "}";

			String FRAGMENT = "#version 330 core" + "\n"

					+ "in vec4 vColor;" + "\n" + "in vec4 vNormal;" + "\n"

					+ "layout(location = 0) out vec4 out_diffuse;" + "\n" + "layout(location = 1) out vec4 out_normal;"
					+ "\n"

					+ "void main(){" + "\n" + "out_diffuse = vColor;" + "\n" + "out_normal = vNormal;" + "\n"

					+ "}" + "\n";

			super.compile(VERTEX, FRAGMENT);

		}

	}

	void setNew(int pointSize) {
		vTo[current] = count;
		current++;
		vFrom[current] = count;
		size[current] = pointSize;
	}

	void flush(int pointSize) {
		bindAndUpload();
		shader.bind();
		int i = 0;
		vTo[current] = count;
		while (i <= current) {
			if (vFrom[i] != vTo[i]) {
				GlHelper.Stencil.setLEQUALreplaceOnPass(i);
				flush(vFrom[i], vTo[i], size[i]);
			}
			i++;
		}
		clear(pointSize);
		glUseProgram(0); // puts an end to the goddamn nvidia errors
	}

	public void bindAndUpload() {
		if (count == 0) {
			return;
		}

		buffer.flip();

		bind();

		glBufferSubData(GL_ARRAY_BUFFER, 0, buffer);

	}

	public void bind() {
		glBindVertexArray(vertexArrayID);

		glBindBuffer(GL_ARRAY_BUFFER, attributeElementID);

		for (int i = 0; i < NR_OF_ATTRIBUTES; i++) {
			glEnableVertexAttribArray(i);
		}
	}

	public void clear(int pointSize) {
		current = 0;
		buffer.clear();
		count = 0;
		size[current] = pointSize;
	}

	public void render(short x, short y, byte nX, byte nY, byte nZ, byte nA, COLOR color, OPACITY opacity) {

		if (count >= MAX_ELEMENTS) {
			return;
		}

		buffer.putShort(x).putShort(y);
		buffer.put(nX).put(nY).put(nZ).put(nA);
		buffer.put(color.red()).put(color.green()).put(color.blue()).put(opacity.get());

		count++;
	}

	public void render(short x, short y, byte red, byte green, byte blue) {

		if (count >= MAX_ELEMENTS) {
			System.err.println("max particles reached");
			return;
		}

		buffer.putShort(x).putShort(y);
		buffer.put(byteZero).put(byteZero).put(byteZero).put(byteZero);
		buffer.put(red).put(green).put(blue).put(byteFull);

		count++;
	}

	public void flush(int from, int to, int size) {
		glPointSize(size);
		glDrawElements(GL11.GL_POINTS, (to - from), GL11.GL_UNSIGNED_INT, from * 4);
	}

	public void dis() {
		shader.dis();
		glBindVertexArray(0);
		glDeleteVertexArrays(vertexArrayID);

		// Dispose the buffer object
		glBindBuffer(GL_ARRAY_BUFFER, 0);
		glDeleteBuffers(attributeElementID);
		MemoryUtil.memFree(buffer);
	}

}

[jakethesnake]

Could this be it:

glBufferData(GL_ARRAY_BUFFER, buffer, GL_STREAM_DRAW);

The buffer at this point is fresh out of:

buffer = MemoryUtil.memAlloc(BUFFER_SIZE);

i.e. it is empty? The limit is 0? I don't know what the wrapper glBufferData does, but if the buffer is empty at this point, could it be like calling glBufferData(GL_ARRAY_BUFFER, 0, GL_STREAM_DRAW), which translates to allocate a buffer of 0 bytes.
Then, each glBufferSubData(GL_ARRAY_BUFFER, 0, buffer); which is called with a non-empty, flipped buffer will always require more space than 0, which will cause the driver to allocate another memory chunk on NVIDIA 1080?