FBO with stencil and AA

[ropelsw]

I need to render a scene offscreen with a resolution (w,h) different from the displayed one and save it as image on disk.
I tried using FBO: my code works, but I don't find a way to modify it adding a stencil buffer and using AA (multisample)... every attempt failed (I tried both adding a separate stencil buffer and using a 24/8 packed depth + stencil...)
Does anybody know how to modify it for stencil and AA ?

working code without stencil and AA:
int framebufferID = EXTFramebufferObject.glGenFramebuffersEXT();// create a new framebuffer
int colorTextureID = GL11.glGenTextures();// and a new texture used as a color buffer
int depthRenderBufferID = EXTFramebufferObject.glGenRenderbuffersEXT();// And finally a new depthbuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, framebufferID);// switch to the new framebuffer
GL11.glBindTexture(GL11.GL_TEXTURE_2D, colorTextureID);// Bind the colorbuffer texture
GL11.glTexParameterf(GL11.GL_TEXTURE_2D, GL11.GL_TEXTURE_MIN_FILTER, GL11.GL_LINEAR);// make it linear GL11.glTexImage2D(GL11.GL_TEXTURE_2D, 0, GL11.GL_RGBA8, width, height, 0,GL11.GL_RGBA, GL11.GL_INT, (ByteBuffer) null);
EXTFramebufferObject.glFramebufferTexture2DEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,GL11.GL_TEXTURE_2D, colorTextureID, 0); // attach it to the framebuffer
// initialize depth renderbuffer
EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, depthRenderBufferID);// bind the depth renderbuffer
EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL14.GL_DEPTH_COMPONENT24, width, height);// get the data space for it
EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_DEPTH_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depthRenderBufferID); // bind it to the renderbuffer
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);
GL11.glBindTexture(GL11.GL_TEXTURE_2D, 0);
EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, framebufferID);      // switch to rendering on our FBO
int framebuffer = EXTFramebufferObject.glCheckFramebufferStatusEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT);
switch (framebuffer) {
....
}
rendermyscene();
Display.update();
GL11.glReadBuffer(EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT);
GL11.glReadPixels(0, 0, width, height, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, buffer);
... save my buffer

[ropelsw]

Ok, I've finally got it to work: FBO + stencil + multisample, blitting to non multisample FBO to be able to get back and save the generated image data.
If anyone was interested, I can send the code or post it somewhere (some suggestions ?)

[kappa]

If the code is small enough (say a single class), then if you like you can post it on the lwjgl wiki and maybe write up a small tutorial for it.

[ropelsw]

I'm not enabled to post anything in the wiki, so here it is the code I'm using to generate an off-screen image using AA and stencil buffer:

boolean ok = false;
// We want FBO and Stencil. The only format really supported anywhere is Depth Buffer + Stencil (24 bit + 8 bit) packed in a single
// 32 bit buffer. Solutions with depth and stencil buffer separated seldom work
if (GLContext.getCapabilities().GL_EXT_framebuffer_object && 
	GLContext.getCapabilities().GL_EXT_packed_depth_stencil
	) {
	System.out.println("Using frame buffer");
	ok = true; // let's be optimist
	
	int maxsamples = 1;
	// enabling AA only if we have multisample capability and are able to blit the results (not readable as is) in a normal (not multisample) buffer
	boolean multi = GLContext.getCapabilities().GL_EXT_framebuffer_multisample && GLContext.getCapabilities().GL_EXT_framebuffer_blit;
	if(multi){
	    // let's use as many samples we can
		maxsamples = GL11.glGetInteger(EXTFramebufferMultisample.GL_MAX_SAMPLES_EXT);
		System.out.println("Using multi" + maxsamples);
	} else {
		System.out.println("No multisamples (AA)");
	}
	
	int fbo_ID = EXTFramebufferObject.glGenFramebuffersEXT(); // we create a new framebuffer
	EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID);// switch to the new framebuffer
	
	// we allocate a depth buffer
	int depth_ID = EXTFramebufferObject.glGenRenderbuffersEXT();
	EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID);	// bind the depth renderbuffer
	if(!multi){
	    // format with packed stencil
		EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, EXTPackedDepthStencil.GL_DEPTH_STENCIL_EXT, width, height);	// get the data space for it
	} else {
	    // same as above, but multisample
		EXTFramebufferMultisample.glRenderbufferStorageMultisampleEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, maxsamples,EXTPackedDepthStencil.GL_DEPTH_STENCIL_EXT, width, height);	// get the data space for it
	}
	// we bind the same buffer twice: once as depth buffer, once as stencil
	EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_DEPTH_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID); // bind it to the renderbuffer
	EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_STENCIL_ATTACHMENT_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, depth_ID); // bind it to the renderbuffer

    // we bind the destination buffer
	int color_ID = EXTFramebufferObject.glGenRenderbuffersEXT();												// and a new texture used as a color buffer
	EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, color_ID);				// bind the depth renderbuffer
	if(!multi){
		EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL11.GL_RGBA, width, height);	// get the data space for it
	} else {
		EXTFramebufferMultisample.glRenderbufferStorageMultisampleEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, maxsamples,GL11.GL_RGBA, width, height);	// get the data space for it
	}
	EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, color_ID); // bind it to the renderbuffer
	EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, 0);				// bind the depth renderbuffer
	EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);

	// FBO render pass
	EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID);		// switch to rendering on our FBO
	
	// we'll check everithing is OK
	int framebuffer = EXTFramebufferObject.glCheckFramebufferStatusEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT);
	switch (framebuffer) {
		case EXTFramebufferObject.GL_FRAMEBUFFER_COMPLETE_EXT:
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_ATTACHMENT_EXT");
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_MISSING_ATTACHMENT_EXT");
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_DIMENSIONS_EXT");
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_DRAW_BUFFER_EXT");
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_FORMATS_EXT");
			break;
		case EXTFramebufferObject.GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER_EXT:
			ok = false;
			System.out.println("FrameBuffer: " + fbo_ID + ", has caused a GL_FRAMEBUFFER_INCOMPLETE_READ_BUFFER_EXT");
			break;
		default:
			ok = false;
			System.out.println("Unexpected reply from glCheckFramebufferStatusEXT: " + framebuffer);
			break;
	}
	int sfbo_ID = 0;
	int scolor_ID = 0;
	if(ok) {
		GL11.glViewport (0, 0, width, height);// set The Current Viewport to the fbo size

		try {
			GL11.glClear(GL11.GL_COLOR_BUFFER_BIT | GL11.GL_DEPTH_BUFFER_BIT | GL11.GL_STENCIL_BUFFER_BIT);
			
			/*
			       ... code where we do our rendering ...
			*/
			
			Display.update();
			
		} catch (Exception e) {
			e.printStackTrace();
		}
		
		// multibuffer isn't directly accessible: we have to blit the results in a single buffer FBO
		if(multi){
		    // we create the single buffer FBO
			sfbo_ID = EXTFramebufferObject.glGenFramebuffersEXT();
			EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, sfbo_ID);
			scolor_ID = EXTFramebufferObject.glGenRenderbuffersEXT();
			EXTFramebufferObject.glBindRenderbufferEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, scolor_ID);
			EXTFramebufferObject.glRenderbufferStorageEXT(EXTFramebufferObject.GL_RENDERBUFFER_EXT, GL11.GL_RGBA, width, height);	// get the data space for it
			EXTFramebufferObject.glFramebufferRenderbufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT,EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT,EXTFramebufferObject.GL_RENDERBUFFER_EXT, scolor_ID); // bind it to the renderbuffer
		    // we set source and destination
			EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferBlit.GL_READ_FRAMEBUFFER_EXT, fbo_ID);
			EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferBlit.GL_DRAW_FRAMEBUFFER_EXT, sfbo_ID);
			// ... and blit
			EXTFramebufferBlit.glBlitFramebufferEXT(0, 0, width, height, 0, 0, width, height, GL11.GL_COLOR_BUFFER_BIT, GL11.GL_NEAREST);
			EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, sfbo_ID);
		} else {
			EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, fbo_ID);
		}
		// whichever operation we did, now the buffer to be read is binded and we'll read it in a ByteBuffer
		GL11.glReadBuffer(EXTFramebufferObject.GL_COLOR_ATTACHMENT0_EXT);
		ByteBuffer buffer = BufferUtils.createByteBuffer(width * height * 3);
		GL11.glReadPixels(0, 0, width, height, GL11.GL_RGB, GL11.GL_UNSIGNED_BYTE, buffer); 
		// we copy the readbuffer in a buffered image
		BufferedImage image = new BufferedImage(width, height, BufferedImage.TYPE_INT_RGB);
		for(int x = 0; x < width; x++) {
		 	for(int y = 0; y < height; y++) {
		 		int i = (x + (width * y)) * 3;
		 		int r = buffer.get(i) & 0xFF;
		 		int g = buffer.get(i + 1) & 0xFF;
		 		int b = buffer.get(i + 2) & 0xFF;
		 		image.setRGB(x, height - (y + 1), (0xFF << 24) | (r << 16) | (g << 8) | b);
		 	}
		}

        // we write out our image, just to see it
		try {
			ImageIO.write(image, "JPG", new File("d:/temp/immagine.jpg"));
		} catch (IOException e) { 
			e.printStackTrace(); 
		}
	    
	}
	// at the end we dispose our buffers...
	EXTFramebufferObject.glBindFramebufferEXT(EXTFramebufferObject.GL_FRAMEBUFFER_EXT, 0);
	EXTFramebufferObject.glDeleteRenderbuffersEXT(depth_ID);
	EXTFramebufferObject.glDeleteRenderbuffersEXT(color_ID);
	EXTFramebufferObject.glDeleteFramebuffersEXT(fbo_ID);
	if(ok && multi){
		EXTFramebufferObject.glDeleteRenderbuffersEXT(scolor_ID);
		EXTFramebufferObject.glDeleteFramebuffersEXT(sfbo_ID);
	}
}