play movies inside your lwjgl apps

[pmlopes]

A couple of months ago i tryed to showcase LWJGL with a video playback feature, it happens that that project died but i wrote some code that is of no value for me now, but could be usefull for anyone who wants some video playback on their apps.

The video playback is based on ffmpeg but the API i wrote in java is quite more simple. You can get all the sources (C++ and Java) and a binary release (win32 only)  but it should compile OK under linux and maybe OSX from here: http://www.scratchydreams.com/projects/lwjgl-mm.7z

Feel free to include the code into lwjgl if you like, it is my small donation

To play a movie in a spinning cube all you need to code is:

package org.lwjgl.mm;

import org.lwjgl.opengl.Display;
import org.lwjgl.opengl.DisplayMode;
import org.lwjgl.opengl.GL11;
import org.lwjgl.opengl.glu.GLU;
import org.lwjgl.input.Keyboard;
import org.lwjgl.Sys;

public class AVDecoderTest {

    private boolean done = false;
    private final String windowTitle = "MM test";
    private DisplayMode displayMode;
    private AVDecoder mm = new AVDecoder("C:\\msys\\devel\\lwjgl-mm\\test\\test-avi.avi");
    private long pts;


    private float xrot;            // X Rotation ( NEW )
    private float yrot;            // Y Rotation ( NEW )
    private float zrot;            // Z Rotation ( NEW )

    private int video;           // Storage For One Texture ( NEW )

    public static void main(String args[]) {
        (new AVDecoderTest()).run();
    }
    public void run() {
        try {
            init();
            while (!done) {
                mainloop();
                render();
                Display.update();
            }
            cleanup();
        }
        catch (Exception e) {
            e.printStackTrace();
            System.exit(0);
        }
    }
    private void mainloop() {
        if(Keyboard.isKeyDown(Keyboard.KEY_ESCAPE)) {       // Exit if Escape is pressed
            done = true;
        }
        if(Display.isCloseRequested()) {                     // Exit if window is closed
            done = true;
        }
    }

    private boolean render() {

        // get the next frame
        if(pts <= Sys.getTime()) {
            if(!AVDecoder.isError(mm.nextFrame())) {
                pts = Sys.getTime() + mm.update();
            }
        }

        GL11.glClear(GL11.GL_COLOR_BUFFER_BIT | GL11.GL_DEPTH_BUFFER_BIT); // Clear The Screen And The Depth Buffer

        GL11.glLoadIdentity(); // Reset The Current Modelview Matrix

        GL11.glTranslatef(0.0f, 0.0f, -5.0f); // Move Into The Screen 5 Units
        GL11.glRotatef(xrot, 1.0f, 0.0f, 0.0f); // Rotate On The X Axis
        GL11.glRotatef(yrot, 0.0f, 1.0f, 0.0f); // Rotate On The Y Axis
        GL11.glRotatef(zrot, 0.0f, 0.0f, 1.0f); // Rotate On The Z Axis
        GL11.glBindTexture(GL11.GL_TEXTURE_2D, video); // Select Our Video frame texture
        GL11.glBegin(GL11.GL_QUADS);
        // Front Face
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(-1.0f, -1.0f, 1.0f); // Bottom Left Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(1.0f, -1.0f, 1.0f); // Bottom Right Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(1.0f, 1.0f, 1.0f); // Top Right Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(-1.0f, 1.0f, 1.0f); // Top Left Of The Texture and Quad
        // Back Face
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(-1.0f, -1.0f, -1.0f); // Bottom Right Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(-1.0f, 1.0f, -1.0f); // Top Right Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(1.0f, 1.0f, -1.0f); // Top Left Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(1.0f, -1.0f, -1.0f); // Bottom Left Of The Texture and Quad
        // Top Face
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(-1.0f, 1.0f, -1.0f); // Top Left Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(-1.0f, 1.0f, 1.0f); // Bottom Left Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(1.0f, 1.0f, 1.0f); // Bottom Right Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(1.0f, 1.0f, -1.0f); // Top Right Of The Texture and Quad
        // Bottom Face
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(-1.0f, -1.0f, -1.0f); // Top Right Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(1.0f, -1.0f, -1.0f); // Top Left Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(1.0f, -1.0f, 1.0f); // Bottom Left Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(-1.0f, -1.0f, 1.0f); // Bottom Right Of The Texture and Quad
        // Right face
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(1.0f, -1.0f, -1.0f); // Bottom Right Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(1.0f, 1.0f, -1.0f); // Top Right Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(1.0f, 1.0f, 1.0f); // Top Left Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(1.0f, -1.0f, 1.0f); // Bottom Left Of The Texture and Quad
        // Left Face
        GL11.glTexCoord2f(0.0f, 0.0f);
        GL11.glVertex3f(-1.0f, -1.0f, -1.0f); // Bottom Left Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 0.0f);
        GL11.glVertex3f(-1.0f, -1.0f, 1.0f); // Bottom Right Of The Texture and Quad
        GL11.glTexCoord2f(1.0f, 1.0f);
        GL11.glVertex3f(-1.0f, 1.0f, 1.0f); // Top Right Of The Texture and Quad
        GL11.glTexCoord2f(0.0f, 1.0f);
        GL11.glVertex3f(-1.0f, 1.0f, -1.0f); // Top Left Of The Texture and Quad
        GL11.glEnd();

//        xrot += 0.3f; // X Axis Rotation
//        yrot += 0.2f; // Y Axis Rotation
//        zrot += 0.4f; // Z Axis Rotation

        return true;
    }
    private void createWindow() throws Exception {

        Display.setFullscreen(false);
        DisplayMode d[] = Display.getAvailableDisplayModes();
        for (DisplayMode aD : d) {
            if (aD.getWidth() == 640
                    && aD.getHeight() == 480
                    && aD.getBitsPerPixel() == 32) {
                displayMode = aD;
                break;
            }
        }
        Display.setDisplayMode(displayMode);
        Display.setTitle(windowTitle);
        Display.create();
    }

    private void init() throws Exception {
        createWindow();
        initGL();

        if(!AVDecoder.isError(mm.open()))
            video = mm.createGLVideoTexture();
    }

    private void initGL() {
        GL11.glEnable(GL11.GL_TEXTURE_2D); // Enable Texture Mapping
        GL11.glShadeModel(GL11.GL_SMOOTH); // Enable Smooth Shading
        GL11.glClearColor(0.0f, 0.0f, 0.0f, 0.0f); // Black Background
        GL11.glClearDepth(1.0f); // Depth Buffer Setup
        GL11.glEnable(GL11.GL_DEPTH_TEST); // Enables Depth Testing
        GL11.glDepthFunc(GL11.GL_LEQUAL); // The Type Of Depth Testing To Do

        GL11.glMatrixMode(GL11.GL_PROJECTION); // Select The Projection Matrix
        GL11.glLoadIdentity(); // Reset The Projection Matrix

        // Calculate The Aspect Ratio Of The Window
        GLU.gluPerspective(45.0f, (float) displayMode.getWidth() / (float) displayMode.getHeight(), 0.1f, 100.0f);
        GL11.glMatrixMode(GL11.GL_MODELVIEW); // Select The Modelview Matrix

        // Really Nice Perspective Calculations
        GL11.glHint(GL11.GL_PERSPECTIVE_CORRECTION_HINT, GL11.GL_NICEST);
    }

    private void cleanup() {
        Display.destroy();
        mm.close();
    }
}

To summarise:

You open a stream:

    private AVDecoder mm = new AVDecoder("C:\\msys\\devel\\lwjgl-mm\\test\\test-avi.avi");

and then you render:

        // get the next frame
        if(pts <= Sys.getTime()) {
            if(!AVDecoder.isError(mm.nextFrame())) {
                pts = Sys.getTime() + mm.update();
            }
        }

[pandur]

thanks for sharing, will definetly check this out.
i wrote something like this using quicktime for java.
do you scale the video to common texture sizes ?
is the performance good enough for scratching videos etc ?

[pmlopes]

Yes the lib scales the texture to a power of 2 so it can be rendered on a surface. I get the audio stream on the C++ side but i haven't implemented the plumbing to openal. It all run quite fast on my laptop winmce2005 + nvidia 7400 go + core-duo 1.8ghz.

It still needs some work, but it's a start!

[wolf_m]

What's the compile cycle and which libs are required, especially on Linux? It doesn't compile out-of-the-box.

[pmlopes]

on linux, windows or mac you need to get a static or dynamic build of ffmeg. ffmpeg compiles using the standard autotools in a GNU way:

./configure --enable-static --disable-shared
make
make install

and my code was supposed to compile with scons the script runs on windows, linux and mac... however since it is so simple you can just call GCC by hand.

generate the java jni header with
javah -h org/lwjgl/mm/AVDecoder.class

and then compile the sources with G++

g++ -shared -o yourdll -D__STDC_CONSTANT_MACROS PacketBuffer.cpp Decoder.cpp MM.cpp

the define is because on gcc 3.4.5 this is how you enable C99 extensions (i think) and of course don't forget to add the path for the include and lib files.

But the best is to either use scons, just call

$ scons

or read the sconstruct script under source.

[wolf_m]

generate the java jni header with
javah -h org/lwjgl/mm/AVDecoder.class

javah -h org/lwjgl/mm/AVDecoder.class 
Usage: javah [options] <classes>

where [options] include:

        -help                 Print this help message and exit
        -classpath <path>     Path from which to load classes
        -bootclasspath <path> Path from which to load bootstrap classes
        -d <dir>              Output directory
        -o <file>             Output file (only one of -d or -o may be used)
        -jni                  Generate JNI-style header file (default)
        -version              Print version information
        -verbose              Enable verbose output
        -force                Always write output files

<classes> are specified with their fully qualified names (for
instance, java.lang.Object).

But the best is to either use scons, just call

$ scons

scons
scons: Reading SConscript files ...
scons: done reading SConscript files.
scons: Building targets ...
g++ -o src/PacketBuffer.os -c -g -fPIC -I/usr/local/include/ffmpeg -I/usr/local/include/ffmpeg/include/ffmpeg -I/usr/local/include/ffmpeg/libavcodec -I/usr/local/include/ffmpeg/libavformat -I/usr/local/include/ffmpeg/libswscale -I/usr/local/include/ffmpeg/libavutil -I/msys/local/jdk/include -I/msys/local/jdk/include/win32 -I/msys/local/jdk/include/linux -Isrc/java src/PacketBuffer.cpp
/usr/local/include/ffmpeg/avcodec.h:2288: warning: 'ImgReSampleContext' is deprecated (declared at /usr/local/include/ffmpeg/avcodec.h:2282)
/usr/local/include/ffmpeg/avcodec.h:2298: warning: 'ImgReSampleContext' is deprecated (declared at /usr/local/include/ffmpeg/avcodec.h:2282)
src/PacketBuffer.cpp: In member function 'void PacketBuffer::dupPacket(AVPacket*, int)':
src/PacketBuffer.cpp:15: error: 'memcpy' was not declared in this scope
src/PacketBuffer.cpp: In member function 'bool PacketBuffer::prepend(AVPacket*)':
src/PacketBuffer.cpp:110: error: 'memcpy' was not declared in this scope
scons: *** [src/PacketBuffer.os] Error 1
scons: building terminated because of errors.

Am I missing some headers? Are your imports broken?

I'm not familiar with C++, by the way (just syntax and basic usage). Neither with JNI.