Rendering of objects on top of markers is complete.

src/ve/ucv/ciens/ccg/nxtar/graphics/CustomPerspectiveCamera.java

@@ -0,0 +1,48 @@
+/*
+ * Copyright (C) 2014 Miguel Angel Astor Romero
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package ve.ucv.ciens.ccg.nxtar.graphics;
+
+import com.badlogic.gdx.graphics.PerspectiveCamera;
+import com.badlogic.gdx.math.Matrix4;
+import com.badlogic.gdx.math.Vector3;
+
+/**
+ * <p>Extension of the standard LibGDX perspective camera that allows setting an
+ * arbitrary projection matrix when updating.</p>
+ */
+public class CustomPerspectiveCamera extends PerspectiveCamera{
+	private final Vector3 tmp = new Vector3();
+
+	public CustomPerspectiveCamera(float fieldOfView, float viewportWidth, float viewportHeight){
+		this.fieldOfView = fieldOfView;
+		this.viewportWidth = viewportWidth;
+		this.viewportHeight = viewportHeight;
+		update();
+	}
+
+	public void update(Matrix4 customProjection, boolean updateFrustum){
+		projection.set(customProjection);
+		view.setToLookAt(position, tmp.set(position).add(direction), up);
+		combined.set(projection);
+		Matrix4.mul(combined.val, view.val);
+
+		if(updateFrustum){
+			invProjectionView.set(combined);
+			Matrix4.inv(invProjectionView.val);
+			frustum.update(invProjectionView);
+		}
+	}
+}

src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java

@@ -15,6 +15,8 @@
  */
 package ve.ucv.ciens.ccg.nxtar.interfaces;
 
+import ve.ucv.ciens.ccg.nxtar.utils.ProjectConstants;
+
 import com.badlogic.gdx.math.Matrix3;
 import com.badlogic.gdx.math.Vector3;
 
@@ -31,9 +33,54 @@ public interface ImageProcessor{
 		public float[] calibrationPoints;
 	}
 
+	/**
+	 * <p>Finds up to {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} markers in the input
+	 * image and returns their codes and pose estimation in the CVMarkerData structure. The
+	 * markers are higlihted in the input image.</p>
+	 * 
+	 * @param frame The JPEG encoded input image.
+	 * @return A data structure containing the processed output image, the
+	 * detected marker codes and their respective locations.
+	 */
 	public MarkerData findMarkersInFrame(byte[] frame);
+
+	/**
+	 * <p>Attempts to detect a checkerboard calibration pattern in the input image.
+	 * If the pattenr is found the method returns an image with the pattern
+	 * highlighted and the spatial location of the calibration points in the 
+	 * output data structure.</p>
+	 * 
+	 * @param frame The JPEG encoded input image.
+	 * @return A data structure containing the processed output image and the
+	 * location of the calibration points. If the pattern was not found, the returnd
+	 * calibration points array is null.
+	 */
 	public CalibrationData findCalibrationPattern(byte[] frame);
+
+	/** 
+	 * <p>Obtains the intrinsic camera parameters necesary for calibration.</p>
+	 */
 	public void calibrateCamera(float[][] calibrationSamples, byte[] frame);
+
+	/**
+	 * <p>Removes camera lens distortion from the input image using the
+	 * camera parameters obtained by the calibrateCamera method.</p>
+	 * 
+	 * @return A JPEG encoded image that is the input image after distortion correction. If the
+	 * camera has not been calibrated or OpenCV failed to load returns null.
+	 */
 	public byte[] undistortFrame(byte[] frame);
+
+	/**
+	 * <p>Indicates if OpenCV has been sucessfully initialized and used
+	 * to obtain the camera parameters for calibration.</p>
+	 * 
+	 * @return True if and only if OpenCV initialized succesfully and calibrateCamera has been called previously.
+	 */
 	public boolean isCameraCalibrated();
+
+	public float getFocalPointX();
+	public float getFocalPointY();
+	public float getCameraCenterX();
+	public float getCameraCenterY();
 }

src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java

@@ -21,6 +21,7 @@ import ve.ucv.ciens.ccg.nxtar.NxtARCore;
 import ve.ucv.ciens.ccg.nxtar.NxtARCore.game_states_t;
 import ve.ucv.ciens.ccg.nxtar.entities.EntityCreatorBase;
 import ve.ucv.ciens.ccg.nxtar.entities.MarkerTestEntityCreator;
+import ve.ucv.ciens.ccg.nxtar.graphics.CustomPerspectiveCamera;
 import ve.ucv.ciens.ccg.nxtar.graphics.RenderParameters;
 import ve.ucv.ciens.ccg.nxtar.interfaces.ImageProcessor.MarkerData;
 import ve.ucv.ciens.ccg.nxtar.network.monitors.MotorEventQueue;
@@ -37,7 +38,6 @@ import com.badlogic.gdx.controllers.Controller;
 import com.badlogic.gdx.controllers.mappings.Ouya;
 import com.badlogic.gdx.graphics.GL20;
 import com.badlogic.gdx.graphics.OrthographicCamera;
-import com.badlogic.gdx.graphics.PerspectiveCamera;
 import com.badlogic.gdx.graphics.Pixmap;
 import com.badlogic.gdx.graphics.Pixmap.Format;
 import com.badlogic.gdx.graphics.Texture;
@@ -47,57 +47,63 @@ import com.badlogic.gdx.graphics.g2d.Sprite;
 import com.badlogic.gdx.graphics.g2d.TextureRegion;
 import com.badlogic.gdx.graphics.glutils.FrameBuffer;
 import com.badlogic.gdx.graphics.glutils.ShaderProgram;
+import com.badlogic.gdx.math.Matrix4;
 import com.badlogic.gdx.math.Vector2;
 import com.badlogic.gdx.math.Vector3;
 
 public class InGameState extends BaseState{
-	private static final String TAG = "IN_GAME_STATE";
+	private static final String TAG                    = "IN_GAME_STATE";
-	private static final String CLASS_NAME = InGameState.class.getSimpleName();
+	private static final String CLASS_NAME             = InGameState.class.getSimpleName();
 	private static final String BACKGROUND_SHADER_PATH = "shaders/bckg/bckg";
+	private static final float  NEAR                   = 0.01f;
+	private static final float  FAR                    = 100.0f;
+	private static final float  FAR_PLUS_NEAR          = FAR + NEAR;
+	private static final float  FAR_LESS_NEAR          = FAR - NEAR;
 
 	// Background related fields.
-	private float uScaling[];
+	private float                   uScaling[];
-	protected Sprite background;
+	protected Sprite                background;
-	private Texture backgroundTexture;
+	private Texture                 backgroundTexture;
-	private ShaderProgram backgroundShader;
+	private ShaderProgram           backgroundShader;
 
 	// 3D rendering fields.
-	private FrameBuffer frameBuffer;
+	private Matrix4                 projectionMatrix;
-	private Sprite frameBufferSprite;
+	private FrameBuffer             frameBuffer;
+	private Sprite                  frameBufferSprite;
 
 	// Game objects.
-	private World gameWorld;
+	private World                   gameWorld;
-	private EntityCreatorBase entityCreator;
+	private EntityCreatorBase       entityCreator;
 
 	// Cameras.
-	private OrthographicCamera camera;
+	private OrthographicCamera      unitaryOrthoCamera;
-	private OrthographicCamera pixelPerfectCamera;
+	private OrthographicCamera      pixelPerfectOrthoCamera;
-	private PerspectiveCamera camera3D;
+	private CustomPerspectiveCamera perspectiveCamera;
 
 	// Video stream graphics.
-	private Texture videoFrameTexture;
+	private Texture                 videoFrameTexture;
-	private Sprite renderableVideoFrame;
+	private Sprite                  renderableVideoFrame;
-	private Pixmap videoFrame;
+	private Pixmap                  videoFrame;
 
 	// Interface buttons.
-	private Texture buttonTexture;
+	private Texture                 buttonTexture;
-	private Texture buttonTexture2;
+	private Texture                 buttonTexture2;
-	private Sprite motorA;
+	private Sprite                  motorA;
-	private Sprite motorB;
+	private Sprite                  motorB;
-	private Sprite motorC;
+	private Sprite                  motorC;
-	private Sprite motorD;
+	private Sprite                  motorD;
-	private Sprite headA;
+	private Sprite                  headA;
-	private Sprite headB;
+	private Sprite                  headB;
-	private Sprite headC;
+	private Sprite                  headC;
 
 	// Button touch helper fields.
-	private boolean[] motorButtonsTouched;
+	private boolean[]               motorButtonsTouched;
-	private int[] motorButtonsPointers;
+	private int[]                   motorButtonsPointers;
-	private boolean[] motorGamepadButtonPressed;
+	private boolean[]               motorGamepadButtonPressed;
 
 	// Monitors.
-	private VideoFrameMonitor frameMonitor;
+	private VideoFrameMonitor       frameMonitor;
-	private MotorEventQueue queue;
+	private MotorEventQueue         queue;
 
 	public InGameState(final NxtARCore core){
 		this.core = core;
@@ -108,8 +114,8 @@ public class InGameState extends BaseState{
 		videoFrame = null;
 
 		// Set up the cameras.
-		pixelPerfectCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
+		pixelPerfectOrthoCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
-		camera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth());
+		unitaryOrthoCamera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth());
 
 		if(!Ouya.runningOnOuya) setUpButtons();
 
@@ -165,8 +171,9 @@ public class InGameState extends BaseState{
 		uScaling[1] = Gdx.graphics.getHeight() > Gdx.graphics.getWidth() ? 16.0f : 9.0f;
 
 		// Set up the 3D rendering.
+		projectionMatrix = new Matrix4().idt();
 		frameBuffer = null;
-		camera3D = null;
+		perspectiveCamera = null;
 		frameBufferSprite = null;
 
 		// Set up the game world.
@@ -177,7 +184,6 @@ public class InGameState extends BaseState{
 		gameWorld.setSystem(new MarkerPositioningSystem());
 		gameWorld.setSystem(new MarkerRenderingSystem(), true);
 		gameWorld.setSystem(new ObjectRenderingSystem(), true);
-
 		gameWorld.initialize();
 	}
 
@@ -191,13 +197,14 @@ public class InGameState extends BaseState{
 		byte[] frame;
 		MarkerData data;
 		TextureRegion region;
+		float focalPointX, focalPointY, cameraCenterX, cameraCenterY;
 
 		// Clear the screen.
 		Gdx.gl.glClearColor(1, 1, 1, 1);
 		Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT);
 
 		// Render the background.
-		core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
+		core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
 		core.batch.begin();{
 			if(backgroundShader != null){
 				core.batch.setShader(backgroundShader);
@@ -213,16 +220,16 @@ public class InGameState extends BaseState{
 		h = frameMonitor.getFrameDimensions().getHeight();
 
 		// Create the 3D perspective camera and the frame buffer object if they don't exist.
-		if(camera3D == null && frameBuffer == null){
+		if(perspectiveCamera == null && frameBuffer == null){
 			frameBuffer = new FrameBuffer(Format.RGBA8888, w, h, true);
 			frameBuffer.getColorBufferTexture().setFilter(TextureFilter.Linear, TextureFilter.Linear);
 
-			camera3D = new PerspectiveCamera(67, w, h);
+			perspectiveCamera = new CustomPerspectiveCamera(67, w, h);
-			camera3D.translate(0.0f, 0.0f, 0.0f);
+			perspectiveCamera.translate(0.0f, 0.0f, 0.0f);
-			camera3D.near = 0.01f;
+			perspectiveCamera.near = NEAR;
-			camera3D.far = 100.0f;
+			perspectiveCamera.far = FAR;
-			camera3D.lookAt(0.0f, 0.0f, -1.0f);
+			perspectiveCamera.lookAt(0.0f, 0.0f, -1.0f);
-			camera3D.update();
+			perspectiveCamera.update();
 		}
 
 		// Apply the undistortion method if the camera has been calibrated already.
@@ -257,13 +264,44 @@ public class InGameState extends BaseState{
 
 			// Set the 3D frame buffer for rendering.
 			frameBuffer.begin();{
+				// Set OpenGL state.
 				Gdx.gl.glDisable(GL20.GL_CULL_FACE);
 				Gdx.gl.glEnable(GL20.GL_DEPTH_TEST);
-				Gdx.gl.glClearColor(1, 1, 1, 0);
+				Gdx.gl.glClearColor(0, 0, 0, 0);
 				Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_DEPTH_BUFFER_BIT);
 
-				RenderParameters.setModelViewProjectionMatrix(camera3D.combined);
+				// Build the projection matrix.
-				RenderParameters.setEyePosition(camera3D.position);
+				focalPointX   = core.cvProc.getFocalPointX();
+				focalPointY   = core.cvProc.getFocalPointY();
+				cameraCenterX = core.cvProc.getCameraCenterX();
+				cameraCenterY = core.cvProc.getCameraCenterY();
+
+				projectionMatrix.val[Matrix4.M00] = -2.0f * focalPointX / w;
+				projectionMatrix.val[Matrix4.M10] = 0.0f;
+				projectionMatrix.val[Matrix4.M20] = 0.0f;
+				projectionMatrix.val[Matrix4.M30] = 0.0f;
+
+				projectionMatrix.val[Matrix4.M01] = 0.0f;
+				projectionMatrix.val[Matrix4.M11] = 2.0f * focalPointY / h;
+				projectionMatrix.val[Matrix4.M21] = 0.0f;
+				projectionMatrix.val[Matrix4.M31] = 0.0f;
+
+				projectionMatrix.val[Matrix4.M02] = 2.0f * cameraCenterX / w - 1.0f;
+				projectionMatrix.val[Matrix4.M12] = 2.0f * cameraCenterY / h - 1.0f;
+				projectionMatrix.val[Matrix4.M22] = -FAR_PLUS_NEAR / FAR_LESS_NEAR;
+				projectionMatrix.val[Matrix4.M32] = -1.0f;
+
+				projectionMatrix.val[Matrix4.M03] = 0.0f;
+				projectionMatrix.val[Matrix4.M13] = 0.0f;
+				projectionMatrix.val[Matrix4.M23] = -2.0f * FAR * NEAR / FAR_LESS_NEAR;
+				projectionMatrix.val[Matrix4.M33] = 0.0f;
+
+				// Set rendering parameters.
+				perspectiveCamera.update(projectionMatrix, true);
+				RenderParameters.setModelViewProjectionMatrix(perspectiveCamera.combined);
+				RenderParameters.setEyePosition(perspectiveCamera.position);
+
+				// Call rendering systems.
 				gameWorld.getSystem(MarkerRenderingSystem.class).setMarkerData(data);
 				gameWorld.getSystem(MarkerRenderingSystem.class).process();
 				gameWorld.getSystem(ObjectRenderingSystem.class).process();
@@ -273,7 +311,7 @@ public class InGameState extends BaseState{
 
 			// Set the frame buffer object texture to a renderable sprite.
 			region = new TextureRegion(frameBuffer.getColorBufferTexture(), 0, 0, frameBuffer.getWidth(), frameBuffer.getHeight());
-			region.flip(true, true);
+			region.flip(false, true);
 			if(frameBufferSprite == null)
 				frameBufferSprite = new Sprite(region);
 			else
@@ -303,9 +341,9 @@ public class InGameState extends BaseState{
 
 			// Set the correct camera for the device.
 			if(!Ouya.runningOnOuya){
-				core.batch.setProjectionMatrix(camera.combined);
+				core.batch.setProjectionMatrix(unitaryOrthoCamera.combined);
 			}else{
-				core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
+				core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
 			}
 
 			// Render the video frame and the frame buffer.
@@ -320,7 +358,7 @@ public class InGameState extends BaseState{
 
 		// Render the interface buttons.
 		if(!Ouya.runningOnOuya){
-			core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
+			core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
 			core.batch.begin();{
 				motorA.draw(core.batch);
 				motorB.draw(core.batch);
@@ -427,7 +465,7 @@ public class InGameState extends BaseState{
 
 		if(!Ouya.runningOnOuya){
 			win2world.set(screenX, screenY, 0.0f);
-			camera.unproject(win2world);
+			unitaryOrthoCamera.unproject(win2world);
 			touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
 
 			Gdx.app.log(TAG, CLASS_NAME + String.format(".touchDown(%d, %d, %d, %d)", screenX, screenY, pointer, button));
@@ -527,7 +565,7 @@ public class InGameState extends BaseState{
 
 		if(!Ouya.runningOnOuya){
 			win2world.set(screenX, screenY, 0.0f);
-			camera.unproject(win2world);
+			unitaryOrthoCamera.unproject(win2world);
 			touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
 
 			Gdx.app.log(TAG, CLASS_NAME + String.format(".touchUp(%d, %d, %d, %d)", screenX, screenY, pointer, button));
@@ -638,7 +676,7 @@ public class InGameState extends BaseState{
 
 		if(!Ouya.runningOnOuya){
 			win2world.set(screenX, screenY, 0.0f);
-			camera.unproject(win2world);
+			unitaryOrthoCamera.unproject(win2world);
 			touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
 
 			if(pointer == motorButtonsPointers[0] && !motorA.getBoundingRectangle().contains(touchPointWorldCoords)){

src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java

@@ -66,6 +66,7 @@ public class MarkerPositioningSystem extends EntityProcessingSystem {
 					Gdx.app.log(TAG, CLASS_NAME + ".process(): Processing marker code " + Integer.toString(markers.markerCodes[i]) + ".");
 					geometry.position.set(markers.translationVectors[i]);
 					geometry.rotation.set(markers.rotationMatrices[i]);
+					break;
 				}
 			}else{
 				Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + ".");

src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java

@@ -87,9 +87,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
 					// Set the geometric transformations.
 					translationMatrix.setToTranslation(geometry.position);
 
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): TRANSLATION:");
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): (" + Float.toString(geometry.position.x) + ", " + Float.toString(geometry.position.y) + ", " + Float.toString(geometry.position.z) + ")");
-
 					rotationMatrix.val[0] = geometry.rotation.val[0];
 					rotationMatrix.val[1] = geometry.rotation.val[1];
 					rotationMatrix.val[2] = geometry.rotation.val[2];
@@ -106,13 +103,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
 					rotationMatrix.val[13] = 0;
 					rotationMatrix.val[14] = 0;
 					rotationMatrix.val[15] = 1;
-					//rotationMatrix.idt();
-
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): ROTATION:");
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[0]) + ", " + Float.toString(rotationMatrix.val[4]) + ", " + Float.toString(rotationMatrix.val[8]) + ", " + Float.toString(rotationMatrix.val[12]) + "|");
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[1]) + ", " + Float.toString(rotationMatrix.val[5]) + ", " + Float.toString(rotationMatrix.val[9]) + ", " + Float.toString(rotationMatrix.val[13]) + "|");
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[2]) + ", " + Float.toString(rotationMatrix.val[6]) + ", " + Float.toString(rotationMatrix.val[10]) + ", " + Float.toString(rotationMatrix.val[14]) + "|");
-					Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[3]) + ", " + Float.toString(rotationMatrix.val[7]) + ", " + Float.toString(rotationMatrix.val[11]) + ", " + Float.toString(rotationMatrix.val[15]) + "|");
 
 					scalingMatrix.setToScaling(geometry.scaling);
 					combinedTransformationMatrix.idt().mul(translationMatrix).mul(rotationMatrix).mul(scalingMatrix);
@@ -123,6 +113,8 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
 						shaderComp.shader.setUniforms();
 						meshComp.model.render(shaderComp.shader.getShaderProgram(), GL20.GL_TRIANGLES);
 					}shaderComp.shader.getShaderProgram().end();
+
+					break;
 				}
 			}else{
 				Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + ".");