diff --git a/src/ve/ucv/ciens/ccg/nxtar/graphics/CustomPerspectiveCamera.java b/src/ve/ucv/ciens/ccg/nxtar/graphics/CustomPerspectiveCamera.java new file mode 100644 index 0000000..bd0f89d --- /dev/null +++ b/src/ve/ucv/ciens/ccg/nxtar/graphics/CustomPerspectiveCamera.java @@ -0,0 +1,48 @@ +/* + * Copyright (C) 2014 Miguel Angel Astor Romero + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package ve.ucv.ciens.ccg.nxtar.graphics; + +import com.badlogic.gdx.graphics.PerspectiveCamera; +import com.badlogic.gdx.math.Matrix4; +import com.badlogic.gdx.math.Vector3; + +/** + *

Extension of the standard LibGDX perspective camera that allows setting an + * arbitrary projection matrix when updating.

+ */ +public class CustomPerspectiveCamera extends PerspectiveCamera{ + private final Vector3 tmp = new Vector3(); + + public CustomPerspectiveCamera(float fieldOfView, float viewportWidth, float viewportHeight){ + this.fieldOfView = fieldOfView; + this.viewportWidth = viewportWidth; + this.viewportHeight = viewportHeight; + update(); + } + + public void update(Matrix4 customProjection, boolean updateFrustum){ + projection.set(customProjection); + view.setToLookAt(position, tmp.set(position).add(direction), up); + combined.set(projection); + Matrix4.mul(combined.val, view.val); + + if(updateFrustum){ + invProjectionView.set(combined); + Matrix4.inv(invProjectionView.val); + frustum.update(invProjectionView); + } + } +} diff --git a/src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java b/src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java index 9d75780..a2effea 100644 --- a/src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java +++ b/src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java @@ -15,6 +15,8 @@ */ package ve.ucv.ciens.ccg.nxtar.interfaces; +import ve.ucv.ciens.ccg.nxtar.utils.ProjectConstants; + import com.badlogic.gdx.math.Matrix3; import com.badlogic.gdx.math.Vector3; @@ -31,9 +33,54 @@ public interface ImageProcessor{ public float[] calibrationPoints; } + /** + *

Finds up to {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} markers in the input + * image and returns their codes and pose estimation in the CVMarkerData structure. The + * markers are higlihted in the input image.

+ * + * @param frame The JPEG encoded input image. + * @return A data structure containing the processed output image, the + * detected marker codes and their respective locations. + */ public MarkerData findMarkersInFrame(byte[] frame); + + /** + *

Attempts to detect a checkerboard calibration pattern in the input image. + * If the pattenr is found the method returns an image with the pattern + * highlighted and the spatial location of the calibration points in the + * output data structure.

+ * + * @param frame The JPEG encoded input image. + * @return A data structure containing the processed output image and the + * location of the calibration points. If the pattern was not found, the returnd + * calibration points array is null. + */ public CalibrationData findCalibrationPattern(byte[] frame); + + /** + *

Obtains the intrinsic camera parameters necesary for calibration.

+ */ public void calibrateCamera(float[][] calibrationSamples, byte[] frame); + + /** + *

Removes camera lens distortion from the input image using the + * camera parameters obtained by the calibrateCamera method.

+ * + * @return A JPEG encoded image that is the input image after distortion correction. If the + * camera has not been calibrated or OpenCV failed to load returns null. + */ public byte[] undistortFrame(byte[] frame); + + /** + *

Indicates if OpenCV has been sucessfully initialized and used + * to obtain the camera parameters for calibration.

+ * + * @return True if and only if OpenCV initialized succesfully and calibrateCamera has been called previously. + */ public boolean isCameraCalibrated(); + + public float getFocalPointX(); + public float getFocalPointY(); + public float getCameraCenterX(); + public float getCameraCenterY(); } diff --git a/src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java b/src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java index 9ed814a..81a0073 100644 --- a/src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java +++ b/src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java @@ -21,6 +21,7 @@ import ve.ucv.ciens.ccg.nxtar.NxtARCore; import ve.ucv.ciens.ccg.nxtar.NxtARCore.game_states_t; import ve.ucv.ciens.ccg.nxtar.entities.EntityCreatorBase; import ve.ucv.ciens.ccg.nxtar.entities.MarkerTestEntityCreator; +import ve.ucv.ciens.ccg.nxtar.graphics.CustomPerspectiveCamera; import ve.ucv.ciens.ccg.nxtar.graphics.RenderParameters; import ve.ucv.ciens.ccg.nxtar.interfaces.ImageProcessor.MarkerData; import ve.ucv.ciens.ccg.nxtar.network.monitors.MotorEventQueue; @@ -37,7 +38,6 @@ import com.badlogic.gdx.controllers.Controller; import com.badlogic.gdx.controllers.mappings.Ouya; import com.badlogic.gdx.graphics.GL20; import com.badlogic.gdx.graphics.OrthographicCamera; -import com.badlogic.gdx.graphics.PerspectiveCamera; import com.badlogic.gdx.graphics.Pixmap; import com.badlogic.gdx.graphics.Pixmap.Format; import com.badlogic.gdx.graphics.Texture; @@ -47,57 +47,63 @@ import com.badlogic.gdx.graphics.g2d.Sprite; import com.badlogic.gdx.graphics.g2d.TextureRegion; import com.badlogic.gdx.graphics.glutils.FrameBuffer; import com.badlogic.gdx.graphics.glutils.ShaderProgram; +import com.badlogic.gdx.math.Matrix4; import com.badlogic.gdx.math.Vector2; import com.badlogic.gdx.math.Vector3; public class InGameState extends BaseState{ - private static final String TAG = "IN_GAME_STATE"; - private static final String CLASS_NAME = InGameState.class.getSimpleName(); + private static final String TAG = "IN_GAME_STATE"; + private static final String CLASS_NAME = InGameState.class.getSimpleName(); private static final String BACKGROUND_SHADER_PATH = "shaders/bckg/bckg"; + private static final float NEAR = 0.01f; + private static final float FAR = 100.0f; + private static final float FAR_PLUS_NEAR = FAR + NEAR; + private static final float FAR_LESS_NEAR = FAR - NEAR; // Background related fields. - private float uScaling[]; - protected Sprite background; - private Texture backgroundTexture; - private ShaderProgram backgroundShader; + private float uScaling[]; + protected Sprite background; + private Texture backgroundTexture; + private ShaderProgram backgroundShader; // 3D rendering fields. - private FrameBuffer frameBuffer; - private Sprite frameBufferSprite; + private Matrix4 projectionMatrix; + private FrameBuffer frameBuffer; + private Sprite frameBufferSprite; // Game objects. - private World gameWorld; - private EntityCreatorBase entityCreator; + private World gameWorld; + private EntityCreatorBase entityCreator; // Cameras. - private OrthographicCamera camera; - private OrthographicCamera pixelPerfectCamera; - private PerspectiveCamera camera3D; + private OrthographicCamera unitaryOrthoCamera; + private OrthographicCamera pixelPerfectOrthoCamera; + private CustomPerspectiveCamera perspectiveCamera; // Video stream graphics. - private Texture videoFrameTexture; - private Sprite renderableVideoFrame; - private Pixmap videoFrame; + private Texture videoFrameTexture; + private Sprite renderableVideoFrame; + private Pixmap videoFrame; // Interface buttons. - private Texture buttonTexture; - private Texture buttonTexture2; - private Sprite motorA; - private Sprite motorB; - private Sprite motorC; - private Sprite motorD; - private Sprite headA; - private Sprite headB; - private Sprite headC; + private Texture buttonTexture; + private Texture buttonTexture2; + private Sprite motorA; + private Sprite motorB; + private Sprite motorC; + private Sprite motorD; + private Sprite headA; + private Sprite headB; + private Sprite headC; // Button touch helper fields. - private boolean[] motorButtonsTouched; - private int[] motorButtonsPointers; - private boolean[] motorGamepadButtonPressed; + private boolean[] motorButtonsTouched; + private int[] motorButtonsPointers; + private boolean[] motorGamepadButtonPressed; // Monitors. - private VideoFrameMonitor frameMonitor; - private MotorEventQueue queue; + private VideoFrameMonitor frameMonitor; + private MotorEventQueue queue; public InGameState(final NxtARCore core){ this.core = core; @@ -108,8 +114,8 @@ public class InGameState extends BaseState{ videoFrame = null; // Set up the cameras. - pixelPerfectCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight()); - camera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth()); + pixelPerfectOrthoCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight()); + unitaryOrthoCamera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth()); if(!Ouya.runningOnOuya) setUpButtons(); @@ -165,8 +171,9 @@ public class InGameState extends BaseState{ uScaling[1] = Gdx.graphics.getHeight() > Gdx.graphics.getWidth() ? 16.0f : 9.0f; // Set up the 3D rendering. + projectionMatrix = new Matrix4().idt(); frameBuffer = null; - camera3D = null; + perspectiveCamera = null; frameBufferSprite = null; // Set up the game world. @@ -177,7 +184,6 @@ public class InGameState extends BaseState{ gameWorld.setSystem(new MarkerPositioningSystem()); gameWorld.setSystem(new MarkerRenderingSystem(), true); gameWorld.setSystem(new ObjectRenderingSystem(), true); - gameWorld.initialize(); } @@ -191,13 +197,14 @@ public class InGameState extends BaseState{ byte[] frame; MarkerData data; TextureRegion region; + float focalPointX, focalPointY, cameraCenterX, cameraCenterY; // Clear the screen. Gdx.gl.glClearColor(1, 1, 1, 1); Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT); // Render the background. - core.batch.setProjectionMatrix(pixelPerfectCamera.combined); + core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined); core.batch.begin();{ if(backgroundShader != null){ core.batch.setShader(backgroundShader); @@ -213,16 +220,16 @@ public class InGameState extends BaseState{ h = frameMonitor.getFrameDimensions().getHeight(); // Create the 3D perspective camera and the frame buffer object if they don't exist. - if(camera3D == null && frameBuffer == null){ + if(perspectiveCamera == null && frameBuffer == null){ frameBuffer = new FrameBuffer(Format.RGBA8888, w, h, true); frameBuffer.getColorBufferTexture().setFilter(TextureFilter.Linear, TextureFilter.Linear); - camera3D = new PerspectiveCamera(67, w, h); - camera3D.translate(0.0f, 0.0f, 0.0f); - camera3D.near = 0.01f; - camera3D.far = 100.0f; - camera3D.lookAt(0.0f, 0.0f, -1.0f); - camera3D.update(); + perspectiveCamera = new CustomPerspectiveCamera(67, w, h); + perspectiveCamera.translate(0.0f, 0.0f, 0.0f); + perspectiveCamera.near = NEAR; + perspectiveCamera.far = FAR; + perspectiveCamera.lookAt(0.0f, 0.0f, -1.0f); + perspectiveCamera.update(); } // Apply the undistortion method if the camera has been calibrated already. @@ -257,13 +264,44 @@ public class InGameState extends BaseState{ // Set the 3D frame buffer for rendering. frameBuffer.begin();{ + // Set OpenGL state. Gdx.gl.glDisable(GL20.GL_CULL_FACE); Gdx.gl.glEnable(GL20.GL_DEPTH_TEST); - Gdx.gl.glClearColor(1, 1, 1, 0); + Gdx.gl.glClearColor(0, 0, 0, 0); Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_DEPTH_BUFFER_BIT); - RenderParameters.setModelViewProjectionMatrix(camera3D.combined); - RenderParameters.setEyePosition(camera3D.position); + // Build the projection matrix. + focalPointX = core.cvProc.getFocalPointX(); + focalPointY = core.cvProc.getFocalPointY(); + cameraCenterX = core.cvProc.getCameraCenterX(); + cameraCenterY = core.cvProc.getCameraCenterY(); + + projectionMatrix.val[Matrix4.M00] = -2.0f * focalPointX / w; + projectionMatrix.val[Matrix4.M10] = 0.0f; + projectionMatrix.val[Matrix4.M20] = 0.0f; + projectionMatrix.val[Matrix4.M30] = 0.0f; + + projectionMatrix.val[Matrix4.M01] = 0.0f; + projectionMatrix.val[Matrix4.M11] = 2.0f * focalPointY / h; + projectionMatrix.val[Matrix4.M21] = 0.0f; + projectionMatrix.val[Matrix4.M31] = 0.0f; + + projectionMatrix.val[Matrix4.M02] = 2.0f * cameraCenterX / w - 1.0f; + projectionMatrix.val[Matrix4.M12] = 2.0f * cameraCenterY / h - 1.0f; + projectionMatrix.val[Matrix4.M22] = -FAR_PLUS_NEAR / FAR_LESS_NEAR; + projectionMatrix.val[Matrix4.M32] = -1.0f; + + projectionMatrix.val[Matrix4.M03] = 0.0f; + projectionMatrix.val[Matrix4.M13] = 0.0f; + projectionMatrix.val[Matrix4.M23] = -2.0f * FAR * NEAR / FAR_LESS_NEAR; + projectionMatrix.val[Matrix4.M33] = 0.0f; + + // Set rendering parameters. + perspectiveCamera.update(projectionMatrix, true); + RenderParameters.setModelViewProjectionMatrix(perspectiveCamera.combined); + RenderParameters.setEyePosition(perspectiveCamera.position); + + // Call rendering systems. gameWorld.getSystem(MarkerRenderingSystem.class).setMarkerData(data); gameWorld.getSystem(MarkerRenderingSystem.class).process(); gameWorld.getSystem(ObjectRenderingSystem.class).process(); @@ -273,7 +311,7 @@ public class InGameState extends BaseState{ // Set the frame buffer object texture to a renderable sprite. region = new TextureRegion(frameBuffer.getColorBufferTexture(), 0, 0, frameBuffer.getWidth(), frameBuffer.getHeight()); - region.flip(true, true); + region.flip(false, true); if(frameBufferSprite == null) frameBufferSprite = new Sprite(region); else @@ -303,9 +341,9 @@ public class InGameState extends BaseState{ // Set the correct camera for the device. if(!Ouya.runningOnOuya){ - core.batch.setProjectionMatrix(camera.combined); + core.batch.setProjectionMatrix(unitaryOrthoCamera.combined); }else{ - core.batch.setProjectionMatrix(pixelPerfectCamera.combined); + core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined); } // Render the video frame and the frame buffer. @@ -320,7 +358,7 @@ public class InGameState extends BaseState{ // Render the interface buttons. if(!Ouya.runningOnOuya){ - core.batch.setProjectionMatrix(pixelPerfectCamera.combined); + core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined); core.batch.begin();{ motorA.draw(core.batch); motorB.draw(core.batch); @@ -427,7 +465,7 @@ public class InGameState extends BaseState{ if(!Ouya.runningOnOuya){ win2world.set(screenX, screenY, 0.0f); - camera.unproject(win2world); + unitaryOrthoCamera.unproject(win2world); touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight()); Gdx.app.log(TAG, CLASS_NAME + String.format(".touchDown(%d, %d, %d, %d)", screenX, screenY, pointer, button)); @@ -527,7 +565,7 @@ public class InGameState extends BaseState{ if(!Ouya.runningOnOuya){ win2world.set(screenX, screenY, 0.0f); - camera.unproject(win2world); + unitaryOrthoCamera.unproject(win2world); touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight()); Gdx.app.log(TAG, CLASS_NAME + String.format(".touchUp(%d, %d, %d, %d)", screenX, screenY, pointer, button)); @@ -638,7 +676,7 @@ public class InGameState extends BaseState{ if(!Ouya.runningOnOuya){ win2world.set(screenX, screenY, 0.0f); - camera.unproject(win2world); + unitaryOrthoCamera.unproject(win2world); touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight()); if(pointer == motorButtonsPointers[0] && !motorA.getBoundingRectangle().contains(touchPointWorldCoords)){ diff --git a/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java b/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java index 3f8a561..f544007 100644 --- a/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java +++ b/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java @@ -66,6 +66,7 @@ public class MarkerPositioningSystem extends EntityProcessingSystem { Gdx.app.log(TAG, CLASS_NAME + ".process(): Processing marker code " + Integer.toString(markers.markerCodes[i]) + "."); geometry.position.set(markers.translationVectors[i]); geometry.rotation.set(markers.rotationMatrices[i]); + break; } }else{ Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + "."); diff --git a/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java b/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java index 2bcf671..2f72ac8 100644 --- a/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java +++ b/src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java @@ -87,9 +87,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem { // Set the geometric transformations. translationMatrix.setToTranslation(geometry.position); - Gdx.app.log(TAG, CLASS_NAME + ".process(): TRANSLATION:"); - Gdx.app.log(TAG, CLASS_NAME + ".process(): (" + Float.toString(geometry.position.x) + ", " + Float.toString(geometry.position.y) + ", " + Float.toString(geometry.position.z) + ")"); - rotationMatrix.val[0] = geometry.rotation.val[0]; rotationMatrix.val[1] = geometry.rotation.val[1]; rotationMatrix.val[2] = geometry.rotation.val[2]; @@ -106,13 +103,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem { rotationMatrix.val[13] = 0; rotationMatrix.val[14] = 0; rotationMatrix.val[15] = 1; - //rotationMatrix.idt(); - - Gdx.app.log(TAG, CLASS_NAME + ".process(): ROTATION:"); - Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[0]) + ", " + Float.toString(rotationMatrix.val[4]) + ", " + Float.toString(rotationMatrix.val[8]) + ", " + Float.toString(rotationMatrix.val[12]) + "|"); - Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[1]) + ", " + Float.toString(rotationMatrix.val[5]) + ", " + Float.toString(rotationMatrix.val[9]) + ", " + Float.toString(rotationMatrix.val[13]) + "|"); - Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[2]) + ", " + Float.toString(rotationMatrix.val[6]) + ", " + Float.toString(rotationMatrix.val[10]) + ", " + Float.toString(rotationMatrix.val[14]) + "|"); - Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[3]) + ", " + Float.toString(rotationMatrix.val[7]) + ", " + Float.toString(rotationMatrix.val[11]) + ", " + Float.toString(rotationMatrix.val[15]) + "|"); scalingMatrix.setToScaling(geometry.scaling); combinedTransformationMatrix.idt().mul(translationMatrix).mul(rotationMatrix).mul(scalingMatrix); @@ -123,6 +113,8 @@ public class MarkerRenderingSystem extends EntityProcessingSystem { shaderComp.shader.setUniforms(); meshComp.model.render(shaderComp.shader.getShaderProgram(), GL20.GL_TRIANGLES); }shaderComp.shader.getShaderProgram().end(); + + break; } }else{ Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + ".");