WallyHackenslacker/NxtAR-core
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Rendering of objects on top of markers is complete.

Browse Source
This commit is contained in:
Wally Hackenslacker
2014-05-15 12:10:14 -04:30
parent a9fa76cb68
commit 87295031dc
5 changed files with 189 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

48
src/ve/ucv/ciens/ccg/nxtar/graphics/CustomPerspectiveCamera.java Normal file
View File

@@ -0,0 +1,48 @@
/*
* Copyright (C) 2014 Miguel Angel Astor Romero
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/
package ve.ucv.ciens.ccg.nxtar.graphics;
import com.badlogic.gdx.graphics.PerspectiveCamera;
import com.badlogic.gdx.math.Matrix4;
import com.badlogic.gdx.math.Vector3;
/**
* <p>Extension of the standard LibGDX perspective camera that allows setting an
* arbitrary projection matrix when updating.</p>
*/
public class CustomPerspectiveCamera extends PerspectiveCamera{
private final Vector3 tmp = new Vector3();
public CustomPerspectiveCamera(float fieldOfView, float viewportWidth, float viewportHeight){
this.fieldOfView = fieldOfView;
this.viewportWidth = viewportWidth;
this.viewportHeight = viewportHeight;
update();
}
public void update(Matrix4 customProjection, boolean updateFrustum){
projection.set(customProjection);
view.setToLookAt(position, tmp.set(position).add(direction), up);
combined.set(projection);
Matrix4.mul(combined.val, view.val);
if(updateFrustum){
invProjectionView.set(combined);
Matrix4.inv(invProjectionView.val);
frustum.update(invProjectionView);
}
}
}

47
src/ve/ucv/ciens/ccg/nxtar/interfaces/ImageProcessor.java
View File

@@ -15,6 +15,8 @@
*/
package ve.ucv.ciens.ccg.nxtar.interfaces;
import ve.ucv.ciens.ccg.nxtar.utils.ProjectConstants;
import com.badlogic.gdx.math.Matrix3;
import com.badlogic.gdx.math.Vector3;
@@ -31,9 +33,54 @@ public interface ImageProcessor{
public float[] calibrationPoints;
}
/**
* <p>Finds up to {@link ProjectConstants.MAXIMUM_NUMBER_OF_MARKERS} markers in the input
* image and returns their codes and pose estimation in the CVMarkerData structure. The
* markers are higlihted in the input image.</p>
*
* @param frame The JPEG encoded input image.
* @return A data structure containing the processed output image, the
* detected marker codes and their respective locations.
*/
public MarkerData findMarkersInFrame(byte[] frame);
/**
* <p>Attempts to detect a checkerboard calibration pattern in the input image.
* If the pattenr is found the method returns an image with the pattern
* highlighted and the spatial location of the calibration points in the
* output data structure.</p>
*
* @param frame The JPEG encoded input image.
* @return A data structure containing the processed output image and the
* location of the calibration points. If the pattern was not found, the returnd
* calibration points array is null.
*/
public CalibrationData findCalibrationPattern(byte[] frame);
/**
* <p>Obtains the intrinsic camera parameters necesary for calibration.</p>
*/
public void calibrateCamera(float[][] calibrationSamples, byte[] frame);
/**
* <p>Removes camera lens distortion from the input image using the
* camera parameters obtained by the calibrateCamera method.</p>
*
* @return A JPEG encoded image that is the input image after distortion correction. If the
* camera has not been calibrated or OpenCV failed to load returns null.
*/
public byte[] undistortFrame(byte[] frame);
/**
* <p>Indicates if OpenCV has been sucessfully initialized and used
* to obtain the camera parameters for calibration.</p>
*
* @return True if and only if OpenCV initialized succesfully and calibrateCamera has been called previously.
*/
public boolean isCameraCalibrated();
public float getFocalPointX();
public float getFocalPointY();
public float getCameraCenterX();
public float getCameraCenterY();
}

144
src/ve/ucv/ciens/ccg/nxtar/states/InGameState.java
View File

@@ -21,6 +21,7 @@ import ve.ucv.ciens.ccg.nxtar.NxtARCore;
import ve.ucv.ciens.ccg.nxtar.NxtARCore.game_states_t;
import ve.ucv.ciens.ccg.nxtar.entities.EntityCreatorBase;
import ve.ucv.ciens.ccg.nxtar.entities.MarkerTestEntityCreator;
import ve.ucv.ciens.ccg.nxtar.graphics.CustomPerspectiveCamera;
import ve.ucv.ciens.ccg.nxtar.graphics.RenderParameters;
import ve.ucv.ciens.ccg.nxtar.interfaces.ImageProcessor.MarkerData;
import ve.ucv.ciens.ccg.nxtar.network.monitors.MotorEventQueue;
@@ -37,7 +38,6 @@ import com.badlogic.gdx.controllers.Controller;
import com.badlogic.gdx.controllers.mappings.Ouya;
import com.badlogic.gdx.graphics.GL20;
import com.badlogic.gdx.graphics.OrthographicCamera;
import com.badlogic.gdx.graphics.PerspectiveCamera;
import com.badlogic.gdx.graphics.Pixmap;
import com.badlogic.gdx.graphics.Pixmap.Format;
import com.badlogic.gdx.graphics.Texture;
@@ -47,57 +47,63 @@ import com.badlogic.gdx.graphics.g2d.Sprite;
import com.badlogic.gdx.graphics.g2d.TextureRegion;
import com.badlogic.gdx.graphics.glutils.FrameBuffer;
import com.badlogic.gdx.graphics.glutils.ShaderProgram;
import com.badlogic.gdx.math.Matrix4;
import com.badlogic.gdx.math.Vector2;
import com.badlogic.gdx.math.Vector3;
public class InGameState extends BaseState{
private static final String TAG = "IN_GAME_STATE";
private static final String CLASS_NAME = InGameState.class.getSimpleName();
private static final String TAG = "IN_GAME_STATE";
private static final String CLASS_NAME = InGameState.class.getSimpleName();
private static final String BACKGROUND_SHADER_PATH = "shaders/bckg/bckg";
private static final float NEAR = 0.01f;
private static final float FAR = 100.0f;
private static final float FAR_PLUS_NEAR = FAR + NEAR;
private static final float FAR_LESS_NEAR = FAR - NEAR;
// Background related fields.
private float uScaling[];
protected Sprite background;
private Texture backgroundTexture;
private ShaderProgram backgroundShader;
private float uScaling[];
protected Sprite background;
private Texture backgroundTexture;
private ShaderProgram backgroundShader;
// 3D rendering fields.
private FrameBuffer frameBuffer;
private Sprite frameBufferSprite;
private Matrix4 projectionMatrix;
private FrameBuffer frameBuffer;
private Sprite frameBufferSprite;
// Game objects.
private World gameWorld;
private EntityCreatorBase entityCreator;
private World gameWorld;
private EntityCreatorBase entityCreator;
// Cameras.
private OrthographicCamera camera;
private OrthographicCamera pixelPerfectCamera;
private PerspectiveCamera camera3D;
private OrthographicCamera unitaryOrthoCamera;
private OrthographicCamera pixelPerfectOrthoCamera;
private CustomPerspectiveCamera perspectiveCamera;
// Video stream graphics.
private Texture videoFrameTexture;
private Sprite renderableVideoFrame;
private Pixmap videoFrame;
private Texture videoFrameTexture;
private Sprite renderableVideoFrame;
private Pixmap videoFrame;
// Interface buttons.
private Texture buttonTexture;
private Texture buttonTexture2;
private Sprite motorA;
private Sprite motorB;
private Sprite motorC;
private Sprite motorD;
private Sprite headA;
private Sprite headB;
private Sprite headC;
private Texture buttonTexture;
private Texture buttonTexture2;
private Sprite motorA;
private Sprite motorB;
private Sprite motorC;
private Sprite motorD;
private Sprite headA;
private Sprite headB;
private Sprite headC;
// Button touch helper fields.
private boolean[] motorButtonsTouched;
private int[] motorButtonsPointers;
private boolean[] motorGamepadButtonPressed;
private boolean[] motorButtonsTouched;
private int[] motorButtonsPointers;
private boolean[] motorGamepadButtonPressed;
// Monitors.
private VideoFrameMonitor frameMonitor;
private MotorEventQueue queue;
private VideoFrameMonitor frameMonitor;
private MotorEventQueue queue;
public InGameState(final NxtARCore core){
this.core = core;
@@ -108,8 +114,8 @@ public class InGameState extends BaseState{
videoFrame = null;
// Set up the cameras.
pixelPerfectCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
camera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth());
pixelPerfectOrthoCamera = new OrthographicCamera(Gdx.graphics.getWidth(), Gdx.graphics.getHeight());
unitaryOrthoCamera = new OrthographicCamera(1.0f, Gdx.graphics.getHeight() / Gdx.graphics.getWidth());
if(!Ouya.runningOnOuya) setUpButtons();
@@ -165,8 +171,9 @@ public class InGameState extends BaseState{
uScaling[1] = Gdx.graphics.getHeight() > Gdx.graphics.getWidth() ? 16.0f : 9.0f;
// Set up the 3D rendering.
projectionMatrix = new Matrix4().idt();
frameBuffer = null;
camera3D = null;
perspectiveCamera = null;
frameBufferSprite = null;
// Set up the game world.
@@ -177,7 +184,6 @@ public class InGameState extends BaseState{
gameWorld.setSystem(new MarkerPositioningSystem());
gameWorld.setSystem(new MarkerRenderingSystem(), true);
gameWorld.setSystem(new ObjectRenderingSystem(), true);
gameWorld.initialize();
}
@@ -191,13 +197,14 @@ public class InGameState extends BaseState{
byte[] frame;
MarkerData data;
TextureRegion region;
float focalPointX, focalPointY, cameraCenterX, cameraCenterY;
// Clear the screen.
Gdx.gl.glClearColor(1, 1, 1, 1);
Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT);
// Render the background.
core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
core.batch.begin();{
if(backgroundShader != null){
core.batch.setShader(backgroundShader);
@@ -213,16 +220,16 @@ public class InGameState extends BaseState{
h = frameMonitor.getFrameDimensions().getHeight();
// Create the 3D perspective camera and the frame buffer object if they don't exist.
if(camera3D == null && frameBuffer == null){
if(perspectiveCamera == null && frameBuffer == null){
frameBuffer = new FrameBuffer(Format.RGBA8888, w, h, true);
frameBuffer.getColorBufferTexture().setFilter(TextureFilter.Linear, TextureFilter.Linear);
camera3D = new PerspectiveCamera(67, w, h);
camera3D.translate(0.0f, 0.0f, 0.0f);
camera3D.near = 0.01f;
camera3D.far = 100.0f;
camera3D.lookAt(0.0f, 0.0f, -1.0f);
camera3D.update();
perspectiveCamera = new CustomPerspectiveCamera(67, w, h);
perspectiveCamera.translate(0.0f, 0.0f, 0.0f);
perspectiveCamera.near = NEAR;
perspectiveCamera.far = FAR;
perspectiveCamera.lookAt(0.0f, 0.0f, -1.0f);
perspectiveCamera.update();
}
// Apply the undistortion method if the camera has been calibrated already.
@@ -257,13 +264,44 @@ public class InGameState extends BaseState{
// Set the 3D frame buffer for rendering.
frameBuffer.begin();{
// Set OpenGL state.
Gdx.gl.glDisable(GL20.GL_CULL_FACE);
Gdx.gl.glEnable(GL20.GL_DEPTH_TEST);
Gdx.gl.glClearColor(1, 1, 1, 0);
Gdx.gl.glClearColor(0, 0, 0, 0);
Gdx.gl.glClear(GL20.GL_COLOR_BUFFER_BIT | GL20.GL_DEPTH_BUFFER_BIT);
RenderParameters.setModelViewProjectionMatrix(camera3D.combined);
RenderParameters.setEyePosition(camera3D.position);
// Build the projection matrix.
focalPointX = core.cvProc.getFocalPointX();
focalPointY = core.cvProc.getFocalPointY();
cameraCenterX = core.cvProc.getCameraCenterX();
cameraCenterY = core.cvProc.getCameraCenterY();
projectionMatrix.val[Matrix4.M00] = -2.0f * focalPointX / w;
projectionMatrix.val[Matrix4.M10] = 0.0f;
projectionMatrix.val[Matrix4.M20] = 0.0f;
projectionMatrix.val[Matrix4.M30] = 0.0f;
projectionMatrix.val[Matrix4.M01] = 0.0f;
projectionMatrix.val[Matrix4.M11] = 2.0f * focalPointY / h;
projectionMatrix.val[Matrix4.M21] = 0.0f;
projectionMatrix.val[Matrix4.M31] = 0.0f;
projectionMatrix.val[Matrix4.M02] = 2.0f * cameraCenterX / w - 1.0f;
projectionMatrix.val[Matrix4.M12] = 2.0f * cameraCenterY / h - 1.0f;
projectionMatrix.val[Matrix4.M22] = -FAR_PLUS_NEAR / FAR_LESS_NEAR;
projectionMatrix.val[Matrix4.M32] = -1.0f;
projectionMatrix.val[Matrix4.M03] = 0.0f;
projectionMatrix.val[Matrix4.M13] = 0.0f;
projectionMatrix.val[Matrix4.M23] = -2.0f * FAR * NEAR / FAR_LESS_NEAR;
projectionMatrix.val[Matrix4.M33] = 0.0f;
// Set rendering parameters.
perspectiveCamera.update(projectionMatrix, true);
RenderParameters.setModelViewProjectionMatrix(perspectiveCamera.combined);
RenderParameters.setEyePosition(perspectiveCamera.position);
// Call rendering systems.
gameWorld.getSystem(MarkerRenderingSystem.class).setMarkerData(data);
gameWorld.getSystem(MarkerRenderingSystem.class).process();
gameWorld.getSystem(ObjectRenderingSystem.class).process();
@@ -273,7 +311,7 @@ public class InGameState extends BaseState{
// Set the frame buffer object texture to a renderable sprite.
region = new TextureRegion(frameBuffer.getColorBufferTexture(), 0, 0, frameBuffer.getWidth(), frameBuffer.getHeight());
region.flip(true, true);
region.flip(false, true);
if(frameBufferSprite == null)
frameBufferSprite = new Sprite(region);
else
@@ -303,9 +341,9 @@ public class InGameState extends BaseState{
// Set the correct camera for the device.
if(!Ouya.runningOnOuya){
core.batch.setProjectionMatrix(camera.combined);
core.batch.setProjectionMatrix(unitaryOrthoCamera.combined);
}else{
core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
}
// Render the video frame and the frame buffer.
@@ -320,7 +358,7 @@ public class InGameState extends BaseState{
// Render the interface buttons.
if(!Ouya.runningOnOuya){
core.batch.setProjectionMatrix(pixelPerfectCamera.combined);
core.batch.setProjectionMatrix(pixelPerfectOrthoCamera.combined);
core.batch.begin();{
motorA.draw(core.batch);
motorB.draw(core.batch);
@@ -427,7 +465,7 @@ public class InGameState extends BaseState{
if(!Ouya.runningOnOuya){
win2world.set(screenX, screenY, 0.0f);
camera.unproject(win2world);
unitaryOrthoCamera.unproject(win2world);
touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
Gdx.app.log(TAG, CLASS_NAME + String.format(".touchDown(%d, %d, %d, %d)", screenX, screenY, pointer, button));
@@ -527,7 +565,7 @@ public class InGameState extends BaseState{
if(!Ouya.runningOnOuya){
win2world.set(screenX, screenY, 0.0f);
camera.unproject(win2world);
unitaryOrthoCamera.unproject(win2world);
touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
Gdx.app.log(TAG, CLASS_NAME + String.format(".touchUp(%d, %d, %d, %d)", screenX, screenY, pointer, button));
@@ -638,7 +676,7 @@ public class InGameState extends BaseState{
if(!Ouya.runningOnOuya){
win2world.set(screenX, screenY, 0.0f);
camera.unproject(win2world);
unitaryOrthoCamera.unproject(win2world);
touchPointWorldCoords.set(win2world.x * Gdx.graphics.getWidth(), win2world.y * Gdx.graphics.getHeight());
if(pointer == motorButtonsPointers[0] && !motorA.getBoundingRectangle().contains(touchPointWorldCoords)){

1
src/ve/ucv/ciens/ccg/nxtar/systems/MarkerPositioningSystem.java
View File

@@ -66,6 +66,7 @@ public class MarkerPositioningSystem extends EntityProcessingSystem {
Gdx.app.log(TAG, CLASS_NAME + ".process(): Processing marker code " + Integer.toString(markers.markerCodes[i]) + ".");
geometry.position.set(markers.translationVectors[i]);
geometry.rotation.set(markers.rotationMatrices[i]);
break;
}
}else{
Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + ".");

12
src/ve/ucv/ciens/ccg/nxtar/systems/MarkerRenderingSystem.java
View File

@@ -87,9 +87,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
// Set the geometric transformations.
translationMatrix.setToTranslation(geometry.position);
Gdx.app.log(TAG, CLASS_NAME + ".process(): TRANSLATION:");
Gdx.app.log(TAG, CLASS_NAME + ".process(): (" + Float.toString(geometry.position.x) + ", " + Float.toString(geometry.position.y) + ", " + Float.toString(geometry.position.z) + ")");
rotationMatrix.val[0] = geometry.rotation.val[0];
rotationMatrix.val[1] = geometry.rotation.val[1];
rotationMatrix.val[2] = geometry.rotation.val[2];
@@ -106,13 +103,6 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
rotationMatrix.val[13] = 0;
rotationMatrix.val[14] = 0;
rotationMatrix.val[15] = 1;
//rotationMatrix.idt();
Gdx.app.log(TAG, CLASS_NAME + ".process(): ROTATION:");
Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[0]) + ", " + Float.toString(rotationMatrix.val[4]) + ", " + Float.toString(rotationMatrix.val[8]) + ", " + Float.toString(rotationMatrix.val[12]) + "|");
Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[1]) + ", " + Float.toString(rotationMatrix.val[5]) + ", " + Float.toString(rotationMatrix.val[9]) + ", " + Float.toString(rotationMatrix.val[13]) + "|");
Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[2]) + ", " + Float.toString(rotationMatrix.val[6]) + ", " + Float.toString(rotationMatrix.val[10]) + ", " + Float.toString(rotationMatrix.val[14]) + "|");
Gdx.app.log(TAG, CLASS_NAME + ".process(): |" + Float.toString(rotationMatrix.val[3]) + ", " + Float.toString(rotationMatrix.val[7]) + ", " + Float.toString(rotationMatrix.val[11]) + ", " + Float.toString(rotationMatrix.val[15]) + "|");
scalingMatrix.setToScaling(geometry.scaling);
combinedTransformationMatrix.idt().mul(translationMatrix).mul(rotationMatrix).mul(scalingMatrix);
@@ -123,6 +113,8 @@ public class MarkerRenderingSystem extends EntityProcessingSystem {
shaderComp.shader.setUniforms();
meshComp.model.render(shaderComp.shader.getShaderProgram(), GL20.GL_TRIANGLES);
}shaderComp.shader.getShaderProgram().end();
break;
}
}else{
Gdx.app.log(TAG, CLASS_NAME + ".process(): Skipping marker number " + Integer.toString(i) + ".");