From 855b25acc20d38dfb98eff8bf73fbc441e174a92 Mon Sep 17 00:00:00 2001 From: Eino-Ville Talvala Date: Fri, 30 Sep 2011 17:53:58 -0700 Subject: [PATCH] Updates to frame transform handling in several source filters. - Affects Camera, Media, and SurfaceTextureSource - Pre-transform source rectangle instead of doing a fragment shader texture coordinate multiply for each pixel. Bug: 5318657 - Add output orientation port to MediaSource. Bug: 5355973 Change-Id: I446a1f0afd64f06638d17010d81c7fc838612feb --- .../java/android/filterfw/core/ShaderProgram.java | 28 ++--- mca/filterfw/jni/jni_shader_program.cpp | 40 ++++---- mca/filterfw/jni/jni_shader_program.h | 40 ++++---- mca/filterpacks/videosrc/java/CameraSource.java | 25 +++-- mca/filterpacks/videosrc/java/MediaSource.java | 113 ++++++++++++++++++--- .../videosrc/java/SurfaceTextureSource.java | 28 +++-- 6 files changed, 189 insertions(+), 85 deletions(-) diff --git a/mca/filterfw/java/android/filterfw/core/ShaderProgram.java b/mca/filterfw/java/android/filterfw/core/ShaderProgram.java index 00d60d1b..a971cb6b 100644 --- a/mca/filterfw/java/android/filterfw/core/ShaderProgram.java +++ b/mca/filterfw/java/android/filterfw/core/ShaderProgram.java @@ -168,25 +168,25 @@ public class ShaderProgram extends Program { } public void setSourceRegion(Quad region) { - setShaderSourceRegion(region.p0.x, region.p0.y, - region.p1.x, region.p1.y, - region.p2.x, region.p2.y, - region.p3.x, region.p3.y); + setSourceRegion(region.p0.x, region.p0.y, + region.p1.x, region.p1.y, + region.p2.x, region.p2.y, + region.p3.x, region.p3.y); } public void setTargetRegion(Quad region) { - setShaderTargetRegion(region.p0.x, region.p0.y, - region.p1.x, region.p1.y, - region.p2.x, region.p2.y, - region.p3.x, region.p3.y); + setTargetRegion(region.p0.x, region.p0.y, + region.p1.x, region.p1.y, + region.p2.x, region.p2.y, + region.p3.x, region.p3.y); } public void setSourceRect(float x, float y, float width, float height) { - setShaderSourceRegion(x, y, x + width, y, x, y + height, x + width, y + height); + setSourceRegion(x, y, x + width, y, x, y + height, x + width, y + height); } public void setTargetRect(float x, float y, float width, float height) { - setShaderTargetRegion(x, y, x + width, y, x, y + height, x + width, y + height); + setTargetRegion(x, y, x + width, y, x, y + height, x + width, y + height); } public void setClearsOutput(boolean clears) { @@ -262,11 +262,11 @@ public class ShaderProgram extends Program { private native Object getUniformValue(String name); - private native boolean setShaderSourceRegion(float x0, float y0, float x1, float y1, - float x2, float y2, float x3, float y3); + public native boolean setSourceRegion(float x0, float y0, float x1, float y1, + float x2, float y2, float x3, float y3); - private native boolean setShaderTargetRegion(float x0, float y0, float x1, float y1, - float x2, float y2, float x3, float y3); + private native boolean setTargetRegion(float x0, float y0, float x1, float y1, + float x2, float y2, float x3, float y3); private static native ShaderProgram nativeCreateIdentity(GLEnvironment glEnv); diff --git a/mca/filterfw/jni/jni_shader_program.cpp b/mca/filterfw/jni/jni_shader_program.cpp index 56c59400..f8048746 100644 --- a/mca/filterfw/jni/jni_shader_program.cpp +++ b/mca/filterfw/jni/jni_shader_program.cpp @@ -140,16 +140,16 @@ jobject Java_android_filterfw_core_ShaderProgram_nativeCreateIdentity(JNIEnv* en return program ? WrapNewObjectInJava(program, env, false) : NULL; } -jboolean Java_android_filterfw_core_ShaderProgram_setShaderSourceRegion(JNIEnv* env, - jobject thiz, - jfloat x0, - jfloat y0, - jfloat x1, - jfloat y1, - jfloat x2, - jfloat y2, - jfloat x3, - jfloat y3) { +jboolean Java_android_filterfw_core_ShaderProgram_setSourceRegion(JNIEnv* env, + jobject thiz, + jfloat x0, + jfloat y0, + jfloat x1, + jfloat y1, + jfloat x2, + jfloat y2, + jfloat x3, + jfloat y3) { ShaderProgram* program = ConvertFromJava(env, thiz); if (program) { program->SetSourceRegion(Quad(Point(x0, y0), Point(x1, y1), Point(x2, y2), Point(x3, y3))); @@ -158,16 +158,16 @@ jboolean Java_android_filterfw_core_ShaderProgram_setShaderSourceRegion(JNIEnv* return JNI_FALSE; } -jboolean Java_android_filterfw_core_ShaderProgram_setShaderTargetRegion(JNIEnv* env, - jobject thiz, - jfloat x0, - jfloat y0, - jfloat x1, - jfloat y1, - jfloat x2, - jfloat y2, - jfloat x3, - jfloat y3) { +jboolean Java_android_filterfw_core_ShaderProgram_setTargetRegion(JNIEnv* env, + jobject thiz, + jfloat x0, + jfloat y0, + jfloat x1, + jfloat y1, + jfloat x2, + jfloat y2, + jfloat x3, + jfloat y3) { ShaderProgram* program = ConvertFromJava(env, thiz); if (program) { program->SetTargetRegion(Quad(Point(x0, y0), Point(x1, y1), Point(x2, y2), Point(x3, y3))); diff --git a/mca/filterfw/jni/jni_shader_program.h b/mca/filterfw/jni/jni_shader_program.h index 68b9d7b7..94a1dd4c 100644 --- a/mca/filterfw/jni/jni_shader_program.h +++ b/mca/filterfw/jni/jni_shader_program.h @@ -61,28 +61,28 @@ Java_android_filterfw_core_ShaderProgram_nativeCreateIdentity(JNIEnv* env, jobject gl_env); JNIEXPORT jboolean JNICALL -Java_android_filterfw_core_ShaderProgram_setShaderSourceRegion(JNIEnv* env, - jobject thiz, - jfloat x0, - jfloat y0, - jfloat x1, - jfloat y1, - jfloat x2, - jfloat y2, - jfloat x3, - jfloat y3); +Java_android_filterfw_core_ShaderProgram_setSourceRegion(JNIEnv* env, + jobject thiz, + jfloat x0, + jfloat y0, + jfloat x1, + jfloat y1, + jfloat x2, + jfloat y2, + jfloat x3, + jfloat y3); JNIEXPORT jboolean JNICALL -Java_android_filterfw_core_ShaderProgram_setShaderTargetRegion(JNIEnv* env, - jobject thiz, - jfloat x0, - jfloat y0, - jfloat x1, - jfloat y1, - jfloat x2, - jfloat y2, - jfloat x3, - jfloat y3); +Java_android_filterfw_core_ShaderProgram_setTargetRegion(JNIEnv* env, + jobject thiz, + jfloat x0, + jfloat y0, + jfloat x1, + jfloat y1, + jfloat x2, + jfloat y2, + jfloat x3, + jfloat y3); JNIEXPORT jboolean JNICALL Java_android_filterfw_core_ShaderProgram_setShaderClearsOutput(JNIEnv* env, diff --git a/mca/filterpacks/videosrc/java/CameraSource.java b/mca/filterpacks/videosrc/java/CameraSource.java index 5abde92c..2c474abd 100644 --- a/mca/filterpacks/videosrc/java/CameraSource.java +++ b/mca/filterpacks/videosrc/java/CameraSource.java @@ -35,6 +35,7 @@ import android.filterfw.format.ImageFormat; import android.graphics.SurfaceTexture; import android.hardware.Camera; import android.os.ConditionVariable; +import android.opengl.Matrix; import java.io.IOException; import java.util.List; @@ -78,7 +79,15 @@ public class CameraSource extends Filter { private SurfaceTexture mSurfaceTexture; private ShaderProgram mFrameExtractor; private MutableFrameFormat mOutputFormat; + private float[] mCameraTransform; + private float[] mMappedCoords; + // These default source coordinates perform the necessary flip + // for converting from OpenGL origin to MFF/Bitmap origin. + private static final float[] mSourceCoords = { 0, 1, 0, 1, + 1, 1, 0, 1, + 0, 0, 0, 1, + 1, 0, 0, 1 }; private static final int NEWFRAME_TIMEOUT = 100; //ms private static final int NEWFRAME_TIMEOUT_REPEAT = 10; @@ -90,12 +99,10 @@ public class CameraSource extends Filter { private static final String mFrameShader = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + - "uniform mat4 camera_transform;\n" + "uniform samplerExternalOES tex_sampler_0;\n" + "varying vec2 v_texcoord;\n" + "void main() {\n" + - " vec2 transformed_texcoord = (camera_transform * vec4(v_texcoord, 0., 1.) ).xy;" + - " gl_FragColor = texture2D(tex_sampler_0, transformed_texcoord);\n" + + " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + "}\n"; private final boolean mLogVerbose; @@ -104,6 +111,7 @@ public class CameraSource extends Filter { public CameraSource(String name) { super(name); mCameraTransform = new float[16]; + mMappedCoords = new float[16]; mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); } @@ -126,9 +134,6 @@ public class CameraSource extends Filter { if (mLogVerbose) Log.v(TAG, "Preparing"); // Compile shader TODO: Move to onGLEnvSomething? mFrameExtractor = new ShaderProgram(context, mFrameShader); - // SurfaceTexture defines (0,0) to be bottom-left. The filter framework defines - // (0,0) as top-left, so do the flip here. - mFrameExtractor.setSourceRect(0, 1, 1, -1); } @Override @@ -187,7 +192,13 @@ public class CameraSource extends Filter { if (mLogVerbose) Log.v(TAG, "Using frame extractor in thread: " + Thread.currentThread()); mSurfaceTexture.getTransformMatrix(mCameraTransform); - mFrameExtractor.setHostValue("camera_transform", mCameraTransform); + Matrix.multiplyMM(mMappedCoords, 0, + mCameraTransform, 0, + mSourceCoords, 0); + mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], + mMappedCoords[4], mMappedCoords[5], + mMappedCoords[8], mMappedCoords[9], + mMappedCoords[12], mMappedCoords[13]); Frame output = context.getFrameManager().newFrame(mOutputFormat); mFrameExtractor.process(mCameraFrame, output); diff --git a/mca/filterpacks/videosrc/java/MediaSource.java b/mca/filterpacks/videosrc/java/MediaSource.java index cb9192bf..76d6502f 100644 --- a/mca/filterpacks/videosrc/java/MediaSource.java +++ b/mca/filterpacks/videosrc/java/MediaSource.java @@ -36,6 +36,7 @@ import android.filterfw.format.ImageFormat; import android.graphics.SurfaceTexture; import android.media.MediaPlayer; import android.os.ConditionVariable; +import android.opengl.Matrix; import android.view.Surface; import java.io.IOException; @@ -54,7 +55,8 @@ public class MediaSource extends Filter { /** User-visible parameters */ /** The source URL for the media source. Can be an http: link to a remote - * resource, or a file: link to a local media file */ + * resource, or a file: link to a local media file + */ @GenerateFieldPort(name = "sourceUrl", hasDefault = true) private String mSourceUrl = ""; @@ -62,30 +64,43 @@ public class MediaSource extends Filter { @GenerateFieldPort(name = "sourceAsset", hasDefault = true) private AssetFileDescriptor mSourceAsset = null; - /** Whether the media source is a URL or an asset file descriptor. Defaults to false. */ + /** Whether the media source is a URL or an asset file descriptor. Defaults + * to false. + */ @GenerateFieldPort(name = "sourceIsUrl", hasDefault = true) private boolean mSelectedIsUrl = false; /** Whether the filter will always wait for a new video frame, or whether it - * will output an old frame again if a new frame isn't available. Defaults to true. + * will output an old frame again if a new frame isn't available. Defaults + * to true. */ @GenerateFinalPort(name = "waitForNewFrame", hasDefault = true) private boolean mWaitForNewFrame = true; - /** Whether the media source should loop automatically or not. Defaults to true. */ + /** Whether the media source should loop automatically or not. Defaults to + * true. + */ @GenerateFieldPort(name = "loop", hasDefault = true) private boolean mLooping = true; - /** Volume control. Currently sound is piped directly to the speakers, so this defaults to mute. */ + /** Volume control. Currently sound is piped directly to the speakers, so + * this defaults to mute. + */ @GenerateFieldPort(name = "volume", hasDefault = true) private float mVolume = 0.f; + /** Orientation. This controls the output orientation of the video. Valid + * values are 0, 90, 180, 270 + */ + @GenerateFieldPort(name = "orientation", hasDefault = true) + private int mOrientation = 0; + private MediaPlayer mMediaPlayer; private GLFrame mMediaFrame; private SurfaceTexture mSurfaceTexture; private ShaderProgram mFrameExtractor; private MutableFrameFormat mOutputFormat; - private float[] mFrameTransform; + private int mWidth, mHeight; // Total timeouts will be PREP_TIMEOUT*PREP_TIMEOUT_REPEAT private static final int PREP_TIMEOUT = 100; // ms @@ -93,21 +108,40 @@ public class MediaSource extends Filter { private static final int NEWFRAME_TIMEOUT = 100; //ms private static final int NEWFRAME_TIMEOUT_REPEAT = 10; + // This is an identity shader; not using the default identity + // shader because reading from a SurfaceTexture requires the + // GL_OES_EGL_image_external extension. private final String mFrameShader = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + - "uniform mat4 frame_transform;\n" + "uniform samplerExternalOES tex_sampler_0;\n" + "varying vec2 v_texcoord;\n" + "void main() {\n" + - " vec2 transformed_texcoord = (frame_transform * vec4(v_texcoord, 0., 1.) ).xy;" + - " gl_FragColor = texture2D(tex_sampler_0, transformed_texcoord);\n" + + " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + "}\n"; + private static final float[] mSourceCoords_0 = { 1, 1, 0, 1, + 0, 1, 0, 1, + 1, 0, 0, 1, + 0, 0, 0, 1 }; + private static final float[] mSourceCoords_90 = { 0, 1, 0, 1, + 0, 0, 0, 1, + 1, 1, 0, 1, + 1, 0, 0, 1 }; + private static final float[] mSourceCoords_180 = { 0, 0, 0, 1, + 1, 0, 0, 1, + 0, 1, 0, 1, + 1, 1, 0, 1 }; + private static final float[] mSourceCoords_270 = { 1, 0, 0, 1, + 1, 1, 0, 1, + 0, 0, 0, 1, + 0, 1, 0, 1 }; + private boolean mGotSize; private boolean mPrepared; private boolean mPlaying; private boolean mNewFrameAvailable; + private boolean mOrientationUpdated; private boolean mPaused; private boolean mCompleted; @@ -117,7 +151,6 @@ public class MediaSource extends Filter { public MediaSource(String name) { super(name); mNewFrameAvailable = false; - mFrameTransform = new float[16]; mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); } @@ -239,9 +272,41 @@ public class MediaSource extends Filter { } mSurfaceTexture.updateTexImage(); - - mSurfaceTexture.getTransformMatrix(mFrameTransform); - mFrameExtractor.setHostValue("frame_transform", mFrameTransform); + mOrientationUpdated = true; + } + if (mOrientationUpdated) { + float[] surfaceTransform = new float[16]; + mSurfaceTexture.getTransformMatrix(surfaceTransform); + + float[] sourceCoords = new float[16]; + switch (mOrientation) { + default: + case 0: + Matrix.multiplyMM(sourceCoords, 0, + surfaceTransform, 0, + mSourceCoords_0, 0); + break; + case 90: + Matrix.multiplyMM(sourceCoords, 0, + surfaceTransform, 0, + mSourceCoords_90, 0); + break; + case 180: + Matrix.multiplyMM(sourceCoords, 0, + surfaceTransform, 0, + mSourceCoords_180, 0); + break; + case 270: + Matrix.multiplyMM(sourceCoords, 0, + surfaceTransform, 0, + mSourceCoords_270, 0); + break; + } + mFrameExtractor.setSourceRegion(sourceCoords[0], sourceCoords[1], + sourceCoords[4], sourceCoords[5], + sourceCoords[8], sourceCoords[9], + sourceCoords[12], sourceCoords[13]); + mOrientationUpdated = false; } Frame output = context.getFrameManager().newFrame(mOutputFormat); @@ -262,6 +327,13 @@ public class MediaSource extends Filter { if (mMediaPlayer.isPlaying()) { mMediaPlayer.stop(); } + mPrepared = false; + mGotSize = false; + mPlaying = false; + mPaused = false; + mCompleted = false; + mNewFrameAvailable = false; + mMediaPlayer.release(); mMediaPlayer = null; mSurfaceTexture.release(); @@ -316,6 +388,13 @@ public class MediaSource extends Filter { if (isOpen()) { mMediaPlayer.setVolume(mVolume, mVolume); } + } else if (name.equals("orientation") && mGotSize) { + if (mOrientation == 0 || mOrientation == 180) { + mOutputFormat.setDimensions(mWidth, mHeight); + } else { + mOutputFormat.setDimensions(mHeight, mWidth); + } + mOrientationUpdated = true; } } @@ -410,7 +489,13 @@ public class MediaSource extends Filter { public void onVideoSizeChanged(MediaPlayer mp, int width, int height) { if (mLogVerbose) Log.v(TAG, "MediaPlayer sent dimensions: " + width + " x " + height); if (!mGotSize) { - mOutputFormat.setDimensions(width, height); + if (mOrientation == 0 || mOrientation == 180) { + mOutputFormat.setDimensions(width, height); + } else { + mOutputFormat.setDimensions(height, width); + } + mWidth = width; + mHeight = height; } else { if (mOutputFormat.getWidth() != width || mOutputFormat.getHeight() != height) { diff --git a/mca/filterpacks/videosrc/java/SurfaceTextureSource.java b/mca/filterpacks/videosrc/java/SurfaceTextureSource.java index f68b87a0..29eaa016 100644 --- a/mca/filterpacks/videosrc/java/SurfaceTextureSource.java +++ b/mca/filterpacks/videosrc/java/SurfaceTextureSource.java @@ -33,6 +33,7 @@ import android.filterfw.format.ImageFormat; import android.graphics.SurfaceTexture; import android.media.MediaPlayer; import android.os.ConditionVariable; +import android.opengl.Matrix; import java.io.IOException; import java.io.FileDescriptor; @@ -115,19 +116,24 @@ public class SurfaceTextureSource extends Filter { private SurfaceTexture mSurfaceTexture; private MutableFrameFormat mOutputFormat; private ConditionVariable mNewFrameAvailable; - private float[] mFrameTransform; private boolean mFirstFrame; + private float[] mFrameTransform; + private float[] mMappedCoords; + // These default source coordinates perform the necessary flip + // for converting from MFF/Bitmap origin to OpenGL origin. + private static final float[] mSourceCoords = { 0, 1, 0, 1, + 1, 1, 0, 1, + 0, 0, 0, 1, + 1, 0, 0, 1 }; // Shader for output private final String mRenderShader = "#extension GL_OES_EGL_image_external : require\n" + "precision mediump float;\n" + - "uniform mat4 frame_transform;\n" + "uniform samplerExternalOES tex_sampler_0;\n" + "varying vec2 v_texcoord;\n" + "void main() {\n" + - " vec2 transformed_texcoord = (frame_transform * vec4(v_texcoord, 0., 1.) ).xy;" + - " gl_FragColor = texture2D(tex_sampler_0, transformed_texcoord);\n" + + " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" + "}\n"; // Variables for logging @@ -139,6 +145,7 @@ public class SurfaceTextureSource extends Filter { super(name); mNewFrameAvailable = new ConditionVariable(); mFrameTransform = new float[16]; + mMappedCoords = new float[16]; } @Override @@ -167,9 +174,6 @@ public class SurfaceTextureSource extends Filter { // Prepare output mFrameExtractor = new ShaderProgram(context, mRenderShader); - // SurfaceTexture defines (0,0) to be bottom-left. The filter framework - // defines (0,0) as top-left, so do the flip here. - mFrameExtractor.setSourceRect(0, 1, 1, -1); } @Override @@ -212,10 +216,14 @@ public class SurfaceTextureSource extends Filter { mSurfaceTexture.updateTexImage(); mSurfaceTexture.getTransformMatrix(mFrameTransform); - + Matrix.multiplyMM(mMappedCoords, 0, + mFrameTransform, 0, + mSourceCoords, 0); + mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1], + mMappedCoords[4], mMappedCoords[5], + mMappedCoords[8], mMappedCoords[9], + mMappedCoords[12], mMappedCoords[13]); // Next, render to output - mFrameExtractor.setHostValue("frame_transform", mFrameTransform); - Frame output = context.getFrameManager().newFrame(mOutputFormat); mFrameExtractor.process(mMediaFrame, output); -- 2.11.0