2 * Copyright (C) 2011 The Android Open Source Project
4 * Licensed under the Apache License, Version 2.0 (the "License");
5 * you may not use this file except in compliance with the License.
6 * You may obtain a copy of the License at
8 * http://www.apache.org/licenses/LICENSE-2.0
10 * Unless required by applicable law or agreed to in writing, software
11 * distributed under the License is distributed on an "AS IS" BASIS,
12 * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
13 * See the License for the specific language governing permissions and
14 * limitations under the License.
17 package android.filterpacks.videosrc;
19 import android.content.Context;
20 import android.content.res.AssetFileDescriptor;
21 import android.filterfw.core.Filter;
22 import android.filterfw.core.FilterContext;
23 import android.filterfw.core.Frame;
24 import android.filterfw.core.FrameFormat;
25 import android.filterfw.core.FrameManager;
26 import android.filterfw.core.GenerateFieldPort;
27 import android.filterfw.core.GenerateFinalPort;
28 import android.filterfw.core.GLFrame;
29 import android.filterfw.core.KeyValueMap;
30 import android.filterfw.core.MutableFrameFormat;
31 import android.filterfw.core.ShaderProgram;
32 import android.filterfw.format.ImageFormat;
33 import android.graphics.SurfaceTexture;
34 import android.media.MediaPlayer;
35 import android.os.ConditionVariable;
36 import android.opengl.Matrix;
38 import java.io.IOException;
39 import java.io.FileDescriptor;
40 import java.lang.IllegalArgumentException;
41 import java.util.List;
44 import android.util.Log;
46 /** <p>A filter that converts textures from a SurfaceTexture object into frames for
47 * processing in the filter framework.</p>
49 * <p>To use, connect up the sourceListener callback, and then when executing
50 * the graph, use the SurfaceTexture object passed to the callback to feed
51 * frames into the filter graph. For example, pass the SurfaceTexture into
53 * android.hardware.Camera.setPreviewTexture(android.graphics.SurfaceTexture)}.
54 * This filter is intended for applications that need for flexibility than the
55 * CameraSource and MediaSource provide. Note that the application needs to
56 * provide width and height information for the SurfaceTextureSource, which it
57 * should obtain from wherever the SurfaceTexture data is coming from to avoid
58 * unnecessary resampling.</p>
62 public class SurfaceTextureSource extends Filter {
64 /** User-visible parameters */
66 /** The callback interface for the sourceListener parameter */
67 public interface SurfaceTextureSourceListener {
68 public void onSurfaceTextureSourceReady(SurfaceTexture source);
70 /** A callback to send the internal SurfaceTexture object to, once it is
71 * created. This callback will be called when the the filter graph is
72 * preparing to execute, but before any processing has actually taken
73 * place. The SurfaceTexture object passed to this callback is the only way
74 * to feed this filter. When the filter graph is shutting down, this
75 * callback will be called again with null as the source.
77 * This callback may be called from an arbitrary thread, so it should not
78 * assume it is running in the UI thread in particular.
80 @GenerateFinalPort(name = "sourceListener")
81 private SurfaceTextureSourceListener mSourceListener;
83 /** The width of the output image frame. If the texture width for the
84 * SurfaceTexture source is known, use it here to minimize resampling. */
85 @GenerateFieldPort(name = "width")
88 /** The height of the output image frame. If the texture height for the
89 * SurfaceTexture source is known, use it here to minimize resampling. */
90 @GenerateFieldPort(name = "height")
93 /** Whether the filter will always wait for a new frame from its
94 * SurfaceTexture, or whether it will output an old frame again if a new
95 * frame isn't available. The filter will always wait for the first frame,
96 * to avoid outputting a blank frame. Defaults to true.
98 @GenerateFieldPort(name = "waitForNewFrame", hasDefault = true)
99 private boolean mWaitForNewFrame = true;
101 /** Maximum timeout before signaling error when waiting for a new frame. Set
102 * this to zero to disable the timeout and wait indefinitely. In milliseconds.
104 @GenerateFieldPort(name = "waitTimeout", hasDefault = true)
105 private int mWaitTimeout = 1000;
107 /** Whether a timeout is an exception-causing failure, or just causes the
110 @GenerateFieldPort(name = "closeOnTimeout", hasDefault = true)
111 private boolean mCloseOnTimeout = false;
113 // Variables for input->output conversion
114 private GLFrame mMediaFrame;
115 private ShaderProgram mFrameExtractor;
116 private SurfaceTexture mSurfaceTexture;
117 private MutableFrameFormat mOutputFormat;
118 private ConditionVariable mNewFrameAvailable;
119 private boolean mFirstFrame;
121 private float[] mFrameTransform;
122 private float[] mMappedCoords;
123 // These default source coordinates perform the necessary flip
124 // for converting from MFF/Bitmap origin to OpenGL origin.
125 private static final float[] mSourceCoords = { 0, 1, 0, 1,
130 private final String mRenderShader =
131 "#extension GL_OES_EGL_image_external : require\n" +
132 "precision mediump float;\n" +
133 "uniform samplerExternalOES tex_sampler_0;\n" +
134 "varying vec2 v_texcoord;\n" +
136 " gl_FragColor = texture2D(tex_sampler_0, v_texcoord);\n" +
139 // Variables for logging
141 private static final String TAG = "SurfaceTextureSource";
142 private static final boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE);
144 public SurfaceTextureSource(String name) {
146 mNewFrameAvailable = new ConditionVariable();
147 mFrameTransform = new float[16];
148 mMappedCoords = new float[16];
152 public void setupPorts() {
154 addOutputPort("video", ImageFormat.create(ImageFormat.COLORSPACE_RGBA,
155 FrameFormat.TARGET_GPU));
158 private void createFormats() {
159 mOutputFormat = ImageFormat.create(mWidth, mHeight,
160 ImageFormat.COLORSPACE_RGBA,
161 FrameFormat.TARGET_GPU);
165 protected void prepare(FilterContext context) {
166 if (mLogVerbose) Log.v(TAG, "Preparing SurfaceTextureSource");
171 mMediaFrame = (GLFrame)context.getFrameManager().newBoundFrame(mOutputFormat,
172 GLFrame.EXTERNAL_TEXTURE,
176 mFrameExtractor = new ShaderProgram(context, mRenderShader);
180 public void open(FilterContext context) {
181 if (mLogVerbose) Log.v(TAG, "Opening SurfaceTextureSource");
182 // Create SurfaceTexture anew each time - it can use substantial memory.
183 mSurfaceTexture = new SurfaceTexture(mMediaFrame.getTextureId());
184 // Connect SurfaceTexture to source
185 mSourceListener.onSurfaceTextureSourceReady(mSurfaceTexture);
186 // Connect SurfaceTexture to callback
187 mSurfaceTexture.setOnFrameAvailableListener(onFrameAvailableListener);
192 public void process(FilterContext context) {
193 if (mLogVerbose) Log.v(TAG, "Processing new frame");
195 // First, get new frame if available
196 if (mWaitForNewFrame || mFirstFrame) {
198 if (mWaitTimeout != 0) {
199 gotNewFrame = mNewFrameAvailable.block(mWaitTimeout);
201 if (!mCloseOnTimeout) {
202 throw new RuntimeException("Timeout waiting for new frame");
204 if (mLogVerbose) Log.v(TAG, "Timeout waiting for a new frame. Closing.");
205 closeOutputPort("video");
210 mNewFrameAvailable.block();
212 mNewFrameAvailable.close();
216 mSurfaceTexture.updateTexImage();
218 mSurfaceTexture.getTransformMatrix(mFrameTransform);
219 Matrix.multiplyMM(mMappedCoords, 0,
222 mFrameExtractor.setSourceRegion(mMappedCoords[0], mMappedCoords[1],
223 mMappedCoords[4], mMappedCoords[5],
224 mMappedCoords[8], mMappedCoords[9],
225 mMappedCoords[12], mMappedCoords[13]);
226 // Next, render to output
227 Frame output = context.getFrameManager().newFrame(mOutputFormat);
228 mFrameExtractor.process(mMediaFrame, output);
230 output.setTimestamp(mSurfaceTexture.getTimestamp());
232 pushOutput("video", output);
237 public void close(FilterContext context) {
238 if (mLogVerbose) Log.v(TAG, "SurfaceTextureSource closed");
239 mSourceListener.onSurfaceTextureSourceReady(null);
240 mSurfaceTexture.release();
241 mSurfaceTexture = null;
245 public void tearDown(FilterContext context) {
246 if (mMediaFrame != null) {
247 mMediaFrame.release();
252 public void fieldPortValueUpdated(String name, FilterContext context) {
253 if (name.equals("width") || name.equals("height") ) {
254 mOutputFormat.setDimensions(mWidth, mHeight);
258 private SurfaceTexture.OnFrameAvailableListener onFrameAvailableListener =
259 new SurfaceTexture.OnFrameAvailableListener() {
260 public void onFrameAvailable(SurfaceTexture surfaceTexture) {
261 if (mLogVerbose) Log.v(TAG, "New frame from SurfaceTextureSource");
262 mNewFrameAvailable.open();