From 1e30a1948335aab752cf22486a80f6f74f309386 Mon Sep 17 00:00:00 2001 From: Eino-Ville Talvala Date: Sun, 21 Aug 2011 13:49:45 -0700 Subject: [PATCH] Several improvements for interacting with Camera app. - Add support for extra library loading in FilterFactory - Add support for listing custom libraries in graph files - Add new SurfaceTargetFilter for output to a Surface managed by the app. Bug: 4510826 Change-Id: Icfd350d8f83627934549c30cdfb52cd1a21d3e72 --- .../java/android/filterfw/core/FilterFactory.java | 43 +++- .../java/android/filterfw/io/TextGraphReader.java | 52 +++-- mca/filterpacks/ui/java/SurfaceTargetFilter.java | 257 +++++++++++++++++++++ 3 files changed, 332 insertions(+), 20 deletions(-) create mode 100644 mca/filterpacks/ui/java/SurfaceTargetFilter.java diff --git a/mca/filterfw/java/android/filterfw/core/FilterFactory.java b/mca/filterfw/java/android/filterfw/core/FilterFactory.java index aad4487a..779df990 100644 --- a/mca/filterfw/java/android/filterfw/core/FilterFactory.java +++ b/mca/filterfw/java/android/filterfw/core/FilterFactory.java @@ -20,6 +20,8 @@ package android.filterfw.core; import android.filterfw.core.Filter; import android.util.Log; +import dalvik.system.PathClassLoader; + import java.lang.reflect.Constructor; import java.lang.ClassLoader; import java.lang.Thread; @@ -33,20 +35,48 @@ public class FilterFactory { private static FilterFactory mSharedFactory; private HashSet mPackages = new HashSet(); - private boolean mLogVerbose; + private static ClassLoader mCurrentClassLoader; + private static HashSet mLibraries; + private static Object mClassLoaderGuard; + + static { + mCurrentClassLoader = Thread.currentThread().getContextClassLoader(); + mLibraries = new HashSet(); + mClassLoaderGuard = new Object(); + } + private static final String TAG = "FilterFactory"; + private static boolean mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); public static FilterFactory sharedFactory() { if (mSharedFactory == null) { mSharedFactory = new FilterFactory(); - mSharedFactory.mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); } return mSharedFactory; } + /** + * Adds a new Java library to the list to be scanned for filters. + * libraryPath must be an absolute path of the jar file. This needs to be + * static because only one classloader per process can open a shared native + * library, which a filter may well have. + */ + public static void addFilterLibrary(String libraryPath) { + if (mLogVerbose) Log.v(TAG, "Adding filter library " + libraryPath); + synchronized(mClassLoaderGuard) { + if (mLibraries.contains(libraryPath)) { + if (mLogVerbose) Log.v(TAG, "Library already added"); + return; + } + mLibraries.add(libraryPath); + // Chain another path loader to the current chain + mCurrentClassLoader = new PathClassLoader(libraryPath, mCurrentClassLoader); + } + } + public void addPackage(String packageName) { if (mLogVerbose) Log.v(TAG, "Adding package " + packageName); - /* + /* TODO: This should use a getPackage call in the caller's context, but no such method exists. Package pkg = Package.getPackage(packageName); if (pkg == null) { throw new IllegalArgumentException("Unknown filter package '" + packageName + "'!"); @@ -58,13 +88,14 @@ public class FilterFactory { public Filter createFilterByClassName(String className, String filterName) { if (mLogVerbose) Log.v(TAG, "Looking up class " + className); Class filterClass = null; - // Get context's classloader; otherwise cannot load non-framework filters - ClassLoader contextClassLoader = Thread.currentThread().getContextClassLoader(); + // Look for the class in the imported packages for (String packageName : mPackages) { try { if (mLogVerbose) Log.v(TAG, "Trying "+packageName + "." + className); - filterClass = contextClassLoader.loadClass(packageName + "." + className); + synchronized(mClassLoaderGuard) { + filterClass = mCurrentClassLoader.loadClass(packageName + "." + className); + } } catch (ClassNotFoundException e) { continue; } diff --git a/mca/filterfw/java/android/filterfw/io/TextGraphReader.java b/mca/filterfw/java/android/filterfw/io/TextGraphReader.java index 015d6c48..366ef822 100644 --- a/mca/filterfw/java/android/filterfw/io/TextGraphReader.java +++ b/mca/filterfw/java/android/filterfw/io/TextGraphReader.java @@ -66,6 +66,19 @@ public class TextGraphReader extends GraphReader { } } + private class AddLibraryCommand implements Command { + private String mLibraryName; + + public AddLibraryCommand(String libraryName) { + mLibraryName = libraryName; + } + + @Override + public void execute(TextGraphReader reader) { + reader.mFactory.addFilterLibrary(mLibraryName); + } + } + private class AllocateFilterCommand implements Command { private String mClassName; private String mFilterName; @@ -159,6 +172,7 @@ public class TextGraphReader extends GraphReader { final Pattern curlyOpenPattern = Pattern.compile("\\{"); final Pattern ignorePattern = Pattern.compile("(\\s+|//[^\\n]*\\n)+"); final Pattern packageNamePattern = Pattern.compile("[a-zA-Z\\.]+"); + final Pattern libraryNamePattern = Pattern.compile("[a-zA-Z\\./:]+"); final Pattern portPattern = Pattern.compile("\\[[a-zA-Z0-9\\-_]+\\]"); final Pattern rightArrowPattern = Pattern.compile("=>"); final Pattern semicolonPattern = Pattern.compile(";"); @@ -166,20 +180,21 @@ public class TextGraphReader extends GraphReader { final int STATE_COMMAND = 0; final int STATE_IMPORT_PKG = 1; - final int STATE_FILTER_CLASS = 2; - final int STATE_FILTER_NAME = 3; - final int STATE_CURLY_OPEN = 4; - final int STATE_PARAMETERS = 5; - final int STATE_CURLY_CLOSE = 6; - final int STATE_SOURCE_FILTERNAME = 7; - final int STATE_SOURCE_PORT = 8; - final int STATE_RIGHT_ARROW = 9; - final int STATE_TARGET_FILTERNAME = 10; - final int STATE_TARGET_PORT = 11; - final int STATE_ASSIGNMENT = 12; - final int STATE_EXTERNAL = 13; - final int STATE_SETTING = 14; - final int STATE_SEMICOLON = 15; + final int STATE_ADD_LIBRARY = 2; + final int STATE_FILTER_CLASS = 3; + final int STATE_FILTER_NAME = 4; + final int STATE_CURLY_OPEN = 5; + final int STATE_PARAMETERS = 6; + final int STATE_CURLY_CLOSE = 7; + final int STATE_SOURCE_FILTERNAME = 8; + final int STATE_SOURCE_PORT = 9; + final int STATE_RIGHT_ARROW = 10; + final int STATE_TARGET_FILTERNAME = 11; + final int STATE_TARGET_PORT = 12; + final int STATE_ASSIGNMENT = 13; + final int STATE_EXTERNAL = 14; + final int STATE_SETTING = 15; + final int STATE_SEMICOLON = 16; int state = STATE_COMMAND; PatternScanner scanner = new PatternScanner(graphString, ignorePattern); @@ -197,6 +212,8 @@ public class TextGraphReader extends GraphReader { String curCommand = scanner.eat(commandPattern, ""); if (curCommand.equals("@import")) { state = STATE_IMPORT_PKG; + } else if (curCommand.equals("@library")) { + state = STATE_ADD_LIBRARY; } else if (curCommand.equals("@filter")) { state = STATE_FILTER_CLASS; } else if (curCommand.equals("@connect")) { @@ -220,6 +237,13 @@ public class TextGraphReader extends GraphReader { break; } + case STATE_ADD_LIBRARY: { + String libraryName = scanner.eat(libraryNamePattern, ""); + mCommands.add(new AddLibraryCommand(libraryName)); + state = STATE_SEMICOLON; + break; + } + case STATE_FILTER_CLASS: curClassName = scanner.eat(wordPattern, ""); state = STATE_FILTER_NAME; diff --git a/mca/filterpacks/ui/java/SurfaceTargetFilter.java b/mca/filterpacks/ui/java/SurfaceTargetFilter.java new file mode 100644 index 00000000..308d1686 --- /dev/null +++ b/mca/filterpacks/ui/java/SurfaceTargetFilter.java @@ -0,0 +1,257 @@ +/* + * Copyright (C) 2011 The Android Open Source Project + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + + +package android.filterpacks.ui; + +import android.filterfw.core.Filter; +import android.filterfw.core.FilterContext; +import android.filterfw.core.Frame; +import android.filterfw.core.FrameFormat; +import android.filterfw.core.GenerateFieldPort; +import android.filterfw.core.GenerateFinalPort; +import android.filterfw.core.GLEnvironment; +import android.filterfw.core.GLFrame; +import android.filterfw.core.KeyValueMap; +import android.filterfw.core.MutableFrameFormat; +import android.filterfw.core.NativeProgram; +import android.filterfw.core.NativeFrame; +import android.filterfw.core.Program; +import android.filterfw.core.ShaderProgram; +import android.filterfw.format.ImageFormat; + +import android.view.Surface; +import android.view.SurfaceHolder; +import android.view.SurfaceView; + +import android.graphics.Rect; + +import android.util.Log; + +/** + * @hide + */ +public class SurfaceTargetFilter extends Filter { + + private final int RENDERMODE_STRETCH = 0; + private final int RENDERMODE_FIT = 1; + private final int RENDERMODE_FILL_CROP = 2; + + /** Required. Sets the destination surface for this node. This assumes that + * higher-level code is ensuring that the surface is valid, and properly + * updates Surface parameters if they change. + */ + @GenerateFinalPort(name = "surface") + private Surface mSurface; + + /** Required. Width of the output surface */ + @GenerateFieldPort(name = "owidth") + private int mScreenWidth; + + /** Required. Height of the output surface */ + @GenerateFieldPort(name = "oheight") + private int mScreenHeight; + + /** Optional. Control how the incoming frames are rendered onto the + * output. Default is FIT. + * RENDERMODE_STRETCH: Just fill the output surfaceView. + * RENDERMODE_FIT: Keep aspect ratio and fit without cropping. May + * have black bars. + * RENDERMODE_FILL_CROP: Keep aspect ratio and fit without black + * bars. May crop. + */ + @GenerateFieldPort(name = "renderMode", hasDefault = true) + private String mRenderModeString; + + private ShaderProgram mProgram; + private GLEnvironment mGlEnv; + private GLFrame mScreen; + private int mRenderMode = RENDERMODE_FIT; + private float mAspectRatio = 1.f; + + private int mSurfaceId = -1; + + private boolean mLogVerbose; + private static final String TAG = "SurfaceRenderFilter"; + + public SurfaceTargetFilter(String name) { + super(name); + + mLogVerbose = Log.isLoggable(TAG, Log.VERBOSE); + } + + @Override + public void setupPorts() { + // Make sure we have a Surface + if (mSurface == null) { + throw new RuntimeException("NULL Surface passed to SurfaceTargetFilter"); + } + + // Add input port + addMaskedInputPort("frame", ImageFormat.create(ImageFormat.COLORSPACE_RGBA)); + } + + public void updateRenderMode() { + if (mRenderModeString != null) { + if (mRenderModeString.equals("stretch")) { + mRenderMode = RENDERMODE_STRETCH; + } else if (mRenderModeString.equals("fit")) { + mRenderMode = RENDERMODE_FIT; + } else if (mRenderModeString.equals("fill_crop")) { + mRenderMode = RENDERMODE_FILL_CROP; + } else { + throw new RuntimeException("Unknown render mode '" + mRenderModeString + "'!"); + } + } + updateTargetRect(); + } + + @Override + public void prepare(FilterContext context) { + mGlEnv = context.getGLEnvironment(); + + // Create identity shader to render, and make sure to render upside-down, as textures + // are stored internally bottom-to-top. + mProgram = ShaderProgram.createIdentity(context); + mProgram.setSourceRect(0, 1, 1, -1); + mProgram.setClearsOutput(true); + mProgram.setClearColor(0.0f, 0.0f, 0.0f); + + MutableFrameFormat screenFormat = ImageFormat.create(mScreenWidth, + mScreenHeight, + ImageFormat.COLORSPACE_RGBA, + FrameFormat.TARGET_GPU); + mScreen = (GLFrame)context.getFrameManager().newBoundFrame(screenFormat, + GLFrame.EXISTING_FBO_BINDING, + 0); + + // Set up cropping + updateRenderMode(); + } + + @Override + public void open(FilterContext context) { + registerSurface(); + } + + @Override + public void process(FilterContext context) { + if (mLogVerbose) Log.v(TAG, "Starting frame processing"); + + // Get input frame + Frame input = pullInput("frame"); + boolean createdFrame = false; + + float currentAspectRatio = (float)input.getFormat().getWidth() / input.getFormat().getHeight(); + if (currentAspectRatio != mAspectRatio) { + if (mLogVerbose) Log.v(TAG, "New aspect ratio: " + currentAspectRatio +", previously: " + mAspectRatio); + mAspectRatio = currentAspectRatio; + updateTargetRect(); + } + + // See if we need to copy to GPU + Frame gpuFrame = null; + if (mLogVerbose) Log.v("SurfaceRenderFilter", "Got input format: " + input.getFormat()); + int target = input.getFormat().getTarget(); + if (target != FrameFormat.TARGET_GPU) { + gpuFrame = context.getFrameManager().duplicateFrameToTarget(input, + FrameFormat.TARGET_GPU); + createdFrame = true; + } else { + gpuFrame = input; + } + + // Activate our surface + mGlEnv.activateSurfaceWithId(mSurfaceId); + + // Process + mProgram.process(gpuFrame, mScreen); + + // And swap buffers + mGlEnv.swapBuffers(); + + if (createdFrame) { + gpuFrame.release(); + } + } + + @Override + public void fieldPortValueUpdated(String name, FilterContext context) { + mScreen.setViewport(0, 0, mScreenWidth, mScreenHeight); + updateTargetRect(); + } + + @Override + public void close(FilterContext context) { + unregisterSurface(); + } + + @Override + public void tearDown(FilterContext context) { + if (mScreen != null) { + mScreen.release(); + } + } + + private void updateTargetRect() { + if (mScreenWidth > 0 && mScreenHeight > 0 && mProgram != null) { + float screenAspectRatio = (float)mScreenWidth / mScreenHeight; + float relativeAspectRatio = screenAspectRatio / mAspectRatio; + + switch (mRenderMode) { + case RENDERMODE_STRETCH: + mProgram.setTargetRect(0, 0, 1, 1); + break; + case RENDERMODE_FIT: + if (relativeAspectRatio > 1.0f) { + // Screen is wider than the camera, scale down X + mProgram.setTargetRect(0.5f - 0.5f / relativeAspectRatio, 0.0f, + 1.0f / relativeAspectRatio, 1.0f); + } else { + // Screen is taller than the camera, scale down Y + mProgram.setTargetRect(0.0f, 0.5f - 0.5f * relativeAspectRatio, + 1.0f, relativeAspectRatio); + } + break; + case RENDERMODE_FILL_CROP: + if (relativeAspectRatio > 1) { + // Screen is wider than the camera, crop in Y + mProgram.setTargetRect(0.0f, 0.5f - 0.5f * relativeAspectRatio, + 1.0f, relativeAspectRatio); + } else { + // Screen is taller than the camera, crop in X + mProgram.setTargetRect(0.5f - 0.5f / relativeAspectRatio, 0.0f, + 1.0f / relativeAspectRatio, 1.0f); + } + break; + } + } + } + + private void registerSurface() { + mSurfaceId = mGlEnv.registerSurface(mSurface); + if (mSurfaceId < 0) { + throw new RuntimeException("Could not register Surface: " + mSurface); + } + } + + private void unregisterSurface() { + if (mSurfaceId > 0) { + mGlEnv.unregisterSurfaceId(mSurfaceId); + } + } + +} -- 2.11.0