From fbe47df2a41e467db92ad86432b8b726e9e79409 Mon Sep 17 00:00:00 2001 From: chaviw Date: Fri, 10 Nov 2017 16:14:49 -0800 Subject: [PATCH] Snapshot Task with the app window crop for Recents Snapshot a specific task and its children instead of taking a snapshot of the whole screen with max and min layer values. This makes it clear what the desired screenshot is instead of trying to determine what the max and min z layers should be. In the process of updating the snapshot code, I split the code that captures a layer from the code that captures the whole screen. This simplified the code so the capture screen doesn't need to invoke the frame calculations since it will be capturing the whole screen anyway. Test: Recents snapshots work correctly without the IME, status, and navigation bar Change-Id: I8776c1ddb9cd8a23a482b045720960702796fd5f --- core/java/android/view/SurfaceControl.java | 28 ++- core/jni/android_view_SurfaceControl.cpp | 43 +++- .../java/com/android/server/wm/DisplayContent.java | 279 +++------------------ .../android/server/wm/TaskSnapshotController.java | 24 +- .../android/server/wm/WindowManagerService.java | 29 +-- services/tests/servicestests/AndroidManifest.xml | 1 + 6 files changed, 132 insertions(+), 272 deletions(-) diff --git a/core/java/android/view/SurfaceControl.java b/core/java/android/view/SurfaceControl.java index aa2f1c1a9e24..3d01ec23b723 100644 --- a/core/java/android/view/SurfaceControl.java +++ b/core/java/android/view/SurfaceControl.java @@ -56,7 +56,9 @@ public class SurfaceControl { Rect sourceCrop, int width, int height, int minLayer, int maxLayer, boolean allLayers, boolean useIdentityTransform); private static native void nativeCaptureLayers(IBinder layerHandleToken, Surface consumer, - int rotation); + Rect sourceCrop, float frameScale); + private static native GraphicBuffer nativeCaptureLayers(IBinder layerHandleToken, + Rect sourceCrop, float frameScale); private static native long nativeCreateTransaction(); private static native long nativeGetNativeTransactionFinalizer(); @@ -1177,14 +1179,24 @@ public class SurfaceControl { * Captures a layer and its children into the provided {@link Surface}. * * @param layerHandleToken The root layer to capture. - * @param consumer The {@link Surface} to capture the layer into. - * @param rotation Apply a custom clockwise rotation to the screenshot, i.e. - * Surface.ROTATION_0,90,180,270. Surfaceflinger will always capture in its - * native portrait orientation by default, so this is useful for returning - * captures that are independent of device orientation. + * @param consumer The {@link Surface} to capture the layer into. + * @param sourceCrop The portion of the root surface to capture; caller may pass in 'new + * Rect()' or null if no cropping is desired. + * @param frameScale The desired scale of the returned buffer; the raw + * screen will be scaled up/down. + */ + public static void captureLayers(IBinder layerHandleToken, Surface consumer, Rect sourceCrop, + float frameScale) { + nativeCaptureLayers(layerHandleToken, consumer, sourceCrop, frameScale); + } + + /** + * Same as {@link #captureLayers(IBinder, Surface, Rect, float)} except this + * captures to a {@link GraphicBuffer} instead of a {@link Surface}. */ - public static void captureLayers(IBinder layerHandleToken, Surface consumer, int rotation) { - nativeCaptureLayers(layerHandleToken, consumer, rotation); + public static GraphicBuffer captureLayersToBuffer(IBinder layerHandleToken, Rect sourceCrop, + float frameScale) { + return nativeCaptureLayers(layerHandleToken, sourceCrop, frameScale); } public static class Transaction implements Closeable { diff --git a/core/jni/android_view_SurfaceControl.cpp b/core/jni/android_view_SurfaceControl.cpp index bb1bfad26dc3..f77e6c4fa7b8 100644 --- a/core/jni/android_view_SurfaceControl.cpp +++ b/core/jni/android_view_SurfaceControl.cpp @@ -291,7 +291,7 @@ static void nativeScreenshot(JNIEnv* env, jclass clazz, jobject displayTokenObj, } static void nativeCaptureLayers(JNIEnv* env, jclass clazz, jobject layerHandleToken, - jobject surfaceObj, int rotation) { + jobject surfaceObj, jobject sourceCropObj, jfloat frameScale) { sp layerHandle = ibinderForJavaObject(env, layerHandleToken); if (layerHandle == NULL) { @@ -303,7 +303,42 @@ static void nativeCaptureLayers(JNIEnv* env, jclass clazz, jobject layerHandleTo return; } - ScreenshotClient::captureLayers(layerHandle, consumer->getIGraphicBufferProducer(), rotation); + Rect sourceCrop; + if (sourceCropObj != NULL) { + sourceCrop = rectFromObj(env, sourceCropObj); + } + + ScreenshotClient::captureLayers(layerHandle, consumer->getIGraphicBufferProducer(), sourceCrop, + frameScale); +} + +static jobject nativeCaptureLayersToBuffer(JNIEnv* env, jclass clazz, jobject layerHandleToken, + jobject sourceCropObj, jfloat frameScale) { + + sp layerHandle = ibinderForJavaObject(env, layerHandleToken); + if (layerHandle == NULL) { + return NULL; + } + + Rect sourceCrop; + if (sourceCropObj != NULL) { + sourceCrop = rectFromObj(env, sourceCropObj); + } + + sp buffer; + status_t res = ScreenshotClient::captureLayersToBuffer(layerHandle, sourceCrop, frameScale, + &buffer); + if (res != NO_ERROR) { + return NULL; + } + + return env->CallStaticObjectMethod(gGraphicBufferClassInfo.clazz, + gGraphicBufferClassInfo.builder, + buffer->getWidth(), + buffer->getHeight(), + buffer->getPixelFormat(), + (jint)buffer->getUsage(), + (jlong)buffer.get()); } static void nativeApplyTransaction(JNIEnv* env, jclass clazz, jlong transactionObj, jboolean sync) { @@ -975,8 +1010,10 @@ static const JNINativeMethod sSurfaceControlMethods[] = { {"nativeScreenshotToBuffer", "(Landroid/os/IBinder;Landroid/graphics/Rect;IIIIZZI)Landroid/graphics/GraphicBuffer;", (void*)nativeScreenshotToBuffer }, - {"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/view/Surface;I)V", + {"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/view/Surface;Landroid/graphics/Rect;F)V", (void*)nativeCaptureLayers }, + {"nativeCaptureLayers", "(Landroid/os/IBinder;Landroid/graphics/Rect;F)Landroid/graphics/GraphicBuffer;", + (void*)nativeCaptureLayersToBuffer }, }; int register_android_view_SurfaceControl(JNIEnv* env) diff --git a/services/core/java/com/android/server/wm/DisplayContent.java b/services/core/java/com/android/server/wm/DisplayContent.java index 56e592208eb6..41348ba4b041 100644 --- a/services/core/java/com/android/server/wm/DisplayContent.java +++ b/services/core/java/com/android/server/wm/DisplayContent.java @@ -2969,227 +2969,30 @@ class DisplayContent extends WindowContainer { - if (!w.mHasSurface) { - return false; - } - if (w.mLayer >= aboveAppLayer) { - return false; - } - if (wallpaperOnly && !w.mIsWallpaper) { - return false; - } - if (w.mIsImWindow) { - return false; - } else if (w.mIsWallpaper) { - // If this is the wallpaper layer and we're only looking for the wallpaper layer - // then the target window state is this one. - if (wallpaperOnly) { - mScreenshotApplicationState.appWin = w; - } - - if (mScreenshotApplicationState.appWin == null) { - // We have not ran across the target window yet, so it is probably behind - // the wallpaper. This can happen when the keyguard is up and all windows - // are moved behind the wallpaper. We don't want to include the wallpaper - // layer in the screenshot as it will cover-up the layer of the target - // window. - return false; - } - // Fall through. The target window is in front of the wallpaper. For this - // case we want to include the wallpaper layer in the screenshot because - // the target window might have some transparent areas. - } else if (appToken != null) { - if (w.mAppToken == null || w.mAppToken.token != appToken) { - // This app window is of no interest if it is not associated with the - // screenshot app. - return false; - } - mScreenshotApplicationState.appWin = w; - } - - // Include this window. - - final WindowStateAnimator winAnim = w.mWinAnimator; - - // Don't include wallpaper in bounds calculation - if (!w.mIsWallpaper && !mutableIncludeFullDisplay.value) { - if (includeDecor) { - final Task task = w.getTask(); - if (task != null) { - task.getBounds(frame); - } else { - - // No task bounds? Too bad! Ain't no screenshot then. - return true; - } - } else { - final Rect wf = w.mFrame; - final Rect cr = w.mContentInsets; - int left = wf.left + cr.left; - int top = wf.top + cr.top; - int right = wf.right - cr.right; - int bottom = wf.bottom - cr.bottom; - frame.union(left, top, right, bottom); - w.getVisibleBounds(stackBounds); - if (!Rect.intersects(frame, stackBounds)) { - // Set frame empty if there's no intersection. - frame.setEmpty(); - } - } - } - - final boolean foundTargetWs = - (w.mAppToken != null && w.mAppToken.token == appToken) - || (mScreenshotApplicationState.appWin != null && wallpaperOnly); - if (foundTargetWs && winAnim.getShown() && winAnim.mLastAlpha > 0f) { - mScreenshotApplicationState.screenshotReady = true; + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); } - - if (w.isObscuringDisplay()){ - return true; - } - return false; - }, true /* traverseTopToBottom */); - - final WindowState appWin = mScreenshotApplicationState.appWin; - final boolean screenshotReady = mScreenshotApplicationState.screenshotReady; - - if (appToken != null && appWin == null) { - // Can't find a window to snapshot. - if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, - "Screenshot: Couldn't find a surface matching " + appToken); return null; } - if (!screenshotReady) { - Slog.i(TAG_WM, "Failed to capture screenshot of " + appToken + - " appWin=" + (appWin == null ? "null" : (appWin + " drawState=" + - appWin.mWinAnimator.mDrawState))); + if (wallpaperOnly && !shouldScreenshotWallpaper()) { return null; } - // Screenshot is ready to be taken. Everything from here below will continue - // through the bottom of the loop and return a value. We only stay in the loop - // because we don't want to release the mWindowMap lock until the screenshot is - // taken. - + int dw = mDisplayInfo.logicalWidth; + int dh = mDisplayInfo.logicalHeight; - if (!mutableIncludeFullDisplay.value) { - // Constrain frame to the screen size. - if (!frame.intersect(0, 0, dw, dh)) { - frame.setEmpty(); - } - } else { - // Caller just wants entire display. - frame.set(0, 0, dw, dh); - } - if (frame.isEmpty()) { + if (dw <= 0 || dh <= 0) { return null; } - if (width < 0) { - width = (int) (frame.width() * frameScale); - } - if (height < 0) { - height = (int) (frame.height() * frameScale); - } - - // Tell surface flinger what part of the image to crop. Take the top - // right part of the application, and crop the larger dimension to fit. - Rect crop = new Rect(frame); - if (width / (float) frame.width() < height / (float) frame.height()) { - int cropWidth = (int)((float)width / (float)height * frame.height()); - crop.right = crop.left + cropWidth; - } else { - int cropHeight = (int)((float)height / (float)width * frame.width()); - crop.bottom = crop.top + cropHeight; - } + final Rect frame = new Rect(0, 0, dw, dh); // The screenshot API does not apply the current screen rotation. int rot = mDisplay.getRotation(); @@ -3198,43 +3001,52 @@ class DisplayContent extends WindowContainer { - final WindowSurfaceController controller = w.mWinAnimator.mSurfaceController; - Slog.i(TAG_WM, w + ": " + w.mLayer - + " animLayer=" + w.mWinAnimator.mAnimLayer - + " surfaceLayer=" + ((controller == null) - ? "null" : controller.getLayer())); - }, false /* traverseTopToBottom */); - } + // SurfaceFlinger is not aware of orientation, so convert our logical + // crop to SurfaceFlinger's portrait orientation. + convertCropForSurfaceFlinger(frame, rot, dw, dh); final ScreenRotationAnimation screenRotationAnimation = mService.mAnimator.getScreenRotationAnimationLocked(DEFAULT_DISPLAY); final boolean inRotation = screenRotationAnimation != null && screenRotationAnimation.isAnimating(); - if (DEBUG_SCREENSHOT && inRotation) Slog.v(TAG_WM, - "Taking screenshot while rotating"); - - // We force pending transactions to flush before taking - // the screenshot by pushing an empty synchronous transaction. - SurfaceControl.openTransaction(); - SurfaceControl.closeTransactionSync(); + if (DEBUG_SCREENSHOT && inRotation) Slog.v(TAG_WM, "Taking screenshot while rotating"); // TODO(b/68392460): We should screenshot Task controls directly // but it's difficult at the moment as the Task doesn't have the // correct size set. - bitmap = screenshoter.screenshot(crop, width, height, 0, 1, - inRotation, rot); + final Bitmap bitmap = SurfaceControl.screenshot(frame, dw, dh, 0, 1, inRotation, rot); if (bitmap == null) { Slog.w(TAG_WM, "Failed to take screenshot"); return null; } + + // Create a copy of the screenshot that is immutable and backed in ashmem. + // This greatly reduces the overhead of passing the bitmap between processes. + final Bitmap ret = bitmap.createAshmemBitmap(config); + bitmap.recycle(); + return ret; } - return bitmap; + } + + private boolean shouldScreenshotWallpaper() { + MutableBoolean screenshotReady = new MutableBoolean(false); + + forAllWindows(w -> { + if (!w.mIsWallpaper) { + return false; + } + + // Found the wallpaper window + final WindowStateAnimator winAnim = w.mWinAnimator; + + if (winAnim.getShown() && winAnim.mLastAlpha > 0f) { + screenshotReady.value = true; + } + + return true; + }, true /* traverseTopToBottom */); + + return screenshotReady.value; } // TODO: Can this use createRotationMatrix()? @@ -3848,15 +3660,6 @@ class DisplayContent extends WindowContainer { - E screenshot(Rect sourceCrop, int width, int height, int minLayer, int maxLayer, - boolean useIdentityTransform, int rotation); - } - SurfaceControl.Builder makeSurface(SurfaceSession s) { return mService.makeSurfaceBuilder(s) .setParent(mWindowingLayer); diff --git a/services/core/java/com/android/server/wm/TaskSnapshotController.java b/services/core/java/com/android/server/wm/TaskSnapshotController.java index e6444179d13c..84e475a25187 100644 --- a/services/core/java/com/android/server/wm/TaskSnapshotController.java +++ b/services/core/java/com/android/server/wm/TaskSnapshotController.java @@ -18,12 +18,12 @@ package com.android.server.wm; import static com.android.server.wm.TaskSnapshotPersister.DISABLE_FULL_SIZED_BITMAPS; import static com.android.server.wm.TaskSnapshotPersister.REDUCED_SCALE; +import static com.android.server.wm.WindowManagerDebugConfig.DEBUG_SCREENSHOT; import static com.android.server.wm.WindowManagerDebugConfig.TAG_WITH_CLASS_NAME; import static com.android.server.wm.WindowManagerDebugConfig.TAG_WM; import android.annotation.Nullable; import android.app.ActivityManager; -import android.app.ActivityManager.StackId; import android.app.ActivityManager.TaskSnapshot; import android.content.pm.PackageManager; import android.graphics.Bitmap; @@ -35,6 +35,7 @@ import android.util.ArraySet; import android.util.Slog; import android.view.DisplayListCanvas; import android.view.RenderNode; +import android.view.SurfaceControl; import android.view.ThreadedRenderer; import android.view.WindowManager.LayoutParams; @@ -210,11 +211,28 @@ class TaskSnapshotController { if (mainWindow == null) { return null; } + if (!mService.mPolicy.isScreenOn()) { + if (DEBUG_SCREENSHOT) { + Slog.i(TAG_WM, "Attempted to take screenshot while display was off."); + } + return null; + } + if (task.getSurfaceControl() == null) { + return null; + } + final boolean isLowRamDevice = ActivityManager.isLowRamDeviceStatic(); final float scaleFraction = isLowRamDevice ? REDUCED_SCALE : 1f; - final GraphicBuffer buffer = top.mDisplayContent.screenshotApplicationsToBuffer(top.token, - -1, -1, false, scaleFraction, false, true); + final Rect taskFrame = new Rect(); + task.getBounds(taskFrame); + + final GraphicBuffer buffer = SurfaceControl.captureLayersToBuffer( + task.getSurfaceControl().getHandle(), taskFrame, scaleFraction); + if (buffer == null || buffer.getWidth() <= 1 || buffer.getHeight() <= 1) { + if (DEBUG_SCREENSHOT) { + Slog.w(TAG_WM, "Failed to take screenshot"); + } return null; } return new TaskSnapshot(buffer, top.getConfiguration().orientation, diff --git a/services/core/java/com/android/server/wm/WindowManagerService.java b/services/core/java/com/android/server/wm/WindowManagerService.java index 1cd40806619f..82f842c0fd85 100644 --- a/services/core/java/com/android/server/wm/WindowManagerService.java +++ b/services/core/java/com/android/server/wm/WindowManagerService.java @@ -3691,9 +3691,8 @@ public class WindowManagerService extends IWindowManager.Stub } try { Trace.traceBegin(TRACE_TAG_WINDOW_MANAGER, "screenshotWallpaper"); - return screenshotApplications(null /* appToken */, DEFAULT_DISPLAY, -1 /* width */, - -1 /* height */, true /* includeFullDisplay */, 1f /* frameScale */, - Bitmap.Config.ARGB_8888, true /* wallpaperOnly */, false /* includeDecor */); + return screenshotApplications(DEFAULT_DISPLAY, Bitmap.Config.ARGB_8888, + true /* wallpaperOnly */); } finally { Trace.traceEnd(TRACE_TAG_WINDOW_MANAGER); } @@ -3712,10 +3711,8 @@ public class WindowManagerService extends IWindowManager.Stub } FgThread.getHandler().post(() -> { - Bitmap bm = screenshotApplications(null /* appToken */, DEFAULT_DISPLAY, - -1 /* width */, -1 /* height */, true /* includeFullDisplay */, - 1f /* frameScale */, Bitmap.Config.ARGB_8888, false /* wallpaperOnly */, - false /* includeDecor */); + Bitmap bm = screenshotApplications(DEFAULT_DISPLAY, Bitmap.Config.ARGB_8888, + false /* wallpaperOnly */); try { receiver.onHandleAssistScreenshot(bm); } catch (RemoteException e) { @@ -3749,29 +3746,21 @@ public class WindowManagerService extends IWindowManager.Stub * In portrait mode, it grabs the full screenshot. * * @param displayId the Display to take a screenshot of. - * @param width the width of the target bitmap - * @param height the height of the target bitmap - * @param includeFullDisplay true if the screen should not be cropped before capture - * @param frameScale the scale to apply to the frame, only used when width = -1 and height = -1 * @param config of the output bitmap * @param wallpaperOnly true if only the wallpaper layer should be included in the screenshot - * @param includeDecor whether to include window decors, like the status or navigation bar - * background of the window */ - private Bitmap screenshotApplications(IBinder appToken, int displayId, int width, - int height, boolean includeFullDisplay, float frameScale, Bitmap.Config config, - boolean wallpaperOnly, boolean includeDecor) { + private Bitmap screenshotApplications(int displayId, Bitmap.Config config, + boolean wallpaperOnly) { final DisplayContent displayContent; synchronized(mWindowMap) { displayContent = mRoot.getDisplayContentOrCreate(displayId); if (displayContent == null) { - if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Screenshot of " + appToken - + ": returning null. No Display for displayId=" + displayId); + if (DEBUG_SCREENSHOT) Slog.i(TAG_WM, "Screenshot returning null. No Display for " + + "displayId=" + displayId); return null; } } - return displayContent.screenshotApplications(appToken, width, height, - includeFullDisplay, frameScale, config, wallpaperOnly, includeDecor); + return displayContent.screenshotDisplay(config, wallpaperOnly); } /** diff --git a/services/tests/servicestests/AndroidManifest.xml b/services/tests/servicestests/AndroidManifest.xml index 9a5ebedcf361..0499bf0eccc7 100644 --- a/services/tests/servicestests/AndroidManifest.xml +++ b/services/tests/servicestests/AndroidManifest.xml @@ -58,6 +58,7 @@ +