How to do Video Filter and Display Rotation in MediaProjection

#android #opengl #mediaprojection #grafika

Video Filter

What should I do if I want to apply effects to the Surface acquired by MediaProjection?

In Android, it is possible to add effects using OpenGL on Surface. In Android, OpenGL is initialized using an API called EGL.

This time, we will implement it based on an application called grafika published by Google. This time, the story of OpenGL and the story of grafika will be omitted for the sake of time, so I will explain a series of flows using grafika.

    fun start(surface: Surface) {

        fun setupEgl(surface: Surface) {
            eglCore = EglCore()
            windowSurface = WindowSurface(eglCore, surface, true).apply {
                makeCurrent()
            }

            // Add Custom Effect (e.g. https://github.com/google/grafika/blob/master/app/src/main/java/com/android/grafika/gles/Texture2dProgram.java)
            videoEffect = Texture2dProgram(
                Texture2dProgram.ProgramType.TEXTURE_EXT
            )
            texId = GlUtil.createTextureObject(GLES11Ext.GL_TEXTURE_EXTERNAL_OES)
            GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MIN_FILTER, GLES20.GL_LINEAR.toFloat())
            GLES20.glTexParameterf(GLES11Ext.GL_TEXTURE_EXTERNAL_OES, GLES20.GL_TEXTURE_MAG_FILTER, GLES20.GL_LINEAR.toFloat())

            sprite2d = Sprite2d(Drawable2d(Drawable2d.Prefab.RECTANGLE))
                .apply {
                    setTexture(texId)
                    setPosition(outputSize.width / 2f, outputSize.height / 2f)
                    setScale(outputSize.width.toFloat(), outputSize.height.toFloat())
                }

            sourceSurfaceTexture = SurfaceTexture(texId).apply {
                setDefaultBufferSize(renderConfig.width, renderConfig.height)
                setOnFrameAvailableListener {
                    synchronized(hasRenderFrame) {
                        hasRenderFrame = true
                    }
                    try {
                        updateTexImage()
                    } catch (e: Exception) {
                        Timber.e(e, "failed to updateTexImage.")
                    }
                }
            }
            sourceSurface = Surface(sourceSurfaceTexture)
        }

        fun setupVirtualDisplay() {
            virtualDisplay = mediaProjection.createVirtualDisplay(
                "Capture Start",
                width,
                height,
                dpi,
                DisplayManager.VIRTUAL_DISPLAY_FLAG_AUTO_MIRROR,
                sourceSurface,
                null,
                null
            )
        }

        setupEgl(surface)
        setupVirtualDisplay()
        renderDisposable = Observable.interval(
            0L,
            1000L / renderConfig.frameRate,
            TimeUnit.MILLISECONDS
        ).observeOn(AndroidSchedulers.from(Looper.myLooper()))
            .subscribe({
                drawFrame()
            }, {
                Timber.e("draw frame error")
            })
        rotationChangeDetector.addListener(onRotationChange)
    }

    private fun drawFrame() {
        synchronized(hasRenderFrame) {
            if (!hasRenderFrame) return
            hasRenderFrame = false
        }
        windowSurface.makeCurrent()
        GLES20.glViewport(0, 0, outputSize.width, outputSize.height)
        sprite2d.draw(videoEffect, tmpMatrix)
        windowSurface.swapBuffers()
    }

setupEgl initializes EglCore of grafika.

windowSurface converts Surface to NativeWindow. In order to handle with OpenGL, once convert GLSurface to NativeWindow for processing on Native side
is needed. NativeWindow and Surface are interchangeable.

sprite2d initializes 2D objects handled by grafika.

sourceSurfaceTexture gets SurfaceTexture from NativeWindow. Here, NativeWindow and SurfaceTexture are converted mutually.

Insert sourceSurface obtained here into createVirtualDisplay.

You have now added a video filter to the screen capture.

Display Rotation

One of the most common problems with MediaProjection is that the aspect ratio becomes strange when rotated.

To avoid this problem, let's rotate using OpenGL.

internal typealias RotationChangeListener = (angle: Int) -> Unit

internal class ScreenRotationChangeDetector @Inject constructor() : SampleApp() {

    private val listeners: MutableList<RotationChangeListener> = mutableListOf()

    fun addListener(listener: RotationChangeListener) {
        listeners.add(listener)
    }

    fun removeListener(listener: RotationChangeListener) {
        listeners.remove(listener)
    }

    fun register(context: Context) {
        val filter = IntentFilter(Intent.ACTION_CONFIGURATION_CHANGED)
        context.registerReceiver(this, filter)
    }

    fun unregister(context: Context) {
        context.unregisterReceiver(this)
    }

    override fun onReceive(context: Context, intent: Intent) {
        val windowManager = context.getSystemService(Context.WINDOW_SERVICE) as WindowManager
        val angle = when (windowManager.defaultDisplay.rotation) {
            Surface.ROTATION_0 -> 0
            Surface.ROTATION_90 -> 90
            Surface.ROTATION_180 -> 180
            Surface.ROTATION_270 -> 270
            else -> throw IllegalArgumentException("unknown rotation.")
        }
        listeners.forEach { it(angle) }
    }

}

    private val onRotationChange: RotationChangeListener = {
        updateRotation(it)
    }

    private fun updateRotation(rotationAngle: Int) {
        val diffAngle = rotationAngle - renderConfig.rotationAngle
        val (w, h) = if (Math.abs(diffAngle) == 270 || Math.abs(diffAngle) == 90) {
            val scale = if (outputSize.width > outputSize.height) {
                outputSize.width.toFloat() / outputSize.height
            } else {
                outputSize.height.toFloat() / outputSize.width
            }
            Pair(outputSize.width * scale, outputSize.height * scale)
        } else {
            Pair(outputSize.width.toFloat(), outputSize.height.toFloat())
        }
        sprite2d.setScale(w, h)
        sprite2d.setRotationZ(diffAngle.toFloat())
    }