Get Video Pixel Data From Exoplayer In Android

Is it possible to grab the pixel data (e.g. as RGB byte array) from a running video within the ExoPlayer? Ideally as the real video resolution and not the size as the shown View. I

Solution 1:

WARNING this solution currently only works on my Android 8 emulator because of this.

Ok, here is my solution. I'm using the SimpleExoPlayer with a custom MediaCodecVideoRenderer. I'm not binding the player to a SimpleExoPlayerView because I'm not allowed to if I want to manually grab the image data. Within my custom MediaCodecVideoRenderer I override the processOutputBuffer and use getOutputImage to get a nice standard Android Image. I then send it through my OpenCV code and then transform it back into an Android Bitmap using the OpenCV Utils whenever I need. Careful this code requires API >= 21. Also this codes does not output any audio.

privatevoidstartPlayer(Context context, Uri uri) {

    BandwidthMeterbandwidthMeter=newDefaultBandwidthMeter();
    TrackSelection.FactoryvideoTrackSelectionFactory=newAdaptiveTrackSelection.Factory(bandwidthMeter);
    TrackSelectortrackSelector=newDefaultTrackSelector(videoTrackSelectionFactory);

    SimpleExoPlayerplayer= ExoPlayerFactory.newSimpleInstance((eventHandler, videoRendererEventListener, audioRendererEventListener, textRendererOutput, metadataRendererOutput) -> {
        returnnewRenderer[]{newCustomMediaCodecVideoRenderer(context, MediaCodecSelector.DEFAULT, 1000, eventHandler, videoRendererEventListener, 100)};
    }, trackSelector);

    DataSource.FactorydataSourceFactory=newDefaultDataSourceFactory(context, Util.getUserAgent(context, context.getPackageName()));
    ExtractorMediaSourcevideoSource=newExtractorMediaSource.Factory(dataSourceFactory).createMediaSource(uri);

    LoopingMediaSourceloopingMediaSource=newLoopingMediaSource(videoSource);

    player.setPlayWhenReady(true);
    player.prepare(loopingMediaSource);

}

classCustomMediaCodecVideoRendererextendsMediaCodecVideoRenderer {
    CustomMediaCodecVideoRenderer(Context context, MediaCodecSelector mediaCodecSelector, long allowedJoiningTimeMs, @Nullable Handler eventHandler, @Nullable VideoRendererEventListener eventListener, int maxDroppedFrameCountToNotify) {
        super(context, mediaCodecSelector, allowedJoiningTimeMs, eventHandler, eventListener, maxDroppedFrameCountToNotify);
    }

    @OverrideprotectedbooleanprocessOutputBuffer(long positionUs, long elapsedRealtimeUs, MediaCodec codec, ByteBuffer buffer, int bufferIndex, int bufferFlags, long bufferPresentationTimeUs, boolean shouldSkip)throws ExoPlaybackException {


        Imageimage= codec.getOutputImage(bufferIndex);

        if (image != null) {
            Log.d(MainActivity.class.getName(), "test");

            Matmat= convertYuv420888ToMat(image, false);
            // TODO do your thing with the OpenCV Mat
        }

        returnsuper.processOutputBuffer(positionUs, elapsedRealtimeUs, codec, buffer, bufferIndex, bufferFlags, bufferPresentationTimeUs, shouldSkip);
    }
}

I found the Image to Mat conversion here.

private Mat convertYuv420888ToMat(Image image, boolean isGreyOnly) {
    intwidth= image.getWidth();
    intheight= image.getHeight();

    Image.PlaneyPlane= image.getPlanes()[0];
    intySize= yPlane.getBuffer().remaining();

    if (isGreyOnly) {
        byte[] data = newbyte[ySize];
        yPlane.getBuffer().get(data, 0, ySize);

        MatgreyMat=newMat(height, width, CvType.CV_8UC1);
        greyMat.put(0, 0, data);

        return greyMat;
    }

    Image.PlaneuPlane= image.getPlanes()[1];
    Image.PlanevPlane= image.getPlanes()[2];

    // be aware that this size does not include the padding at the end, if there is any// (e.g. if pixel stride is 2 the size is ySize / 2 - 1)intuSize= uPlane.getBuffer().remaining();
    intvSize= vPlane.getBuffer().remaining();

    byte[] data = newbyte[ySize + (ySize/2)];

    yPlane.getBuffer().get(data, 0, ySize);

    ByteBufferub= uPlane.getBuffer();
    ByteBuffervb= vPlane.getBuffer();

    intuvPixelStride= uPlane.getPixelStride(); //stride guaranteed to be the same for u and v planesif (uvPixelStride == 1) {
        uPlane.getBuffer().get(data, ySize, uSize);
        vPlane.getBuffer().get(data, ySize + uSize, vSize);

        MatyuvMat=newMat(height + (height / 2), width, CvType.CV_8UC1);
        yuvMat.put(0, 0, data);
        MatrgbMat=newMat(height, width, CvType.CV_8UC3);
        Imgproc.cvtColor(yuvMat, rgbMat, Imgproc.COLOR_YUV2RGB_I420, 3);
        yuvMat.release();
        return rgbMat;
    }

    // if pixel stride is 2 there is padding between each pixel// converting it to NV21 by filling the gaps of the v plane with the u values
    vb.get(data, ySize, vSize);
    for (inti=0; i < uSize; i += 2) {
        data[ySize + i + 1] = ub.get(i);
    }

    MatyuvMat=newMat(height + (height / 2), width, CvType.CV_8UC1);
    yuvMat.put(0, 0, data);
    MatrgbMat=newMat(height, width, CvType.CV_8UC3);
    Imgproc.cvtColor(yuvMat, rgbMat, Imgproc.COLOR_YUV2RGB_NV21, 3);
    yuvMat.release();
    return rgbMat;
}