diff --git a/app/src/main/java/com/limelight/Game.java b/app/src/main/java/com/limelight/Game.java index 75f501e4..042bffaf 100644 --- a/app/src/main/java/com/limelight/Game.java +++ b/app/src/main/java/com/limelight/Game.java @@ -405,9 +405,9 @@ public class Game extends Activity implements SurfaceHolder.Callback, this); // Don't stream HDR if the decoder can't support it - if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported()) { + if (willStreamHdr && !decoderRenderer.isHevcMain10Hdr10Supported() && !decoderRenderer.isAv1Main10Supported()) { willStreamHdr = false; - Toast.makeText(this, "Decoder does not support HEVC Main10HDR10", Toast.LENGTH_LONG).show(); + Toast.makeText(this, "Decoder does not support HDR10 profile", Toast.LENGTH_LONG).show(); } // Display a message to the user if HEVC was forced on but we still didn't find a decoder @@ -415,6 +415,23 @@ public class Game extends Activity implements SurfaceHolder.Callback, Toast.makeText(this, "No HEVC decoder found.\nFalling back to H.264.", Toast.LENGTH_LONG).show(); } + // TODO: Display a message to the user if HEVC was forced on but we still didn't find a decoder + + // H.264 is always supported + int supportedVideoFormats = MoonBridge.VIDEO_FORMAT_H264; + if (decoderRenderer.isHevcSupported()) { + supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_H265; + if (decoderRenderer.isHevcMain10Hdr10Supported()) { + supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_H265_MAIN10; + } + } + if (decoderRenderer.isAv1Supported()) { + supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_AV1_MAIN8; + if (decoderRenderer.isAv1Main10Supported()) { + supportedVideoFormats |= MoonBridge.VIDEO_FORMAT_AV1_MAIN10; + } + } + int gamepadMask = ControllerHandler.getAttachedControllerMask(this); if (!prefConfig.multiController) { // Always set gamepad 1 present for when multi-controller is @@ -464,7 +481,8 @@ public class Game extends Activity implements SurfaceHolder.Callback, .setMaxPacketSize(1392) .setRemoteConfiguration(StreamConfiguration.STREAM_CFG_AUTO) // NvConnection will perform LAN and VPN detection .setHevcBitratePercentageMultiplier(75) - .setHevcSupported(decoderRenderer.isHevcSupported()) + .setAv1BitratePercentageMultiplier(60) + .setSupportedVideoFormats(supportedVideoFormats) .setEnableHdr(willStreamHdr) .setAttachedGamepadMask(gamepadMask) .setClientRefreshRateX100((int)(displayRefreshRate * 100)) diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java index 55006ce3..fb6eb8a9 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecDecoderRenderer.java @@ -46,6 +46,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C private MediaCodecInfo avcDecoder; private MediaCodecInfo hevcDecoder; + private MediaCodecInfo av1Decoder; private byte[] vpsBuffer; private byte[] spsBuffer; @@ -64,7 +65,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C private boolean needsSpsBitstreamFixup, isExynos4; private boolean adaptivePlayback, directSubmit, fusedIdrFrame; private boolean constrainedHighProfile; - private boolean refFrameInvalidationAvc, refFrameInvalidationHevc; + private boolean refFrameInvalidationAvc, refFrameInvalidationHevc, refFrameInvalidationAv1; private byte optimalSlicesPerFrame; private boolean refFrameInvalidationActive; private int initialWidth, initialHeight; @@ -231,6 +232,10 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C return hevcDecoderInfo; } + private MediaCodecInfo findAv1Decoder(PreferenceConfiguration prefs) { + return MediaCodecHelper.findProbableSafeDecoder("video/av01", -1); + } + public void setRenderTarget(SurfaceHolder renderTarget) { this.renderTarget = renderTarget; } @@ -269,6 +274,14 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C LimeLog.info("No HEVC decoder found"); } + av1Decoder = findAv1Decoder(prefs); + if (av1Decoder != null) { + LimeLog.info("Selected AV1 decoder: "+av1Decoder.getName()); + } + else { + LimeLog.info("No AV1 decoder found"); + } + // Set attributes that are queried in getCapabilities(). This must be done here // because getCapabilities() may be called before setup() in current versions of the common // library. The limitation of this is that we don't know whether we're using HEVC or AVC. @@ -299,6 +312,14 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C LimeLog.info("Decoder "+hevcDecoder.getName()+" wants "+hevcOptimalSlicesPerFrame+" slices per frame"); } + if (av1Decoder != null) { + refFrameInvalidationAv1 = MediaCodecHelper.decoderSupportsRefFrameInvalidationAv1(av1Decoder); + + if (refFrameInvalidationAv1) { + LimeLog.info("Decoder "+av1Decoder.getName()+" will use reference frame invalidation for AV1"); + } + } + // Use the larger of the two slices per frame preferences optimalSlicesPerFrame = (byte)Math.max(avcOptimalSlicesPerFrame, hevcOptimalSlicesPerFrame); LimeLog.info("Requesting "+optimalSlicesPerFrame+" slices per frame"); @@ -332,6 +353,25 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C return false; } + public boolean isAv1Supported() { + return av1Decoder != null; + } + + public boolean isAv1Main10Supported() { + if (av1Decoder == null) { + return false; + } + + for (MediaCodecInfo.CodecProfileLevel profileLevel : av1Decoder.getCapabilitiesForType("video/av01").profileLevels) { + if (profileLevel.profile == MediaCodecInfo.CodecProfileLevel.AV1ProfileMain10HDR10) { + LimeLog.info("AV1 decoder "+av1Decoder.getName()+" supports AV1 Main 10 HDR10"); + return true; + } + } + + return false; + } + public int getPreferredColorSpace() { // Default to Rec 709 which is probably better supported on modern devices. // @@ -340,7 +380,7 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C // an HEVC decoder, we will use Rec 709 (even for H.264) since we can't choose a // colorspace by codec (and it's probably safe to say a SoC with HEVC decoding is // plenty modern enough to handle H.264 VUI colorspace info). - if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O || hevcDecoder != null) { + if (Build.VERSION.SDK_INT >= Build.VERSION_CODES.O || hevcDecoder != null || av1Decoder != null) { return MoonBridge.COLORSPACE_REC_709; } else { @@ -551,6 +591,17 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C refFrameInvalidationActive = refFrameInvalidationHevc; } + else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_AV1) != 0) { + mimeType = "video/av01"; + selectedDecoderInfo = av1Decoder; + + if (av1Decoder == null) { + LimeLog.severe("No available AV1 decoder!"); + return -2; + } + + refFrameInvalidationActive = refFrameInvalidationAv1; + } else { // Unknown format LimeLog.severe("Unknown format"); @@ -1324,6 +1375,8 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C decoder = avcDecoder.getName(); } else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_H265) != 0) { decoder = hevcDecoder.getName(); + } else if ((videoFormat & MoonBridge.VIDEO_FORMAT_MASK_AV1) != 0) { + decoder = av1Decoder.getName(); } else { decoder = "(unknown)"; } @@ -1670,6 +1723,9 @@ public class MediaCodecDecoderRenderer extends VideoDecoderRenderer implements C if (refFrameInvalidationHevc) { capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC; } + if (refFrameInvalidationAv1) { + capabilities |= MoonBridge.CAPABILITY_REFERENCE_FRAME_INVALIDATION_AV1; + } // Enable direct submit on supported hardware if (directSubmit) { diff --git a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java index a092cab7..a80afb0c 100644 --- a/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java +++ b/app/src/main/java/com/limelight/binding/video/MediaCodecHelper.java @@ -691,6 +691,17 @@ public class MediaCodecHelper { return isDecoderInList(refFrameInvalidationHevcPrefixes, decoderInfo.getName()); } + public static boolean decoderSupportsRefFrameInvalidationAv1(MediaCodecInfo decoderInfo) { + // We'll use the same heuristics as HEVC for now + if (decoderSupportsAndroidRLowLatency(decoderInfo, "video/av01") || + decoderSupportsKnownVendorLowLatencyOption(decoderInfo.getName())) { + LimeLog.info("Enabling AV1 RFI based on low latency option support"); + return true; + } + + return false; + } + public static boolean decoderIsWhitelistedForHevc(MediaCodecInfo decoderInfo) { // Google didn't have official support for HEVC (or more importantly, a CTS test) until // Lollipop. I've seen some MediaTek devices on 4.4 crash when attempting to use HEVC, diff --git a/app/src/main/java/com/limelight/nvstream/NvConnection.java b/app/src/main/java/com/limelight/nvstream/NvConnection.java index 6f28fa5b..e1589496 100644 --- a/app/src/main/java/com/limelight/nvstream/NvConnection.java +++ b/app/src/main/java/com/limelight/nvstream/NvConnection.java @@ -259,8 +259,8 @@ public class NvConnection { return false; } else if ((context.streamConfig.getWidth() > 4096 || context.streamConfig.getHeight() > 4096) && - !context.streamConfig.getHevcSupported()) { - context.connListener.displayMessage("Your streaming device must support HEVC to stream at resolutions above 4K."); + (context.streamConfig.getSupportedVideoFormats() & ~MoonBridge.VIDEO_FORMAT_MASK_H264) == 0) { + context.connListener.displayMessage("Your streaming device must support HEVC or AV1 to stream at resolutions above 4K."); return false; } else if (context.streamConfig.getHeight() >= 2160 && !h.supports4K(serverInfo)) { @@ -429,9 +429,10 @@ public class NvConnection { context.streamConfig.getRefreshRate(), context.streamConfig.getBitrate(), context.negotiatedPacketSize, context.negotiatedRemoteStreaming, context.streamConfig.getAudioConfiguration().toInt(), - context.streamConfig.getHevcSupported(), + context.streamConfig.getSupportedVideoFormats(), context.negotiatedHdr, context.streamConfig.getHevcBitratePercentageMultiplier(), + context.streamConfig.getAv1BitratePercentageMultiplier(), context.streamConfig.getClientRefreshRateX100(), context.streamConfig.getEncryptionFlags(), context.riKey.getEncoded(), ib.array(), diff --git a/app/src/main/java/com/limelight/nvstream/StreamConfiguration.java b/app/src/main/java/com/limelight/nvstream/StreamConfiguration.java index ccb356fa..4809e881 100644 --- a/app/src/main/java/com/limelight/nvstream/StreamConfiguration.java +++ b/app/src/main/java/com/limelight/nvstream/StreamConfiguration.java @@ -22,8 +22,9 @@ public class StreamConfiguration { private int maxPacketSize; private int remote; private MoonBridge.AudioConfiguration audioConfiguration; - private boolean supportsHevc; + private int supportedVideoFormats; private int hevcBitratePercentageMultiplier; + private int av1BitratePercentageMultiplier; private boolean enableHdr; private int attachedGamepadMask; private int encryptionFlags; @@ -90,6 +91,11 @@ public class StreamConfiguration { return this; } + public StreamConfiguration.Builder setAv1BitratePercentageMultiplier(int multiplier) { + config.av1BitratePercentageMultiplier = multiplier; + return this; + } + public StreamConfiguration.Builder setEnableHdr(boolean enableHdr) { config.enableHdr = enableHdr; return this; @@ -135,8 +141,8 @@ public class StreamConfiguration { return this; } - public StreamConfiguration.Builder setHevcSupported(boolean supportsHevc) { - config.supportsHevc = supportsHevc; + public StreamConfiguration.Builder setSupportedVideoFormats(int supportedVideoFormats) { + config.supportedVideoFormats = supportedVideoFormats; return this; } @@ -168,8 +174,7 @@ public class StreamConfiguration { this.sops = true; this.enableAdaptiveResolution = false; this.audioConfiguration = MoonBridge.AUDIO_CONFIGURATION_STEREO; - this.supportsHevc = false; - this.enableHdr = false; + this.supportedVideoFormats = MoonBridge.VIDEO_FORMAT_H264; this.attachedGamepadMask = 0; } @@ -221,14 +226,18 @@ public class StreamConfiguration { return audioConfiguration; } - public boolean getHevcSupported() { - return supportsHevc; + public int getSupportedVideoFormats() { + return supportedVideoFormats; } public int getHevcBitratePercentageMultiplier() { return hevcBitratePercentageMultiplier; } + public int getAv1BitratePercentageMultiplier() { + return av1BitratePercentageMultiplier; + } + public boolean getEnableHdr() { return enableHdr; } diff --git a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java index 4de84356..c827ef94 100644 --- a/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java +++ b/app/src/main/java/com/limelight/nvstream/jni/MoonBridge.java @@ -14,9 +14,13 @@ public class MoonBridge { public static final int VIDEO_FORMAT_H264 = 0x0001; public static final int VIDEO_FORMAT_H265 = 0x0100; public static final int VIDEO_FORMAT_H265_MAIN10 = 0x0200; + public static final int VIDEO_FORMAT_AV1_MAIN8 = 0x1000; + public static final int VIDEO_FORMAT_AV1_MAIN10 = 0x2000; - public static final int VIDEO_FORMAT_MASK_H264 = 0x00FF; - public static final int VIDEO_FORMAT_MASK_H265 = 0xFF00; + public static final int VIDEO_FORMAT_MASK_H264 = 0x000F; + public static final int VIDEO_FORMAT_MASK_H265 = 0x0F00; + public static final int VIDEO_FORMAT_MASK_AV1 = 0xF000; + public static final int VIDEO_FORMAT_MASK_10BIT = 0x2200; public static final int ENCFLG_NONE = 0; public static final int ENCFLG_AUDIO = 1; @@ -40,6 +44,7 @@ public class MoonBridge { public static final int CAPABILITY_DIRECT_SUBMIT = 1; public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_AVC = 2; public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_HEVC = 4; + public static final int CAPABILITY_REFERENCE_FRAME_INVALIDATION_AV1 = 0x40; public static final int DR_OK = 0; public static final int DR_NEED_IDR = -1; @@ -340,9 +345,10 @@ public class MoonBridge { String rtspSessionUrl, int width, int height, int fps, int bitrate, int packetSize, int streamingRemotely, - int audioConfiguration, boolean supportsHevc, + int audioConfiguration, int supportedVideoFormats, boolean enableHdr, int hevcBitratePercentageMultiplier, + int av1BitratePercentageMultiplier, int clientRefreshRateX100, int encryptionFlags, byte[] riAesKey, byte[] riAesIv, diff --git a/app/src/main/jni/moonlight-core/callbacks.c b/app/src/main/jni/moonlight-core/callbacks.c index f0bd76f4..0587561b 100644 --- a/app/src/main/jni/moonlight-core/callbacks.c +++ b/app/src/main/jni/moonlight-core/callbacks.c @@ -432,9 +432,10 @@ Java_com_limelight_nvstream_jni_MoonBridge_startConnection(JNIEnv *env, jclass c jstring rtspSessionUrl, jint width, jint height, jint fps, jint bitrate, jint packetSize, jint streamingRemotely, - jint audioConfiguration, jboolean supportsHevc, + jint audioConfiguration, jint supportedVideoFormats, jboolean enableHdr, jint hevcBitratePercentageMultiplier, + jint av1BitratePercentageMultiplier, jint clientRefreshRateX100, jint encryptionFlags, jbyteArray riAesKey, jbyteArray riAesIv, @@ -454,9 +455,10 @@ Java_com_limelight_nvstream_jni_MoonBridge_startConnection(JNIEnv *env, jclass c .packetSize = packetSize, .streamingRemotely = streamingRemotely, .audioConfiguration = audioConfiguration, - .supportsHevc = supportsHevc, + .supportedVideoFormats = supportedVideoFormats, .enableHdr = enableHdr, .hevcBitratePercentageMultiplier = hevcBitratePercentageMultiplier, + .av1BitratePercentageMultiplier = av1BitratePercentageMultiplier, .clientRefreshRateX100 = clientRefreshRateX100, .encryptionFlags = encryptionFlags, .colorSpace = colorSpace, diff --git a/app/src/main/jni/moonlight-core/moonlight-common-c b/app/src/main/jni/moonlight-core/moonlight-common-c index c0792168..e36bde4a 160000 --- a/app/src/main/jni/moonlight-core/moonlight-common-c +++ b/app/src/main/jni/moonlight-core/moonlight-common-c @@ -1 +1 @@ -Subproject commit c0792168f5a7ed48fc6feeb7fce01b83df405df2 +Subproject commit e36bde4acce48c21c79a57fb29727d96fdae6503