From e6af9df142f60db996bd9daefd59b8b7a22d6739 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 29 Oct 2013 20:18:22 -0400 Subject: [PATCH] Rewrite video support so that it actually works. Add a janky NAL parser dissector. --- AndroidManifest.xml | 1 + LuaScripts/NALParser.lua | 62 ++++++ src/com/limelight/Connection.java | 12 +- src/com/limelight/nvstream/NvVideoStream.java | 192 ++++++++---------- .../nvstream/av/AvBufferDescriptor.java | 14 ++ .../limelight/nvstream/av/AvDecodeUnit.java | 34 ++++ src/com/limelight/nvstream/av/AvPacket.java | 18 ++ src/com/limelight/nvstream/av/AvParser.java | 131 ++++++++++++ 8 files changed, 350 insertions(+), 114 deletions(-) create mode 100644 LuaScripts/NALParser.lua create mode 100644 src/com/limelight/nvstream/av/AvBufferDescriptor.java create mode 100644 src/com/limelight/nvstream/av/AvDecodeUnit.java create mode 100644 src/com/limelight/nvstream/av/AvPacket.java create mode 100644 src/com/limelight/nvstream/av/AvParser.java diff --git a/AndroidManifest.xml b/AndroidManifest.xml index dc47bbfc..1318cd0f 100644 --- a/AndroidManifest.xml +++ b/AndroidManifest.xml @@ -27,6 +27,7 @@ = 0) - { - ByteBuffer buf = decoderInputBuffers[inputIndex]; - - buf.clear(); - buf.put(firstFrame); - - decoder.queueInputBuffer(inputIndex, - 0, firstFrame.length, - 0, 0); - frameIndex++; - } + videoDecoder.configure(videoFormat, surface, null, 0); + videoDecoder.setVideoScalingMode(MediaCodec.VIDEO_SCALING_MODE_SCALE_TO_FIT); + videoDecoder.start(); + videoDecoderInputBuffers = videoDecoder.getInputBuffers(); final RTPSession session = new RTPSession(rtp, rtcp); session.addParticipant(new Participant(host, RTP_PORT, RTCP_PORT)); - //session.RTPSessionRegister(NvVideoStream.this, null, null); // Ping thread new Thread(new Runnable() { @@ -129,64 +120,103 @@ public class NvVideoStream implements RTPAppIntf { } }).start(); - // Receive thread + // Decoder thread new Thread(new Runnable() { @Override public void run() { - byte[] packet = new byte[1500]; - - // Send PING every 100 ms + // Read the decode units generated from the RTP stream for (;;) { - DatagramPacket dp = new DatagramPacket(packet, 0, packet.length); - + AvDecodeUnit du; try { - rtp.receive(dp); - } catch (IOException e) { + du = parser.getNextDecodeUnit(); + } catch (InterruptedException e) { e.printStackTrace(); - break; + return; } - System.out.println("in receiveData"); - int inputIndex = decoder.dequeueInputBuffer(-1); - if (inputIndex >= 0) + switch (du.getType()) { - ByteBuffer buf = decoderInputBuffers[inputIndex]; - NvVideoPacket nvVideo = new NvVideoPacket(dp.getData()); - - buf.clear(); - buf.put(nvVideo.data); - - System.out.println(nvVideo); - if (nvVideo.length == 0xc803) { - decoder.queueInputBuffer(inputIndex, - 0, nvVideo.length, - 0, 0); - frameIndex++; - } else { - decoder.queueInputBuffer(inputIndex, - 0, 0, - 0, 0); + case AvDecodeUnit.TYPE_H264: + { + int inputIndex = videoDecoder.dequeueInputBuffer(-1); + if (inputIndex >= 0) + { + ByteBuffer buf = videoDecoderInputBuffers[inputIndex]; + + // Clear old input data + buf.clear(); + + // Copy data from our buffer list into the input buffer + for (AvBufferDescriptor desc : du.getBufferList()) + { + buf.put(desc.data, desc.offset, desc.length); + } + + videoDecoder.queueInputBuffer(inputIndex, + 0, du.getDataLength(), + 0, 0); + } } + break; + + default: + { + System.out.println("Unknown decode unit type"); + } + break; } } } }).start(); + // Receive thread + new Thread(new Runnable() { + + @Override + public void run() { + byte[] buffer = new byte[1500]; + AvBufferDescriptor desc = new AvBufferDescriptor(null, 0, 0); + + for (;;) + { + DatagramPacket packet = new DatagramPacket(buffer, buffer.length); + + try { + rtp.receive(packet); + } catch (IOException e) { + // TODO Auto-generated catch block + e.printStackTrace(); + return; + } + + desc.length = packet.getLength(); + desc.offset = packet.getOffset(); + desc.data = packet.getData(); + + // Skip the RTP header + desc.offset += 12; + desc.length -= 12; + + // Give the data to the AV parser + parser.addInputData(new AvPacket(desc)); + + } + } + + }).start(); + for (;;) { BufferInfo info = new BufferInfo(); - System.out.println("dequeuing outputbuffer"); - int outIndex = decoder.dequeueOutputBuffer(info, -1); - System.out.println("done dequeuing output buffer"); + int outIndex = videoDecoder.dequeueOutputBuffer(info, -1); switch (outIndex) { case MediaCodec.INFO_OUTPUT_BUFFERS_CHANGED: System.out.println("Output buffers changed"); break; case MediaCodec.INFO_OUTPUT_FORMAT_CHANGED: System.out.println("Output format changed"); - //decoderOutputFormat = decoder.getOutputFormat(); - System.out.println("New output Format: " + decoder.getOutputFormat()); + System.out.println("New output Format: " + videoDecoder.getOutputFormat()); break; case MediaCodec.INFO_TRY_AGAIN_LATER: System.out.println("Try again later"); @@ -195,28 +225,13 @@ public class NvVideoStream implements RTPAppIntf { break; } if (outIndex >= 0) { - System.out.println("releasing output buffer"); - decoder.releaseOutputBuffer(outIndex, true); - System.out.println("output buffer released"); + videoDecoder.releaseOutputBuffer(outIndex, true); } } } }).start(); } - - @Override - public void receiveData(DataFrame frame, Participant participant) { - } - - @Override - public void userEvent(int type, Participant[] participant) { - } - - @Override - public int frameSize(int payloadType) { - return 1; - } /** * Generates the presentation time for frame N, in microseconds. @@ -224,33 +239,4 @@ public class NvVideoStream implements RTPAppIntf { private static long computePresentationTime(int frameIndex) { return 132 + frameIndex * 1000000 / FRAME_RATE; } - - class NvVideoPacket { - byte[] preamble; - short length; - byte[] extra; - byte[] data; - - public NvVideoPacket(byte[] payload) - { - ByteBuffer bb = ByteBuffer.wrap(payload).order(ByteOrder.LITTLE_ENDIAN); - - preamble = new byte[12+16]; - extra = new byte[38]; - - bb.get(preamble); - length = bb.getShort(); - bb.get(extra); - data = new byte[length]; - - if (bb.remaining() + length <= payload.length) - bb.get(data); - } - - public String toString() - { - return "";//String.format("Length: %d | %02x %02x %02x %02x %02x %02x %02x %02x", - //length, data[0], data[1], data[2], data[3], data[4], data[5], data[6], data[7]); - } - } } diff --git a/src/com/limelight/nvstream/av/AvBufferDescriptor.java b/src/com/limelight/nvstream/av/AvBufferDescriptor.java new file mode 100644 index 00000000..73090237 --- /dev/null +++ b/src/com/limelight/nvstream/av/AvBufferDescriptor.java @@ -0,0 +1,14 @@ +package com.limelight.nvstream.av; + +public class AvBufferDescriptor { + public byte[] data; + public int offset; + public int length; + + public AvBufferDescriptor(byte[] data, int offset, int length) + { + this.data = data; + this.offset = offset; + this.length = length; + } +} diff --git a/src/com/limelight/nvstream/av/AvDecodeUnit.java b/src/com/limelight/nvstream/av/AvDecodeUnit.java new file mode 100644 index 00000000..693503dc --- /dev/null +++ b/src/com/limelight/nvstream/av/AvDecodeUnit.java @@ -0,0 +1,34 @@ +package com.limelight.nvstream.av; + +import java.util.List; + +public class AvDecodeUnit { + public static final int TYPE_UNKNOWN = 0; + public static final int TYPE_H264 = 1; + + private int type; + private List bufferList; + private int dataLength; + + public AvDecodeUnit(int type, List bufferList, int dataLength) + { + this.type = type; + this.bufferList = bufferList; + this.dataLength = dataLength; + } + + public int getType() + { + return type; + } + + public List getBufferList() + { + return bufferList; + } + + public int getDataLength() + { + return dataLength; + } +} diff --git a/src/com/limelight/nvstream/av/AvPacket.java b/src/com/limelight/nvstream/av/AvPacket.java new file mode 100644 index 00000000..cf49aedb --- /dev/null +++ b/src/com/limelight/nvstream/av/AvPacket.java @@ -0,0 +1,18 @@ +package com.limelight.nvstream.av; + +public class AvPacket { + private AvBufferDescriptor buffer; + + public AvPacket(AvBufferDescriptor rtpPayload) + { + byte[] data = new byte[rtpPayload.length]; + System.arraycopy(rtpPayload.data, rtpPayload.offset, data, 0, rtpPayload.length); + buffer = new AvBufferDescriptor(data, 0, data.length); + } + + public AvBufferDescriptor getPayload() + { + int payloadOffset = buffer.offset+56; + return new AvBufferDescriptor(buffer.data, payloadOffset, buffer.length-payloadOffset); + } +} diff --git a/src/com/limelight/nvstream/av/AvParser.java b/src/com/limelight/nvstream/av/AvParser.java new file mode 100644 index 00000000..0878831d --- /dev/null +++ b/src/com/limelight/nvstream/av/AvParser.java @@ -0,0 +1,131 @@ +package com.limelight.nvstream.av; + +import java.util.LinkedList; +import java.util.List; +import java.util.concurrent.LinkedBlockingQueue; + +public class AvParser { + + // Current NAL state + private LinkedList nalDataChain; + private int nalDataLength; + + private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(); + + private void reassembleNal() + { + // This is the start of a new NAL + if (nalDataChain != null && nalDataLength != 0) + { + // Construct the H264 decode unit + AvDecodeUnit du = new AvDecodeUnit(AvDecodeUnit.TYPE_H264, nalDataChain, nalDataLength); + decodedUnits.add(du); + + // Clear old state + nalDataChain = null; + nalDataLength = 0; + } + } + + public void addInputData(AvPacket packet) + { + AvBufferDescriptor payload = packet.getPayload(); + AvBufferDescriptor location = new AvBufferDescriptor(payload.data, payload.offset, payload.length); + + while (location.length != 0) + { + // Remember the start of the NAL data in this packet + int start = location.offset; + + // Check for the start sequence + if (H264NAL.hasStartSequence(location)) + { + // Reassemble any pending NAL + reassembleNal(); + + // Setup state for the new NAL + nalDataChain = new LinkedList(); + nalDataLength = 0; + + // Skip the start sequence and the type byte + location.length -= 5; + location.offset += 5; + } + + // If there's a NAL assembly in progress, add the current data + if (nalDataChain != null) + { + // FIXME: This is a hack to make parsing full packets + // take less time. We assume if they don't start with + // a NAL start sequence, they're full of NAL data + if (payload.length == 968) + { + location.offset += location.length; + location.length = 0; + } + else + { + System.out.println("Using slow parsing case"); + while (location.length != 0) + { + // Check if this should end the current NAL + if (H264NAL.hasStartSequence(location)) + { + break; + } + else + { + // This byte is part of the NAL data + location.offset++; + location.length--; + } + } + } + + // Add a buffer descriptor describing the NAL data in this packet + nalDataChain.add(new AvBufferDescriptor(location.data, start, location.offset-start)); + nalDataLength += location.offset-start; + } + else + { + // Otherwise, skip the data + location.offset++; + location.length--; + } + } + } + + public AvDecodeUnit getNextDecodeUnit() throws InterruptedException + { + return decodedUnits.take(); + } +} + +class H264NAL { + public static boolean shouldTerminateNal(AvBufferDescriptor buffer) + { + if (buffer.length < 4) + return false; + + if (buffer.data[buffer.offset] != 0x00 || + buffer.data[buffer.offset+1] != 0x00 || + buffer.data[buffer.offset+2] != 0x00) + { + return false; + } + + return true; + } + + public static boolean hasStartSequence(AvBufferDescriptor buffer) + { + // NAL start sequence is 00 00 00 01 + if (!shouldTerminateNal(buffer)) + return false; + + if (buffer.data[buffer.offset+3] != 0x01) + return false; + + return true; + } +}