From 4eaf68ae58c3795c4aec474b2de9386d307d6a14 Mon Sep 17 00:00:00 2001 From: Cameron Gutman Date: Tue, 12 Nov 2013 02:04:08 -0500 Subject: [PATCH] Lower the buffer size of the Opus decoder based on the data we're actual receiving. Handle invocations of the decoder with null data for packet loss indication. --- jni/nv_opus_dec.c | 4 +- jni/nv_opus_dec_jni.c | 17 ++++-- libs/armeabi-v7a/libnv_opus_dec.so | Bin 148716 -> 152812 bytes .../av/audio/AvAudioDepacketizer.java | 54 +++++++++--------- 4 files changed, 42 insertions(+), 33 deletions(-) diff --git a/jni/nv_opus_dec.c b/jni/nv_opus_dec.c index 60425aec..c3514a82 100644 --- a/jni/nv_opus_dec.c +++ b/jni/nv_opus_dec.c @@ -30,7 +30,7 @@ int nv_opus_get_channel_count(void) { // This number assumes 2 channels at 48 KHz int nv_opus_get_max_out_shorts(void) { - return 5760*2; + return 512*nv_opus_get_channel_count(); } // The Opus stream is 48 KHz @@ -48,7 +48,7 @@ int nv_opus_decode(unsigned char* indata, int inlen, short* outpcmdata) { // Decoding to 16-bit PCM with FEC off // Maximum length assuming 48KHz sample rate err = opus_decode(decoder, indata, inlen, - outpcmdata, 5760, 0); + outpcmdata, 512, 0); return err; } diff --git a/jni/nv_opus_dec_jni.c b/jni/nv_opus_dec_jni.c index bb99568b..1df69883 100644 --- a/jni/nv_opus_dec_jni.c +++ b/jni/nv_opus_dec_jni.c @@ -1,5 +1,6 @@ #include "nv_opus_dec.h" +#include #include // This function must be called before @@ -48,13 +49,19 @@ Java_com_limelight_nvstream_av_audio_OpusDecoder_decode( jbyte* jni_input_data; jshort* jni_pcm_data; - jni_input_data = (*env)->GetByteArrayElements(env, indata, 0); - jni_pcm_data = (*env)->GetShortArrayElements(env, outpcmdata, 0); + jni_pcm_data = (*env)->GetShortArrayElements(env, outpcmdata, 0); + if (indata != NULL) { + jni_input_data = (*env)->GetByteArrayElements(env, indata, 0); - ret = nv_opus_decode(&jni_input_data[inoff], inlen, jni_pcm_data); + ret = nv_opus_decode(&jni_input_data[inoff], inlen, jni_pcm_data); + + // The input data isn't changed so it can be safely aborted + (*env)->ReleaseByteArrayElements(env, indata, jni_input_data, JNI_ABORT); + } + else { + ret = nv_opus_decode(NULL, 0, jni_pcm_data); + } - // The input data isn't changed so it can be safely aborted - (*env)->ReleaseByteArrayElements(env, indata, jni_input_data, JNI_ABORT); (*env)->ReleaseShortArrayElements(env, outpcmdata, jni_pcm_data, 0); return ret; diff --git a/libs/armeabi-v7a/libnv_opus_dec.so b/libs/armeabi-v7a/libnv_opus_dec.so index 4f8be2974dcd277ee42fb4f04ea4696b1a0b65c9..1c02515a399ab84ba3f12b6485bd54bcf1d4f850 100644 GIT binary patch delta 6731 zcmaFU$@yj_=LAK@z==vuc04vr3?R(Pz`(#Dz`(%Bz`*dtn~C8`Gz2TKFfgdFFfgz| z<(U{57!LR{F&u~m;fagPv~K)oWPsrx3=9k)%)-FH0K!2}OHvS~O=e;YQ{3PNkrHBH zXK(^3OJD*k6=V=%@M2(Ku$WxQs9ev*z`y`fF^z$N!9bINfrEj8L5YE#VJ!m#gS-O7 zIUxC?3=9ma3JeSyQ1zD>7#QX_GcY`W@}Dsm4*17<`x^A$x+Ef#Cy^xeu5b7#<+; zzcDi~Fd*3@&ceXpgT&WoVPIH-#CK<5VCd0-#1zOQVJr*`Q*;;@mO$N7#KKU|V4=&v zAOjWXVPRkh(Pd!Bf$|ryFfbg@WnfT%^7peaFg(yjM9l>j28Jg{@-J8z7+CZe7!E+y z|7T%fIHJeEAOhtJvobL3&}U#!f%0`&>rn-qSs57aASnnz<0pgigaN{UdNlq_Rt5$Y zB>4?!{G+T43?4}GH$es<3%q7!V2ChbU~quC@Ee#z|i2!z+eECp90eVpAjD2i`f|%?)*dWH?cEJUd|>cIKhvB z!3L`5GCKpqnaL;F<81 zU`c?K6aow%|8GO%AL3wO;7FLv#UWSE@Q#sT1=Ks&IT#o^{xgD#ZE)fEl!Jj`#XCgw zeFJ&y9V5dXsQUjL3=B)&Bjoux85oYdN2C{7P6mcE?-?03FhSCb8XDi2lcApB!h1%B zHBbdMoD2*#pBUj;%nOYl&dI=l?9XIQ28I(z>8X&Dfq~;QBSQ()f%Rzo9!>@Zna_+2 z2~hbNob{*zOF$MPCCIg$3=A*6GBU709k>mRf0z?BYR_^qFnIi6geSsVAoV{O8RkHf z#tTl=4D%7>LgXCti<5z& z@!h#lvv7R?7Xt%w;*I2DU@-WH2*N}(ekRC(e~j=_wh)b9$;H6n@Q)FmEt}BzU8wvp zhDlru3`_nY;(Rd|1H%@ibiNslf0&Dbfdwg_UgBb4(D{$ZU3a(`7&<2Z0Rsbr$Yfn9bw+{7p;GpYe3Pe2 zsnka^F^DrTFuZ`$Z=m!CDE$RW|A5kepfp1aM4t+jwt&($P}%`X`#|XcC>;W&BcODB z43v=qr8A&(4wRk)r58Zy3sCwBlxB&AID`XATR>?WDD42HJ)pD?ln#K<^$Z~p215jt zj)Bq%P&x%lXF%y3C|v-hYoK%klx~629Z6~&Q=s$$FkR2UumsFtU|0dA*FfnF zPwZ2_fi zptJ*&c7f6!P}&Dd2SDi%C>;T%W1w^blun6fst2e345&a3lrDhMB~ZEoO4mT?1}NPE zr8}T>50st&rKdpY8Blr-lwJU(m&DgYf^ttB6N9KI1H%;%t-`>tC7y{vjER9^2Z&~5 zVAunqMHv_lfM^*8h9e+aj)CC>h*o4^I0K@U7#J>qXm19FDM z7#@J=NCt){AUcMD;YB@&k;uUC21F+@Fnj>f84L_xKy)Sp!w(Rh&A{*nL>DkHusqOx z%JN|TQ@#iN4>&&Td%DAciGkr*|09)CJ&$&L*z**|-hjkjfy7>b#GdhC&(nICh6zX# z9Z2j3Bz6T7y8wxufy7QgVn-mc1CZDr2)5EA2L$Jl1rpl;#-4mpUZUOs)J2>Bl!2lD z!NLFkUvW4vF)({qyGUn9WN>Vh{cwDLf%`T)CI%*j0(XwB|G)qK@4&>d*ZW(8L4mvQ z*8kss{bzCe|NX;%##M}N3T{jvHob)EPGFoNvzK9ig&h-v`Bq^!b}v>p9yk9l3XqD>J}i1?w>Qd;iQ)hEYy1CyKd}G*_Y+WjdcW?=5AO1AcYmlX z5OrhDU{GRl3v|2uW2TW?quT+=|KE@O2g!K3 zUH-vP%%t4lrr@^2o%Ml+o2HwQ+yC#||2McPXDGM@x)}+&UH-ul%%Hr&-N9||ix2G# z43nQIcrX@j)>Uk0WU8>+yiG}nk@4*0lgiy8dR=-8|QAGdrX6 z=0LxER>q#q2ZNt8Gk%!7GJG}Tm&u_K(-|jkejCxu$Y`{=FzPfT7Wn>TuWn`GKdF`s*OpI-t zGuLV}Gp?LGb;D=IoXMRVKZ~ZUWMtUm$H>6K07-*#|Meg4*gSEQJ_n=0=7YPW85yr{ zzPtM|7n{daMh26qn~xnk#=-b~bMN_TM#dSNIWGn?F+SKFd)c0iv263)o9CHWBYYVd z{%kJ1v!9XCc(UQW)r=gQ@7@z;Vr-uL@qs*J^<=?^@{Glk4Ij>D^ait?Cx3jX%o*X% z$PnVj$S`HH?4y20qs?m{m9ubeaA0I$m?3Frvsw4~V z!ThHStQlT2C@grq!C?N=O;!xA7jFN^!Z@3manAH!4n}pxN!zz_Ft#%@?%!_9!?>A| z(RunuUdCoo!z2G+hw-*Pa!^?CSc9SA5xc;GhdSFk`531&Ns5FpGPnpZFnB03FciG| z|9_1n1H&6h28KJ+Ed?3vEfWv_f6XSa;312`g2yZj3m(QE{{NcCVD3{8pIM;w0XxHj zhY=t(3=1CeBtXR31X>;hZ{H}$$j;2TVEaxH#^ubCC%hQJV-5#m85j&i7#S}6CJ8j` zW?)z|y;6eFpRsrPNeM<{#)j#CB^ae8D-ZsEt)e*PDF?%Xhg`0fCAjzh>vjMCzY;L$eZ<2s z_bIzz^8*Ei1&_J+{(t@Re($3X{d-^huV-LTFx~!0i!q3a?S?TU!vW*%R=SK{j7&C0 z)7$hIt(aU4wja`CG-G6p+5S_X(VdY=;Ma5~L&hk^;_Y({8KW5)Ww!q`Vk~E7)Sq5x z#wgEdHhrQQ<9w$7JGUE|GZu+3ew@D4n^B(e<@TfAjKYkpU*0eSJVLVXC;e zooy0h6(i%j>5Y>aFLOOP!@w}-3M>Fse-7_}Hha~A!7&C1jAh=rly5h#j5sUv;* z+^LNFnf{#G9yg7Vje}8bd+I#KVphhJ+ixyrjAmy1wB2+$qctPrrs;(%7;iHsZ1-Ks zSkK7FGyUu;MtMe-?Qd5xx-&9v+HSgrv7M8tW$pHTTN(E;Gp$*2I?857fmCEK^{ zW<18s_}O=O+AeyH(V0b3bK?Kk2?=wb+BD34n!+&mDTiR&1CSq-rx)I0)MixQ zKKB-*JtLD#%k+D<84od4G*4f2hw&3*)by!$8H*Whw{zZO6y{*G*e?5wF`A8OMbq@E zH;i?RjoW{|VQdy+*hc)Tlu8?@;+yzk+{b!wm?(of_kh5OiT7xQ-a@sQK*W0KvG2{p{F}#rnB|-*<9U4pw8#I_0&ge2R)aWuX+|glTP|;&z5Yb~| z_@c|ikf6uJ5TM7zV4%mupuoVzAflJT$YH?5a6+Gnp+%30K|`O3;ej3#!wfwph8(aO z0fscS)r>L5Objx{Obj`OObjwcObk~HnHYKunfMtz3?buHAYZSDpMK1isb4Ub2|T(B z3Is_828KPc)6?vjg6p9pTNfCZz!L=)nvgMl&@ed2@u1G>4^1Y9Em}+r3R+AIJz7i* z1zJoD9w4?R69a=b6T=TJCWZqVObjg=ObitoObmB4m>BkGFfo8SvK<;s3`;ba7!K$# zF=S{kF-*~6Vo168lnHcVv zGcmldVq##hW@0d~Vq(}~#l*m2&BSoUl!;-21|(QO9s}VAatzzI+cTYHV!SZD(TPc( zbH#r~1`sxwUg*puJbjN76A$MN=oHG2=@*@t-;4xhh zWal4{EGSX}rt`TlDRY2kU;;pT!7lP~VG?8vm>%cCq|Oq+#2^E)Zw^Fs9aQuN*d71_ Cu8Q#h delta 6422 zcmaE}lJiX`=LAJY$B9Z#c6Y3q7(kenfq{WRfPsOLfq~(PH50=VZwOXkVPH^UVPIf` z$}=%AFdVRDVmROn!V?#nY3=yW$N|n`Bw}K4222^7bq|?Fq9(kBN-VODis*&!A=5M zFpH6ap%zKOJw^tGMkKy069Yr50s}(=)B&NO00M;-0|SE+0|P@Y69a>cCIbTpn9t78 z!o)&h9i3V2!ZoV3=B_@6nti4VEBT>=VNAIsF{3{NuF`T zSQzRVRCE~_ zWS{~)EDQ_|x(p0CQ2qiI28K1d3=9fT{(cq)hBLZ|sJg(yz;FRc{sjvI!v|dkh67Ob z|5+FqHs~=hh(P(mtPBiG^cfgbpnM(HdQ<^tRtAO>ND4yG_{ppc3~!L+>(Tf#Ss54v z3=kGHwn zDH{XB3ll^T`>-)Ed_dwSvoSDic?ZhDkSH!-V_;DE&B!o=5yB6xWn*Bt^MjFL3RIwr zje+684@QOwQ2ul_28N1XjPNK}3^MQ&Biw=O*%%lyJ~J}pFhLC5!^Xg1GI=MfydWqm z$3UI=h>d|EX7Wo``}zzQ1_l$TS_yUrh6Qd63@K1mM(hj>Eba^p3!wZEb_RwC-V6*T zP<|Ra1H%z-M07T>Gce@%GB6lG<)?u3|7V0p_hNPih78N2 z!NAb630!7^OQJXq28K6p5rLn>!N70_$<%5N28M#ko$UIIdnWH>mpAJ9#0W3N)^jj0 zumnI#4grRb|F@y>4{{opz8l~FfdGbkC5l*WMJ6v9^qtJP6mb@?-?03K%-0zjc?4!P|vXEJtM;! zr~(^K28N7JjPNYxg~ku(WMDw{XEG-P!xp6URLIG|@Z}RDLkZM@^=SMaP6h^!&y4WG zbp~fWs=yMEg-8i4Vqhql{F7_p=8fEM7#R&W zxAQ*aVC0*eDdx^7FnObxvM3h=149J^1H)7X28JmN3=EB+u$uf*%$$*Pv!Zx4BdaI_ z0|U?Gg%Xn)1t*(I`ZJ15Zk3!a$j!jO0Ii=G3K$p|geU7tsWb9V4wbTJzzPpmYP2Zh_JrP`U?7Pk_=>p!5PT4XTd83 z4{-nsl;(iaJWyHyN{c{g2`DWCr4^vGia!%WJt*;MKm~N5v;mYhfzlRG+6GEHKxr2! z?E$5IpmYF~4uR4UP&x)mCqU^Gf2MkH>d$}*m1n zcmkqh7#LpEgBXbn3~xYm5(C2r5S_um@C8I?GBErA(b)_Pe?W8r0|Uzg-KQ)M=0D|o z(Eotr!@j3G444=gj`crMKGpMR$A>*nVeAb^>=j7t1xV}}AND-0hiRCAB+-GyZa`vJ zAh8RO*cnLd1SEC@5<38i?SWt`J#s*B9$6r<4Pfla7v&}D4L}{V`A-=b`X3zp|Nj+- z0TTnWceRUjhC~L(M%fR0_ZPTt(_>;_QYdiu|6&qQ;O_mc!Jxohcy zKjSJ!Hw8DQ53626btW**klD+yzebOV!F;Q*8@m^)8;_g+mn{KREA!X1H_AWOd_w&Ca0Z_W%2_|H>JLZeDJee=rm?DL1(( zxb1LP&QNf3bklOX{DUKzL3x9FfZN^|AKDoVCQSaV;L2FG*-^2bk*P;-^C=}EM#f8% z?<$8g?wM?=Qq8zv^F|eZCPw+m2i1}pIVY>CFK4{J`K)?4BhwAl&1#y{SQukB@6~l? z=j{3Q|22n$?#r&toTk$`8FeQww0q3>Vsoo~G!xT@qmv&wsxg{O=5*T6sJ8i}Q#>Q% z#mSm3+KeYRhq}ZwGGiV0}X!Az5&FqXGn^XPrSs4p9Uk!fF%=mTk-tg6oHzsFB zOlO?2nK81Nkkzs)!BLfQqBvs1&*MB%?^U_WF9E_%$ zukMm&WW2NaRPjB_{3UJPbpJhHj)vOODP_2!K? z&oi;w_%bp)*xY(&KO>{XWY>GE8F@EZ^Q4pUG2S z#WNaA{`#t%QD$@Q>(zX$D@+*~1UAe5+RVuKXY$40%8Z{kfBh}b$Y`-y^zUv)M&r#F z|E*_cblKj>#OTk+_+k1*X2$c3d!{dBVH9TEK7A_-qcP*o=`UFrr5TrPXJuua&CIxP z`a%vyb;jx2PjWD}Gcz999>~MEnUT?BIx8Pzv#8mT|F6S%TOTDq-3NuD*KPt$`&dj)U`&kjj<;;?ItQZ*{ z=rJ%Hh-F|95n*Jw?3*Oeu$zHl!t`DVMt{ah)9*?!8Z)*|7nEd_W~`g8E6FI%STWsI zlJPa8?Q}ya#&AaU?VVDL$t;X~+uzDDPUd3F+}^0k=*`F|HvOy?qqva3-v6(E-tT?% zp?~j-|Md(ECaT-NX)y*dvAt1dWVoWd-A|X%i;*cnY5FWZMk}TWh3(h$7|j?N9k%lt zFuF4`efc>(%#bmPv10p1L&j)EM#b&C#*F36j3(1t%^2kwt*0+FW1P>#vTM7WIb)Fs z|$hRVw^vHWe=mi zdblg2y1GbEbdnVbo@9nl9VR_?d0SK?VkcgWJXW7@1g@ zdJb-vn#5Se$oP5s)X9vOxqj?nVA!ySfnmz-?L|`s6#?_22+Y?tZ)-y5+PJg(HQJ#@=JL76bcSgqT+dbDXwsSJgn7jSbR>pnIOb2Fe zpR|+FjEU*VlGah4R{IY%Ge#ZA)vQLT`7?yN0Fo5FgUp*7Uga7;t3=ALs=PqVs zXxTpZ6ytqP#?#YluP{n8F5W)#3S&Pbqy2WxYmCk;jC#{^Z!wB9YD{mv#i-4wvwh<& zMter4h?41_ZZjTY>M5SS?GEE7#`x(g?=luMI&GJ|$0*FfXuI9;8DlgX)1IQ~eQy}+ z7~8hl*$*#d#F})o)nO(r`lwq<=+&jq<&|!w)?sh7Ed5 z3>kV%3^rgj0t{(ts~L5SnHb&}F)`Q}GBLa{WMWui$ixt1$i&YeVF(#|0r`4`|MX+N zO#OnsOyGe^P#{P$Ffi=#ou1~$6kHzwQMrJD2|VVlq6rz@1&!!}9Pi=F#Bf5BiJ?V{ ziQ$7L6GMy^6N7^m6N7{n6GMz96T<~9CWaGQObin=m>5Ddm>4`Xm>AY*FfsIKFfo97 zWf2-o3?&*&3=?#i7%ViH7*ceY7z{vi8cYl=rc4a=79fB5GBFfrFfn9kFfrtqGBH$W zFfl|}Ffj;NGBMO>FfkaIGBK3+GBN0wGBM2XWn%C!Wn!=~Wnys9U}Csn%Ea)*l!>9n zjEOVQFr6=iNtpvQncx7@3wDuD2$LYA!}PciCUq7ECI%UZeRCk9>!6}H H!1e$D*z*NO diff --git a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java index beffca91..13a78f4b 100644 --- a/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java +++ b/src/com/limelight/nvstream/av/audio/AvAudioDepacketizer.java @@ -11,7 +11,7 @@ public class AvAudioDepacketizer { private LinkedBlockingQueue decodedUnits = new LinkedBlockingQueue(15); - private AvShortBufferPool pool = new AvShortBufferPool(512); + private AvShortBufferPool pool = new AvShortBufferPool(OpusDecoder.getMaxOutputShorts()); // Sequencing state private short lastSequenceNumber; @@ -21,34 +21,11 @@ public class AvAudioDepacketizer { pool.purge(); } - public void decodeInputData(AvRtpPacket packet) + private void decodeData(byte[] data, int off, int len) { - short seq = packet.getSequenceNumber(); - - if (packet.getPacketType() != 97) { - // Only type 97 is audio - return; - } - - // Toss out the current NAL if we receive a packet that is - // out of sequence - if (lastSequenceNumber != 0 && - (short)(lastSequenceNumber + 1) != seq) - { - System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); - - // Tell the decoder about this packet loss - //OpusDecoder.decode(null, 0, 0, null); - } - - lastSequenceNumber = seq; - - // This is all the depacketizing we need to do - AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); - // Submit this data to the decoder short[] pcmData = pool.allocate(); - int decodeLen = OpusDecoder.decode(rtpPayload.data, rtpPayload.offset, rtpPayload.length, pcmData); + int decodeLen = OpusDecoder.decode(data, off, len, pcmData); if (decodeLen > 0) { // Return value of decode is frames decoded per channel @@ -66,6 +43,31 @@ public class AvAudioDepacketizer { } } + public void decodeInputData(AvRtpPacket packet) + { + short seq = packet.getSequenceNumber(); + + if (packet.getPacketType() != 97) { + // Only type 97 is audio + return; + } + + // Toss out the current NAL if we receive a packet that is + // out of sequence + if (lastSequenceNumber != 0 && + (short)(lastSequenceNumber + 1) != seq) + { + System.out.println("Received OOS audio data (expected "+(lastSequenceNumber + 1)+", got "+seq+")"); + decodeData(null, 0, 0); + } + + lastSequenceNumber = seq; + + // This is all the depacketizing we need to do + AvByteBufferDescriptor rtpPayload = packet.getNewPayloadDescriptor(); + decodeData(rtpPayload.data, rtpPayload.offset, rtpPayload.length); + } + public void releaseBuffer(AvShortBufferDescriptor decodedData) { pool.free(decodedData.data);