All of Statistics: A Concise Course in Statistical Inference (draft) (Springer Texts in Statistics)

All of Statistics: A Concise Course in Statistical Inference Brief Contents 1. Introduction…………………………………………………………11 Part I Probability 2. Probability…………………………………………………………...21 3. Random Variables…………………………….…………………….37 4. Expectation…………………………………………………………..69 5. Equalities…………………………………………………………….85 6. Convergence of Random Variables………………………………...89 Part II Statistical Inference 7. Models, Statistical Inference and Learning………………………105 8. Estimating the CDF and Statistical Functionals…………………117 9. The Bootstrap………………………………………………………129 10. Parametric Inference……………………………………………..145 11. Hypothesis Testing and p-values…………………………………179 12. Bayesian Inference………………………………………………..205 13. Statistical Decision Theory……………………………………….227 Part III Statistical Models and Methods

14. Linear Regression………………………………………………...245 15. Multivariate Models……………………………………………...269 16. Inference about Independence…………………………………..279 17. Undirected Graphs and Conditional Independence……………297 18. Loglinear Models…………………………………………………309 19. Causal Inference………………………………………………….327 20. Directed Graphs………………………………………………….343 21. Nonparametric curve Estimation……………………………….359 22. Smoothing Using Orthogonal Functions..………………………393 23. Classification……………………………………………………..425 24. Stochastic Processes………………………………………………473 25. Simulation Methods………………………………………………505 Appendix Fundamental Concepts in Inference


  

    !#"%$&%')(*,+.-0/12!34+5-6(*879:"?/9@#"%=A/B"%CD3E!9:%FG=+5G7,9:!/B"H/+.$5+5(JIK"%G= -@(*"H(*+5-@(*+5CL-M'N!9O-P(*F=,LG#(*-+.G-P(D"H(:+.-@(:+.CQ-LRSCL!T7F,(:L9U-:CQ+.QGCLVWQ-:7XLCQ+Y"%$5$5I=B"H(D" T+.G[Z +.G>"%G=8T>"%CD+.G,\$5]"%9@G+.G,E^_RBT>"H(*,LT>"H(*+5CL-LR"%G=K9@L$."H(*Q=`=+5-:CQ+.7$5+.GQ-La)+.-b/X[!3 CL(:t3EG<euCL"%$.CQF$.F,-S"%G=v"0$.+q(:(:$.r$5+.G[Z ]"%9n"%$5!L/,9*",a&wnU79:_;[+5!F-b3EG<eb$.Q=!\%'x7,9:!/B"H/+.$5+5(JIh"%G=?-@(*"H(*+5-@(*+5CL-+5-9:LyEF+59:Q=oa 0(*_g2(n+.--@F+5(*"%/$56'N!9b"%=,;H"%GCQL=8FG=Q9:!9:"%=FB"i(*L-m"%G=8!9:"%=FB"H(:z-@(:F=LG#(:-La {j|(:+.T%Rx-P(D"H(:+.-P(*+.CQ-d9@L-:L"%9:CDcer"%-MCL%G[Z =FC_(*L=Œ+5Gh-P(D"H(:+.-@(:+.CQ-r=Q7B"%9@(:T>QG#(*-reb+5$.n=B"H(*"tT+.G+5Gd"HG=vT>"%CD+.G,m$5]"%9@G+.G\9:Z -:L"%9:CD\es"%-‰CL%G=FC_(*L=t+.G\CL!T7F,(:L9‘-@CL+.QGCL)=Q7B"%9@(:T>QG#(*-La&’2(D"H(:+.-P(*+.CQ+Y"%G,-j(*%F!#( (*B"i(vCL%T>7F[(*L9>-:CQ+.LG#(:+.-@(:-he“L9: 9@L+5GE;%LG#(*+5G(:KebQL$”a–•r!T>7,F,(*Q9h-:CQ+.QGE(:+.-P(*(*%F!#(b(*B"H(b-P(D"H(:+.-P(*+.CL"%$j(:L!9PIŒ=+5=Go—˜(n"%77$qIh(*U(*,L+.97,9:!/$5LT-La +.G%-U"%9:ŒCDB"HG!+.G,a’2(*"H(*+5-@(:+.CL+."%G-tG<e™9:QCL!%G+.ŽQv(*B"H(MCL!T7F,(:L9O-:CQ+.QG[Z (*+5-@(*-r"%9:mTh"H32+5G\G7,F,(*Q9r-:CQ+.QGE(:+.-P(*-rG,ef9@LCL%!G+.ŽQ (*0!QGL9:"%$.+q(JIv%'-@(*"H(*+5-@(:+.C]"H$1(*,L!9PI?"%G=8T>_(*2=!$5!%I%a&•r$._;!Q9m=B"i(D"OT>+5G+.G,O"%$qZ !!9@+5(*,T>-z"H9:UT>%9:M-@C]"%$."%/$.M(:B"%GA-@(*"H(*+5-@(*+5CL+."%G-6_;!L9z(:!F!A7X!-:-@+./$5%aUB!9:T>"%$ -@(*"H(*+5-@(*+5C]"%$#(:L!9PId+5-xT!9:)7XL9@;H"%-@+5;%r(*B"HGMCL!T7F,(:L9-:CL+5LG#(*+5-@(:-šB"%=O9@]"%$5+.ŽLQ=oa‰›œ$.$ "%!9@LU-@(:F=LG#(:-\eb8=L"%$xeb+5(: (*"%GB"H$5I2-:+5-6%'“=B"H(D"Œ-@!F$5=A/XOerQ$.$š%9:!FG,=L= +.G?/B"%-@+.Cz79@!/B"%/,+.$.+q(JIv"HG=?Th"H(:LT>"H(:+.C]"H$j-@(*"H(*+5-@(:+.CL-QaSm-@+.GO'W"%GC_Iv(*2!$5-s$.+53%zGQF[Z ž!ž


ž  

    ! 9*"H$1G_(*-QR/12!-P(*+.G,M"%G=Œ-@F77X!9@(r;!LC_(*!9T>"%CD+5GL-)eb+5(:!F,(F,G=L9@-@(*"%G=+5GO/B"%-@+.C -@(*"H(*+5-@(:+.CL-+5-$.+53%z=!+5GO/9:"%+.G?-:F,9:!Q9@I?/XQ'N!9@z32G,eb+5G<e (*F-@t"O/B"HG=B"%+5=oa "rF,(>ebL9@ C]"%Gu-P(*F=QG#(*-Œ$.L"%9:G /B"%-@+.C479@!/B"%/,+.$.+q(JIc"%G= -@(D"i(*+.-P(*+5CL-hyEF+5CD3E$5$ I# wm<ebQ9:Ha\›s(6$.]"H-@(]RX(*B"i(6es"H-6TdI`CL!GCQ$.F-@+.!G`ebLG TdI8CL%T>7F[(*L9œ-:CL+5LGCQOCL!$ Z $.L"%!FQ-‘3%Q7,("%-:3E+.G,nT& %'( L9@)CL"%G0k1-@LG=tTdI6-P(*F=,LG#(*-‡(*!_("s![2=0"FG=Q9PZ -@(*"%G=+5G8%'rT2=L9@G-P(D"H(:+.-@(:+.CQ-zy2F,+.CD3E$5$I #*) U(JI[7,+.C]"H$&Th"H(:LT>"H(:+.C]"H$š-@(*"H(*+5-@(*+5C CL%F9:-@`-@71QG=-h(:[ TMFCD (*+5T>?!Gc(*Q=+.!F,-h"%G,= FG,+.G-@7+.9@+.G (*%7+.CQ-4VNCL!FG#(:+.G TQ(*,[=-QR(Jer=+5T>QG-:+5!GB"%$1+.G#(*Q!9*"%$5-Q(:C%a ^>"H((:tg[71QG-:\%'šCQVW/X2%(*-P(*9*"H77+.G,RoCLF,9@;!UQ-@(*+5Th"i(*+.%GoRj!9:"%7+5C]"%$T>2=Q$.-œ_(*CHa ^aU’[?km-@Q( !F,(b(:>9@L=Q-:+.%G`!F,9nF,G=L9@!9*"H=FB"H(:t%G!9:-nCL%F9:-@t!G879@!/B"%/+5$.+q(JIv"%G,=KT>"H(*Z T>"H(*+5C]"%$‘-P(D"H(:+.-@(:+.CQ-Lan+.-m/12!38"%9:%-:t'N9@!T (*B"i(6CL%F9:-@%a +nL9:t+.-œ"h-:FTT>"%9@I?%' (*,\T>"%+5Gv'N]"H(:F9:Q-nH'(:+.-/X[%3Xa ž abœ/12!3U+.-)-:F+q(D"%/,$.m'N%9“%G!9:-rFG=Q9:!9:"%=FB"H(:L-s+5GvT>"H(*oR2-@(*"H(*+5-@(:+.CL-“"%G= CQ!T>7,F,(*Q9“-@CL+.QGCL6"%-“e“L$5$j"%-“!9*"H=FB"H(:œ-@(:F=QGE(:-+.G>CL!T7F,(:L9r-@CL+5LGCQ6"%G= %(:L9yEFB"HGE(:+5(*"H(*+q;!zpBL$5=-La abkvCQg,"%T>7,$.%RrG!G7B"H9*"%TQ(:9:+.C?9@L!9@L-:-@+.!GjR/12%(*-P(*9:"%77+5GRr=,LG[Z -@+5(JIvQ-@(:+.T>"H(*+5!G?"%G=8!9*"H7+.CL"%$jT2=L$5-La , abkOB"];%4!T+5(:(:L= (*!7+5CL->+.G 7,9:!/B"H/+.$5+5(JI(*"H(v=G%(v7,$Y"]I "CLQG#(*9*"H$œ9:%$. +5G-@(*"H(*+5-@(*+5C]"%$)+.G['NL9:QGCLHaA %9d_g,"%T7$5%R&CL%FG#(*+.G, T>_(*2=-M"H9:h;2+.9P(*FB"H$.$5I "%/,-:LG#(La - abkJG !LG,L9*"H$”R)kt(*9PI(*A"];!!+5= /XL$."%/X!9:+5G4(*Q=+.!F,-CL"%$.CQF$Y"i(*+.%G-M+.Gc'W"];!!9U%' QT>7"%-:+5ŽL+.G,OCL!GCQL7,(:-La . abk1CL_(*9:+5CS+.G,'NQ9:LG,CL)/1_'N!9:)7B"%9:"%T>_(*9@+.C&+.G['NL9:QGCLHax+5-+5-‘(* !7,71!-@+5(:&%',T!-P(-P(D"H(:+.-P(*+.CQ-‘/12!3E-‡/F,(‰kj/XL$5+._;!S+5(‡+5-o(*)9:+5!#(‘es"]Iz(*n=b+5(La /"%9*"%TQ(:9:+5C\T2=L$5-m"H9:dF,G9:L"%$.+5-@(*+5C\"%G=K7XL="%!!!+5C]"%$5$5IhFGGB"H(:F9*"H$”a\0V +n<e e“!F$5=>e“œ3EG<e (*nQ;%L9@IE(:+.GU"H/1!F[(“(:m=,+.-@(:9:+5/F,(*+5!G_g[CLQ7,(s'N!9S!Gm!9 (Je“M7B"%9:"%TQ(*Q9:1- #i^dk&+5G#(*9:2=FCQm-P(D"H(:+.-P(*+.CL"%$B'NFGC_(*+.%GB"%$.-)"%G,=v/12%(:-@(*9:"%77+5G ;%L9@I?L"%9:$qIŒ"%G,=`-P(*F=QG#(*-bpBG=?(*+5-y2F,+5(*0GB"i(*F9:"%$”a $ ':x+59:-@(MQ9:T 45/&9:%/B"%/+5$.+5(J6I ) "HG7= ':’[LCQ!G= ‰L9@T 2 abkt"%/B"%G,=!Gc(*?F-@FB"%3 4’2(D"H(:+.-P(*+.CQ8- )8"%77,9:#"%CDja’[!T-P(*F=,LG#(*-\!G$qIK(D"H3%O(*OpB9@-@(tB"%$q'r"HG=+q(

ž, e“!F$.=\/Xr"bCL9@+.T)+q'[(:QI0=,+.=\G%(‰-:Qs"%G#I0-@(*"H(*+5-@(:+.C]"H$%(*Q!9@I%ašBF9P(*Q9:T!9:HR 79@!/B"%/+5$.+q(JI`+5-6T!9:OQG#"%!+5GhebQG-P(*F=,LG#(*-\C]"%G -@LU+5(z7,F,(z(*Œe“!9:34+5G (*,\CQ!G#(*g[(m%'-@(*"H(*+5-@(*+5CL-Qa
ab CQ!F9:-@ T>"%G#It9@L-@F$5(:-x"%9@r-P(D"H(:L=teb+q(*%F,( 79@[%' a`Œ/+5/$.+5!!9*"H7+.CO9@Q'NQ9:LG,CLL-U"i(d(*,vLG=H'b]"%CDCDB"H7,(*Q9O71!+5G#( (*,\-P(*F=,LG#(m(:"%779@!79:+."H(*z-@!F9@CLL-Qa a 6GdTdIze“L/,-:+5(:"%9@)pB$5L-‡eb+5(:  CL2=Seb+.CDM-P(*F=,LG#(*-šCL"%GdF,-:“'N%9=%+.Gm"%$.$ (*,œCL%T>7F[(*+.G,a +ne“Q;%L9QR,(*m/12!3O+5-)G,%(“(:+.Q=>(*  "%G=v"HGEICQ!T>7,F,(*+5G $Y"HG!FB"%%6C]"%G8/X\F,-:L=ja mpB9@-@(“7B"%9P(r%'1(*n(*g[(“+.-SCQ!GCLQ9:GQ=veb+5(:>79@!/B"%/,+.$.+q(JId(:L!9PI!RE(*b'N!9@Th"%$ $Y"%G,!FB"%!zH'xFG,CLL9P(D"%+5G#(JIŒeb+.CD8+.-s(:0/B"%-:+5-b%'x-P(D"H(:+.-P(*+.CL"%$o+5G,'NL9@LGCQ%aS\/B"%-:+5C 79:%/$.QT(*"H(ber0-@(:F=,I?+5G87,9:!/B"H/+.$5+5(JI>+.- %     !"#$&% !('*)+-,,+.0/213 !"451% !('0%6!(#7+,8'9:51;'0= )-'*?4+,#@

)-:LCQ!G=\7B"%9P(‡%'[(:S/12!3m+.-‡"%/X!F,(‡-P(D"H(:+.-@(:+.CL"%$%+5G,'NL9@LGCQ“"HG=t+q(*-‘CQ$.!-@&CL!F-@+.G-QR =B"H(*"0T+.G,+.Gz"HG=UTh"%CD,+.G$5]"%9@G+.G,ab/B"H-:+.C7,9:!/$5LT %'X-@(D"i(*+.-P(*+5C]"%$[+.G,'NQ9:LG,CL +.-s(:0+.G#;!Q9:-:\%'79:%/B"%/+5$.+5(JI %   A51B'0(:%(:L9>"%F,(:!9:->"%G=c(*71%+.G#(9:]"H=L9@-(:%(:L9UF-@Q'NF$ 9:_'NL9@LGCQL-Laukter!F,$.= L-:7XLCQ+Y"%$5$5I$5+.3H?(:TLG#(:+.!G(*,`/X[%32-/#I M6
9:2%("%G= ’[CDQ9@;2+.-@ V  ^m"%G,=
9@+.TT>_(:(m"%G=A’E(*+.9@Ž]"%3HL9UV ž 6 ^b'N9:%T eb,+.CD kn"%=B"%7[(*L= T>"%G#Iv_g,"%T7$5L-n"%G=8_g[CLQ9:CQ+.-:Q-La

ž
.

#7,C#7)-, N L 6   )+#7'* !(F 






2’ (D"H(:+.-P(*+.CQ+Y"%G,-b"%G=KCQ!T7F,(*Q9b-:CL+5LG#(*+5-@(:-m%' (:LG8F-:t=+ 1L9@LG#(n$Y"%G,!FB"%!z'N!9(: -*"%TK(:+.G,a +mQ9:4+5-v"=+5CQ(*+5!GB"%9PI(*B"i(v(*K9:]"H=L9vT>"]Ices"%G#(Œ(:9:_(*F9@G(* (*9@!F!,!F,(b(*0CQ!F9:-@%a 

#7,C#7)-,

L-P(*+.T>"H(:+.!G CL$."%-:-@+5pBCL"H(*+5!G CL$5F-@(:L9:+5G =B"H(*" CL"H(*Q-m"HG=?CL!G,p=LGCQd+5GE(:L9P;%"H$.eb+q(*?!FB"%9:"%G#(*QL-b!G?'N9:LyEFQGCQI8/XLB"];2+.%9 FG,+5'N!9@T/X!FG=,-m!G?79@!/B"%/,+.$.+q(JIhH'xQ9:9:%9:





       "!

%$

ž2

 

    !

'#7'* FI?H6'


-6$

@9 ]"%$oGEFTd/1Q9:+.G['  xV 1^ +5G,pBTMFT %x(:0$Y"%9@!L-P(bG2F,TM/1Q9h-:F,CD`(*"H( šV 1^“'N%9b"%$.$  (:+.G3>%'‰(*+5-b"%-(*0T+.G+5TMFT %'

-:F,7  šV 1^ -@F79:QTMFT %š(*0-@Th"%$5$.Q-@(GEFTM/XL9 v-@FCD`(:B"H(  x V 1^r'N!9b"%$5$  (:+.G3>%'‰(*+5-b"%-(*0T>"ig[+.TMFT %'

V 1 " ž ^ V #" ^ %$&$&$' , ž )+ * ! ) -, )2# ( ,/. -0 , 0 =+? 
"%TTh"t'NF,GCQ(:+.!G768 ;: 3)V54x^ 9 !F[(*CL%T> @ A -:"%T>7,$.6-@7B"%CLhVW-@Q(n%'!F,(:CL!TL-*^  _;!LG#(MVW-@F/-:_(œH' A ^ B xV @ ^ +5G=+.CL"H(*%9r'NFGC_(*+.%GDC ž +5' @ E "%G=  H(*Q9@eb+.-@ F“VG6^ 79@!/B"%/,+.$.+q(JI%'x_;!QGEH(  I I GEFTM/XL9b%'7X!+.G#(*-+5G?-:Q(  JLK CQFTMF$."H(*+q;!z=+5-@(*9@+./F[(*+.%Gh'NFGC_(*+.%G

MK 79@!/B"%/,+.$.+q(JI=LG-@+5(JIVN!9Th"H-:-D^“'NF,GCQ(:+.!G ONPJ B"%-b=+5-@(:9:+./,F,(*+5!GQJ ONR

B"%-b=QG-:+q(J7I

4S "%G= B"];!0(*,\-:"%T\=,+.-@(:9:+5/F,(*+5!G TNPJ -:"%T>7,$.6%'x-:+5ŽL  'N9:!TUJ V -P(D"%G="%9:=`wm!9@Th"H$j79@!/B"%/+5$.+q(JIh=QG-:+q(JI W -P(D"%G="%9:=`wm!9@Th"H$j=+5-@(*9@+./F[(*+.%Gv'NFGCQ0 (:+.!G F7,71QY9 4yEFB"HGE(:+.$5z%'[ZV  ž ^r+”a %\a W V ž " 4š^ X : ? ]6V A^ 4^6_ JhV` 1^ g,7XLC_(*Q=`;H"%$.F,VWT]"%G ^rH'x9:"%G=!T;%"H9:+Y"H/$. ]6VGa,V A^@^ 4 6 aV` 1^ ? JhV` 1^ g,7XLC_(*Q=`;H"%$.F,VWT]"%G ^rH'a,V A^ bUV A^ ;H"%9@+Y"%GCQz%'9*"%G=,!T ;H"%9:+."%/$5 ' V M^ CQ;H"%9@+Y"%G,CLz/XQ(Je“LLG "HG= =B"i(D" -:"%T>7,$.6-@+.ŽL  c CQ!G#;!L9@!LG,CLd+5G?79:!/"%/+.$5+5(JI "ed CQ!G#;!L9@!LG,CLd+5G?=+.-P(*9:+5/F,(:+.!G f g"eid h CQ!G#;!L9@!LG,CLd+5G?y2F"%=9*"i(*+.CzT]"HG TjPZV k 2lDm ^ V " k‡o^ npl f ZV  ž ^ 



































  



























  









 "





ž
FI?H'


-6$

'# '0 '0#$IT>"\'NF,GCQ(:+.!GŒ+5-s+.-s=,QpBGQ=?/EI#3“VG4š^ 46\9 8  : 0 < 0>= ? 'N!9 4   a&kl' 4 ž (*QG3)V54x^ 4 V54 " ž ^3)V54 " ž ^aSkl'  +.-n"%G8+.G#(*Q!L9s(:LG 3)V  ^ 4 V Q" ž ^  a ’[!Tz-:7XLCQ+Y"%$1;%"H$.FQ-m"H9:&%3)V ž ^ 4 ž "%G= 3)V ž n ^ 4 &a 






  
   





   


     

  !#"%$&"('*)*',+.-/'*01+325467$8+32(467$8+3':9;$&)#?%$8>#4A@B&DCE?%$&=H?(=594I!+J$8'*=Z+3 +3254/$8=%$&):-[0!'*0\8@]9I#6N5?(+34I$&):>##Q',+32560K1^254O0_+J$&_+3'*=(>`Na#':=#=$˜!4;$&)%=E?56¦"d4 \Ï ¼ Òx+3/4IU&4!-4­U#4=5VE?5=5N5!4S5':9I+J$8"5)*4P0Q4CE?54I=594/&@ +3&0Q0Q4I0 2T25&0Q4H):'*6':+Q'*=5>RN(Q#Nd#!+Q'*#=`+Q4=5S50A+3i$˜9I#=50!+3$&=m':=E+Q4!N5Q4­+J$8+Q'*#=50i)*4;$8S +3m+ 2ž 0!9J25[#)*0`&@c':=(@B4!4=59I4&ë +32(4/@B!4CE?54I=E+Q'*0_+H$8=5S|+Q254 Z$;-#4I0Q' $8=³0!9J25G&)*0KL4/S54I@B4IcS('*0Q9I?50Q0!'*#=|?5=2Z$;-[0&@ #!S54!'*=5>/+3254I0Q4R&" !49­+30Z':0
® Ï Ð  Ò Ï ÐÛ#Ò Ü Û  K `%#ž9I#=  #" !4I9I+30 @BQ&6 4K `×&\4­~($&6N5):4&V(',@ 2ž4O2%$;U#4˜$i9)*$&0Q0\&@xÛ Ú Nd4#N5):4O$&=5S 2ž4 2\$&=#4I=54Q$&)EŒ„ׇ+QQ?54+Q2%$8+ \Ï ¼ L + / MÒO® \Ï ¼ L +`Ò  \Ï ¼ L MÒ­Kb^2(47!?5)*4I0M&@\N5Q&"%$&"5':)*':+.-}$&N5N():-m+Q³4­U#4=#':U&4= +32%$¨+A-&#?b25$U&4P0!Na8+30A'*0=5&+  K $.=+Q25'*09;$80Q4&VE+32(4cS('SPa4Q4I=594c"d4I+ 2ž44= ZÏ ¼ L +`Ò$&=5S \Ï +,L ¼ Ò1':0D#" + S5G4I0Q=  +9J2%$8=5>#4 +3254/N5!#"%$&"('*)*',+.-b8@ ¼ K `×!#6 +Q254 S5.4 -5=5':+Q'*#=Ó&@˜9I#=5S5',+3'*&=%$&)HN(Q#"%$8"5'*):':+.- 2\4 9$&= 2T!':+Q4 ZÏ ¼ +`Ò® \Ï ¼ L +`Ò \Ï +`Òc$8=5S$&):0Q ZÏ ¼ +`Ò\® \Ï +'L ¼ Ò žÏ ¼ Ò­K  @F+34= Vd+Q2540!4i@B#Q6i?5) $&4/>#',U#4˜?50 $`9#=&':U#4I0 -#&?+Q254P&N5Na&!+3?(=5':+.-M+3&44IN7-&#?5S5[##0 2T',+39J2Ã+3/+3254T&+Q254?5=5&Na4I=54S $

2$







,





,



Ü'


   

S5[#IK