From 1b177f5aeceb7d4225bb127506a25fc34635369d Mon Sep 17 00:00:00 2001 From: root Date: Sat, 9 Jul 2016 11:59:07 +0800 Subject: [PATCH 1/9] new file: dist/ngxtop-0.0.2.tar.gz new file: ngxtop.egg-info/PKG-INFO new file: ngxtop.egg-info/SOURCES.txt new file: ngxtop.egg-info/dependency_links.txt new file: ngxtop.egg-info/entry_points.txt new file: ngxtop.egg-info/requires.txt new file: ngxtop.egg-info/top_level.txt modified: ngxtop/config_parser.py modified: ngxtop/ngxtop.py new file: tmp/ngxtop.log --- dist/ngxtop-0.0.2.tar.gz | Bin 0 -> 9884 bytes ngxtop.egg-info/PKG-INFO | 175 +++++++++++ ngxtop.egg-info/SOURCES.txt | 13 + ngxtop.egg-info/dependency_links.txt | 1 + ngxtop.egg-info/entry_points.txt | 3 + ngxtop.egg-info/requires.txt | 3 + ngxtop.egg-info/top_level.txt | 1 + ngxtop/config_parser.py | 30 +- ngxtop/ngxtop.py | 20 +- tmp/ngxtop.log | 436 +++++++++++++++++++++++++++ 10 files changed, 662 insertions(+), 20 deletions(-) create mode 100644 dist/ngxtop-0.0.2.tar.gz create mode 100644 ngxtop.egg-info/PKG-INFO create mode 100644 ngxtop.egg-info/SOURCES.txt create mode 100644 ngxtop.egg-info/dependency_links.txt create mode 100644 ngxtop.egg-info/entry_points.txt create mode 100644 ngxtop.egg-info/requires.txt create mode 100644 ngxtop.egg-info/top_level.txt create mode 100644 tmp/ngxtop.log diff --git a/dist/ngxtop-0.0.2.tar.gz b/dist/ngxtop-0.0.2.tar.gz new file mode 100644 index 0000000000000000000000000000000000000000..cafc5b65b718e1070042819196003bda5451407d GIT binary patch literal 9884 zcmcJVMN=G%qC|0bmmpz)A-Dtx?wUY=z~Jt|-QC^Y-90$L2Par?4THN6KKJ_zZ}E1g zs;kbft2a#(gNC;55~zs)GqZMa<*>K(bailK<7DS#=Vo^`c7{6_vGG{%XnM)&Ls;$> zO(lpTxo@tkZ@XvQs{Kv~jB8n6w|Qaug`}{dQmUxt@4E1^eu+g*FRq3w)Hc6Pa7a&r zZUi@lWr5OSelzZzejF7$0SpC`W?#=M6FopxUv8BJeO$Gva5 z`%t~qP$|}GbBw@xo37g|a7oqsrG(qT~g>8G@j|u-7PA)=n8=*pM z-K8+T`9FW+odliUbO)=Hzbr;U5_T*ZCr7;LP|xCZ4=i@OPdRkCxYYa)wAPvee-ZBh=R3%$Dd={$d!q9PY*h6v zfZ~2AW?`e8Z&STV{`RQp8rAXJWM`58kYZ)_A6pQLIITL~H z$?QVBwU5J!=rZv#unHn0-P?^J>k(2T)IUOtfK*p68V7lzEj6W>cZ`U%JR9oI65r3W zg>LtVghj)2Hv6bzL#{^9FHHT>K7=*M_zxBbhIFfPA-B#Vxy>*BRZW?eUY`Lri$971QUGrJZ?~%rM>~` z7A$QIxs*=Z`|nw#5UA*-5b)F(ffr;3PZF^W01C~iZCIm=arPp!`+>y?PDQQVXbgh0 z3^;X`LLnev$s!hcXX_(Ie0US(Be7e(x-m-ZQD$u67AG}51#Ld4S|r>#_vH~v9EYsz zStN5r`uoZHqIl&JqbPmj4g9zfE1oSXF9=-5Mb1tpExE&+~?#d=( zUj)m@Z&CR@J|!^DqQ?wI!j(9i$-9tchS~OBZ59zOV9;#xB41NJBF6hmlhhc+9dVM) zMR!Jy-Xo@DfDL$}agEIS&1naj8e<7@D$-_am$~}lpWY_Gt1Zlhnh}4BXkRSgqJnaR zr_?Majl=P`uCR8YHr;e?%zMkWCCC;V6Km zvFQ3R+wP3tb2JDz4#w(C1H0f3rr=V}S*FXJ9f5w`AzJdhZVdPwkD^IM?Q!-K zFp~>)G$Zw?F!8 zWaPE-#&{xKFUo!@bGw?YMYVuuE9Og9+Bcbi3J0aV8Ff<8Lkn|;CLUthKQ7J~2k*JH ze>W3TTZ7H0T{c}Oy!}mvcbXIis^bb#eDugATx|kTvmDeqa5A#V-v{( zlHOyMDQK|59;`%EYcna2D*AB5T-XK6A3Q4N=;W@U9KZ=KL(NrZ%hk?Nbk$tc;3u zdgH7ovQey6TXd{XU7Z3EGSdt{whBq68-pn9UE~*HQ^=*sj3;F*Y^y@6&tx@N%Wr?_RO8I^MsXU($pd*~snm z7sM}T2V{S8J!%IRE7mI=RiZRo)kg8-Q6J07fPH_ci{TrLcYp;efUyp%8o5H_I5G@! z58v-_s&JU^WBK#JdOw@)BH`*>k`d1J__8C1m5yIsKN~0rl=);x7BtmW1+7$8GllW2 zoyoDH&H^!K1{ef&w;_Ur_#;^P=#^@Qu@Y7Q3tA?TBmC(^g_Dymn*~`{T~zZBlUbsf zO$WCe%kk2;bnmcPi=e)bwfqj>v5k!Z2B>2ZDo3x}T%LjW8O{+Ke0!K}V2v^(o4W89 zQ&Q5lsvl(i%p+3-;>a&{oFAD3e4!$6uLCAE5~{qYG1y=PKlwLRb_ylNNv+z!R+EZR*g4*U4> zVXisuUhge+w=~8$%TYYl-$On=y#$AQ*fQS6jj6lnhF^c(`!;N%@K*@;z-l~F!|Q>8 zm2{5%8H+`@_C5Ry^vdji9Jn z_HsC7GJg81W<_5lcrd*@s**TP1ENt}49iSuh+=LqRv?-_o~5!RGV1bf9}_sjKt1cH zPI68KR2V=J;V!%el(@gCrs~aAxtt@+8-o$3_PTM*I^fnP5{%(@TvXYYyE;QLbNZnd zW7vd!vA%b*>Y>V`KU7vb_IlCvRcxl1f;=ePFyUbc?q7w&4B|v0tP3-P{qcWmx{u*y zp0V4<|1;ETKeg+}D<&g}L%0uwL`wcs9d^Wxl^n7e{kmPuZgJ?Yury8dOwI0FK*drF zGCj&g22}6-w&o3$F)VO&mwL3-ZTEKrk|j1;BFw1iGUE*!r<%|%JE+Oavsl4=KHtD4 zdVcW!L#GxL>6h!BxP&!`kDVskXeph#QewHT5DfeRsoD5-WePtJ+67Q+UVV8^)rYnUvTnV1ekklerO^`fG)38*QJPtkXpiF5_ga zE|Mp(NCIJIpQtG6fpQ;7~zQrj+j3EMt)w&pP@l!+Cs{+qk?A_ z%yqi^MIt>VfvME7ZO9?U18U`Z{wUSPfea`silJ2QuLxo+3XcvYh$)s9EqF-n_n7J=px|icpNW%UlYozlu!;S7W{|kILgHw|46KYW6oPZFU>a!AY53XTCNGU!4PS6q$ANQE^OYpWT2cX=y=Sf{&XV{~Gzr*6t4&^#l^L zQ(=|DaE3&B?z3=1?u4C94!}bvoa+$2hmD%y!%@t}$!1QnPl0YvNcx#fSyhHV?mTeQ z7ulX74OkVH^mQ<>ct?3+hj}D=3`426DG6{;fM>0{W{K$``8s+>AzA5vAtB_*Qi*iU zoB@j7dwOibWSw^|^BWJz%&Uk6Is=`uY<*3t3++qMdR3pHRT-mVRBt!WLpvOOANs@u zWS6U4_@<)0+Td~!8cmlv{H(JNRhMU&)NKxl@qaw8vuktHglw3loWKQ@ zA?aQ}YJaY=vt1LJzCAi-Zmzz^K~h#_YooM;?7sjl+s z>iqi5FoEvsEX1GoUNS1K$aG8Kx6q562TG+imGouf)vI-Szes^|4eaX@o~NUQ z6%W(G&2QVTqx8bqseFtEX1UnG;&VmM9*ps)hk^9UTqj@S1#C4wtWD(##8Mf6R$#or zCp{xLszG~34gKECQfVbmAG&De=~Bgt_t&mTrcy#JqfF6z@uWRfuv~GWWqW~1nHv%D zH^|&8#vMzt-$iNJQOA~serTD)i{}MrhD0=MXviHqMypOMB>khBC}=4%pz|txA@(&7 zVq@kCWJg>43BT587iHL#Kpr6DKN6I`j0H9#7bA%lh<7v}JCGm(kfN2ASz^|< zh@*BQ4%;|x{Kg_kVD`xEze)UhSxRVEiqOSBaebCN^WFbv6qvtQV#y`ED5nGun@%Um zZt)T0mxH1MSF7d|4tmYJ$qON|3;&7!V|UW0!oOqDZJLpS;7*C!7|pMO78p2x8N|a( z4P)VFEhaW(jxdBN4<$aZjL&ui+>8-Ts4R8 z4fZjF;wLhnG56G|+hPa>t=V*MssbvXb& z_>>WKS>8DAcmJo*a*Nt9OWw~L@X^fQP?vrYs?EU(wUitY0dO_g9;$aWo=8a6!4}M} zQ+eF04*q;Jx1Nz)yAcHaZrck35iBhJe13 z!$myx7_TnTrYq8iW9^_Cg`dvUgQ@*Ph$DbW3YX}kz{}*RN3#w?$20a)Pq0ihW26R~M6TuF8&`m(85jQm|S@atW_H9IOY{N0KDg|s}S4u%oEkg%drG~-7a zp6S8!GCW^8RgpApf2md^QEL6bT`iy=&kSz{Pg!R5oeAkZammP{cg3xmo~hXpg6`Qg zVe9{%czeZYTuoKnllI3(`z{@al&kJ49Nrj%SsBsebC^DmN ze$xfyClA$*eE6}|Fs&Q1zoK9h5M4I-OY4yFg-#q7euXb@vyft_vU3+)M{BmNr7P}4 z8P!)4?NE@c^Thk#VUnPuH-q`Pll>CB2%0gj7!Lk{+u=bk%6~yXYCH|CO$w-A!vn^Q zjxn;r?`*5G8Q1~FiMwjOpbNg?C_k2StI<<9Ves+#oEIdyK!#gP)WA==#Q3QBN1U&D z6a5kJB{}_76@PJ*D3#Y6QNzXzV?+$Qq}QrhR_k%ylLR_Vn>(yY{f(tOXN9GPm+zL% zSvhe_&&Z62lJZG{Hrb;p+l7Xg;}z2oO^}=;b&BRw9yg&C+GJWLjuyYRC#uiErt%&Z zUp_IL;gsjsy#$uBgf$*21@dYRjxiFR65S%`3c;G;l%sAK$oZry(5vVNhO(CN0R=p# zhc;7TbfMy!T7LN{3Z_H#X-7_)!qK?7W_UJknM^suk4HG*OQvnkhqtI2nO%)M)T8am zjdn~{fafj6vZo#q3HvRUj17Csm`&BDI$-rng4Y;-p(AQQeq5UTcl!lT^t#~YdGm5% zd91dVVOHPQB(f)!l;DRtt3{hVXu!=T8mKM-rctAP>T>TaCB5BfKoa+H9rmk~;w6KX zb1?V6*~wqpdy@_Dmy#{yzSEiapZjKXfuB!8kli`Kx14!65O?aeVT0-l*j6*MbX~z* z)JFTQ>zZ&l+&n$c%k`1ij8ij(h)@ydw73c>n?bNt*0ly->Z4Ej&qUQW86sk0bIo7FS2OoL}p6x4H%cPB7O#{#12`aK=s4jNCQ(WaU=?fIyjaawN8ZJ62 z>Gt@R^lj2~)HaOnzd-Z}nL5leDvz9qotJ>fuQCf6IUxwUCPKJSSHdWv&`%$ES zq#2uU&rBc{y|s%@zGYTMh6aPq51tl6U`G#C!#Z7+E^B!G9lNi6qSet%Wtgi-L#Q%6 zBM9VdRuhzl582q$-8D)1WpR*-8@T)+*z7~D-QgW8)!b5Cox!9P+GwT_5_a=|xK$1R zHOG<5%wCA8)U!@{lF4a*Z1;Lh1MNXEP-0crYDoF-C0wRRQfLy*=Od0gNa@jVE6pI*`3 zd0aacgQ?V1D_&{892c6?^jHw>DJbOScK7_2lg%fOFZAgx;Kk%pN2+Hk?O9i{qA5m~ zu*i02Z)iLR0u6zV$~D-O8n3n@)%U>3P&jL=*|gJJ*;U{iT1Xgm*_+b$?niT_Bajr0 z3Cr?EE>szJ;%oJK{2Y61H=++LE}|2iyPi8wXJf6zszj|xwkAXgrp$2hP?kf-vioKd z%zg=KC^S%_IjcT$9vX}JOO&X!X>M6zpcfaj+suzm#e4(}UY`_31~P<6IgfZT2N?7Wi~Kcz0VWO6++fE%HM-CBKF@f?Mcg zv5{Tk@aj@qrKMi+cki#j>-s9++t=WJ&I2e+!N`HFiQv2>){aGofaw zAosA^2nw{O{odh85oSt;m09+gj5Y!brCeA=#+|E;3^v*$OvCZ&u=!Tt4(Nn&b;4Ta zex_artjkBs@nk4Z+#_BsP(>3D+fvuWb997Ew%kvfsFr2_aPJ0uy|@)OzsX%85ZAyk zbgGKk-G}EvtcXqw_oNHDEh(#|lRUK8dEVGC4~Y9Kp`7dCR5^RR2Zj30&l_uNpy*qRTn7$af)q0Xygsd9~$6683*!$K{YF2 z-C}*!C0>uE&wZ?kj>% zV@qU&W&G{{J%ENP&-x#Y0l%sLnX~b~FCZBj`9XM(B*hR`9F#@LIDTx0Zh6x*xv`t6 zkp{eu`9QA$m1~wWk^*~{GwC?l>RL>q;pEJ^ZwX5!d!sFuJvOt%?kW`5HIvOAd{;5> zD-ZgiT-6ZdTCk|UnQ66Ytt1<{0w+k~YOXW`As3mIkFY&^1sjh4h$dlf;S3EO8n{{{ z7(&ZA@UVnDQu~U=jplxx&$>~G*v*Ku62-KIsYquO(h*;@I1t%MnY7J0l?&D`bm=1d zM$q#o7dm7V^nz^?11}aX4Lro`uU6ar@)l_ zZ=c5YD3E@!fLTzU!VSU*A>h#M!jEu_%Y9r%LsgLPo`KwSBVoPKlR_lM zsm1RO2&BiIb7SBd2veVXf58q{I~unoAT>n;CG|&a#7s3s)tsw=oNk!gQv1_6NV)#|nw`MU=jI%j z&+-ZWG0MGBz#b7x7XD+~V-OX#<#(SU<&W(m7j;4ue^IylOC-Er*F8U9>Go#H+4xYl z-?0LpE%>2U8q|(437~)lNm6@$Hdm(`NTqJZIqeAh@@^vy zND~xHPr)4EjX#Ztu%Fg0`!M~bL}QyGdG(8SW3v&v<$K(E zK2HMZ8;?^Q@+xBaAr0B>?Pm^OU+rT0JQe;}QmLEFi8LI}-|_~C9eRUDp9Q>@H?}Xe zrsJ_2y4+hPo3kS#>sFd*X~}qnz|14u%Y&s?@b<< z)|*;6aYOE!AK|Rq*FexlI*1n$iWk)>P2>gT&;T zL$jtL2motuTok$HPy*Mrbxli^EnY~}n6~1svW%Tr=-6ylEE1a{&%RjKxF0%hx8dLj z7B8M@twXKSYVBL=V%=bci;dZJ^;U~pzO}3y&K*$QU#M%_GF2SP!T<2b@~m6unl-vE zXjalUjZN*=u(yhakf`E(TA5L%*M7IjEMSS>?ueJ!5M~EA-GC@J5!2lgDQ|B;bGrM4 zV-~gs1X@(pR;@0TCM)*5LLg{htH~snd-n@;$JmBqriYiOV%55uj4|&m9hnt8vSw{z z8T(tJ#)grt;1TsaeCA z3A+4y7oq1axWl}Q%rUca7qoCeWV0((v#wVcrP&JAo*qBR<5i?z#M4e*dk8W|6k2(C zN#9iJD9r-)ViTItV6+Q?ScvA~I2QdoW#D!CM8E`2`u7VdgEqVOS$J5>5(9X0X zY>fCQm1@x6gspV)chN5qVCKobfKm0>s$Z?=j}?qX*`r~B$NeS1@~OY@h&gT;bFgUAvw z7FYBpZZqM;EgIK6?4=peU@dan?l~nG0qnexz2{zvc6(`&B8$z%sM zy64QNK@a|icuR;T_r|2FF{e5j*}eK(%$X4<-i4b8i)&W$B{%+cijby8%y#dZlbd{3 zTwJ3(;7F#Kh-}TX*142lZQEkn3)aIwL{P>R%-bGeNLJUWl@ip4nE%r?*Jb>g|cPv^Nl^sW|X zm9SRGcM>uMEZ*&=1?|_x9tW=)pG%4tXzg6rf9Q2d!3L>$f#v;v+kcCJ@UZm870kl= z=IqZc-(X!Yb@MO>%Hy}|xk>&}5E&Q+GoU#f67|U2$0IEs*~6;jRi@8Z-}Akf%G@ki zA=dm)*$77c_N3inEpQ+CQEXTfM=kM^) z`~>zeQF_0o8p~@j5iBV-G4N9XFHA$|82ub3Ta%Rqi(B$<&xNha@1{tOw}&2h+*mw0 z+rN&^z~po=>e^u~|4HGKRQ|x~=K6Ba_Fp5O3hYrtep1#P_HCYDtY^K5t>wuIz}l00 zzp&rO0L=P2P)FM6jQhSIEIoGNHy8@c-#Y>H;u?tNo{8zS+o=Yg)95s5fF_Hxbn2~R z>LtsBniU6t@7NG@z&C7a)$C&#Mm1OK&?7`Tsl`Z@5>AA>Nq|@Z=k9G4K&0UF;B6E@ zsNhTSf@e0`AC8y}-{Q#W2K=hdG&Cg$Jbo5#+;ER3(t$-%#>j5~TRVyI6ET3LokG=m z3~fT%!-9^1Cx0YLZGn!q@5{~HnAeMF@aYM_6Y6g)U6KUd-K>XRxc_!yGOL<678 zGTQUgIdvyRuE&XQ*zf&w2jH|*T6g?LlQLD3{+Df)nH$d>yYVuJCSj^Az4I*Je~%w{ z3VC*gb7j1U%liq%o@4d(OJq18Pa%beY(X&1(OrZQH>JMH}WPFd2aC~=Y zOl`NJ^yWd5>qrRTkop5xMdI-_V7Hf*;KFc~n8IxqQ9;dZ#U%KWF_}fOA z)wUV=>s+?jjvv9UJIQiA<_k~$?yKR5pYRR&Ed{K=!n7|>2-YsN{{-VBfJu_gi{Kz| z(B1gZK91}Ji4%{ky#j=C^@pJz$qhtY7x)}$4x!NLd(F@o9nU3Um6~Rbm#iwOXzf?57YcW8aO1McC_EUofp|obNNH;q?b9n%}(d%f%*bl{FwD0GL*1Ww0bK8 z^3P_oYY_R;8c4Eoz&6#q(L=n81UALI%Oq@AOX@aucnV%KyC7&U@x!HNk=uWG+e>VQ znejRU#D}i)YYx9D@&%_20fmn)3MQi%3L}S(gIxQ=5jIpw`#Ko?+^#)#>tlGImMc;- zF`%j`ns0~uwS gj#(4x<=kmi-^`hR{2y>3r5uK+4foIwM+Oh~KT1k%KL7v# literal 0 HcmV?d00001 diff --git a/ngxtop.egg-info/PKG-INFO b/ngxtop.egg-info/PKG-INFO new file mode 100644 index 0000000..ad75766 --- /dev/null +++ b/ngxtop.egg-info/PKG-INFO @@ -0,0 +1,175 @@ +Metadata-Version: 1.1 +Name: ngxtop +Version: 0.0.2 +Summary: Real-time metrics for nginx server +Home-page: https://github.com/lebinh/ngxtop +Author: Binh Le +Author-email: lebinh.it@gmail.com +License: MIT +Description: ================================================================ + ``ngxtop`` - **real-time** metrics for nginx server (and others) + ================================================================ + + **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server. + So you can tell what is happening with your server in real-time. + + ``ngxtop`` is designed to run in a short-period time just like the ``top`` command for troubleshooting and monitoring + your Nginx server at the moment. If you need a long running monitoring process or storing your webserver stats in external + monitoring / graphing system, you can try `Luameter `_. + + ``ngxtop`` tries to determine the correct location and format of nginx access log file by default, so you can just run + ``ngxtop`` and having a close look at all requests coming to your nginx server. But it does not limit you to nginx + and the default top view. ``ngxtop`` is flexible enough for you to configure and change most of its behaviours. + You can query for different things, specify your log and format, even parse remote Apache common access log with ease. + See sample usages below for some ideas about what you can do with it. + + Installation + ------------ + + :: + + pip install ngxtop + + + Note: ``ngxtop`` is primarily developed and tested with python2 but also supports python3. + + Usage + ----- + + :: + + Usage: + ngxtop [options] + ngxtop [options] (print|top|avg|sum) + ngxtop info + + Options: + -l , --access-log access log file to parse. + -f , --log-format log format as specify in log_format directive. + --no-follow ngxtop default behavior is to ignore current lines in log + and only watch for new lines as they are written to the access log. + Use this flag to tell ngxtop to process the current content of the access log instead. + -t , --interval report interval when running in follow mode [default: 2.0] + + -g , --group-by group by variable [default: request_path] + -w , --having having clause [default: 1] + -o , --order-by order of output for default query [default: count] + -n , --limit limit the number of records included in report for top command [default: 10] + -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output + + -v, --verbose more verbose output + -d, --debug print every line and parsed record + -h, --help print this help message. + --version print version information. + + Advanced / experimental options: + -c , --config allow ngxtop to parse nginx config file for log format and location. + -i , --filter filter in, records satisfied given expression are processed. + -p , --pre-filter in-filter expression to check in pre-parsing phase. + + Samples + ------- + + Default output + ~~~~~~~~~~~~~~ + + :: + + $ ngxtop + running for 411 seconds, 64332 records processed: 156.60 req/sec + + Summary: + | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |---------+------------------+-------+-------+-------+-------| + | 64332 | 2775.251 | 61262 | 2994 | 71 | 5 | + + Detailed: + | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |------------------------------------------+---------+------------------+-------+-------+-------+-------| + | /abc/xyz/xxxx | 20946 | 434.693 | 20935 | 0 | 11 | 0 | + | /xxxxx.json | 5633 | 1483.723 | 5633 | 0 | 0 | 0 | + | /xxxxx/xxx/xxxxxxxxxxxxx | 3629 | 6835.499 | 3626 | 0 | 3 | 0 | + | /xxxxx/xxx/xxxxxxxx | 3627 | 15971.885 | 3623 | 0 | 4 | 0 | + | /xxxxx/xxx/xxxxxxx | 3624 | 7830.236 | 3621 | 0 | 3 | 0 | + | /static/js/minified/utils.min.js | 3031 | 1781.155 | 2104 | 927 | 0 | 0 | + | /static/js/minified/xxxxxxx.min.v1.js | 2889 | 2210.235 | 2068 | 821 | 0 | 0 | + | /static/tracking/js/xxxxxxxx.js | 2594 | 1325.681 | 1927 | 667 | 0 | 0 | + | /xxxxx/xxx.html | 2521 | 573.597 | 2520 | 0 | 1 | 0 | + | /xxxxx/xxxx.json | 1840 | 800.542 | 1839 | 0 | 1 | 0 | + + View top source IPs of clients + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ngxtop top remote_addr + running for 20 seconds, 3215 records processed: 159.62 req/sec + + top remote_addr + | remote_addr | count | + |-----------------+---------| + | 118.173.177.161 | 20 | + | 110.78.145.3 | 16 | + | 171.7.153.7 | 16 | + | 180.183.67.155 | 16 | + | 183.89.65.9 | 16 | + | 202.28.182.5 | 16 | + | 1.47.170.12 | 15 | + | 119.46.184.2 | 15 | + | 125.26.135.219 | 15 | + | 125.26.213.203 | 15 | + + List 4xx or 5xx responses together with HTTP referer + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ngxtop -i 'status >= 400' print request status http_referer + running for 2 seconds, 28 records processed: 13.95 req/sec + + request, status, http_referer: + | request | status | http_referer | + |-----------+----------+----------------| + | - | 400 | - | + + Parse apache log from remote server with `common` format + ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ + + :: + + $ ssh user@remote_server tail -f /var/log/apache2/access.log | ngxtop -f common + running for 20 seconds, 1068 records processed: 53.01 req/sec + + Summary: + | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |---------+------------------+-------+-------+-------+-------| + | 1068 | 28026.763 | 1029 | 20 | 19 | 0 | + + Detailed: + | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | + |------------------------------------------+---------+------------------+-------+-------+-------+-------| + | /xxxxxxxxxx | 199 | 55150.402 | 199 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxx | 167 | 47591.826 | 167 | 0 | 0 | 0 | + | /xxxxxxxxxxxxx/xxxxxx | 25 | 7432.200 | 25 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxxx | 22 | 698.727 | 22 | 0 | 0 | 0 | + | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxx | 19 | 7431.632 | 19 | 0 | 0 | 0 | + | /xxxxx/xxxxx/ | 18 | 7840.889 | 18 | 0 | 0 | 0 | + | /xxxxxxxx/xxxxxxxxxxxxxxxxx | 15 | 7356.000 | 15 | 0 | 0 | 0 | + | /xxxxxxxxxxx/xxxxxxxx | 15 | 9978.800 | 15 | 0 | 0 | 0 | + | /xxxxx/ | 14 | 0.000 | 0 | 14 | 0 | 0 | + | /xxxxxxxxxx/xxxxxxxx/xxxxx | 13 | 20530.154 | 13 | 0 | 0 | 0 | + + +Keywords: cli monitoring nginx system +Platform: UNKNOWN +Classifier: Development Status :: 4 - Beta +Classifier: License :: OSI Approved :: MIT License +Classifier: Environment :: Console +Classifier: Intended Audience :: Developers +Classifier: Intended Audience :: System Administrators +Classifier: Programming Language :: Python :: 2 +Classifier: Programming Language :: Python :: 2.6 +Classifier: Programming Language :: Python :: 2.7 +Classifier: Programming Language :: Python :: 3 +Classifier: Programming Language :: Python :: 3.2 +Classifier: Programming Language :: Python :: 3.3 diff --git a/ngxtop.egg-info/SOURCES.txt b/ngxtop.egg-info/SOURCES.txt new file mode 100644 index 0000000..0eb294a --- /dev/null +++ b/ngxtop.egg-info/SOURCES.txt @@ -0,0 +1,13 @@ +README.rst +setup.cfg +setup.py +ngxtop/__init__.py +ngxtop/config_parser.py +ngxtop/ngxtop.py +ngxtop/utils.py +ngxtop.egg-info/PKG-INFO +ngxtop.egg-info/SOURCES.txt +ngxtop.egg-info/dependency_links.txt +ngxtop.egg-info/entry_points.txt +ngxtop.egg-info/requires.txt +ngxtop.egg-info/top_level.txt \ No newline at end of file diff --git a/ngxtop.egg-info/dependency_links.txt b/ngxtop.egg-info/dependency_links.txt new file mode 100644 index 0000000..8b13789 --- /dev/null +++ b/ngxtop.egg-info/dependency_links.txt @@ -0,0 +1 @@ + diff --git a/ngxtop.egg-info/entry_points.txt b/ngxtop.egg-info/entry_points.txt new file mode 100644 index 0000000..f8a510e --- /dev/null +++ b/ngxtop.egg-info/entry_points.txt @@ -0,0 +1,3 @@ +[console_scripts] +ngxtop = ngxtop.ngxtop:main + diff --git a/ngxtop.egg-info/requires.txt b/ngxtop.egg-info/requires.txt new file mode 100644 index 0000000..5691c21 --- /dev/null +++ b/ngxtop.egg-info/requires.txt @@ -0,0 +1,3 @@ +docopt +tabulate +pyparsing diff --git a/ngxtop.egg-info/top_level.txt b/ngxtop.egg-info/top_level.txt new file mode 100644 index 0000000..103e23a --- /dev/null +++ b/ngxtop.egg-info/top_level.txt @@ -0,0 +1 @@ +ngxtop diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 0d9f00a..f0f852c 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -98,8 +98,11 @@ def detect_log_config(arguments): if not os.path.exists(config): error_exit('Nginx config file not found: %s' % config) - with open(config) as f: - config_str = f.read() + with open(config) as f1: + config_str1 = f1.read() + with open("/data/app/bvc_cdn_cache_ngx_lua/conf.d/nginx_log_format.conf") as f2: + config_str2 = f2.read() + config_str = config_str1 + config_str2 access_logs = dict(get_access_logs(config_str)) if not access_logs: error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config) @@ -111,7 +114,8 @@ def detect_log_config(arguments): return log_path, LOG_FORMAT_COMBINED if format_name not in log_formats: error_exit('Incorrect format name set in config for access log file "%s"' % log_path) - return log_path, log_formats[format_name] + return log_path, log_formats + #return log_path, log_formats['mincdn_cache_hit_log'] # multiple access logs configured, offer to select one print('Multiple access logs detected in configuration:') @@ -122,20 +126,22 @@ def detect_log_config(arguments): return log_path, log_formats[format_name] -def build_pattern(log_format): +def build_pattern(log_formats): """ Build regular expression to parse given format. :param log_format: format string to parse :return: regular expression to parse given format """ - if log_format == 'combined': - log_format = LOG_FORMAT_COMBINED - elif log_format == 'common': - log_format = LOG_FORMAT_COMMON - pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) - pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) - return re.compile(pattern) - + #if log_format == 'combined': + #log_format = LOG_FORMAT_COMBINED + #elif log_format == 'common': + #log_format = LOG_FORMAT_COMMON + pattern_list = [] + for key in log_formats: + pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_formats[key]) + pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) + pattern_list.append(re.compile(pattern)) + return pattern_list def extract_variables(log_format): """ diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 8667b8b..111d266 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -171,6 +171,7 @@ def parse_request_path(record): def parse_status_type(record): + logging.info ('record[status]: %s', record['status']) return record['status'] // 100 if 'status' in record else None @@ -182,8 +183,8 @@ def to_float(value): return float(value) if value and value != '-' else 0.0 -def parse_log(lines, pattern): - matches = (pattern.match(l) for l in lines) +def parse_log(lines, pattern_list): + matches = (pattern.match(l) for l in lines for pattern in pattern_list) records = (m.groupdict() for m in matches if m is not None) records = map_field('status', to_int, records) records = add_field('status_type', parse_status_type, records) @@ -253,12 +254,11 @@ def count(self): # =============== # Log processing # =============== -def process_log(lines, pattern, processor, arguments): +def process_log(lines, pattern_list, processor, arguments): pre_filer_exp = arguments['--pre-filter'] if pre_filer_exp: lines = (line for line in lines if eval(pre_filer_exp, {}, dict(line=line))) - - records = parse_log(lines, pattern) + records = parse_log(lines, pattern_list) filter_exp = arguments['--filter'] if filter_exp: @@ -364,10 +364,10 @@ def process(arguments): return source = build_source(access_log, arguments) - pattern = build_pattern(log_format) + pattern_list = build_pattern(log_format) processor = build_processor(arguments) setup_reporter(processor, arguments) - process_log(source, pattern, processor, arguments) + process_log(source, pattern_list, processor, arguments) def main(): @@ -378,7 +378,11 @@ def main(): log_level = logging.INFO if args['--debug']: log_level = logging.DEBUG - logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s') + logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s', + datefmt='%a, %d %b %Y %H:%M:%S', + filename='/root/ngxtop/tmp/ngxtop.log', + filemode='w' + ) logging.debug('arguments:\n%s', args) try: diff --git a/tmp/ngxtop.log b/tmp/ngxtop.log new file mode 100644 index 0000000..1cbf64e --- /dev/null +++ b/tmp/ngxtop.log @@ -0,0 +1,436 @@ +DEBUG: arguments: +{'--a': None, + '--access-log': None, + '--config': '/data/app/bvc_cdn_cache_ngx_lua/nginx.conf', + '--debug': True, + '--filter': None, + '--group-by': 'request_path', + '--having': '1', + '--help': False, + '--interval': '0.1', + '--limit': '40', + '--log-format': 'combined', + '--no-follow': False, + '--order-by': 'count', + '--pre-filter': None, + '--verbose': False, + '--version': False, + '': [], + '': [], + 'avg': False, + 'info': False, + 'print': False, + 'query': False, + 'sum': False, + 'top': False} +INFO: access_log: /dev/shm/logger +INFO: log_format: {'mincdn_cache_miss_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||MISS', 'mincdn_log': '$http_host||$forward_ip||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$upstream_cache_status||$upstream_status||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||$upstream_response_time||$upstream_addr', 'mincdn_cache_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$upstream_status||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||$upstream_response_time||cache', 'mincdn_cache_hit_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||HIT'} +INFO: query for "Summary:": + SELECT + count(1) AS count, + avg(bytes_sent) AS avg_bytes_sent, + count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', + count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', + count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + FROM log + ORDER BY count DESC + LIMIT 40 +INFO: query for "Detailed:": + SELECT + request_path, + count(1) AS count, + avg(bytes_sent) AS avg_bytes_sent, + count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', + count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', + count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', + count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' + FROM log + GROUP BY request_path + HAVING 1 + ORDER BY count DESC + LIMIT 40 +INFO: sqlite init: create table log (bytes_sent,status_type,request_path) +INFO: sqlite insert: insert into log (bytes_sent,status_type,request_path) values (:bytes_sent,:status_type,:request_path) +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 400 +INFO: record[status]: 403 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 400 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 200 +INFO: record[status]: 400 +INFO: record[status]: 403 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 200 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 +INFO: record[status]: 206 From 7c3950edabcde4d4b51ed3209a53d961099a8f39 Mon Sep 17 00:00:00 2001 From: root Date: Sat, 9 Jul 2016 14:21:03 +0800 Subject: [PATCH 2/9] deleted: tmp/ngxtop.log --- tmp/ngxtop.log | 436 ------------------------------------------------- 1 file changed, 436 deletions(-) delete mode 100644 tmp/ngxtop.log diff --git a/tmp/ngxtop.log b/tmp/ngxtop.log deleted file mode 100644 index 1cbf64e..0000000 --- a/tmp/ngxtop.log +++ /dev/null @@ -1,436 +0,0 @@ -DEBUG: arguments: -{'--a': None, - '--access-log': None, - '--config': '/data/app/bvc_cdn_cache_ngx_lua/nginx.conf', - '--debug': True, - '--filter': None, - '--group-by': 'request_path', - '--having': '1', - '--help': False, - '--interval': '0.1', - '--limit': '40', - '--log-format': 'combined', - '--no-follow': False, - '--order-by': 'count', - '--pre-filter': None, - '--verbose': False, - '--version': False, - '': [], - '': [], - 'avg': False, - 'info': False, - 'print': False, - 'query': False, - 'sum': False, - 'top': False} -INFO: access_log: /dev/shm/logger -INFO: log_format: {'mincdn_cache_miss_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||MISS', 'mincdn_log': '$http_host||$forward_ip||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$upstream_cache_status||$upstream_status||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||$upstream_response_time||$upstream_addr', 'mincdn_cache_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$upstream_status||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||$upstream_response_time||cache', 'mincdn_cache_hit_log': '$http_host||$remote_addr||$msec||$status||$request_length||$bytes_sent||"$request"||"$http_referer"||"$http_user_agent"||$request_time||$cookie_DedeUserID.$cookie_sid$sent_http_logdata||HIT'} -INFO: query for "Summary:": - SELECT - count(1) AS count, - avg(bytes_sent) AS avg_bytes_sent, - count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', - count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', - count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' - FROM log - ORDER BY count DESC - LIMIT 40 -INFO: query for "Detailed:": - SELECT - request_path, - count(1) AS count, - avg(bytes_sent) AS avg_bytes_sent, - count(CASE WHEN status_type = 2 THEN 1 END) AS '2xx', - count(CASE WHEN status_type = 3 THEN 1 END) AS '3xx', - count(CASE WHEN status_type = 4 THEN 1 END) AS '4xx', - count(CASE WHEN status_type = 5 THEN 1 END) AS '5xx' - FROM log - GROUP BY request_path - HAVING 1 - ORDER BY count DESC - LIMIT 40 -INFO: sqlite init: create table log (bytes_sent,status_type,request_path) -INFO: sqlite insert: insert into log (bytes_sent,status_type,request_path) values (:bytes_sent,:status_type,:request_path) -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 400 -INFO: record[status]: 403 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 400 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 200 -INFO: record[status]: 400 -INFO: record[status]: 403 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 200 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 -INFO: record[status]: 206 From 0ba4dec149ab6c01364aeda8e491039216cbc37d Mon Sep 17 00:00:00 2001 From: fei <734377597@@qq.com> Date: Mon, 11 Jul 2016 17:48:50 +0800 Subject: [PATCH 3/9] modified: ngxtop/config_parser.py modified: ngxtop/ngxtop.py add ngxtop ability to support muti log_path and muti log_format --- dist/ngxtop-0.0.2.tar.gz | Bin 9884 -> 0 bytes ngxtop/config_parser.py | 110 ++++++++++++++++++++++++++------------- ngxtop/ngxtop.py | 17 +++--- 3 files changed, 85 insertions(+), 42 deletions(-) delete mode 100644 dist/ngxtop-0.0.2.tar.gz diff --git a/dist/ngxtop-0.0.2.tar.gz b/dist/ngxtop-0.0.2.tar.gz deleted file mode 100644 index cafc5b65b718e1070042819196003bda5451407d..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 9884 zcmcJVMN=G%qC|0bmmpz)A-Dtx?wUY=z~Jt|-QC^Y-90$L2Par?4THN6KKJ_zZ}E1g zs;kbft2a#(gNC;55~zs)GqZMa<*>K(bailK<7DS#=Vo^`c7{6_vGG{%XnM)&Ls;$> zO(lpTxo@tkZ@XvQs{Kv~jB8n6w|Qaug`}{dQmUxt@4E1^eu+g*FRq3w)Hc6Pa7a&r zZUi@lWr5OSelzZzejF7$0SpC`W?#=M6FopxUv8BJeO$Gva5 z`%t~qP$|}GbBw@xo37g|a7oqsrG(qT~g>8G@j|u-7PA)=n8=*pM z-K8+T`9FW+odliUbO)=Hzbr;U5_T*ZCr7;LP|xCZ4=i@OPdRkCxYYa)wAPvee-ZBh=R3%$Dd={$d!q9PY*h6v zfZ~2AW?`e8Z&STV{`RQp8rAXJWM`58kYZ)_A6pQLIITL~H z$?QVBwU5J!=rZv#unHn0-P?^J>k(2T)IUOtfK*p68V7lzEj6W>cZ`U%JR9oI65r3W zg>LtVghj)2Hv6bzL#{^9FHHT>K7=*M_zxBbhIFfPA-B#Vxy>*BRZW?eUY`Lri$971QUGrJZ?~%rM>~` z7A$QIxs*=Z`|nw#5UA*-5b)F(ffr;3PZF^W01C~iZCIm=arPp!`+>y?PDQQVXbgh0 z3^;X`LLnev$s!hcXX_(Ie0US(Be7e(x-m-ZQD$u67AG}51#Ld4S|r>#_vH~v9EYsz zStN5r`uoZHqIl&JqbPmj4g9zfE1oSXF9=-5Mb1tpExE&+~?#d=( zUj)m@Z&CR@J|!^DqQ?wI!j(9i$-9tchS~OBZ59zOV9;#xB41NJBF6hmlhhc+9dVM) zMR!Jy-Xo@DfDL$}agEIS&1naj8e<7@D$-_am$~}lpWY_Gt1Zlhnh}4BXkRSgqJnaR zr_?Majl=P`uCR8YHr;e?%zMkWCCC;V6Km zvFQ3R+wP3tb2JDz4#w(C1H0f3rr=V}S*FXJ9f5w`AzJdhZVdPwkD^IM?Q!-K zFp~>)G$Zw?F!8 zWaPE-#&{xKFUo!@bGw?YMYVuuE9Og9+Bcbi3J0aV8Ff<8Lkn|;CLUthKQ7J~2k*JH ze>W3TTZ7H0T{c}Oy!}mvcbXIis^bb#eDugATx|kTvmDeqa5A#V-v{( zlHOyMDQK|59;`%EYcna2D*AB5T-XK6A3Q4N=;W@U9KZ=KL(NrZ%hk?Nbk$tc;3u zdgH7ovQey6TXd{XU7Z3EGSdt{whBq68-pn9UE~*HQ^=*sj3;F*Y^y@6&tx@N%Wr?_RO8I^MsXU($pd*~snm z7sM}T2V{S8J!%IRE7mI=RiZRo)kg8-Q6J07fPH_ci{TrLcYp;efUyp%8o5H_I5G@! z58v-_s&JU^WBK#JdOw@)BH`*>k`d1J__8C1m5yIsKN~0rl=);x7BtmW1+7$8GllW2 zoyoDH&H^!K1{ef&w;_Ur_#;^P=#^@Qu@Y7Q3tA?TBmC(^g_Dymn*~`{T~zZBlUbsf zO$WCe%kk2;bnmcPi=e)bwfqj>v5k!Z2B>2ZDo3x}T%LjW8O{+Ke0!K}V2v^(o4W89 zQ&Q5lsvl(i%p+3-;>a&{oFAD3e4!$6uLCAE5~{qYG1y=PKlwLRb_ylNNv+z!R+EZR*g4*U4> zVXisuUhge+w=~8$%TYYl-$On=y#$AQ*fQS6jj6lnhF^c(`!;N%@K*@;z-l~F!|Q>8 zm2{5%8H+`@_C5Ry^vdji9Jn z_HsC7GJg81W<_5lcrd*@s**TP1ENt}49iSuh+=LqRv?-_o~5!RGV1bf9}_sjKt1cH zPI68KR2V=J;V!%el(@gCrs~aAxtt@+8-o$3_PTM*I^fnP5{%(@TvXYYyE;QLbNZnd zW7vd!vA%b*>Y>V`KU7vb_IlCvRcxl1f;=ePFyUbc?q7w&4B|v0tP3-P{qcWmx{u*y zp0V4<|1;ETKeg+}D<&g}L%0uwL`wcs9d^Wxl^n7e{kmPuZgJ?Yury8dOwI0FK*drF zGCj&g22}6-w&o3$F)VO&mwL3-ZTEKrk|j1;BFw1iGUE*!r<%|%JE+Oavsl4=KHtD4 zdVcW!L#GxL>6h!BxP&!`kDVskXeph#QewHT5DfeRsoD5-WePtJ+67Q+UVV8^)rYnUvTnV1ekklerO^`fG)38*QJPtkXpiF5_ga zE|Mp(NCIJIpQtG6fpQ;7~zQrj+j3EMt)w&pP@l!+Cs{+qk?A_ z%yqi^MIt>VfvME7ZO9?U18U`Z{wUSPfea`silJ2QuLxo+3XcvYh$)s9EqF-n_n7J=px|icpNW%UlYozlu!;S7W{|kILgHw|46KYW6oPZFU>a!AY53XTCNGU!4PS6q$ANQE^OYpWT2cX=y=Sf{&XV{~Gzr*6t4&^#l^L zQ(=|DaE3&B?z3=1?u4C94!}bvoa+$2hmD%y!%@t}$!1QnPl0YvNcx#fSyhHV?mTeQ z7ulX74OkVH^mQ<>ct?3+hj}D=3`426DG6{;fM>0{W{K$``8s+>AzA5vAtB_*Qi*iU zoB@j7dwOibWSw^|^BWJz%&Uk6Is=`uY<*3t3++qMdR3pHRT-mVRBt!WLpvOOANs@u zWS6U4_@<)0+Td~!8cmlv{H(JNRhMU&)NKxl@qaw8vuktHglw3loWKQ@ zA?aQ}YJaY=vt1LJzCAi-Zmzz^K~h#_YooM;?7sjl+s z>iqi5FoEvsEX1GoUNS1K$aG8Kx6q562TG+imGouf)vI-Szes^|4eaX@o~NUQ z6%W(G&2QVTqx8bqseFtEX1UnG;&VmM9*ps)hk^9UTqj@S1#C4wtWD(##8Mf6R$#or zCp{xLszG~34gKECQfVbmAG&De=~Bgt_t&mTrcy#JqfF6z@uWRfuv~GWWqW~1nHv%D zH^|&8#vMzt-$iNJQOA~serTD)i{}MrhD0=MXviHqMypOMB>khBC}=4%pz|txA@(&7 zVq@kCWJg>43BT587iHL#Kpr6DKN6I`j0H9#7bA%lh<7v}JCGm(kfN2ASz^|< zh@*BQ4%;|x{Kg_kVD`xEze)UhSxRVEiqOSBaebCN^WFbv6qvtQV#y`ED5nGun@%Um zZt)T0mxH1MSF7d|4tmYJ$qON|3;&7!V|UW0!oOqDZJLpS;7*C!7|pMO78p2x8N|a( z4P)VFEhaW(jxdBN4<$aZjL&ui+>8-Ts4R8 z4fZjF;wLhnG56G|+hPa>t=V*MssbvXb& z_>>WKS>8DAcmJo*a*Nt9OWw~L@X^fQP?vrYs?EU(wUitY0dO_g9;$aWo=8a6!4}M} zQ+eF04*q;Jx1Nz)yAcHaZrck35iBhJe13 z!$myx7_TnTrYq8iW9^_Cg`dvUgQ@*Ph$DbW3YX}kz{}*RN3#w?$20a)Pq0ihW26R~M6TuF8&`m(85jQm|S@atW_H9IOY{N0KDg|s}S4u%oEkg%drG~-7a zp6S8!GCW^8RgpApf2md^QEL6bT`iy=&kSz{Pg!R5oeAkZammP{cg3xmo~hXpg6`Qg zVe9{%czeZYTuoKnllI3(`z{@al&kJ49Nrj%SsBsebC^DmN ze$xfyClA$*eE6}|Fs&Q1zoK9h5M4I-OY4yFg-#q7euXb@vyft_vU3+)M{BmNr7P}4 z8P!)4?NE@c^Thk#VUnPuH-q`Pll>CB2%0gj7!Lk{+u=bk%6~yXYCH|CO$w-A!vn^Q zjxn;r?`*5G8Q1~FiMwjOpbNg?C_k2StI<<9Ves+#oEIdyK!#gP)WA==#Q3QBN1U&D z6a5kJB{}_76@PJ*D3#Y6QNzXzV?+$Qq}QrhR_k%ylLR_Vn>(yY{f(tOXN9GPm+zL% zSvhe_&&Z62lJZG{Hrb;p+l7Xg;}z2oO^}=;b&BRw9yg&C+GJWLjuyYRC#uiErt%&Z zUp_IL;gsjsy#$uBgf$*21@dYRjxiFR65S%`3c;G;l%sAK$oZry(5vVNhO(CN0R=p# zhc;7TbfMy!T7LN{3Z_H#X-7_)!qK?7W_UJknM^suk4HG*OQvnkhqtI2nO%)M)T8am zjdn~{fafj6vZo#q3HvRUj17Csm`&BDI$-rng4Y;-p(AQQeq5UTcl!lT^t#~YdGm5% zd91dVVOHPQB(f)!l;DRtt3{hVXu!=T8mKM-rctAP>T>TaCB5BfKoa+H9rmk~;w6KX zb1?V6*~wqpdy@_Dmy#{yzSEiapZjKXfuB!8kli`Kx14!65O?aeVT0-l*j6*MbX~z* z)JFTQ>zZ&l+&n$c%k`1ij8ij(h)@ydw73c>n?bNt*0ly->Z4Ej&qUQW86sk0bIo7FS2OoL}p6x4H%cPB7O#{#12`aK=s4jNCQ(WaU=?fIyjaawN8ZJ62 z>Gt@R^lj2~)HaOnzd-Z}nL5leDvz9qotJ>fuQCf6IUxwUCPKJSSHdWv&`%$ES zq#2uU&rBc{y|s%@zGYTMh6aPq51tl6U`G#C!#Z7+E^B!G9lNi6qSet%Wtgi-L#Q%6 zBM9VdRuhzl582q$-8D)1WpR*-8@T)+*z7~D-QgW8)!b5Cox!9P+GwT_5_a=|xK$1R zHOG<5%wCA8)U!@{lF4a*Z1;Lh1MNXEP-0crYDoF-C0wRRQfLy*=Od0gNa@jVE6pI*`3 zd0aacgQ?V1D_&{892c6?^jHw>DJbOScK7_2lg%fOFZAgx;Kk%pN2+Hk?O9i{qA5m~ zu*i02Z)iLR0u6zV$~D-O8n3n@)%U>3P&jL=*|gJJ*;U{iT1Xgm*_+b$?niT_Bajr0 z3Cr?EE>szJ;%oJK{2Y61H=++LE}|2iyPi8wXJf6zszj|xwkAXgrp$2hP?kf-vioKd z%zg=KC^S%_IjcT$9vX}JOO&X!X>M6zpcfaj+suzm#e4(}UY`_31~P<6IgfZT2N?7Wi~Kcz0VWO6++fE%HM-CBKF@f?Mcg zv5{Tk@aj@qrKMi+cki#j>-s9++t=WJ&I2e+!N`HFiQv2>){aGofaw zAosA^2nw{O{odh85oSt;m09+gj5Y!brCeA=#+|E;3^v*$OvCZ&u=!Tt4(Nn&b;4Ta zex_artjkBs@nk4Z+#_BsP(>3D+fvuWb997Ew%kvfsFr2_aPJ0uy|@)OzsX%85ZAyk zbgGKk-G}EvtcXqw_oNHDEh(#|lRUK8dEVGC4~Y9Kp`7dCR5^RR2Zj30&l_uNpy*qRTn7$af)q0Xygsd9~$6683*!$K{YF2 z-C}*!C0>uE&wZ?kj>% zV@qU&W&G{{J%ENP&-x#Y0l%sLnX~b~FCZBj`9XM(B*hR`9F#@LIDTx0Zh6x*xv`t6 zkp{eu`9QA$m1~wWk^*~{GwC?l>RL>q;pEJ^ZwX5!d!sFuJvOt%?kW`5HIvOAd{;5> zD-ZgiT-6ZdTCk|UnQ66Ytt1<{0w+k~YOXW`As3mIkFY&^1sjh4h$dlf;S3EO8n{{{ z7(&ZA@UVnDQu~U=jplxx&$>~G*v*Ku62-KIsYquO(h*;@I1t%MnY7J0l?&D`bm=1d zM$q#o7dm7V^nz^?11}aX4Lro`uU6ar@)l_ zZ=c5YD3E@!fLTzU!VSU*A>h#M!jEu_%Y9r%LsgLPo`KwSBVoPKlR_lM zsm1RO2&BiIb7SBd2veVXf58q{I~unoAT>n;CG|&a#7s3s)tsw=oNk!gQv1_6NV)#|nw`MU=jI%j z&+-ZWG0MGBz#b7x7XD+~V-OX#<#(SU<&W(m7j;4ue^IylOC-Er*F8U9>Go#H+4xYl z-?0LpE%>2U8q|(437~)lNm6@$Hdm(`NTqJZIqeAh@@^vy zND~xHPr)4EjX#Ztu%Fg0`!M~bL}QyGdG(8SW3v&v<$K(E zK2HMZ8;?^Q@+xBaAr0B>?Pm^OU+rT0JQe;}QmLEFi8LI}-|_~C9eRUDp9Q>@H?}Xe zrsJ_2y4+hPo3kS#>sFd*X~}qnz|14u%Y&s?@b<< z)|*;6aYOE!AK|Rq*FexlI*1n$iWk)>P2>gT&;T zL$jtL2motuTok$HPy*Mrbxli^EnY~}n6~1svW%Tr=-6ylEE1a{&%RjKxF0%hx8dLj z7B8M@twXKSYVBL=V%=bci;dZJ^;U~pzO}3y&K*$QU#M%_GF2SP!T<2b@~m6unl-vE zXjalUjZN*=u(yhakf`E(TA5L%*M7IjEMSS>?ueJ!5M~EA-GC@J5!2lgDQ|B;bGrM4 zV-~gs1X@(pR;@0TCM)*5LLg{htH~snd-n@;$JmBqriYiOV%55uj4|&m9hnt8vSw{z z8T(tJ#)grt;1TsaeCA z3A+4y7oq1axWl}Q%rUca7qoCeWV0((v#wVcrP&JAo*qBR<5i?z#M4e*dk8W|6k2(C zN#9iJD9r-)ViTItV6+Q?ScvA~I2QdoW#D!CM8E`2`u7VdgEqVOS$J5>5(9X0X zY>fCQm1@x6gspV)chN5qVCKobfKm0>s$Z?=j}?qX*`r~B$NeS1@~OY@h&gT;bFgUAvw z7FYBpZZqM;EgIK6?4=peU@dan?l~nG0qnexz2{zvc6(`&B8$z%sM zy64QNK@a|icuR;T_r|2FF{e5j*}eK(%$X4<-i4b8i)&W$B{%+cijby8%y#dZlbd{3 zTwJ3(;7F#Kh-}TX*142lZQEkn3)aIwL{P>R%-bGeNLJUWl@ip4nE%r?*Jb>g|cPv^Nl^sW|X zm9SRGcM>uMEZ*&=1?|_x9tW=)pG%4tXzg6rf9Q2d!3L>$f#v;v+kcCJ@UZm870kl= z=IqZc-(X!Yb@MO>%Hy}|xk>&}5E&Q+GoU#f67|U2$0IEs*~6;jRi@8Z-}Akf%G@ki zA=dm)*$77c_N3inEpQ+CQEXTfM=kM^) z`~>zeQF_0o8p~@j5iBV-G4N9XFHA$|82ub3Ta%Rqi(B$<&xNha@1{tOw}&2h+*mw0 z+rN&^z~po=>e^u~|4HGKRQ|x~=K6Ba_Fp5O3hYrtep1#P_HCYDtY^K5t>wuIz}l00 zzp&rO0L=P2P)FM6jQhSIEIoGNHy8@c-#Y>H;u?tNo{8zS+o=Yg)95s5fF_Hxbn2~R z>LtsBniU6t@7NG@z&C7a)$C&#Mm1OK&?7`Tsl`Z@5>AA>Nq|@Z=k9G4K&0UF;B6E@ zsNhTSf@e0`AC8y}-{Q#W2K=hdG&Cg$Jbo5#+;ER3(t$-%#>j5~TRVyI6ET3LokG=m z3~fT%!-9^1Cx0YLZGn!q@5{~HnAeMF@aYM_6Y6g)U6KUd-K>XRxc_!yGOL<678 zGTQUgIdvyRuE&XQ*zf&w2jH|*T6g?LlQLD3{+Df)nH$d>yYVuJCSj^Az4I*Je~%w{ z3VC*gb7j1U%liq%o@4d(OJq18Pa%beY(X&1(OrZQH>JMH}WPFd2aC~=Y zOl`NJ^yWd5>qrRTkop5xMdI-_V7Hf*;KFc~n8IxqQ9;dZ#U%KWF_}fOA z)wUV=>s+?jjvv9UJIQiA<_k~$?yKR5pYRR&Ed{K=!n7|>2-YsN{{-VBfJu_gi{Kz| z(B1gZK91}Ji4%{ky#j=C^@pJz$qhtY7x)}$4x!NLd(F@o9nU3Um6~Rbm#iwOXzf?57YcW8aO1McC_EUofp|obNNH;q?b9n%}(d%f%*bl{FwD0GL*1Ww0bK8 z^3P_oYY_R;8c4Eoz&6#q(L=n81UALI%Oq@AOX@aucnV%KyC7&U@x!HNk=uWG+e>VQ znejRU#D}i)YYx9D@&%_20fmn)3MQi%3L}S(gIxQ=5jIpw`#Ko?+^#)#>tlGImMc;- zF`%j`ns0~uwS gj#(4x<=kmi-^`hR{2y>3r5uK+4foIwM+Oh~KT1k%KL7v# diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index f0f852c..119eaa2 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -4,6 +4,8 @@ import os import re import subprocess +import glob + from pyparsing import Literal, Word, ZeroOrMore, OneOrMore, Group, \ printables, quotedString, pythonStyleComment, removeQuotes @@ -59,17 +61,24 @@ def get_access_logs(config): access_log = Literal("access_log") + ZeroOrMore(parameter) + semicolon access_log.ignore(pythonStyleComment) + access_logs_dict = {} for directive in access_log.searchString(config).asList(): path = directive[1] if path == 'off' or path.startswith('syslog:'): # nothing to process here continue - - format_name = 'combined' + access_logs_dict[path] = ['combined'] + #format_name = 'combined' if len(directive) > 2 and '=' not in directive[2]: - format_name = directive[2] + #format_name = directive[2] + if directive[2] not in access_logs_dict[path]: + if 'combined' in access_logs_dict[path]: + access_logs_dict[path] = [directive[2]] + else: + (access_logs_dict[path]).append(directive[2]) + return access_logs_dict - yield path, format_name + #yield path, format_name def get_log_formats(config): @@ -86,6 +95,32 @@ def get_log_formats(config): format_string = ''.join(directive[2]) yield name, format_string +def get_config_str(config): + """ + Parse config for include_format directives + """ + # include path + include = Literal("include") + ZeroOrMore(parameter) + semicolon + include.ignore(pythonStyleComment) + + config_str = '' + config_str_list = [] + path_list = [config] + with open(config) as f1: + config_str_list.append(f1.read()) + for directive in include.searchString(config_str_list[0]).asList(): + path = directive[1] + path_list.append (path) + for path in path_list: + if path == config: + continue + file_list = glob.glob(path) + for file_path in file_list: + with open(file_path) as f2: + config_str_list.append(f2.read()) + config_str = '\n'.join(config_str_list) + return config_str + def detect_log_config(arguments): """ @@ -98,50 +133,55 @@ def detect_log_config(arguments): if not os.path.exists(config): error_exit('Nginx config file not found: %s' % config) - with open(config) as f1: - config_str1 = f1.read() - with open("/data/app/bvc_cdn_cache_ngx_lua/conf.d/nginx_log_format.conf") as f2: - config_str2 = f2.read() - config_str = config_str1 + config_str2 - access_logs = dict(get_access_logs(config_str)) - if not access_logs: + config_str = get_config_str(config) + #access_logs = dict(get_access_logs(config_str)) + access_logs_dict = get_access_logs(config_str) + if len(access_logs_dict) == 0: error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config) - log_formats = dict(get_log_formats(config_str)) - if len(access_logs) == 1: - log_path, format_name = list(access_logs.items())[0] - if format_name == 'combined': - return log_path, LOG_FORMAT_COMBINED - if format_name not in log_formats: - error_exit('Incorrect format name set in config for access log file "%s"' % log_path) - return log_path, log_formats - #return log_path, log_formats['mincdn_cache_hit_log'] + log_formats_dict = dict(get_log_formats(config_str)) + if len(access_logs_dict) == 1: + #log_path, format_name = list(access_logs.items())[0] + for log_path in access_logs_dict: + if access_logs_dict[log_path] == ['combined']: + return log_path, LOG_FORMAT_COMBINED + for format_name in access_logs_dict[log_path]: + if format_name not in log_formats_dict: + error_exit('Incorrect format name set in config for access log file "%s"' % log_path) + return log_path, log_formats_dict # multiple access logs configured, offer to select one print('Multiple access logs detected in configuration:') - log_path = choose_one(list(access_logs.keys()), 'Select access log file to process: ') - format_name = access_logs[log_path] - if format_name not in log_formats: - error_exit('Incorrect format name set in config for access log file "%s"' % log_path) - return log_path, log_formats[format_name] + log_path = choose_one(list(access_logs_dict.keys()), 'Select access log file to process: ') + format_name_list = access_logs_dict[log_path] + for format_name in format_name_list: + if format_name not in log_formats_dict: + error_exit('Incorrect format name set in config for access log file "%s"' % log_path) + return log_path, dict(log_formats_dict[log_path]) -def build_pattern(log_formats): +def build_pattern(log_formats_dict, arguments): """ Build regular expression to parse given format. :param log_format: format string to parse :return: regular expression to parse given format """ - #if log_format == 'combined': - #log_format = LOG_FORMAT_COMBINED - #elif log_format == 'common': - #log_format = LOG_FORMAT_COMMON - pattern_list = [] - for key in log_formats: - pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_formats[key]) + log_format = arguments['--log-format'] + if len(log_formats_dict) == 0: + if log_format == 'combined': + log_format = LOG_FORMAT_COMBINED + elif log_format == 'common': + log_format = LOG_FORMAT_COMMON + pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_format) pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) - pattern_list.append(re.compile(pattern)) - return pattern_list + return re.compile(pattern) + else: + pattern_list = [] + for key in log_formats_dict: + pattern = re.sub(REGEX_SPECIAL_CHARS, r'\\\1', log_formats_dict[key]) + pattern = re.sub(REGEX_LOG_FORMAT_VARIABLE, '(?P<\\1>.*)', pattern) + pattern_list.append(re.compile(pattern)) + return pattern_list def extract_variables(log_format): """ diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 111d266..3e8aa64 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -183,8 +183,11 @@ def to_float(value): return float(value) if value and value != '-' else 0.0 -def parse_log(lines, pattern_list): - matches = (pattern.match(l) for l in lines for pattern in pattern_list) +def parse_log(lines, pattern): + if isinstance(pattern, list): + matches = (pattern_value.match(l) for l in lines for pattern_value in pattern) + else: + matches = (pattern.match(l) for l in lines ) records = (m.groupdict() for m in matches if m is not None) records = map_field('status', to_int, records) records = add_field('status_type', parse_status_type, records) @@ -254,11 +257,11 @@ def count(self): # =============== # Log processing # =============== -def process_log(lines, pattern_list, processor, arguments): +def process_log(lines, pattern, processor, arguments): pre_filer_exp = arguments['--pre-filter'] if pre_filer_exp: lines = (line for line in lines if eval(pre_filer_exp, {}, dict(line=line))) - records = parse_log(lines, pattern_list) + records = parse_log(lines, pattern) filter_exp = arguments['--filter'] if filter_exp: @@ -344,7 +347,7 @@ def print_report(sig, frame): def process(arguments): access_log = arguments['--access-log'] - log_format = arguments['--log-format'] + #log_format = arguments['--log-format'] if access_log is None and not sys.stdin.isatty(): # assume logs can be fetched directly from stdin when piped access_log = 'stdin' @@ -364,10 +367,10 @@ def process(arguments): return source = build_source(access_log, arguments) - pattern_list = build_pattern(log_format) + pattern = build_pattern(log_format, arguments) processor = build_processor(arguments) setup_reporter(processor, arguments) - process_log(source, pattern_list, processor, arguments) + process_log(source, pattern, processor, arguments) def main(): From 2dca53ed552068310c9db2104167c50f810c4f4e Mon Sep 17 00:00:00 2001 From: fei <734377597@@qq.com> Date: Mon, 11 Jul 2016 17:51:05 +0800 Subject: [PATCH 4/9] deleted: ngxtop.egg-info/PKG-INFO deleted: ngxtop.egg-info/SOURCES.txt deleted: ngxtop.egg-info/dependency_links.txt deleted: ngxtop.egg-info/entry_points.txt deleted: ngxtop.egg-info/requires.txt deleted: ngxtop.egg-info/top_level.txt --- ngxtop.egg-info/PKG-INFO | 175 --------------------------- ngxtop.egg-info/SOURCES.txt | 13 -- ngxtop.egg-info/dependency_links.txt | 1 - ngxtop.egg-info/entry_points.txt | 3 - ngxtop.egg-info/requires.txt | 3 - ngxtop.egg-info/top_level.txt | 1 - 6 files changed, 196 deletions(-) delete mode 100644 ngxtop.egg-info/PKG-INFO delete mode 100644 ngxtop.egg-info/SOURCES.txt delete mode 100644 ngxtop.egg-info/dependency_links.txt delete mode 100644 ngxtop.egg-info/entry_points.txt delete mode 100644 ngxtop.egg-info/requires.txt delete mode 100644 ngxtop.egg-info/top_level.txt diff --git a/ngxtop.egg-info/PKG-INFO b/ngxtop.egg-info/PKG-INFO deleted file mode 100644 index ad75766..0000000 --- a/ngxtop.egg-info/PKG-INFO +++ /dev/null @@ -1,175 +0,0 @@ -Metadata-Version: 1.1 -Name: ngxtop -Version: 0.0.2 -Summary: Real-time metrics for nginx server -Home-page: https://github.com/lebinh/ngxtop -Author: Binh Le -Author-email: lebinh.it@gmail.com -License: MIT -Description: ================================================================ - ``ngxtop`` - **real-time** metrics for nginx server (and others) - ================================================================ - - **ngxtop** parses your nginx access log and outputs useful, ``top``-like, metrics of your nginx server. - So you can tell what is happening with your server in real-time. - - ``ngxtop`` is designed to run in a short-period time just like the ``top`` command for troubleshooting and monitoring - your Nginx server at the moment. If you need a long running monitoring process or storing your webserver stats in external - monitoring / graphing system, you can try `Luameter `_. - - ``ngxtop`` tries to determine the correct location and format of nginx access log file by default, so you can just run - ``ngxtop`` and having a close look at all requests coming to your nginx server. But it does not limit you to nginx - and the default top view. ``ngxtop`` is flexible enough for you to configure and change most of its behaviours. - You can query for different things, specify your log and format, even parse remote Apache common access log with ease. - See sample usages below for some ideas about what you can do with it. - - Installation - ------------ - - :: - - pip install ngxtop - - - Note: ``ngxtop`` is primarily developed and tested with python2 but also supports python3. - - Usage - ----- - - :: - - Usage: - ngxtop [options] - ngxtop [options] (print|top|avg|sum) - ngxtop info - - Options: - -l , --access-log access log file to parse. - -f , --log-format log format as specify in log_format directive. - --no-follow ngxtop default behavior is to ignore current lines in log - and only watch for new lines as they are written to the access log. - Use this flag to tell ngxtop to process the current content of the access log instead. - -t , --interval report interval when running in follow mode [default: 2.0] - - -g , --group-by group by variable [default: request_path] - -w , --having having clause [default: 1] - -o , --order-by order of output for default query [default: count] - -n , --limit limit the number of records included in report for top command [default: 10] - -a ..., --a ... add exp (must be aggregation exp: sum, avg, min, max, etc.) into output - - -v, --verbose more verbose output - -d, --debug print every line and parsed record - -h, --help print this help message. - --version print version information. - - Advanced / experimental options: - -c , --config allow ngxtop to parse nginx config file for log format and location. - -i , --filter filter in, records satisfied given expression are processed. - -p , --pre-filter in-filter expression to check in pre-parsing phase. - - Samples - ------- - - Default output - ~~~~~~~~~~~~~~ - - :: - - $ ngxtop - running for 411 seconds, 64332 records processed: 156.60 req/sec - - Summary: - | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | - |---------+------------------+-------+-------+-------+-------| - | 64332 | 2775.251 | 61262 | 2994 | 71 | 5 | - - Detailed: - | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | - |------------------------------------------+---------+------------------+-------+-------+-------+-------| - | /abc/xyz/xxxx | 20946 | 434.693 | 20935 | 0 | 11 | 0 | - | /xxxxx.json | 5633 | 1483.723 | 5633 | 0 | 0 | 0 | - | /xxxxx/xxx/xxxxxxxxxxxxx | 3629 | 6835.499 | 3626 | 0 | 3 | 0 | - | /xxxxx/xxx/xxxxxxxx | 3627 | 15971.885 | 3623 | 0 | 4 | 0 | - | /xxxxx/xxx/xxxxxxx | 3624 | 7830.236 | 3621 | 0 | 3 | 0 | - | /static/js/minified/utils.min.js | 3031 | 1781.155 | 2104 | 927 | 0 | 0 | - | /static/js/minified/xxxxxxx.min.v1.js | 2889 | 2210.235 | 2068 | 821 | 0 | 0 | - | /static/tracking/js/xxxxxxxx.js | 2594 | 1325.681 | 1927 | 667 | 0 | 0 | - | /xxxxx/xxx.html | 2521 | 573.597 | 2520 | 0 | 1 | 0 | - | /xxxxx/xxxx.json | 1840 | 800.542 | 1839 | 0 | 1 | 0 | - - View top source IPs of clients - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :: - - $ ngxtop top remote_addr - running for 20 seconds, 3215 records processed: 159.62 req/sec - - top remote_addr - | remote_addr | count | - |-----------------+---------| - | 118.173.177.161 | 20 | - | 110.78.145.3 | 16 | - | 171.7.153.7 | 16 | - | 180.183.67.155 | 16 | - | 183.89.65.9 | 16 | - | 202.28.182.5 | 16 | - | 1.47.170.12 | 15 | - | 119.46.184.2 | 15 | - | 125.26.135.219 | 15 | - | 125.26.213.203 | 15 | - - List 4xx or 5xx responses together with HTTP referer - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :: - - $ ngxtop -i 'status >= 400' print request status http_referer - running for 2 seconds, 28 records processed: 13.95 req/sec - - request, status, http_referer: - | request | status | http_referer | - |-----------+----------+----------------| - | - | 400 | - | - - Parse apache log from remote server with `common` format - ~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~~ - - :: - - $ ssh user@remote_server tail -f /var/log/apache2/access.log | ngxtop -f common - running for 20 seconds, 1068 records processed: 53.01 req/sec - - Summary: - | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | - |---------+------------------+-------+-------+-------+-------| - | 1068 | 28026.763 | 1029 | 20 | 19 | 0 | - - Detailed: - | request_path | count | avg_bytes_sent | 2xx | 3xx | 4xx | 5xx | - |------------------------------------------+---------+------------------+-------+-------+-------+-------| - | /xxxxxxxxxx | 199 | 55150.402 | 199 | 0 | 0 | 0 | - | /xxxxxxxx/xxxxx | 167 | 47591.826 | 167 | 0 | 0 | 0 | - | /xxxxxxxxxxxxx/xxxxxx | 25 | 7432.200 | 25 | 0 | 0 | 0 | - | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxxx | 22 | 698.727 | 22 | 0 | 0 | 0 | - | /xxxx/xxxxx/x/xxxxxxxxxxxxx/xxxxxx | 19 | 7431.632 | 19 | 0 | 0 | 0 | - | /xxxxx/xxxxx/ | 18 | 7840.889 | 18 | 0 | 0 | 0 | - | /xxxxxxxx/xxxxxxxxxxxxxxxxx | 15 | 7356.000 | 15 | 0 | 0 | 0 | - | /xxxxxxxxxxx/xxxxxxxx | 15 | 9978.800 | 15 | 0 | 0 | 0 | - | /xxxxx/ | 14 | 0.000 | 0 | 14 | 0 | 0 | - | /xxxxxxxxxx/xxxxxxxx/xxxxx | 13 | 20530.154 | 13 | 0 | 0 | 0 | - - -Keywords: cli monitoring nginx system -Platform: UNKNOWN -Classifier: Development Status :: 4 - Beta -Classifier: License :: OSI Approved :: MIT License -Classifier: Environment :: Console -Classifier: Intended Audience :: Developers -Classifier: Intended Audience :: System Administrators -Classifier: Programming Language :: Python :: 2 -Classifier: Programming Language :: Python :: 2.6 -Classifier: Programming Language :: Python :: 2.7 -Classifier: Programming Language :: Python :: 3 -Classifier: Programming Language :: Python :: 3.2 -Classifier: Programming Language :: Python :: 3.3 diff --git a/ngxtop.egg-info/SOURCES.txt b/ngxtop.egg-info/SOURCES.txt deleted file mode 100644 index 0eb294a..0000000 --- a/ngxtop.egg-info/SOURCES.txt +++ /dev/null @@ -1,13 +0,0 @@ -README.rst -setup.cfg -setup.py -ngxtop/__init__.py -ngxtop/config_parser.py -ngxtop/ngxtop.py -ngxtop/utils.py -ngxtop.egg-info/PKG-INFO -ngxtop.egg-info/SOURCES.txt -ngxtop.egg-info/dependency_links.txt -ngxtop.egg-info/entry_points.txt -ngxtop.egg-info/requires.txt -ngxtop.egg-info/top_level.txt \ No newline at end of file diff --git a/ngxtop.egg-info/dependency_links.txt b/ngxtop.egg-info/dependency_links.txt deleted file mode 100644 index 8b13789..0000000 --- a/ngxtop.egg-info/dependency_links.txt +++ /dev/null @@ -1 +0,0 @@ - diff --git a/ngxtop.egg-info/entry_points.txt b/ngxtop.egg-info/entry_points.txt deleted file mode 100644 index f8a510e..0000000 --- a/ngxtop.egg-info/entry_points.txt +++ /dev/null @@ -1,3 +0,0 @@ -[console_scripts] -ngxtop = ngxtop.ngxtop:main - diff --git a/ngxtop.egg-info/requires.txt b/ngxtop.egg-info/requires.txt deleted file mode 100644 index 5691c21..0000000 --- a/ngxtop.egg-info/requires.txt +++ /dev/null @@ -1,3 +0,0 @@ -docopt -tabulate -pyparsing diff --git a/ngxtop.egg-info/top_level.txt b/ngxtop.egg-info/top_level.txt deleted file mode 100644 index 103e23a..0000000 --- a/ngxtop.egg-info/top_level.txt +++ /dev/null @@ -1 +0,0 @@ -ngxtop From fb7303102992a7ac1e945dd62c971f47c3b3c3ee Mon Sep 17 00:00:00 2001 From: fei <734377597@@qq.com> Date: Mon, 11 Jul 2016 17:54:54 +0800 Subject: [PATCH 5/9] modified: config_parser.py modified: ngxtop.py delete some useless note --- ngxtop/config_parser.py | 5 ----- ngxtop/ngxtop.py | 7 +------ 2 files changed, 1 insertion(+), 11 deletions(-) diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 119eaa2..429f165 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -68,9 +68,7 @@ def get_access_logs(config): # nothing to process here continue access_logs_dict[path] = ['combined'] - #format_name = 'combined' if len(directive) > 2 and '=' not in directive[2]: - #format_name = directive[2] if directive[2] not in access_logs_dict[path]: if 'combined' in access_logs_dict[path]: access_logs_dict[path] = [directive[2]] @@ -78,7 +76,6 @@ def get_access_logs(config): (access_logs_dict[path]).append(directive[2]) return access_logs_dict - #yield path, format_name def get_log_formats(config): @@ -134,14 +131,12 @@ def detect_log_config(arguments): error_exit('Nginx config file not found: %s' % config) config_str = get_config_str(config) - #access_logs = dict(get_access_logs(config_str)) access_logs_dict = get_access_logs(config_str) if len(access_logs_dict) == 0: error_exit('Access log file is not provided and ngxtop cannot detect it from your config file (%s).' % config) log_formats_dict = dict(get_log_formats(config_str)) if len(access_logs_dict) == 1: - #log_path, format_name = list(access_logs.items())[0] for log_path in access_logs_dict: if access_logs_dict[log_path] == ['combined']: return log_path, LOG_FORMAT_COMBINED diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 3e8aa64..13a75aa 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -347,7 +347,6 @@ def print_report(sig, frame): def process(arguments): access_log = arguments['--access-log'] - #log_format = arguments['--log-format'] if access_log is None and not sys.stdin.isatty(): # assume logs can be fetched directly from stdin when piped access_log = 'stdin' @@ -381,11 +380,7 @@ def main(): log_level = logging.INFO if args['--debug']: log_level = logging.DEBUG - logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s', - datefmt='%a, %d %b %Y %H:%M:%S', - filename='/root/ngxtop/tmp/ngxtop.log', - filemode='w' - ) + logging.basicConfig(level=log_level, format='%(levelname)s: %(message)s') logging.debug('arguments:\n%s', args) try: From 2afaa05fbc6ed69ac27c2b1183f5dcc963efb644 Mon Sep 17 00:00:00 2001 From: fei <734377597@@qq.com> Date: Mon, 11 Jul 2016 21:58:32 +0800 Subject: [PATCH 6/9] modified: ngxtop/ngxtop.py delete records[status] logging --- ngxtop/ngxtop.py | 1 - 1 file changed, 1 deletion(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 13a75aa..99b1af9 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -171,7 +171,6 @@ def parse_request_path(record): def parse_status_type(record): - logging.info ('record[status]: %s', record['status']) return record['status'] // 100 if 'status' in record else None From 945fb48e8c124f76af201c7e0b713ec7447ab54c Mon Sep 17 00:00:00 2001 From: huangxiaofei Date: Sat, 6 Aug 2016 11:02:14 +0800 Subject: [PATCH 7/9] modified: ngxtop/ngxtop.py --- ngxtop/ngxtop.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 99b1af9..0a44149 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -351,9 +351,9 @@ def process(arguments): access_log = 'stdin' if access_log is None: access_log, log_format = detect_log_config(arguments) + logging.info('log_format: %s', log_format) logging.info('access_log: %s', access_log) - logging.info('log_format: %s', log_format) if access_log != 'stdin' and not os.path.exists(access_log): error_exit('access log file "%s" does not exist' % access_log) From f8fbfd398c6f9c65b01dd42b8809e2e416325e67 Mon Sep 17 00:00:00 2001 From: huangxiaofei Date: Sat, 6 Aug 2016 12:29:41 +0800 Subject: [PATCH 8/9] modified: ngxtop/config_parser.py modified: ngxtop/ngxtop.py --- ngxtop/config_parser.py | 4 +++- ngxtop/ngxtop.py | 9 +++++---- 2 files changed, 8 insertions(+), 5 deletions(-) diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 429f165..169c0c3 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -139,7 +139,9 @@ def detect_log_config(arguments): if len(access_logs_dict) == 1: for log_path in access_logs_dict: if access_logs_dict[log_path] == ['combined']: - return log_path, LOG_FORMAT_COMBINED + log_formats_dict.clear() + log_formats_dict['combined'] = LOG_FORMAT_COMBINED + return log_path, log_formats_dict for format_name in access_logs_dict[log_path]: if format_name not in log_formats_dict: error_exit('Incorrect format name set in config for access log file "%s"' % log_path) diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 0a44149..911556e 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -346,12 +346,13 @@ def print_report(sig, frame): def process(arguments): access_log = arguments['--access-log'] + log_formats_dict = {} if access_log is None and not sys.stdin.isatty(): # assume logs can be fetched directly from stdin when piped access_log = 'stdin' if access_log is None: - access_log, log_format = detect_log_config(arguments) - logging.info('log_format: %s', log_format) + access_log, log_formats_dict = detect_log_config(arguments) + logging.info('log_format: %s', log_formats_dict) logging.info('access_log: %s', access_log) if access_log != 'stdin' and not os.path.exists(access_log): @@ -360,12 +361,12 @@ def process(arguments): if arguments['info']: print('nginx configuration file:\n ', detect_config_path()) print('access log file:\n ', access_log) - print('access log format:\n ', log_format) + print('access log format:\n ', log_formats_dict) print('available variables:\n ', ', '.join(sorted(extract_variables(log_format)))) return source = build_source(access_log, arguments) - pattern = build_pattern(log_format, arguments) + pattern = build_pattern(log_formats_dict, arguments) processor = build_processor(arguments) setup_reporter(processor, arguments) process_log(source, pattern, processor, arguments) From b35c6a980700a1f57ac8b6a4fcf65d4173b6ff58 Mon Sep 17 00:00:00 2001 From: huangxiaofei Date: Sat, 6 Aug 2016 12:40:44 +0800 Subject: [PATCH 9/9] modified: ngxtop/config_parser.py modified: ngxtop/ngxtop.py fix bug when type 'ngxtop info' --- ngxtop/config_parser.py | 11 ++++++----- ngxtop/ngxtop.py | 2 +- 2 files changed, 7 insertions(+), 6 deletions(-) diff --git a/ngxtop/config_parser.py b/ngxtop/config_parser.py index 169c0c3..17a4244 100644 --- a/ngxtop/config_parser.py +++ b/ngxtop/config_parser.py @@ -180,14 +180,15 @@ def build_pattern(log_formats_dict, arguments): pattern_list.append(re.compile(pattern)) return pattern_list -def extract_variables(log_format): +def extract_variables(log_formats_dict): """ Extract all variables from a log format string. :param log_format: format string to extract :return: iterator over all variables in given format string """ - if log_format == 'combined': - log_format = LOG_FORMAT_COMBINED - for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format): - yield match + for log_format in log_formats_dict: + if log_format == 'combined': + log_format = LOG_FORMAT_COMBINED + for match in re.findall(REGEX_LOG_FORMAT_VARIABLE, log_format): + yield match diff --git a/ngxtop/ngxtop.py b/ngxtop/ngxtop.py index 911556e..e356372 100644 --- a/ngxtop/ngxtop.py +++ b/ngxtop/ngxtop.py @@ -362,7 +362,7 @@ def process(arguments): print('nginx configuration file:\n ', detect_config_path()) print('access log file:\n ', access_log) print('access log format:\n ', log_formats_dict) - print('available variables:\n ', ', '.join(sorted(extract_variables(log_format)))) + print('available variables:\n ', ', '.join(sorted(extract_variables(log_formats_dict)))) return source = build_source(access_log, arguments)