From 6eae0bf45685b47df6cbe96fe598f433f03d0cd8 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Mon, 1 Mar 2021 12:17:01 +0100 Subject: [PATCH 01/20] Added SSL demo certificatore including reservations IP --- deploy/demo/ssl/f7t_internal.crt | 32 ++++++++++++++++---------------- 1 file changed, 16 insertions(+), 16 deletions(-) diff --git a/deploy/demo/ssl/f7t_internal.crt b/deploy/demo/ssl/f7t_internal.crt index 3d8c4477..52ddead3 100644 --- a/deploy/demo/ssl/f7t_internal.crt +++ b/deploy/demo/ssl/f7t_internal.crt @@ -1,8 +1,8 @@ -----BEGIN CERTIFICATE----- -MIIFojCCA4qgAwIBAgIUPPBK8HvPTkdgfjFWQq8DVOPyTb0wDQYJKoZIhvcNAQEL +MIIFqDCCA5CgAwIBAgIUDl8L8XPoiiqkxAbFgkG93x5Q2ZQwDQYJKoZIhvcNAQEL BQAwZjELMAkGA1UEBhMCQ0gxDzANBgNVBAgMBlRpY2lubzEPMA0GA1UEBwwGTHVn YW5vMQ0wCwYDVQQKDARDU0NTMRIwEAYDVQQLDAlGaXJlY1JFU1QxEjAQBgNVBAMM -CTEyNy4wLjAuMTAeFw0yMDExMTMxNjEwMjRaFw0zMDExMTExNjEwMjRaMGYxCzAJ +CTEyNy4wLjAuMTAeFw0yMTAzMDExMTAwNTZaFw0zMTAyMjcxMTAwNTZaMGYxCzAJ BgNVBAYTAkNIMQ8wDQYDVQQIDAZUaWNpbm8xDzANBgNVBAcMBkx1Z2FubzENMAsG A1UECgwEQ1NDUzESMBAGA1UECwwJRmlyZWNSRVNUMRIwEAYDVQQDDAkxMjcuMC4w LjEwggIiMA0GCSqGSIb3DQEBAQUAA4ICDwAwggIKAoICAQC/dGgWlhOE2NL22+zU @@ -16,18 +16,18 @@ GjWiij2O8kcZ58x7q+Y43t5u3xTyl03gVPHliyjVpC78f5AlEH40cGUSyTztb8c5 OQ2Y9t3R2XiN6w+fa7SuxyDuHCLIcP4qKBaDgB1kbFCf+cFMNC1ze810pkFTNX2J 0G6zHuWcAXPVVCaKkf1QrKOgBzoCwJmvYBQ2QSn5/M9tVCLhyA8jiP2+a1eKWEVJ sW/L8Sx5ur1Cy6wGkA3L1m2DHi/gXNYNU5TSIDGX+y5JYCSpXVVOvcPxSwx3m4BC -CmaczBdQn6bF0q0nAsgQJq1qowIDAQABo0gwRjBEBgNVHREEPTA7gglsb2NhbGhv -c3SHBMCo3AuHBMCo3AmHBMCo3ASHBMCo3AWHBMCo3AaHBMCo3AeHBMCo3CiHBH8A -AAEwDQYJKoZIhvcNAQELBQADggIBADbmLjRBLtAKYgOabjo120nI0rtXOja6Na5t -2hgnlce/h4/Ir79761Ox3UkFF9D8vQSDibvWUyOqWoAqS7UgZ6wk0JZhk+3Xig4a -z0ArkzTneDl/M7C1e02GAx4JWxcZb+ET0sSKDWGOJATZCWsXaE6GhCZO0FDHH512 -zI0/DUUAfPtTZqBmwdzcdE1QRO6cO/gNUJHIRdi+yTuCMB0Mlj1t2nYrF7JPCetL -SkmSvpAtAgqE3D0oOs0GqlyzY2BNwwMVRT02wnEvPCY4lHtXgOtRcD1W2GUZ+1e4 -7xD+WvIc+BS+gRGYjPdB3j+yBo2xb2Nbbb6LUdeNBUqovRrB77IxXBLICwx2tgip -fFPQPAFtSrHbrbLm4GwMGqjPs8bEDJtdWggDThoE6gD9QrYa74YY0thX8CXKy2wn -NYbhF2ICSp3wn4mfxHgmntT394sLi5Aah4Pk33gN1qv/fOj2SMT0B2NEtNIL4MQQ -WkoSy6bDugUNRinTqMLvUqgmp7sNOhAZdN/7tjIsezWj4Ykvq8p/BKSs2nhnuFtw -27wcdHJEvFfAcnK21WNQQ5hKQMfxYrWnZ2VNQ0VK4kr6X5SxEwvLf4eu6A8D5EI7 -Yru1VRH1edGjWQ8P9HTllhhrC7/QaZJYDO4Bf90YRLNqF7TCCp8kDHd6OiDJHWng -MoxMs8E+ +CmaczBdQn6bF0q0nAsgQJq1qowIDAQABo04wTDBKBgNVHREEQzBBgglsb2NhbGhv +c3SHBMCo3AuHBMCo3AmHBMCo3ASHBMCo3AWHBMCo3AaHBMCo3AeHBMCo3AiHBMCo +3CiHBH8AAAEwDQYJKoZIhvcNAQELBQADggIBAD977LacT/KqhCajm4GayB+Bucyw +chit/H5D1YmMfedod1xDhI+T6pS0jILYlaUqtTCJQMae6FSGWg1BgY67yJZOsRQM +2d5l8aII28zd7Ku8GAMW94HkhmHTu2EEW2w+yjdkB82L71GOakx7MrcOLNzx2WiI +DRxvTSiQRORtr+lErORaFp86aBBAHQrCvy0ImUfvJwWdA6BpEz6OAz0iw7kOoE8q +36nyRsurk43drPdELm4UxTzwUsgl4Ml2q3NSBBEaR1eOhvav6Oan3hdaRshUkA+T +lkiC+MTgbiFp60QKgsXzSE2LFd9D4y3effWj2B5SYxKBjVKcn7OfNTg7EMvkUxeY +C0NQqnYjtFAXatMIMzIR6SWwPDnyqi5SAseG1xc2696/EbnUQ8+e5+WnzIIG2w2N +9j9s7kQKFwjiF23coOmdgE5aLDYUNe5e4ChE+TBR31JEWVFqP8DqzDPzOmRkSa+U +LaJTyncDR5a1r3GLeFjQ7oi/1tT6jryYzUX5TclXlZZ5LXEYhbClPY0yKH1NOTgG +IQiU3yw9TW+LEUooWEVmfuvdW4dGEzoYyTzss7xR+sQBgCA8bcm3+MTd0AsWpHs+ +ucKw0sGevdl1OWDEJ76uZYkEZGReDnxqoWoTQIWtTINrcoJ2d0m6RdLpp4hdm6kN +pn2lF3QeQZkntTKf -----END CERTIFICATE----- From a80e29b596f28e95f8624e48b38a5035b86792f2 Mon Sep 17 00:00:00 2001 From: Felipe Date: Tue, 2 Mar 2021 11:43:50 +0100 Subject: [PATCH 02/20] enabling nohome plugin for slurm --- deploy/test-build/cluster/Dockerfile | 17 +++++++++--- .../test-build/cluster/slurm/plugstack.conf | 1 + .../plugstack.conf.d/cscs-slurm-nohome.conf | 1 + .../cluster/spank/nohome-1.1-1.x86_64.rpm | Bin 0 -> 11604 bytes deploy/test-build/cluster/spank/test.sh | 25 ++++++++++++++++++ deploy/test-build/docker-compose.yml | 5 +++- 6 files changed, 45 insertions(+), 4 deletions(-) create mode 100644 deploy/test-build/cluster/slurm/plugstack.conf create mode 100644 deploy/test-build/cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf create mode 100644 deploy/test-build/cluster/spank/nohome-1.1-1.x86_64.rpm create mode 100755 deploy/test-build/cluster/spank/test.sh diff --git a/deploy/test-build/cluster/Dockerfile b/deploy/test-build/cluster/Dockerfile index 27136b4c..23baa125 100644 --- a/deploy/test-build/cluster/Dockerfile +++ b/deploy/test-build/cluster/Dockerfile @@ -50,6 +50,19 @@ RUN set -x \ && chown -R slurm:slurm /var/log/slurm \ && chown -R slurm:slurm /var/spool/slurm* + +ADD cluster/slurm/*.conf /etc/slurm/ +ADD cluster/slurm/*.sh / +RUN chmod 644 /etc/slurm/* && chmod 755 /*.sh + +RUN mkdir /etc/slurm/plugstack.conf.d +RUN chmod 755 /etc/slurm/plugstack.conf.d +ADD cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf /etc/slurm/plugstack.conf.d/ + +RUN mkdir /spank +ADD cluster/spank/nohome-1.*.rpm /spank/ +RUN rpm -i /spank/nohome-*.rpm + RUN set -x \ && /sbin/create-munge-key @@ -59,10 +72,8 @@ ADD cluster/supervisord.conf /etc/supervisord.conf RUN chown -R munge:munge /var/log/munge && chmod 755 /var/log/munge && chmod 755 /run/munge -ADD cluster/slurm/*.conf /etc/slurm/ -ADD cluster/slurm/*.sh / -RUN chmod 644 /etc/slurm/* && chmod 755 /*.sh +RUN chmod 755 /*.sh RUN useradd -m -s /bin/bash test1 && useradd -m -s /bin/bash test2 RUN echo 'test1:test11' | chpasswd && echo 'test2:test22' | chpasswd diff --git a/deploy/test-build/cluster/slurm/plugstack.conf b/deploy/test-build/cluster/slurm/plugstack.conf new file mode 100644 index 00000000..ffe10a91 --- /dev/null +++ b/deploy/test-build/cluster/slurm/plugstack.conf @@ -0,0 +1 @@ +include plugstack.conf.d/* diff --git a/deploy/test-build/cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf b/deploy/test-build/cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf new file mode 100644 index 00000000..bde65e06 --- /dev/null +++ b/deploy/test-build/cluster/slurm/plugstack.conf.d/cscs-slurm-nohome.conf @@ -0,0 +1 @@ +optional /opt/cscs/nohome/nohome.so diff --git a/deploy/test-build/cluster/spank/nohome-1.1-1.x86_64.rpm b/deploy/test-build/cluster/spank/nohome-1.1-1.x86_64.rpm new file mode 100644 index 0000000000000000000000000000000000000000..f706929a7162bc2ea21156d1258251f6ab997705 GIT binary patch literal 11604 zcmeHsc{tSH`}fEaLbfDZW2cN+%tBOz$*wGA&#ZPc%#b~l>}f$NQI<-j$WjQ|w|taH z$eNuJAzJvo-)hwQ&pG$G&-Gb*@OTD zx#}OzJ_8^JAl@v#03cuwp#uWgvo8V&>LURG?AgBp2j)cV$i3k*qNF`Aa2ns?=LkmK}VF?;kEex7OB2YC6Bq|1hrJx9CBtnaT!D^ti zFj^D>1_$`i)FR-J2qXrDC1VH}ga%auP0~UD6G9-8FDIO%HB#n^PQG?U6}}<%_d_6* zo^93N$KMG2jlkas{Efih2>gw}-w6DTz~2b`jlkas{Efih2>gw}|L+mlu8C}IZCwTe z0n{`gkbkS29D+bNPXJv<0Rie7oVoylW5|GT0|GX%?g7?Gz`Dr}7N2Et9u}Wtaj@P3 zjtA=~pl&yde`fImEdB*Ru&(07;$WQx^zX{zO8|m;CX25D2ug0e;AAZ0ua>4 zvG{Kmf5PIMEDqL)fc0^JzCn2j2(Tv(&=xFJHL^J98|1Al&I=$o{xys50uYQ3SbqWS z`TlUWQ!Ea~0T|C_$KqfdfbnduEDpvP04P`;t$^e#tYP|00{WsaUDQ#|C#^-{&(DAaV-D=|2y(n9LS4p-o)Z~ z06}}8=Co~p96->19YApZMt}HrwGZUR00;cB1G25RW9h*-fqHwE9^5y`!8`!`u|rwh z89>lpg~eT19P|s0Cjtnphn;o)9Ap*;^9tM_g~h+IIF-f0JOcIJEDo+8OOhjm*HK{~2P79}r!jg!Z5G<02Bw#TbBn*m( z!ePh+6q$42xFX_NNOx{|IoElgQkvE zQB}dBi5^TheCR;f85AS&;5k5y5v0?`||i!dsI5$sEH_ow;N0vNFWP(ny_4;myW1cG7RAs8co0nE64t5^{L zVxniQZ)^-@Qi7NeLo+-Si-rOD1U0ih24%Pt{3&E8jpzwXt_J;+nrhHMia*1H<_kqa zRjlo;PpPUw$-%w^9}g12+dCKvxc8!v)u6`$JiN)q29AIer1N1kf=EVSs6+yhLdFsi zB!HvH;3XsCw8&^2a5Ygx1cgdb`}18PQ-}d>XFPnVG&QI%jR|Ei{XOV(;KT91H+C!F zI|>LJAdn3~oXyXSeTIB~K5r=x0t%@C ze9;J0DvnAfV=yQxhNOkjAW(@Y430`55YSp!0vUy(kjNU_6NCA({rhHX>z6C=F8CW4 zV&?$d&9R?+cXvQ!*~EZOCVVn?l=4oP>x^UH!Jd?3 zbj>h|7JSLzqNg3=YLlHX#jb@zO7CsJy4QPb!>y-!TXSuXgCJi9cl^*z50;X*57F4v zyYhB#=2-a8+S7Yx+-%JgjWP#^oJxB_71@4;%QrjYBfm_`@`kmit^Od94=t4{#knY6 z7n}bz_9kCvCMqhBBZg16d0=N~Rcqd{e+mRfQY^I9AdcOM%{|i<7O9UKnQu{6{|Ij= zy@!fUC|^nay6&eAhwgKhO^XowiBf+#mWOgS%H#X#{@c>OQbc+e=l)^G-3eyR;&)IC zTu!x!O6A;VsHO}Ag>`2pDiDMAQ6I^ZRS(a7a7sE@HxE^eIJvMA z)$t%??`I$7%e>u@5%u@Wj|>7)H=VmRWxPfp0@E0 z*G$9j-$&00Uzs<_%+ZY_XON2;U?0BPAdD{*ybaMLsZ<~8b507;%?f+;vF#4M-+H9( zUHQ31-i$*>7LCVSo6gh9w;F{SOURBp`RzzU`f;B- zqRNfpJ$nK>xBRkx&1g&Y?vX%Ezu+I6K9o{)vLbg)wuk@3b?Q?8v6`%xBUwAGN)~@h zCl)FdyVcZ6@?yGg3W!~yE6Mz}MpjW#f5K09?iBUa`};I4$il7|xUYi2WI z-H1|>0)Ahcu1N9W+Ksm&Gel3Pqq^toR>N{Q`99`nUrB(@?~%%C;ulP(UXZ&X8n?hB zD*f?LgL`XTb!9{oXU#jdD%A3PX{F?AFE_Q0Pnml?lJgEizI$KB4D5ebddwjGY?=l& zSq=;LA9_duIaVjsZ+qi3vN4@n8(ZJB zG7t3*jfa%2chi;HZF>PC(7fT+yd{`c_n84&FQ$fmazzu;L6rySko{_4ddu|_X!*95zz{KUCifn=dLp{W<_ow@6KYLYk`(ASox<+`oT zsPVW4Z@Km&xus7&_->Yo(P!s~?fvnY`0~}Np{#y!YI7z=DAk0cSAQi~hD!yM~{vSoM+@ z7+p3|>9r!e@89?}*UW^x?&_e%9z@jZX@3irAKojwv!f_Xk}+=g`QXzRd*plvmU+KA zR!i?LqVHb!^;tl};Hvj>ptgt_eW9<-lkWpky@%Gj2|vpc?UV6Qu$A+trtl54k43g$ zs*dK@A6!{jm@8$bh8d*Ei5M^Uox2rVO^MsDf!efYIuu9w1yJ63Ro!63D~MUA4o6=- z{?X%eM#Pp@Ov%xG(P61C4_W!GFxx#*dd6pr?{(alGpm+_7qmuJHY>k+(g7t|m+U|J zE>51(FMM<2b}8vTuTR!S4Yzj@58HKr+p5J^KSy8GWh$2XN-O1Vn^ z!U3UO+)NI={frUI)gi0ac0^ivxGl59`2HE)TnCq_U-)z7Y~4IYDlg3#!Mf72Cq^Y) zB+0ev0<+~%&7=NmkA^di0v>qFYF&ytr~AU&N3m6M?PfQ72_L%eO{Qbn;zrQaWBk3h zS&)`Ox__#snY^QA;+w+}L5kIJ`jl|Zz6ytl=!D?etkZgUSIg^`; z8(zl+=j_X4|2bOd)$_iNrhlJclyPv!hH0{(=EdPp+HI~vVW}<+opo=D{AW5~QbayvtRUw;bYNt2@in)|@my%q3w)r$4`-^sY1}m7qZV*wXE)lj{+) zs0=TEO(G8l_R3X^c1z8PNG$MkiJF<)miGh(hCZ zpJD%^`IacJw_xT=+QruR+UHA?$h&rdvd z6>;Gc=_)t@LG}P*NV&?Sc@5>+HGJFM^Hn2iuml8AE$q-#1aGPIB~>B=xfAc;Cuw9Q zPEb$ zefZ~|6bLJ3W4pqGzEJU#L!XPt)W_Y*k2Nm7huqiU*mH6sm~5=3w46K^)T-ZhUvkfb z50`#LocgR&9DK|G5+q);^Pt$Ad$OE|=BM82=nk>NeFkFd$jC~*-U;mP167}9Ll2uS zn&%%mp)!3*W{~P~v%7ZH@2QM^!|A2@iu+vcaT+pBp#dxFY?pO;ANAAg)I?3S`GyvR zTH*qX`t6PeXZYWG$as*)SWZo`k%OOC8B~7ABlA$$@b;o%ru0p==n+~hUh7T@lAX)3 z_qjJEwsz_Om+i$*xt8?Lqvxm z9p=UsKHm_$e^t$Sua_)Z^X`{7!SAC?cdG5>emJksC0j0@kY+ILJJtJMjSp3RyLLzk zp~LGbI(tg7sIT|F09~S?D)nhHUg~(w;ROt1Vlwe*;1A?7|J_sJ!?L!5+KaT41y%MZ zAUnE$#@p>Wy81NOVscf1_G$EJ@u|b8bv~u2_oX*qa{9X-*ewv&d&YNxOtkSc@~6wI zx|cr7mW*29vWE@q(`szo(fFp4>r z!Gl4Y@rR}Cr3K4Q-s?+MKRf9jmf`b{VM2D*XwdkLZnwj?KfdlGj-SPgL_>${9arka z8xV1HnMwHpCfr!?vX<%(e?FA9kvXW$QB* z$QQwg&`^5U4(0f-LNvq|rpU9mwu5U*oa&fPn}!1OtrfS%qLjCxc2V2Jf zvElapP%OwDDZP66Yf|)`dIFulxMs9kLKffaleX1N#hg6d&%p z9`@ZQTE@YoM0G)KgT40E*7TI%&Q5f0RL9wn4#)y$XpqY9zS|`ouw~mF`!c+<{JeE@ z84nu+gWI~Tj8(=`haJ>?8gK9Ferb%GG%nVdl(vyfpBj5ItvfyKB(HpIrb-)^qnFhy zv&MY3wv>NE@C72hem+?Bb=1u?oSZEqUmNXwDF37Bxte54rG@^ddq3-_l7`$SGEbv}vtA-so5 zP#%>b<0nRWPD|cqd-+qOMlgBpgIN9NOSCz+cg8E`+y{r-I~M&9sN)mqjM>PR0|HOf z>fG#fbz;xBemh*L?kLi}2@k?1o0+=UwS=kpZZaC&Ba*(Vbeh{GIjb)CjQN{;HQ>4~ z{D8W0Cp!hggr25P3%SfrNOrFEna4e9Hnzu>J#_k>GrjqCZ`_EQ!xR3>N>@u^QI|L4 zBB!(4!{NKL+c`WT+4^&@1Fz;Fu77t*lz^}Po-0Wxuo=9_ue>#DFD*+-ckeZr@M(Ex zVldeIf$mwO;@sz88YK<&mK5_>{KxECb+x&aWvGjQUUN{&MDrRf;$w=`v7LF3tadL< zJTn|v&Q<%86w-)4fBf!&773h5PhYfi`)`XRwmz52I4{wvp|Y#BdKR^!8v=N_C--#k z72>aaFMDI%-`c{n7KwLokI<>_7|^dZNtGD+EkCMHY702CBB}Fzw#ejZy!j8z*@rkc zoBU0IhsHx!%^mlWogIC|MsF87w-3}v#9QrHJ0)|@Q(-AM)#`cp@e&*P_}?y^Ucc>)d!y(=y|?6n9R@z^cDQv%&f9~#4+PB% z|BwhSxFpO&zM604z`=Z-ko!aAvv6nkR-%|Y>V@phN1G`E(|@{Z!C+~z9=ztMK*cvxOQ1s&B+UX zjTpFEP{*Yem(VAD)`CWcMJWRjS4?5`&6T-n58KHJD3ib!O?fvnKN9c(+2g{5z@>oa zn(O%}kCq72@cx>R)J>9WRLB-?ae4E43L;UjcUQCpSr_!Z$< zBV1+L>e%VAtm=YJ6Fu2bkhSK0htfycu0o z!1mZTz0HYM8$>KGBx*e$k&3_KC@r#j9KWhq*P8Z8$z{zqOz*(hssnYGOr%EQKLR%? zD^nTg?c$^ literal 0 HcmV?d00001 diff --git a/deploy/test-build/cluster/spank/test.sh b/deploy/test-build/cluster/spank/test.sh new file mode 100755 index 00000000..568962eb --- /dev/null +++ b/deploy/test-build/cluster/spank/test.sh @@ -0,0 +1,25 @@ +#!/bin/bash + +# TODO: this test fails if root is group owner of some other ownership variations, a more sophisticated check that looks at folder contents should be implemented + +# with enabled $HOME the home directory is owned by the user +ENABLED_HOME=$(sshpass -p test11 ssh test1@127.0.0.1 -p 2223 srun stat -c %G /home/test1) +echo $ENABLED_HOME +if [[ $ENABLED_HOME = "test1" ]] +then + echo PASS +else + echo FAILED +fi + +# with disabled $HOME, the home directory is now group owned by root +DISABLED_HOME=$(sshpass -p test11 ssh test1@127.0.0.1 -p 2223 srun --nohome stat -c %G /home/test1) + +if [[ $DISABLED_HOME = "root" ]] +then + echo PASS +else + echo FAILED +fi + + diff --git a/deploy/test-build/docker-compose.yml b/deploy/test-build/docker-compose.yml index 4d015e94..a8109bed 100644 --- a/deploy/test-build/docker-compose.yml +++ b/deploy/test-build/docker-compose.yml @@ -89,6 +89,9 @@ services: hostname: cluster ports: - "2223:22" + # enables mount, and unshare needed by the drophome SPANK plugin + cap_add: + - SYS_ADMIN minio: # runs on private network so "cluster" can reach it @@ -126,4 +129,4 @@ services: ports: - "9090:8080" environment: - SWAGGER_JSON: /tmp/openapi.yaml \ No newline at end of file + SWAGGER_JSON: /tmp/openapi.yaml From 7946056ee27b9835f53fc7f1f99aef19bef360a1 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Wed, 3 Mar 2021 11:21:25 +0100 Subject: [PATCH 03/20] Environment variables bool casting --- src/certificator/certificator.py | 6 +++--- src/common/cscs_api_common.py | 6 +++--- src/compute/compute.py | 4 ++-- src/reservations/reservations.py | 4 ++-- src/status/status.py | 4 ++-- src/storage/storage.py | 8 ++++---- src/tasks/tasks.py | 4 ++-- src/utilities/utilities.py | 4 ++-- 8 files changed, 20 insertions(+), 20 deletions(-) diff --git a/src/certificator/certificator.py b/src/certificator/certificator.py index 8b7af330..2195d072 100644 --- a/src/certificator/certificator.py +++ b/src/certificator/certificator.py @@ -25,12 +25,12 @@ CERTIFICATOR_PORT = os.environ.get("F7T_CERTIFICATOR_PORT", 5000) # OPA endpoint -OPA_USE = os.environ.get("F7T_OPA_USE",False) +OPA_USE = os.environ.get("F7T_OPA_USE",False) == "True" OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"') POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -41,7 +41,7 @@ realm_pubkey = '-----BEGIN PUBLIC KEY-----\n' + realm_pubkey + '\n-----END PUBLIC KEY-----' realm_pubkey_type = os.environ.get("F7T_REALM_RSA_TYPE").strip('\'"') -debug = os.environ.get("F7T_DEBUG_MODE", False) +debug = os.environ.get("F7T_DEBUG_MODE", False) == "True" app = Flask(__name__) diff --git a/src/common/cscs_api_common.py b/src/common/cscs_api_common.py index 24d9a221..96204e1c 100644 --- a/src/common/cscs_api_common.py +++ b/src/common/cscs_api_common.py @@ -41,15 +41,15 @@ CERTIFICATOR_URL = os.environ.get("F7T_CERTIFICATOR_URL") TASKS_URL = os.environ.get("F7T_TASKS_URL") -F7T_SSH_CERTIFICATE_WRAPPER = os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", None) +F7T_SSH_CERTIFICATE_WRAPPER = os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", None) == "True" # OPA endpoint -OPA_USE = os.environ.get("F7T_OPA_USE",False) +OPA_USE = os.environ.get("F7T_OPA_USE",False) == "True" OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"') POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") diff --git a/src/compute/compute.py b/src/compute/compute.py index 16dac2a8..d38641c0 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -37,7 +37,7 @@ COMPUTE_PORT = os.environ.get("F7T_COMPUTE_PORT", 5000) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -68,7 +68,7 @@ # max content length for upload in bytes app.config['MAX_CONTENT_LENGTH'] = int(MAX_FILE_SIZE) * 1024 * 1024 -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" def is_jobid(jobid): diff --git a/src/reservations/reservations.py b/src/reservations/reservations.py index fdd7a871..5300d6bb 100644 --- a/src/reservations/reservations.py +++ b/src/reservations/reservations.py @@ -31,13 +31,13 @@ TIMEOUT = os.environ.get("F7T_UTILITIES_TIMEOUT", 5) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") RESERVATION_CMD = os.environ.get("F7T_RESERVATION_CMD", "rsvmgmt") -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" app = Flask(__name__) diff --git a/src/status/status.py b/src/status/status.py index 0feea74e..8121dd0d 100644 --- a/src/status/status.py +++ b/src/status/status.py @@ -34,7 +34,7 @@ SERVICES_DICT = {} ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", "False") == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -47,7 +47,7 @@ OBJECT_STORAGE=os.environ.get("F7T_OBJECT_STORAGE") # debug on console -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", "False") == "True" app = Flask(__name__) diff --git a/src/storage/storage.py b/src/storage/storage.py index 158d72fd..5c3b7a66 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -66,7 +66,7 @@ XFER_PARTITION = os.environ.get("F7T_XFER_PARTITION", "").strip('\'"') # --account parameter needed in sbatch? -USE_SLURM_ACCOUNT = os.environ.get("F7T_USE_SLURM_ACCOUNT", False) +USE_SLURM_ACCOUNT = os.environ.get("F7T_USE_SLURM_ACCOUNT", False) == "True" # Machine used for external transfers @@ -96,11 +96,11 @@ CERT_CIPHER_KEY = os.environ.get("F7T_CERT_CIPHER_KEY", "").strip('\'"').encode('utf-8') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") # verify signed SSL certificates -SSL_SIGNED = os.environ.get("F7T_SSL_SIGNED", False) +SSL_SIGNED = os.environ.get("F7T_SSL_SIGNED", False) == "True" # aynchronous tasks: upload & download --> http://TASKS_URL # {task_id : AsyncTask} @@ -112,7 +112,7 @@ uploaded_files = {} # debug on console -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" app = Flask(__name__) diff --git a/src/tasks/tasks.py b/src/tasks/tasks.py index 8fa39591..3c87640b 100644 --- a/src/tasks/tasks.py +++ b/src/tasks/tasks.py @@ -29,7 +29,7 @@ PERSIST_PWD = os.environ.get("F7T_PERSIST_PWD") ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -39,7 +39,7 @@ # expire time in seconds, for download/upload: default 30 days + 24 hours = 2678400 secs STORAGE_TASK_EXP_TIME = os.environ.get("F7T_STORAGE_TASK_EXP_TIME", 2678400) -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" # task dict, key is the task_id tasks = {} diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index 7ea50c72..b0ae1b8d 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -31,13 +31,13 @@ # internal machines for file operations SYS_INTERNALS = os.environ.get("F7T_SYSTEMS_INTERNAL_UTILITIES").strip('\'"').split(";") -debug = os.environ.get("F7T_DEBUG_MODE", None) +debug = os.environ.get("F7T_DEBUG_MODE", False) == "True" #max file size for upload/download in MB MAX_FILE_SIZE=int(os.environ.get("F7T_UTILITIES_MAX_FILE_SIZE")) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) +USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") From ff1b3bb6159fc48a44cf37f42a86413b48017959 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Thu, 4 Mar 2021 10:51:44 +0100 Subject: [PATCH 04/20] Using UTILITIES system to check files or directories --- src/storage/storage.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index 158d72fd..3cec64c6 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -53,11 +53,15 @@ # Job machine where to send xfer-internal jobs (must be defined in SYSTEMS_PUBLIC) STORAGE_JOBS_MACHINE = os.environ.get("F7T_STORAGE_JOBS_MACHINE").strip('\'"') + + # SYSTEMS_PUBLIC: list of allowed systems # remove quotes and split into array SYSTEMS_PUBLIC = os.environ.get("F7T_SYSTEMS_PUBLIC").strip('\'"').split(";") # internal machines to submit/query jobs SYS_INTERNALS = os.environ.get("F7T_SYSTEMS_INTERNAL_COMPUTE").strip('\'"').split(";") +# internal machines for small operations +SYS_INTERNALS_UTILITIES = os.environ.get("F7T_SYSTEMS_INTERNAL_UTILITIES").strip('\'"').split(";") ###### ENV VAR FOR DETECT TECHNOLOGY OF STAGING AREA: OBJECT_STORAGE = os.environ.get("F7T_OBJECT_STORAGE", "").strip('\'"') @@ -734,7 +738,7 @@ def internal_operation(request, command): auth_header = request.headers[AUTH_HEADER_NAME] system_idx = SYSTEMS_PUBLIC.index(STORAGE_JOBS_MACHINE) - system_addr = SYS_INTERNALS[system_idx] + system_addr = SYS_INTERNALS_UTILITIES[system_idx] system_name = STORAGE_JOBS_MACHINE try: From 60c8e7b79dbf946201c922a421b9ee0538367699 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Thu, 4 Mar 2021 15:45:59 +0100 Subject: [PATCH 05/20] openapi: updated version --- doc/openapi/firecrest-api.yaml | 2 +- doc/openapi/firecrest-developers-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index a99ebdb4..70230642 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.0 + version: 1.7.1-beta1 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index 0320b6e8..9c656f7e 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.0 + version: 1.7.1-beta1 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a From d44e71b0f1398660de86363a760d617df5264afb Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Thu, 4 Mar 2021 15:47:14 +0100 Subject: [PATCH 06/20] Added get_boolean_var function --- src/certificator/certificator.py | 19 ++++++++++++++++--- src/common/cscs_api_common.py | 27 ++++++++++++++++++++++----- src/compute/compute.py | 7 ++++--- src/reservations/reservations.py | 6 +++--- src/status/status.py | 6 +++--- src/storage/storage.py | 10 +++++----- src/tasks/tasks.py | 6 +++--- src/utilities/utilities.py | 8 ++++---- 8 files changed, 60 insertions(+), 29 deletions(-) diff --git a/src/certificator/certificator.py b/src/certificator/certificator.py index 2195d072..53d40150 100644 --- a/src/certificator/certificator.py +++ b/src/certificator/certificator.py @@ -14,6 +14,19 @@ import base64 import requests +# Checks if an environment variable injected to F7T is a valid True value +# var <- str +# returns -> boolean +def get_boolean_var(var): + + # ensure variable to be a string + var = str(var) + # True, true or TRUE + # Yes, yes or YES + # 1 + + return var.upper() == "TRUE" or var.upper() == "YES" or var == "1" + AUTH_HEADER_NAME = 'Authorization' AUTH_AUDIENCE = os.environ.get("F7T_AUTH_TOKEN_AUD", '').strip('\'"') @@ -25,12 +38,12 @@ CERTIFICATOR_PORT = os.environ.get("F7T_CERTIFICATOR_PORT", 5000) # OPA endpoint -OPA_USE = os.environ.get("F7T_OPA_USE",False) == "True" +OPA_USE = get_boolean_var(os.environ.get("F7T_OPA_USE",False)) OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"') POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -41,7 +54,7 @@ realm_pubkey = '-----BEGIN PUBLIC KEY-----\n' + realm_pubkey + '\n-----END PUBLIC KEY-----' realm_pubkey_type = os.environ.get("F7T_REALM_RSA_TYPE").strip('\'"') -debug = os.environ.get("F7T_DEBUG_MODE", False) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) app = Flask(__name__) diff --git a/src/common/cscs_api_common.py b/src/common/cscs_api_common.py index 96204e1c..49c35c6e 100644 --- a/src/common/cscs_api_common.py +++ b/src/common/cscs_api_common.py @@ -20,7 +20,21 @@ import io import time -debug = os.environ.get("F7T_DEBUG_MODE", None) +# Checks if an environment variable injected to F7T is a valid True value +# var <- str +# returns -> boolean +def get_boolean_var(var): + + # ensure variable to be a string + var = str(var) + # True, true or TRUE + # Yes, yes or YES + # 1 + + return var.upper() == "TRUE" or var.upper() == "YES" or var == "1" + + +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) AUTH_HEADER_NAME = 'Authorization' @@ -41,15 +55,15 @@ CERTIFICATOR_URL = os.environ.get("F7T_CERTIFICATOR_URL") TASKS_URL = os.environ.get("F7T_TASKS_URL") -F7T_SSH_CERTIFICATE_WRAPPER = os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", None) == "True" +F7T_SSH_CERTIFICATE_WRAPPER = get_boolean_var(os.environ.get("F7T_SSH_CERTIFICATE_WRAPPER", False)) # OPA endpoint -OPA_USE = os.environ.get("F7T_OPA_USE",False) == "True" +OPA_USE = get_boolean_var(os.environ.get("F7T_OPA_USE",False)) OPA_URL = os.environ.get("F7T_OPA_URL","http://localhost:8181").strip('\'"') POLICY_PATH = os.environ.get("F7T_POLICY_PATH","v1/data/f7t/authz").strip('\'"') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -759,4 +773,7 @@ def check_command_error(error_str, error_code, service_msg): header = {"X-Permission-Denied": "User does not have permissions to access path"} return {"description": service_msg, "status_code": 400, "header": header} header = {"X-Error": error_str} - return {"description": service_msg, "error": error_str, "status_code": 400, "header": header} \ No newline at end of file + return {"description": service_msg, "error": error_str, "status_code": 400, "header": header} + + + diff --git a/src/compute/compute.py b/src/compute/compute.py index d38641c0..5476ff1b 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -11,7 +11,8 @@ import async_task from cscs_api_common import check_auth_header, get_username, \ - exec_remote_command, create_task, update_task, clean_err_output, in_str, is_valid_file + exec_remote_command, create_task, update_task, clean_err_output, \ + in_str, is_valid_file, get_boolean_var from job_time import check_sacctTime @@ -37,7 +38,7 @@ COMPUTE_PORT = os.environ.get("F7T_COMPUTE_PORT", 5000) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -68,7 +69,7 @@ # max content length for upload in bytes app.config['MAX_CONTENT_LENGTH'] = int(MAX_FILE_SIZE) * 1024 * 1024 -debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) def is_jobid(jobid): diff --git a/src/reservations/reservations.py b/src/reservations/reservations.py index 5300d6bb..9f1d3033 100644 --- a/src/reservations/reservations.py +++ b/src/reservations/reservations.py @@ -12,7 +12,7 @@ import os import logging from logging.handlers import TimedRotatingFileHandler -from cscs_api_common import check_auth_header, exec_remote_command, in_str +from cscs_api_common import check_auth_header, exec_remote_command, in_str, get_boolean_var import re import datetime @@ -31,13 +31,13 @@ TIMEOUT = os.environ.get("F7T_UTILITIES_TIMEOUT", 5) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") RESERVATION_CMD = os.environ.get("F7T_RESERVATION_CMD", "rsvmgmt") -debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) app = Flask(__name__) diff --git a/src/status/status.py b/src/status/status.py index 8121dd0d..86b482b2 100644 --- a/src/status/status.py +++ b/src/status/status.py @@ -11,7 +11,7 @@ import multiprocessing as mp # common modules -from cscs_api_common import check_auth_header +from cscs_api_common import check_auth_header, get_boolean_var import paramiko import socket @@ -34,7 +34,7 @@ SERVICES_DICT = {} ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", "False") == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -47,7 +47,7 @@ OBJECT_STORAGE=os.environ.get("F7T_OBJECT_STORAGE") # debug on console -debug = os.environ.get("F7T_DEBUG_MODE", "False") == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) app = Flask(__name__) diff --git a/src/storage/storage.py b/src/storage/storage.py index 5c3b7a66..61a3b6cc 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -21,7 +21,7 @@ from cscs_api_common import exec_remote_command from cscs_api_common import create_certificate from cscs_api_common import in_str -from cscs_api_common import is_valid_file, is_valid_dir, check_command_error +from cscs_api_common import is_valid_file, is_valid_dir, check_command_error, get_boolean_var # job_time_checker for correct SLURM job time in /xfer-internal tasks import job_time @@ -66,7 +66,7 @@ XFER_PARTITION = os.environ.get("F7T_XFER_PARTITION", "").strip('\'"') # --account parameter needed in sbatch? -USE_SLURM_ACCOUNT = os.environ.get("F7T_USE_SLURM_ACCOUNT", False) == "True" +USE_SLURM_ACCOUNT = get_boolean_var(os.environ.get("F7T_USE_SLURM_ACCOUNT", False)) # Machine used for external transfers @@ -96,11 +96,11 @@ CERT_CIPHER_KEY = os.environ.get("F7T_CERT_CIPHER_KEY", "").strip('\'"').encode('utf-8') ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") # verify signed SSL certificates -SSL_SIGNED = os.environ.get("F7T_SSL_SIGNED", False) == "True" +SSL_SIGNED = get_boolean_var(os.environ.get("F7T_SSL_SIGNED", False)) # aynchronous tasks: upload & download --> http://TASKS_URL # {task_id : AsyncTask} @@ -112,7 +112,7 @@ uploaded_files = {} # debug on console -debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) app = Flask(__name__) diff --git a/src/tasks/tasks.py b/src/tasks/tasks.py index 3c87640b..2cae649b 100644 --- a/src/tasks/tasks.py +++ b/src/tasks/tasks.py @@ -12,7 +12,7 @@ import os import logging from logging.handlers import TimedRotatingFileHandler -from cscs_api_common import check_auth_header, get_username, check_header +from cscs_api_common import check_auth_header, get_username, check_header, get_boolean_var import tasks_persistence as persistence AUTH_HEADER_NAME = 'Authorization' @@ -29,7 +29,7 @@ PERSIST_PWD = os.environ.get("F7T_PERSIST_PWD") ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") @@ -39,7 +39,7 @@ # expire time in seconds, for download/upload: default 30 days + 24 hours = 2678400 secs STORAGE_TASK_EXP_TIME = os.environ.get("F7T_STORAGE_TASK_EXP_TIME", 2678400) -debug = os.environ.get("F7T_DEBUG_MODE", None) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) # task dict, key is the task_id tasks = {} diff --git a/src/utilities/utilities.py b/src/utilities/utilities.py index b0ae1b8d..2022cbf2 100644 --- a/src/utilities/utilities.py +++ b/src/utilities/utilities.py @@ -13,9 +13,9 @@ from math import ceil -from cscs_api_common import check_auth_header, get_username,exec_remote_command, parse_io_error, check_command_error, in_str -import base64 +from cscs_api_common import check_auth_header, get_username,exec_remote_command, parse_io_error, check_command_error, in_str, get_boolean_var import io +import base64 CERTIFICATOR_URL = os.environ.get("F7T_CERTIFICATOR_URL") @@ -31,13 +31,13 @@ # internal machines for file operations SYS_INTERNALS = os.environ.get("F7T_SYSTEMS_INTERNAL_UTILITIES").strip('\'"').split(";") -debug = os.environ.get("F7T_DEBUG_MODE", False) == "True" +debug = get_boolean_var(os.environ.get("F7T_DEBUG_MODE", False)) #max file size for upload/download in MB MAX_FILE_SIZE=int(os.environ.get("F7T_UTILITIES_MAX_FILE_SIZE")) ### SSL parameters -USE_SSL = os.environ.get("F7T_USE_SSL", False) == "True" +USE_SSL = get_boolean_var(os.environ.get("F7T_USE_SSL", False)) SSL_CRT = os.environ.get("F7T_SSL_CRT", "") SSL_KEY = os.environ.get("F7T_SSL_KEY", "") From 2df84633e5e6aafdd280a5f1368b1070d041812f Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Thu, 4 Mar 2021 16:08:17 +0100 Subject: [PATCH 07/20] openapi: version updated --- doc/openapi/firecrest-api.yaml | 2 +- doc/openapi/firecrest-developers-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index a99ebdb4..7ffaff74 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.0 + version: 1.7.1-beta2 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index 0320b6e8..bb1740ee 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.0 + version: 1.7.1-beta2 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a From 1f64bf0cafeb74e39ef3dfd05dc67282b7aecd83 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 08:56:22 +0100 Subject: [PATCH 08/20] Change type argument from str to object --- src/certificator/certificator.py | 2 +- src/common/cscs_api_common.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/src/certificator/certificator.py b/src/certificator/certificator.py index 53d40150..efa4e20b 100644 --- a/src/certificator/certificator.py +++ b/src/certificator/certificator.py @@ -15,7 +15,7 @@ import requests # Checks if an environment variable injected to F7T is a valid True value -# var <- str +# var <- object # returns -> boolean def get_boolean_var(var): diff --git a/src/common/cscs_api_common.py b/src/common/cscs_api_common.py index 49c35c6e..b55fe4b0 100644 --- a/src/common/cscs_api_common.py +++ b/src/common/cscs_api_common.py @@ -21,7 +21,7 @@ import time # Checks if an environment variable injected to F7T is a valid True value -# var <- str +# var <- object # returns -> boolean def get_boolean_var(var): From b4dece91687e248effc9b204af96a6a1aca4b86c Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 13:40:13 +0100 Subject: [PATCH 09/20] cryptography package updated --- deploy/docker/base/requirements.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/deploy/docker/base/requirements.txt b/deploy/docker/base/requirements.txt index 3b9675b2..0bac5391 100644 --- a/deploy/docker/base/requirements.txt +++ b/deploy/docker/base/requirements.txt @@ -1,4 +1,4 @@ -cryptography==2.8 +cryptography==3.4.6 Flask==1.1.2 PyJWT==1.7.1 requests==2.22.0 \ No newline at end of file From b2fdcf68cdd21946beb1b4111112181824b53e78 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 13:40:35 +0100 Subject: [PATCH 10/20] updated beta version --- doc/openapi/firecrest-api.yaml | 2 +- doc/openapi/firecrest-developers-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index 7ffaff74..40063d3c 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta2 + version: 1.7.1-beta3 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index bb1740ee..226cba52 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta2 + version: 1.7.1-beta3 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a From 67e8d48936ee7c0dfb9a46549eda305711ad7427 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 16:35:50 +0100 Subject: [PATCH 11/20] openapi: version update --- doc/openapi/firecrest-api.yaml | 2 +- doc/openapi/firecrest-developers-api.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc/openapi/firecrest-api.yaml b/doc/openapi/firecrest-api.yaml index 7ffaff74..7b8d41ef 100644 --- a/doc/openapi/firecrest-api.yaml +++ b/doc/openapi/firecrest-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta2 + version: 1.7.1-beta4 title: FirecREST Developers API description: > This API specification is intended for FirecREST developers only. There're some endpoints that are not available in the public version for client developers. diff --git a/doc/openapi/firecrest-developers-api.yaml b/doc/openapi/firecrest-developers-api.yaml index bb1740ee..8f49f233 100644 --- a/doc/openapi/firecrest-developers-api.yaml +++ b/doc/openapi/firecrest-developers-api.yaml @@ -9,7 +9,7 @@ servers: - url: 'http://FIRECREST_URL' - url: 'https://FIRECREST_URL' info: - version: 1.7.1-beta2 + version: 1.7.1-beta4 title: FirecREST API description: > FirecREST platform, a RESTful Services Gateway to HPC resources, is a From 80e18d57836d8d94e3faacb34433691b15e2465c Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 16:38:27 +0100 Subject: [PATCH 12/20] cast time() to int --- src/storage/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index ea44e7dd..b2cbce80 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -214,7 +214,7 @@ def os_to_fs(task_id): # Therefore, using delete_object_after a few minutes (in this case 5 minutes) will trigger internal staging area # mechanism to delete the file automatically and without a need of a connection - staging.delete_object_after(containername=username,prefix=task_id,objectname=objectname, ttl = time.time()+600) + staging.delete_object_after(containername=username,prefix=task_id,objectname=objectname, ttl = int(time.time())+600) # if error, should be prepared for try again else: @@ -463,7 +463,7 @@ def invalidate_request(): # error = staging.delete_object(containername,prefix,objectname) # replacing delete_object by delete_object_after 5 minutes - error = staging.delete_object_after(containername=containername, prefix=prefix, objectname=objectname, ttl=time.time()+600) + error = staging.delete_object_after(containername=containername, prefix=prefix, objectname=objectname, ttl=int(time.time())+600) if error == -1: return jsonify(error="Could not invalidate URL"), 400 From e1bc11cad7fd80e6bafd26c17e0ad0822102961d Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 16:39:23 +0100 Subject: [PATCH 13/20] Change to X-Delete-At --- src/storage/swiftOS.py | 19 ++++++++++++------- 1 file changed, 12 insertions(+), 7 deletions(-) diff --git a/src/storage/swiftOS.py b/src/storage/swiftOS.py index 1100911a..4bb3a697 100644 --- a/src/storage/swiftOS.py +++ b/src/storage/swiftOS.py @@ -9,6 +9,7 @@ import requests import keystone from time import time +from datetime import datetime import hmac from hashlib import sha1 @@ -138,6 +139,7 @@ def is_object_created(self,containername,prefix,objectname): try: req = requests.head(url, headers={"X-Auth-Token": self.auth}) + logging.info(req.headers) headers = req.headers # if Content-Lenght == 0, then object doesn't exist @@ -292,23 +294,26 @@ def list_objects(self,containername,prefix=None): # sets time to live (TTL) for an object in SWIFT def delete_object_after(self,containername,prefix,objectname,ttl): - swift_account_url = "{swift_url}/{containername}/{prefix}/{objectname}".format( - swift_url=self.url, containername=containername, prefix=prefix, objectname=objectname) + swift_account_url = f"{self.url}/{containername}/{prefix}/{objectname}" - header = {'X-Delete-After': "{}".format(ttl), "X-Auth-Token": self.auth} + header = {"X-Delete-At": str(ttl), "X-Auth-Token": self.auth} try: - logging.info("Setting {seconds} [s] as X-Delete-After".format(seconds=ttl)) + logging.info(f"Setting {ttl} [s] as X-Delete-At") req = requests.post(swift_account_url, headers=header) if not req.ok: - logging.error("Object couldn't be marked as X-Delete-After") + logging.error("Object couldn't be marked as X-Delete-At") + logging.error(req.text) return -1 + date_ttl = datetime.fromtimestamp(ttl).strftime("%Y-%m-%dT%H:%M:%S") + + logging.info(f"Object was marked as to be deleted at {date_ttl}") return 0 except Exception as e: - logging.error("Object couldn't be marked as X-Delete-After") + logging.error("Object couldn't be marked as X-Delete-At") logging.error(e) return -1 @@ -321,7 +326,7 @@ def delete_object(self,containername,prefix,objectname): try: - logging.info("Deleting object: {}/{}/{}".format(containername,prefix,objectname)) + logging.info(f"Deleting object: {containername}/{prefix}/{objectname}") req = requests.delete(swift_account_url, headers=header) From 451bf941275aec9f7440a865e69cdf2fe54be01e Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 16:40:03 +0100 Subject: [PATCH 14/20] same output than swiftOS --- src/storage/s3v4OS.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/storage/s3v4OS.py b/src/storage/s3v4OS.py index 213f07c3..e906beb6 100644 --- a/src/storage/s3v4OS.py +++ b/src/storage/s3v4OS.py @@ -666,7 +666,7 @@ def delete_object_after(self,containername,prefix,objectname,ttl): resp = requests.put(url, data=body, headers=headers) if resp.ok: - logging.info("Object marked as delete-at succesfully") + logging.info(f"Object was marked as to be deleted at {_delete_at_iso}") return 0 From b4f8c9b93b23f7d18d92a84f74af7db5d488ce6b Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Fri, 5 Mar 2021 17:03:00 +0100 Subject: [PATCH 15/20] Adapted deletion time for download tasks --- src/storage/storage.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index b2cbce80..cbeee7f6 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -372,7 +372,8 @@ def download_task(auth_header,system_name, system_addr,sourcePath,task_id): # if succesfully created: temp_url in task with success status update_task(task_id, auth_header, async_task.ST_UPL_END, temp_url) - retval = staging.delete_object_after(containername=container_name,prefix=object_prefix,objectname=object_name,ttl=STORAGE_TEMPURL_EXP_TIME) + # marked deletion from here to STORAGE_TEMPURL_EXP_TIME (default 30 days) + retval = staging.delete_object_after(containername=container_name,prefix=object_prefix,objectname=object_name,ttl=int(time.time()) + STORAGE_TEMPURL_EXP_TIME) if retval == 0: app.logger.info("Setting {seconds} [s] as X-Delete-After".format(seconds=STORAGE_TEMPURL_EXP_TIME)) From d13350aced1fdcc484a53b6d8b30c2c7b3f2f4c8 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Mon, 8 Mar 2021 10:32:51 +0100 Subject: [PATCH 16/20] upgraded pip for testing --- ci/pre-prod/run_tests.yml | 3 +++ 1 file changed, 3 insertions(+) diff --git a/ci/pre-prod/run_tests.yml b/ci/pre-prod/run_tests.yml index e16cb563..2ce2edc0 100644 --- a/ci/pre-prod/run_tests.yml +++ b/ci/pre-prod/run_tests.yml @@ -20,6 +20,9 @@ #environment: # PATH: "{{ ansible_env.PATH }}:/home/centos/.local/bin" tasks: + - name: Upgrade pip3 + shell: pip3 install --user --upgrade pip==21.0.1 + - name: Install pytest modules shell: pip3 install --user -r requirements.txt args: From 1fd05317def3ca67a99756a1053e20c7cc461593 Mon Sep 17 00:00:00 2001 From: jpdorsch Date: Mon, 8 Mar 2021 19:03:07 +0100 Subject: [PATCH 17/20] replaced X-Delete-After log messages --- src/storage/storage.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/storage/storage.py b/src/storage/storage.py index cbeee7f6..ae572c88 100644 --- a/src/storage/storage.py +++ b/src/storage/storage.py @@ -376,9 +376,9 @@ def download_task(auth_header,system_name, system_addr,sourcePath,task_id): retval = staging.delete_object_after(containername=container_name,prefix=object_prefix,objectname=object_name,ttl=int(time.time()) + STORAGE_TEMPURL_EXP_TIME) if retval == 0: - app.logger.info("Setting {seconds} [s] as X-Delete-After".format(seconds=STORAGE_TEMPURL_EXP_TIME)) + app.logger.info("Setting {seconds} [s] as X-Delete-At".format(seconds=STORAGE_TEMPURL_EXP_TIME)) else: - app.logger.error("Object couldn't be marked as X-Delete-After") + app.logger.error("Object couldn't be marked as X-Delete-At") From 4afff51c79e441664daf4ececa315d26cb616878 Mon Sep 17 00:00:00 2001 From: fcruzcscs Date: Wed, 17 Mar 2021 14:49:12 +0100 Subject: [PATCH 18/20] Update compute.py adding a traceback --- src/compute/compute.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/compute/compute.py b/src/compute/compute.py index 6d7ce708..7113a0bf 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -9,6 +9,7 @@ from logging.handlers import TimedRotatingFileHandler import threading import async_task +import traceback from cscs_api_common import check_auth_header, get_username, \ exec_remote_command, create_task, update_task, clean_err_output, \ @@ -201,6 +202,7 @@ def submit_job_task(auth_header, system_name, system_addr, job_file, job_dir, ta except Exception as e: app.logger.error(type(e)) app.logger.error(e) + traceback.print_exc(file=sys.stdout) update_task(task_id, auth_header, async_task.ERROR, e.message) From ce2592805a7900d7c55f71fe39ba3f864066ee49 Mon Sep 17 00:00:00 2001 From: Tomas Aliaga Date: Wed, 17 Mar 2021 15:06:27 +0100 Subject: [PATCH 19/20] Try to get some info --- src/compute/compute.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) diff --git a/src/compute/compute.py b/src/compute/compute.py index 7113a0bf..e6342528 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -10,6 +10,7 @@ import threading import async_task import traceback +import sys from cscs_api_common import check_auth_header, get_username, \ exec_remote_command, create_task, update_task, clean_err_output, \ @@ -196,14 +197,14 @@ def submit_job_task(auth_header, system_name, system_addr, job_file, job_dir, ta update_task(task_id, auth_header, async_task.SUCCESS, job_extra_info, True) except IOError as e: - app.logger.error(e.filename) + app.logger.error(e.filename, exc_info=True, stack_info=True) app.logger.error(e.strerror) update_task(task_id, auth_header,async_task.ERROR, e.message) except Exception as e: - app.logger.error(type(e)) + app.logger.error(type(e), exc_info=True, stack_info=True) app.logger.error(e) traceback.print_exc(file=sys.stdout) - update_task(task_id, auth_header, async_task.ERROR, e.message) + update_task(task_id, auth_header, async_task.ERROR) @@ -877,7 +878,7 @@ def cancel_job(jobid): action = f"scancel -v {jobid}" try: - # obtain new task from TASKS microservice + # obtain new task from TASKS microservice. task_id = create_task(auth_header,service="compute") # if error in creating task: From 9d2fd93c400881d99047aa6b4d2e25372495a9f0 Mon Sep 17 00:00:00 2001 From: Tomas Aliaga Date: Wed, 17 Mar 2021 15:59:24 +0100 Subject: [PATCH 20/20] Adjust parsing of scontrol show in compute.py --- src/compute/compute.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/src/compute/compute.py b/src/compute/compute.py index e6342528..bea0cd2a 100644 --- a/src/compute/compute.py +++ b/src/compute/compute.py @@ -246,13 +246,13 @@ def get_slurm_files(auth_header, system_name, system_addr, task_id,job_info,outp # if it's ok, we can add information control_resp = resp["msg"] + # tokens are expected to be space-separated and with a k=v form. See man scontrol show control_list = control_resp.split() + control_dict = { value.split("=")[0] : value.split("=")[1] for value in control_list if "=" in value } - control_dict = { value.split("=")[0] : value.split("=")[1] for value in control_list } - - control_info["job_file_out"] = control_dict["StdOut"] - control_info["job_file_err"] = control_dict["StdErr"] - control_info["job_file"] = control_dict["Command"] + control_info["job_file_out"] = control_dict.get("StdOut", "stdout-file-not-found") + control_info["job_file_err"] = control_dict.get("StdErr", "stderr-file-not-found") + control_info["job_file"] = control_dict.get("Command", "command-not-found") control_info["job_data_out"] = "" control_info["job_data_err"] = "" # if all fine: