From 3eb51698c8f2a20c19bfbf4a8b6525a7cddfbf92 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:34:19 +0000 Subject: [PATCH 01/10] clean up some LiRA code --- research/mi_lira_2021/README.md | 6 +++--- research/mi_lira_2021/inference.py | 30 +++++++----------------------- research/mi_lira_2021/plot.py | 5 +++-- research/mi_lira_2021/train.py | 5 ++--- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/research/mi_lira_2021/README.md b/research/mi_lira_2021/README.md index 7cb30e17..65eddfe2 100644 --- a/research/mi_lira_2021/README.md +++ b/research/mi_lira_2021/README.md @@ -2,9 +2,9 @@ This directory contains code to reproduce our paper: -**"Membership Inference Attacks From First Principles"** -https://arxiv.org/abs/2112.03570 -by Nicholas Carlini, Steve Chien, Milad Nasr, Shuang Song, Andreas Terzis, and Florian Tramer. +**"Membership Inference Attacks From First Principles"**
+https://arxiv.org/abs/2112.03570
+by Nicholas Carlini, Steve Chien, Milad Nasr, Shuang Song, Andreas Terzis, and Florian Tramèr. ### INSTALLING diff --git a/research/mi_lira_2021/inference.py b/research/mi_lira_2021/inference.py index 11ad696a..7fd3f814 100644 --- a/research/mi_lira_2021/inference.py +++ b/research/mi_lira_2021/inference.py @@ -12,32 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools -import os -from typing import Callable import json - +import os import re -import jax -import jax.numpy as jn -import numpy as np -import tensorflow as tf # For data augmentation. -import tensorflow_datasets as tfds -from absl import app, flags -from tqdm import tqdm, trange -import pickle -from functools import partial +import numpy as np import objax -from objax.jaxboard import SummaryWriter, Summary -from objax.util import EasyDict -from objax.zoo import convnet, wide_resnet - -from dataset import DataSet +import tensorflow as tf # For data augmentation. +from absl import app +from absl import flags -from train import MemModule, network +from train import MemModule +from train import network -from collections import defaultdict FLAGS = flags.FLAGS @@ -142,9 +129,6 @@ def features(model, xbatch, ybatch): flags.DEFINE_string('dataset', 'cifar10', 'Dataset.') flags.DEFINE_string('logdir', 'experiments/', 'Directory where to save checkpoints and tensorboard data.') flags.DEFINE_string('regex', '.*experiment.*', 'keep files when matching') - flags.DEFINE_bool('random_labels', False, 'use random labels.') flags.DEFINE_integer('dataset_size', 50000, 'size of dataset.') flags.DEFINE_integer('from_epoch', None, 'which epoch to load from.') - flags.DEFINE_integer('seed_mod', None, 'keep mod seed.') - flags.DEFINE_integer('modulus', 8, 'modulus.') app.run(main) diff --git a/research/mi_lira_2021/plot.py b/research/mi_lira_2021/plot.py index 435125ce..422b18e4 100644 --- a/research/mi_lira_2021/plot.py +++ b/research/mi_lira_2021/plot.py @@ -220,5 +220,6 @@ def fig_fpr_tpr(): plt.show() -load_data("exp/cifar10/") -fig_fpr_tpr() +if __name__ == '__main__': + load_data("exp/cifar10/") + fig_fpr_tpr() diff --git a/research/mi_lira_2021/train.py b/research/mi_lira_2021/train.py index 19ff0e3c..dd0b901d 100644 --- a/research/mi_lira_2021/train.py +++ b/research/mi_lira_2021/train.py @@ -24,12 +24,11 @@ import tensorflow as tf # For data augmentation. import tensorflow_datasets as tfds from absl import app, flags -from tqdm import tqdm, trange import objax from objax.jaxboard import SummaryWriter, Summary from objax.util import EasyDict -from objax.zoo import convnet, wide_resnet, dnnet +from objax.zoo import convnet, wide_resnet from dataset import DataSet @@ -269,7 +268,7 @@ def main(argv): logdir = "experiment-"+str(seed) logdir = os.path.join(FLAGS.logdir, logdir) - if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz"%10)): + if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz"%FLAGS.epochs)): print(f"run {FLAGS.expid} already completed.") return else: From 7f81648a1e4168e62d9b29ddc1ef3a4ffd1e2848 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:38:19 +0000 Subject: [PATCH 02/10] Add code to reproduce Truth Serum --- research/mi_poison_2022/README.md | 118 +++++++++++ research/mi_poison_2022/fprtpr.png | Bin 0 -> 32936 bytes research/mi_poison_2022/logs/.keep | 0 research/mi_poison_2022/plot_poison.py | 107 ++++++++++ research/mi_poison_2022/scripts/train_demo.sh | 30 +++ .../scripts/train_demo_multigpu.sh | 32 +++ research/mi_poison_2022/train_poison.py | 198 ++++++++++++++++++ 7 files changed, 485 insertions(+) create mode 100644 research/mi_poison_2022/README.md create mode 100644 research/mi_poison_2022/fprtpr.png create mode 100644 research/mi_poison_2022/logs/.keep create mode 100644 research/mi_poison_2022/plot_poison.py create mode 100644 research/mi_poison_2022/scripts/train_demo.sh create mode 100644 research/mi_poison_2022/scripts/train_demo_multigpu.sh create mode 100644 research/mi_poison_2022/train_poison.py diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md new file mode 100644 index 00000000..071c209b --- /dev/null +++ b/research/mi_poison_2022/README.md @@ -0,0 +1,118 @@ +## Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets + +This directory contains code to reproduce results from the paper: + +**"Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets"**
+https://arxiv.org/abs/2204.00032
+by Florian Tramèr, Reza Shokri, Ayrton San Joaquin, Hoang Le, Matthew Jagielski, Sanghyun Hong and Nicholas Carlini + +### INSTALLING + +The experiments in this directory are built on top of the [LiRA +membership inference attack](../mi_lira_2021). + +After following the [installation instructions](../mi_lira_2021#installing) +for LiRa, make sure the attack code is on your `PYTHONPATH`: + +```bash +export PYTHONPATH="${PYTHONPATH}:../mi_lira_2021" +``` + + +### RUNNING THE CODE + +#### 1. Train the models + +The first step in our attack is to train shadow models, with some data points +targeted by a poisoning attack. You can train 16 shadow models +with the command + +> bash scripts/train_demo.sh + +or if you have multiple GPUs on your machine and want to train these models +in parallel, then modify and run + +> bash scripts/train_demo_multigpu.sh + +This will train 16 CIFAR-10 wide ResNet models to ~91% accuracy each, with +250 points targeted for poisoning. For each of these 250 targeted points, the +attacker adds 8 mislabeled poisoned copies of the point into the training set. +The training run will output a bunch of files under the directory exp/cifar10 with structure: + +``` +exp/cifar10/ +- xtrain.npy +- ytain.npy +- poison_pos.npy +- experiment_N_of_16 +-- hparams.json +-- keep.npy +-- ckpt/ +--- 0000000100.npz +-- tb/ +``` + +The following flags control the poisoning attack: +- `num_poison_targets (default=250)`. The number of targeted points. +- `poison_reps (default=8)`. The number of replicas per poison. +- `poison_pos_seed (default=0)`. The random seed to use to choose the target points. + +We recommend that `num_poison_targets * poison_reps < 5000` on CIFAR-10, as +otherwise the poisons introduce too much label noise and the model's +accuracy (and the attack's success rate) will be degraded. + +#### 2. Perform inference and compute scores + +Exactly as for LiRA, we then evaluate the models on the entire CIFAR-10 dataset, +and generate logit-scaled membership inference scores. +See [here](../mi_lira_2021#2-perform-inference) +and [here](../mi_lira_2021#3-compute-membership-inference-scores) +for details. + +```bash +python3 -m inference --logdir=exp/cifar10/ +python3 -m score exp/cifar10/ +``` + +### PLOTTING THE RESULTS + +Finally we can generate pretty pictures, and run the plotting code + +```bash +python3 plot_poison.py +``` + +which should give (something like) the following output + + +![Log-log ROC Curve for all attacks](fprtpr.png "Log-log ROC Curve") + +``` +Attack No poison + AUC 0.7025, Accuracy 0.6258, TPR@0.1%FPR of 0.0544 +Attack No poison (Global threshold) + AUC 0.6191, Accuracy 0.6173, TPR@0.1%FPR of 0.0012 +Attack With poison + AUC 0.9943, Accuracy 0.9653, TPR@0.1%FPR of 0.4945 +Attack With poison (Global threshold) + AUC 0.9922, Accuracy 0.9603, TPR@0.1%FPR of 0.3930 +``` + +where the baselines are LiRA and a simple global threshold on the +membership scores, both without poisoning. +With poisoning, both LiRA and the global threshold attack are boosted +significantly. Note that because we only train a few models, we use +the fixed variance variant of LiRA. + +### Citation + +You can cite this paper with + +``` +@article{tramer2022truth, + title={Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets}, + author={Tramer, Florian and Shokri, Reza and Joaquin, Ayrton San and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, + journal={arXiv preprint arXiv:2204.00032}, + year={2022} +} +``` diff --git a/research/mi_poison_2022/fprtpr.png b/research/mi_poison_2022/fprtpr.png new file mode 100644 index 0000000000000000000000000000000000000000..a870cb96e91c44f568ee83c02ef636a49d94fcfa GIT binary patch literal 32936 zcmafbbySsU)bF87x;q4(X)5F6(%>Qx2z*r)1swzeB?*4J z$s>O5urEe%5gd0FV^0Kv&&D_w4n-XCI@~#gABrZkSd-r7*~~ znuE27_0R|3?Nv5Un$$F|6qv<$&n+xezFgaX)vo=}um%rLlde1}rUBjV!E$)gR_F6# zNfI2Qq=~<#lKYoE!=aLeTeJEJBiat6s~h)0hep`)WS-M!lpp>|)JZ2qtN z!w!TuH9W!#0(z7ED~)UstI5~m8RDMo2+NDJ12LZ;bZ0+4$C7g!^b8CrR1PJ-dWA_y zm}pXGsI46m6N9_`#)wBLC`j7N+q>8LP1ZU!@+I-ZiO47^`v(TlU7p_DCn?TB$ZF*x9k|uD-jgs-eL)I5;>kH1uL?dwanzQNPN7 z7;zGLHO+WF>lmb?qeFAq_NmmF$$MmCB0_~Vw4k71?y0=IyyWvMIz-RcCrA5r-F=IT zl-6&kECNsUb8gmmU7b7gJZz5hH8nEA#>2xaE-sdpmoM_o#hz~_Le+aE{Dm%P!c|!Y z@gz^wY&k1i)5mhVO*HX0KGrW%TgECY)N!2Df88nkzFkcloZWw)%O{W8kS4 z^_p0OcHhJx_PgDFvY;@L2v58*W$4+S^ZVZPhYmf0jpbb*>q|>*%qxnnf8hQ5bLJ~DqrN5zj zODa>~X_a50n6v*~#ZhTi6P{>5(rb|I7aF1O2N8p>IYsOe?4bDskOKcMhL#L`O|<**DXc zbAK5_(8l@PsT0NS4ERd|PcuC;(-)3IYH4j9T3A@O7}sih|)3(wDIirTBR)|n{t3od6Mt7H)Mu@rcMZf}kl{T0HS(8Oq{ zkB2B8Gs+3Q-Lri^OR+K_^=eI;e7U#azP$AJ3Ib&&vdyi2y?^|Y#QevsKe6mEMD(7S z2HdT_HPn_l5?CW0cpglIDk>_t$6`uuC80CD7k}+SA{UmM!Ww^u}iR@ZaBKn!ua!+Io8EP&ddZC}!Ase1(e%HxAad znrg&R!*?Wm+)B2tF*tj!CD*=JnUh+2BJ5m8u>1q3O1coT+VM<8F-JRLdM+2DD2^B@ zzayJtP^HgyV+f@>WaTp-y2am=lA^b=vdR#!LVYHgJ8qNp<+w%zE9vmi{m=RN!VfxMT4d(Ui<&;D${$(=6y6%Xlgozm_btl% zTiaWV`DIsECN@|%E@NY3%l=$_mmP>IR1-2+FtI3$UgD;7(ivN+x!HcTgpEQFhpBVZ z?q?kXRZ1-uje%B_R4rQv%4OQOfTJ&bTQ-}1SeIc|ARJmlT=Ta*DueguYcUp642#?Qw z`t_#xS}F3v`h6^V-^GeFKi|TXYRdAQu@$^?C#<32HV+Tao=bxh!COq7NQ0^np;E`a z?30yjzdc+fI>q4Qwcps>VZ%MNXo!DURxVY%=S>dRRzU+K{ zGDjv@_uMexk2xg=XXjrQa+=hLAd&NRkNw99ygWSN(Awr~@bQ;Zv7P0enI?+oFJ!sK zXecDd*HQzb#&j$MvYyW&Bxv6_6(5;T!eXNKVKxC zd)-rwAc{?pR9NTm&u~(0!1(iVUg|RcoTJKPbEtQ9l>t#4s;X9qqVNI(kHWGy6hl4y zvHmY1jd^`E2M2*l&B~v0U3)@`-cSGCCr(f-wH%(ORM&lpy!^?2OycT3s=gv$ZYkxL z)XU$=23iTXLx>`!gX*sKyM#x%n3+Rxm}(+0lbRFCem!107nihL;!i2P)k5ud&Y5vJ zj~X_s&=WMCJ^a=&rf_Shj%;H($8gC*n0R?G#g6%8o?NiS!chI_Z7}YGUmpHPj_lHR zX(E2zC+6|iKkRpi^s6%qOe2r%?}ZBep~L5=eW3KOe3as%O%4Qu=a$Ub0YY9|FXuj2 z_!47s+&j^fq=zgBM1}*G)oSqLjBk%8Pdtd^>M!^3{skO~W1|FApVl9sElTY8bV>=< zGZ$5xpyD&}78Imj?v2*o*VfBUvfz?EoM1&?yRSrSfsGKD*BL@x`$6^7o_(H}4+_x1 z$go(D3_0R(!=D11&XJI>?8oFUXeFkt?kKXCq-YP(@R@E4Io|SfesD$_%Sw8N$d9F@ zmr&FG$-&lr!~ZcW^Oia8C^uUkvDvBq8%tTAxF>=kp>g?I=A3<-Z{_NZWnl`{_iZ>A z-;IxtPq(;PK+o*&?=PCDiMt;;3uPR_6G3u<;Z1!<%13L+)`P242qpu)zE` zWA^L1{l&QsnSNQ>Zp4zuDaNGKWUva)-=(*rA9|4QVeN)4((!}v_jhY)>pQUOe zg5g98N=khT3(d=A1k+ZXyoy@5*ff79F3s34<`<*Q^gF13-L%|eUw*oC9X!TOYiJNt zSWG16E+tLbQQO^f`p4A1Be98L`)vP3*Jp3%8uCv;$&teq5gD^}+=w1oGaSslCQ%uE zM!VBLM>gXbf<;HpFI$LcWVep0TxY9!e%IctTs06@_`OYGIetAKc+kEbqeaFRVZqs# zS?=}yU1s=BW{TqLB_7MxR2;YoF4k;&fWS-+v#As_E$H zR!lu#@(2aLP22v5NEGMAVM46`<($6Q%o?6%ZX?sL`=^6Kuo@IC^O)^so!;%j`w_#v z3O=6Wd(IempWD~^Sp<9AJ*Vn_+mj&RhJv5k=W?eiYZ4nO3GtbJ8IfUpd$)gU1vhPX^%V&-%Vo*1zrL?U5#nCA4 zjfN-M6W#b!Cu#q#&+B^T@#Fak6}5??acU*U`ns}5qMjuWyZ8vq_t53lQvGl+{VDK~ ztM1pRz<#2?%6q<$6ja)7P4`~+TbmM7R*lQ=r>4@gsFoPAYATDM9MDv#ua@jqbXj#m zbvWQlxxT(YbKn2{VlC{Y5UZou;{C2SY0OH`NVr)Y?8yYrvhRu?|_GI=|s_%ik3u743txhDLlnUY;8VXR@K?_il< z?xVv_^kuH!i>~vMM37o>{ER_qFXO9gg!Ux6&hFe-Hlmm}ygVp(I(Z6;OrGc2oN61G z@=>b2;t9#hG70a@e`WSy&1dUy0+vIMbF(D?=LAGNihsG*n4!o_ETW`zB>1od@5 z#*0)!-6Pfm4cDUBT3qRq@1gLL)31t}wbIjlOESm<2&2VIv<%iQyWCn#!iGQYE`HqP zs5+lUuq3wI7aF?RlK@UHI0=#qZjc<|;3GF%VO7-DP~l;3O@`YhbGH5wB7MoC8Y`LD zuM`|c%-)vWUzPp%ouks0$Au!pHhpJeXv2LfM-Hn+As?}-E2qe*HzR? zV$|}6HGfx~ah3<8?sE=26TO3wH4GHy`PKN!c%env_+?t!+gQX5nF9gvlusi(o>@jY~LM~ti2QWUl{@d znzi*w=e7Uq5YG?PI;59rCG* zRg2{`?fq7b>+>3v{dDofmTq&!@@`*W8owW|UiXh0;XqBF=-c~nma(^7GUG2tHEZ-`B9bTlFAMP34>9sO3p~;nbP9P&Ag9wJ2`ht*gbw<(E^S=RCC6Xgb)eet+ z(CE%5K@Zz&Tv`P0@a@bX5!THc&7$FN>^JYMi8Mw3*j5QNOMou|i^;iFr)M18XlQ6+ z5)#cisvjBtd)c>u_^9@Qi=6DS?F+zEf7bE($bTx<=ch2a&?-R#$fFVO zr&eRhY17Nz%S04RHFyAusvX`5w&1is+b#bkP#=kc`H{MB*o1Mkb32E8)o2&vE3(K; z-H9zI?iq3;H4~V7v)#~0$ROF%=DAZmQ|B|FZQ#W&p{PB<&e=kiZgbn?tFZ`+Mb`_J zf7>0UZ3(yk`B7QdzhjoH@p@>RJJj*)a8FoznKAkk2?}7|H$E@!z8-&T%TQHOf%C@b z6Xz{V5i(@?LqRB5hho*4t7Vm1HypCG=s9)Q6iB=fqq0iMVg9Zmw4&iT`Np5~&yN#Q z>~QN70iEGm#~;EHV;HC?sVDUhTOMaf`HO%k^l`$|g;yw8OYfF}4)ehL5}j$nh=P#| z0q)q^$s_k=p3X&m$}1^+cyVSPua6cYyMS09)(*%sk=fMeFpFqB2{CEHiP0~rt~Kj; zvE2AbURfCnVQFu_V50cg5SB~1W;jvkz-yiGVf)Rso#oc_*Hb)w=!mRQ+;46-sm14~ zlnD1+t4TWxfF^%`z5RlFs^QIHje5IVVuRnGL*_eEVJUzY$mV;=Um3En2vRP7@$}&3 z!>XtoD>-|(S7}-QuGq0My^W%^sraV@KlGF@ijO&!#MLCa4m5NAGv9YAYKzp~IT3MC0HfU3W4ic+9rNEXhY}tVphN(Xm%im=`bI*|FbS)xoFt zjDEj{pM><^6HW?Xp4f=PeJg}#PFCY*M(KbkJW4*qx7WLi*(Ss1+=bhqbRPVJ0`0 zgt#~}7Z)x80fCpbC22mqOoZ9NXFY^(D&2WS1!@v}#zKhctejHLq4$(}xaYT3BF{-h15?cvjDgkuuLI29oQ z%uQy>77|Ynz^iq?#){9XxSwn+>ti8nUC#odl)m+qcD9e`MGr*Ho_Wtk zI8z>&P)hFcNFS`>2eTH{cAaP2#=Gq_=oq?PmAK!%iJXg7lu(S%s>5j<%IATTlmGWZjD;xrkwm?1?^r@8dxdqm_2vKe~rZ zrd>MqJV!Wdd*Z|COcSRDC&E9`6qR`C`9kH0ylOuGHmgsll;st!^>S{}E~D)y!#< zur)jFw*}TShS620R}FU-f)nEz0^0ZcKkIbp#=UalNzSmmm{)LVr^8AH$Zv$!QQ9#^ zCy-xPmk8pLoUU%Fh;dUACPq?|^Wv98nX8^Dnd>-l&uxK^^mKIfzQ^`7vgMkbRZ)ri z*CjY@FEb)Pk?h)YN(n3eITb+O3f|N84)<}BHygQJKoAcuWu83>LP@p?Av-`K}VvA1X6(a=GzHd|XHGhcUi z(5NjD<_In>DJcNyUAQ@R&|1&HU{oxua789LW8Kk@`7!&}hJiKT57w{pOaYB7jM;3e zON(ntjOL%cH&zv~Bp>fEn13W2_tqnlY!u}RXKD3LE;ddV9(DTn=Sb+;+Z0R;qPutR znsx`}RPY&ydwfCnJ571lETGsV8cx*to_Db3+KJjH^_TPbxyZbZ(({Lbu(lP~9{Wuj zd~MQkyi4#V!}9p$GqEfuYbe_Y74NiJZ3EW_Fd(BZxdlOHgGax|L$O)ceI@7O!gKa> zEwTIhs-taGMSJ2tEBmut{)Lai!e-M$9hBi3AM&4`vAdxEIpCq z$8$xbuya$yVkjyqZVMt-Z08kB0vJ>lYt1}_|&%qH~RmE-I`RqfvoR*&M z7zfWVP;3R(ZDtlhLJ+enE}ddbpBe4;RAuu?R(nm)8ig$gm@MVesv2ES-H94N2cT5H zDKARppb{!S&%q5QOB|yN0}p762_oh9P*}yqjqUvs0Up6D2^#|xEig~VCHnhI_;L69 z{)uo?$*D93ld;YEjTJ>a>4lU8w_A63-M>w*7@W52Az&N6uHp=$+rd`kD;whAMsV=* zMgU#nPJfl6Sc`bFJx;xSP04Q_Ub>x~Ow5vuZCGm>Qc_}kDA{GA3R zY!DRy0umuY&0BXWz_(+d-&=mgLvM@bPl|uTLjW~)un_PKbKAM>l8fqm?x6L?K|eVe znWdv+w8isF@|#L=4!JtbBr8}m&7Gs%@pYmgu))I{-3(MajSg`U$=+^oLg8J&($RnUwxOZ#o&WFrva-eUY;8Sx zJ#59+V;086FHhWs`LK?b*>o;6n$nI|Va~CZ_T&>wu^m$Z%+N2A9DV#MrWb=P^pc!N zZ1id(C^-s;LFm%&VkTcY2^^ViE-b)3GkBxOMwFgwzhN)OH_E+s(&n+m)9U@EjOnTGCe7h|v*%@E-={z-l0h`0Yp~z2l)D zQ%OlltKTV?`8z+15Lu=sx1%qE8U_NDo}dk|q#hGfWQQFtWd%~dpo#hRQG@X6>5Id? zQaoPr6`px>$@`o=usD;>+vy|;Xl3G>hl^L?hK`Po99&%1DU45dXG)Y_;8h+tYax6A z(Y(N?eF3s4IB#+QmD<|cw7?z<3AvYjgUMz%LGjr$v48*mxqlgmUf|O#yG25ZGvfRk!*Z|$S~M8UI#1W zGS~m^AUzIHSA`A5X z!(rBVjXqpOJ)=$7euYb{ahi50>%MxbzP_^fgx$P{5JxZv2Zw;XAO4LShkNtfC*PMz zv$C>OR8<`hC+YyJ0`vL;UFZG#_dO%2w*>9SsVZzhaH;p$RI;_VN4la=Y41u(O40=I zSNamOIA%#)vyv7F8b3>Z$knuV{{eZ}_Zu6|&IlhhSBd?OFyGfEQEbYLU#gg;1O|Gs z5thH!DnopJt_}{yQf8j)d!^|Qf5=D7R2lMr>hK|4SXlVB)-3~NHu}>iQ#vts1VA|a zE?-AS&eu6Pg3b#(nN8_QNkrgD-hiebSypyS#BDV$I2Zx0XJ1bb0lA6gm+<9lJmQ%Z_@owW|C-22M3O(&(qWO z_x9^9E-p4d6e%OCeP$+g0;5a<*lF)2&%lx%YOov27}gg~nhYfheV<< zvTF5KdPnGG`OG_`#?tx2hBr4h6J?a3MIQb8>sf8|2KCWYuA;20tmmu74|4MIU!@JS zwUdqg^Bc>FV%&kGLSz2o7eJt6k*;8OzC>kvY-qghRSLBY=2TyH6&~!VF=`d zumi@ng6yuWaczU2GsPNBgikP%r2Nk;As;aP;Y08KVjV;u9=oi(#X+u+V%5~ctr1dt zB}d&4v$JTC{r3zEUd^3VSMyL&Qw#6DlOP3eci1taH8UyJ`KS#PpkP@J`!jCNdcnC~ zDgYYyp#3jV&3K%rZT&1QmV0lDY9#Rj~_n{4-b2`C$hCoOky(cJnEVa z+;6n(MddN752h}eUM=q8lV3V^K@z`-a7nUPYIVPBY6l|9UJ{&t{05{-%4^oJq{xON z6G=K6i?mg^IU;cr!$9|Gy=vLwVWM8SUhhRa`MH*-dE%FcZ278bmpg1tSvsi~>^MqdOC{eDi4k^15K zyfaeRqs3`c8Bfj~D64(1nD@9H8fUEZ=acErvhlS;%JZr`>`Ots#NFmGVJhm=iiQ|> zrJsa>kfEcio5`ODHn#F9`j1pULcc)hleYB zur2wK2cHDmFpI8Jv6P{Xr42aBAiVU>-ltp~>*e& z38@Yb4{!cG-hmPbZgzfAbvca#BDsKVJA`#R&2q7cvXvq|>_o0>PMTXvAFLsqH3m#@ z0!!UYDveofDL26eJz{AgGZm>C`p2ZKtZe(}Q4y4-w<9qr(P!X^bApYlXl?436|sLL z&cRbR@b^}x&c&rajb(ifF;Jb7im3{FA8~fPJo>;Woe*rm0N7c3xOdU)!e6|wUhfGR zuMe!;EDU6FH4^OBNl6kAXM8TLqWHwzINqlEL{D)11NQ8>(F~ogUMx{8PG}!VJ*N;s zI3m|TVo79>JhagC`_damh>}3L9)!AO(owE=cX}O>l93S=cyY!oC`kOU*#+Pm%4D7l zAs$|=W==z!I@7?$-MXX<-I($;)CywV`8E@WUL!SXT`8VJb3?QG-EiqjjL({*{dMV^ z2!yT_Z@p77J;Q2Z8jao8&CS`-lH!$E;NMk-b=Cm4zkO`sM`~-{`wP9$_Y)bU`k)Q( znblVmF^8{znN@e`K1WTQ#dVI$n9<$r{h&ws41>UhfgiuWO=_o=9{2^Im}`)Zmc;<@=~J_=IP2k!^} z4pSowe4G>gx}TGz(QXAK7`R(a9QsvKyND|m6tV9e*4W;t7Y=@z%Z0F9W}@BN+Oix@ zp#Pthb^BZ4N&?8VU1C?*pdW0&The77GmtV>b|Ge>L*_ahv=AR=X5^HVFsrMpi<@Nv z{t!U`K*OZI+4j%Ez6xjZst72xLv2}oiwnW3t{U2|gU9160e*)lDNodTH|qu8Phz8n zARr6`8~pzL`x&?GaOdXd9aG8Y9q`Y$vL$_Qzv8z@*c?s_8?;6ea45WgzQpnixURh2 z=CY~s&osnrF2sF3F7yF(H85E$H7HFASD56m6IRuFEAKpDCW^5ttO$42HM4rp*?)HM z@dNnJtsNa`h?CRPhiz~5{1@|mncxxdD0pPu-337GX*K18-%HgqL)X_W-Nopu!dfeF zocI57s2}}6F>9J^`oufmm1cl_G@tiDj4}rWao99JJanX~66ZS**Dy0X+u!MXEb+klc1xi?n8khLm4t8am-Ph# zB3^0aY`U>nNwlJmdp>^b&=z%iu;L`)6O$4_)aRAZ{hssQJzusP;&$ri-T>cw!>~ZA zNK2z2L`JG$@c*ob5gu+tGWJbQ)@qBo@;l&%JB|H$$x?my_YIJ3x)>Hxm(6u_LU|uH z%W{xGkJhU}#)|IYsT7x$Z5~g_3o#wWjti0p@A2uXRK>@N~t zUtPHArq}6zpuwJ}$nnc~QFrYZ2ayN}OB{`%KQgM)4;|hV98))ZR|Au;pKV}~ti?*Q z0THV|R~J8t>FGaL?@l3;2zn27K(tJceY51D(ctCvoHo!Ub1QV|7NN)P4BNNeT_m=8 z5gIDTou1}IXu?bTb0S;R6y{tB`r8>VZ zvd>Lz=1i-lTmG)i5lSbYucVm^pNB=Jo+Cj{dP~!G#pLeom2kzT=HgE}>^Lk4Tgs#4 zlkdrT=iX59o#H6m`r1LvD|$S$`gS0tnM|J&l(woV>A{gmN9^mq^pWq?WkdaI5Ax)A zC}yKW^s1^_Fp-+X4SESOpo(Ii+v+)#w8Bn$sT8$M$kC-~i2a*!tjx+#LV=11y3VIm zlIX6mf^-@SA0Gk07bxjYZz-rF2JIV@Gc$3(3aVdjdO&OS#)$q0?OhmPwT|oCjIoId z%r*IRKme8`mND*Qs3Mt3clUE=%EUcclVY5z^Kt@-zOce9Wk6-F*X))LjZhV=QPs@I zzh`_2FQ+XS=cr!-QMN3!L z=IrR1q>+iqm&@O4Zom4p$(Yfpsi}eF2#AWhHM{9RriD{ZiI{PZi^ZYdNI=UloFBA8b&9}M6)Ugv-7hjwoqn8I3Uyo7(Yd zy`rylt?>IPG|W`y=gb7aGAcGUEH{_oZBx^JHR{)$9Z|;(ld(_?ym|oD_Gzf|=PSo? z&VoP!s?ZT&jpkP{DQ!59_($|_$Uc17VTPqr>FPZBTk{>tR?AvYSbMv~@%7n1rPA_J zYez>MsJc*m!*}c4eSC)byZ?ovpkb=ek6ATZRhHZ+P(jBY)zV|7)-HjW;pMGSI^C}} zZ6j%JZqAVK-n%Q4ehw?n5|Br$_ddmviwF-dugBgTXKQP#3jOb+VMd$1xr)QnQ$NT8HmzJ^vcw8BzYyj9-^dL|A&$psPME(Lo#|)kaN(0g(h#It3&y}S1^Yi<&c6~KW zw|Q`Y4`Y&%naM0Bma&|hlY{T(<|gTX7I%gx9I~=1&)da*9)?AD2*IZ9qutpuq#hG+ zJR$lPKYh;&6%Fl;{RB#9{`g(g&?mqn<3==>`PF_$} zhNZ8s4_SD~r`o`Nq7rr@1C3z1&O(Ou)-AN8x5m%^eoc3GcMlE=L!;m|8R(p{AI}JZ zVvUBF%$35sb?X)ka}Qh;=shNam6f$;Z7pxv2Zq1&HDCpxvXPOISTJ7oZfnw+Q^`|1 zH(6N~YPDaXVctk;S0)E>3rG?KkW-c^XlY}Mi&?*~bu*S}=E!Ml6E{u)g8SU=-Cv@X zzW46dd^-s5(7$3^Tg4DPv6TD*jx(&~<>l*tjyC`7Rn;N)_Pl}k-Mjq|3GTi;qeTQG zASQCNr5KgA%OW9H%7vKN0}tqH?@E(1GU*Sx_rkZIt-i#INoMMVIv$!&(-Yk|M%L6Hk7FXvEDRxWyeeT4|F&@Hou)O1ur!qCy? zu%Q2$O9i0^l#fsE{3&;)@`k?*#xj%PI-6d=-m7nD@GB#BCnqQWbMdoV5T4%tBY@Mx z56X-Xajr`S=nlni*t*=q@jQ4+ASga!C*Z8^uSJ zwOv=Tgq@N7&eqnhs89zT+7cZF-tWiegKs%gwMWb;GDP!RhDMvSgLI}xFB)Ef)vKMErfgmwX9?K4U9)D!Qyr}4-`7HeIBAQATh zQrr+$Pm!Ye~Lvjo<(mgXXBmgjb zkUI+$mu*w&JVHCWuS%cOoy75RnV>5p(LVv5STK@o{5jk}1cS`3^jxi={Om^~fBNRn zZn>oF92|{WB0ILh!=s~nkVpprt8HWy)qQo22i8!EZ;*lldTK-9?;oF`CSgSOhje$# zfKf#lB8w={iu&B)V{^FPD?Iu9>Yw=cgM&Bj8*%~XyJf9`7b1YTH}>{MPh4S}AqA3e zi**)Zx!I|yq}Dbzw0~%mON)xa-~u-}IIy7!A!AiY5p-+I7WVSy*Rgb?W|vGo-Ytk7 zxIcXwRCb{=Q3s^vczfd6;W`QhpJ|j^_ay~f01JVHVdm!M6FRanPW$Z-GuBgm^H}b3 zlfHbJ@|QuMF-iZDbuH+E7vk%7xj$i1!NvO_?E7f)buYy45h_`Yre2WHD2 zlr&?(#Eg%}{jI2Lt1zrHN2ctNIxwgxmay@3E}&S%z>LW;5d{;4BEJir0H2F{{#TYQ z6@T6oR%E~9FUMas@dTT$fv&Dp{LJVWUthlG`cl-?RW%C01=#%Ll5su z>+Koi#36@0%AM`=nc5B`Vf+d+L@O?B(T<}g>4bdi6}DwDr?334XOiE(eG_ut`@V47 zst?`P&JGw;%X?359-itZ#z?1!GATB=3U#LN&!e#uep4BYsQG=VSC3EI)q~?urNgZSmv|^n^u;XoU)_Ta+&|x0 zkx(+B&T9JBffy*<;mNM<*3QmN85zb0xsvyw+EVbFe*(O%_mG{0w0Jw68B)V?Z>a9$ z(q&XMGIQ~VYH_4xA(XXqG{Um#kpnV4tz|zh!Q|Kn50@e?QTz_R%0W6S@?l}iO^6UO zZAS*%cPqc_oWb2nSG9k>Uv<<&dZcZ@k~3CWQf=1pidtXT_m?gB^qSiqAdW*NM=)i2 z9>If2`ki2FYHD(DbN55JL0Xjc%P!Rx-3-7mv6yz_u0j3Hhr>IdG$3t$6qJdJj?$GD zaF^lGRAYN>e*UzTK`fes$ZB)kp!K|ixsxWO{W)rg?kPv(2RRaRU0vvWm}&bjgmY6l z0q_kC1TaCNh)7STypz_|4&`6qS_b@SgF173XmW@iKqaV9)?w>00d0cEh20xbTgw*^ z5O4_CPu|#=8ngzv?x4%0+*|^E{do%C1ig#deD>-PG%V|p6gHhPm(r3FD{ztykB^-J zD(b0`l7_yvmL9nF#&0m@$#CMxfBQ`sIPL$~PlqG7^76qDaewlp599zK0nxCND`-IP zjMfW|j*1FyX%U5uv$3@`uvML+4BJTh{M*g)%F6#*r~O19@VY%Ix2asIahH?I{qby3 z!eHB|AHgVS?!UHF5k1od@|h$42?+^I;Y_z~DgU^wlk*^~Yt;2)Gk%d$d?delP)|tNxg_mjkJTudv zbXDF0IB$J#zDmq}9np9IH22#~N$i|AxQ+UopJ0*-0H^5;>&$%wH~^1$1VXH7+mi_D zp^uG^9{jnu5LMgYsAMCOLni9&nHgE+dhFli_c1et2M=0@fNmXIp+C<-z$hID4L5jd z2k@@^g9mk!{t5~TJqru3jDLqt*#DQyvSn9s!R_tsr4qDOcxUJT@g=umz2B*mX;(m- zS#frmN#Z9ISP$q>^}q!U&dXy2gf#d1YhgJBGg)kMibaBTmn~)+S0Y}ZU1-V|)OrQf zQq{`ye~};=Sne;>fV~ool)_$%hryLY$G+aj?@SVnOp_EQf+KYm>g^*VBi@r-x56!< zr~T=^zL2Rn&|ef35wWk;4MdE?zz71ZL}m^E3O{+7w>J<)clEpbnwTP1(Z$6j3gW4d zyWWcB?@HR4dsZ(N;=s;RA8`n z)W>ao{hADC3sPRaQi30eGUBQASOpawc}Zg2h~mP+&>OTOabkDxrjCx@FRZM*pDP`B zADek=(c(FqzT2CB{{pOSZ52R=Q^ICeU?oZJ?38SS<0TYI04~8(#MwChJ1{X356|Ab zun8v(qk(!6M=R)jG^{aE1b)<^yvyLc+qC_EjzT z)VI-LxYSei=1H(%(<5C=r8ue(fM<7n-yndj(B68GI{>!a`qmaYWI8@|`s)5@AfOUJ zq_S&-78Tv}JNq7eFANjuha-aDzgJtqF2we zBq(1QsrOjFQ*|}xBf7`Ip?vh56qPn>ET(X!-5@X8Tw9x2nBw(>J-Af`ot>T9q9x@E zJ#}Qv(jDl3OWa<8`w*P+#Tt#ZLOAoBbydDTRQjc(NPIA>0w17MS| z>n=xSG+7!B81w(SeHqB74K_k1>;;7hl2zskNzphfZzgdEgG5v}RCpVZ!Ly?;7{?i1 zN9D~?M4@rl3!GT1aSbHcTp4=-4^>DXXdfh-qOeLzc4o%j)7JQTaI$~yBzZUxkVbY4 zhgmHxEfEBfx>NcbF_ju)Y~-+Ez>d{<%n6C1dSf1{DsEXL^YN{Qnd#|o?)y)*^wf$_ z!uOJZYuykc1BWp(85ggF4FmCwgFD9Q{s}kLyj7YWcCxN%ujGP`L2zy;Q3;!d%;+e< zIVr!BU@Z>Y=C+QG79-D}5EBBGseq!*5GXhDWE}hEQ#d6HRX!eM5CP{miPU#6LHt0l za_T|uLHDOO2rqWEN9K9J`z-+}HJLa1VFtG+xZ+M6(ILsl=Ybq zBX08F>b}vgDqixQZ(IIk4U0Z@5Ot?pc&v&YQ^c-O9~Q#d_vKmJMdv8Spg$;pM_rTs zYm6H)0eV^n`P_LlInWB~DgjFb>@iro;m|8vo+g*GLp$#r=ayGuW<*Da1dm=GZB;=5 zN?>3hvR#Pb0~af_vn$$r&bSlFRySQe86>>k7$cu*=tE(5{R2bG&hTQz6NZ}u93xh zOamn|{M|b#I(m9R?>#+W*S9a=+_HfCy6nlf4`|TC6+mo4sLBn4JDd+F5e(*ZZY{%- zin>vPB%sd-r27p?5lb7JFvv~_cK5?;Bku_|43gX4zJ1&M=!@brfB$~4I2jli)Ds%u zVNMP{Qcg}z!gvh$tOX8#61{m7d9!c;e3%4uQBda)-rYg-
)pw`qkHAS?v_{1+? zGX@;qt9dR{Q(MsX09ge~HFNGGms?5}7%t4Vz=F2UPxs>|%VtbLi5`LZ#$gTz6@v&u zQ}A9azyAtR8thkbQyOH-s^mV>o3hi_ghH(H_;KwxoqI#6tuQO3*z5!43n!9;>3a_< zJv{Wl;2_Ap^eL}jt79{xBcS?Oz%wAFit>tzWT8*^zZa(52sYyT0!@^bf^$uU$EaAHXZAX9-1%YpyQ-gwDyt z7Eo{UPIvT%47i)WayZ6=P{>)xmedWKuJ9)@;1ht$&I^aD|wlNSs z7${L+cYnj7p&|8LmYX+kg6b`}Mo@mnTUU&Vv&;smfr1{Zez{aE4`1d#0n%JtT*be% zgv*4T7c9Yw9~a%>0?wXs*M|=QYARxLC|=&dp?uu=nXj+ln8Nd4Bex*<6RE-~2Vn`& zWKaN{f;m2(c?T<<-=fx|3*g8r9^=rw=U3DpK74?)-Expofln&v;bFIPbw%n)$QY)0 zT_sMnav}r7m*0a3>~SzRr$ds>zskIx->)^$tX0*ix|cY|rokH~0oo@|nuY z$)Um?z=DQ|j0w7k&;krZ;^%tjMZSQGgBD0)rOfqN$-lS}0yrCy58B!I>DRzZWb6Pa zEYfMx)g=XuE_ubx);1jGT}gOC7)Sszifauk3{t`Ab2H6=7zcs8(F7dBPO;j0pDZ9K zh!Q)PQfY07gNuvAxoK%>>$|(raK=;Qk2C_Q`9vwO*faA~2HLzaVp5o~wW}Ped}D2? z7T?eS`QbE9-TJ2oD}G}awN4#Fr5f2tZ420oCB$3)z*6`s(S447A2bJHrhJeZW9G1+ zAiYaUN(u(Tj)rLW-hTlKQo!Zu;&}6rWMs?lscg|i$4QVQLgV8#iezN>{c_>sE5@&J zn5l#uh(Nc<$OAA2isdFSjbRYqkRKR;L>AFJ`yE}?I|g|ziJQlNL3MV0vSkU$G048c z&>Grwfcn_jZX*Bx{oD!x*K9<_)3ui8mHZvqd3lKtVzSzq%)#foIsY`O())>16-STh zLuwPcdsRCP=RZ1rx*x9JtLPSg{`~p(l_oqyQfg}0|57-Rr(qCF@BBp}71g`pz7A<3 zV9WAML||rFgZ820?4#%V|Ccy!uXTHSJb99`N7n**DlmBd6WD$SI6790A75TxdVF6J zB)d$negYP%&eR4hb_mB%9Dk~!DDwx^3HCH9MJ!U+-2h?j$xA~sGqcLX5@u9XR3aJ( zCw_CVe>hLM0EHgby!FakN96Q*z|!%nyY+K_2Ob(0PY6*cOaYA=xs3lACapHGOc zF^Py!fc*%iW}HPQl7PMsEYNz_k0Nk4&|W$w70YekG&I2He=q}{#|5bdk&Hq86)zqQ6$(>NvAY2JgNi0#$CzcprYniX&ZIKp*1Hb_{>^J$@IIc5V)Jjnf!+%KrE zz5$OK2Fhy}nC0viUqnPjDOr-sm9TPQ26R8yO~FvOpJ|*amZ8U_Jbo(LDIje8gu6J33{~5h-t2yG+*L`BaF1>=Ux!;U}Lzg-&|wQ`ZZf1LWtn@4{Z@?G#DBN%W>ialFfq5-6Lm!`TD8TGzf8 z%21}Ffn>^%BC{yTkf@EK%#|@pBy-9bkz|MxO2!Ih4k?Kw6h#?RWJoeph!FaIPy2lD z`|j@@uJ=3FIoCe54eP(wdhX}Gf7A1v*{;pSyaalvwk_&+4i--wHE+#4WP*1U?i$_0 ztHl5#qrB-bm|unN&8~y5RrhCs|pf;T$% zbY6Ko=B>;Nbng^mOGNS_O~jvL)J|OJ0#3yaGKha={bJi*`5o$vlck^9-%|Y_8a_JR zq8Yy%n)&gsI9($U@Nsd3Q?mAB%$zbCEnVFG&HtO^&jP1TEcgU{!&uRM&r#5Ad9@g; z09m-qe7Ay*S$*HWs097TXfN=t@?D6Es)ce~_-fHGrp%eqqSxeh;sz7A8l+3(VtMBD zT{W5v2ZJM5F;9~*78=`l72h*K&O`$c9K2-E1+){&P)d3_9l!yia|JjW`sfjn+!6{2 zCH}>Y{FW=L#Z3YbwIFqZ{y^hq2=|Foepr7Rp2ABRX2Z8a!F8t;7OoT#5dnJ{geIHA z_TcG>-(M64I!cD|nG{_I7%|RNeR3rd43sD<;N0?&U|}}nug3w4e?P69#dHoGeLQ3z zpv{CmSOu04EjDx+^9u_r)~sQ`wgBC&x~XYdXEI-_iu_R0RG0IU`uDtR*ItN;iJ|XN zmXTRmQBl#*pUpaoMsowo1*U6tJv~+e?V@YLj7)^7974&fHJyMp zO2QAmo9^5wcIx}Yr@uHkIV<7J#%}3!vaA!>tsp8KWFmuug9QFYTLqO}{Ok&lY5R?h z?Ook8ckTL!v4!Y$83jy=>{wvrkZ4~}S63%2jGCe0DoP4!Vjd;eU3OIG1D6ep3qLLF zRCK09ED+AbT}u~}&;!F!gEBMPOF*+k`mlEE$s0aTIJ4g#kIA#cNGE_t@*TfKZm68_ zn8FpYN1^k(PnSoxDTlr9B}dfv*EaQ~{Z^5?Aq>0BwW9UTK0bDJ2c2b3n}OFeqfSvV zO3R^=YIrYXIx#s($UOqcqq?$(*XX2}K6Dp*`|L-_e@4$%qLa&U@d21ySXg*!?#FTR zZ2<$}rcPn3cgb4By~0jejp9l`RXih^u=Qel<3yPFF(a&ZID8zghVs6`iv=@v8n9zD z@LMS7Zx`vj+xx!0|L=GkUQFS0X#GVde^W%E=kl?V2JQ6s+4(b}rn~n77_x_6KU*I$AIzQ7Ng{dtRK;VP- z(>?QA^2ziI9_5Fx3J3FJ>biXSGLe{Ezi*iXrJGzb40+6*J}lO>nVVh4-zsVi#oe*| z)@_3u#JGI9hN|k=`T?gZh}QG5TM}RoEi_>#007|CzE$B~w{CtQE6~Kmis&7!av(c1WZnO4*0|$no z!D9fGSqVBW2(eo7##~vf&))4{zd2+&0Jfq?|Iv^>`*s)XP$>FDRF5vmc}%d4%u#qF zL($n{(?o5pX~azrDuZEd>R*ZCQBs_3{I4 z$WS9tTK((S&Oh$|oDRulcq6B$qAmR7aOBl@ahOi=2I_!7uE#x4g=~9+8%xx~HoD^y zw1EL+uyh}3i-4Iv+PPvj1doqHow4(gmn517L9k=C)gdeybLQfLg6B{;R^Z;?#_0gx ze4Fws_sHW!6I0U_n>TN^%Rw$&nb$X7GHp&OZrytA5|2_0a@#o2ajfLzd^T#1g()pB zFRaWP1&($lZgfmik}jiwkd#z3+Td8E8^r?MjzM27KiIBOEZvy!B3M&<$K#Gx1V!~? zIsnz`P!iC#`rtRZu<&pVD=YAva%PWA#`F?YFMQKesj^^uo05|j(A6`ecKKx$06KU` znDC6Srma9LVMe&@%>rlw+afO8BhydeqJvoRV8Op|gpKaj7 z2i+I7#Mg2_ynfSg``hS*`}THr1XRH*W{IREIjQ{zx3{KHg`Q((KI1iONbSMC6fgb7 zaH!OM88xjXWYd~(hkr@a{=U9PD?AzV+X85{-h8^9n;Qu`AxT%$Z5&J4m~92j^EZTJ z=S#V`7vl)6x*L9aiz^GBEBQVJ}iONYhsb#-qjFLDj{?b`=d-Ugz?x|c*hjM<`4{IB3T3D-Ped)3dM z)58;*H3|K~;GP{N#l?iyrYyTkoZ{tAtd_mbN0EF%!afoD05ehYomf~0gxi2bonA> zzE!ohgTp2OQy9rE01shnPk<9!)@=u>MOkk7r@M1I zpl+6l0-UPn2VjA_Hpi^^qPS@xt~kSQXE6Ip4h{`;@bR_4m*Nu>WAKG5(Q{{waH!Y8 ze^zH~;cIvzA#*S(Y>P36k=Fd`J@PlSb-pVXt-VFN;^9#C$00{!+ui5M_-SPq=S7dU z6_-W!Az&Z4QBsZ_QW>{zpM&^CT$~vbQM=5H+R*ChWI=A?-50NK1=l;dAfZc8X%Xejdx#>d5bt9a~ek=iX4h zgi8vl?u8mC^?+|PQCz{+ggvBM{LK%5p8|q{v{;n}%Wt*X11tsNq+_9KZJpN858(+A ztg81fA%bD!Q`u!=vNnX7ixy>{sziS^j!4k9wVE9eB@wWJ4852hp#+nlQ~`d0ccDYG zflUt4mn;CkPVS#2aWcK9u$PPcoXlOhZryUAOc>a$h{t8{x^uiQG4v)bC(;4`#aV_t zizjx|>oCyE%NJMu%t#D5M6*kLH6Y@YvX^h{;2EK$rKKNWh(EFBmf;RK7#n=Oq^JE$ zL6$I6*hVOeBz2x`?1bGU`vr0-bE1<(AF9ANz+i%Ty^X<$v3OkJQvM_Qoo>wn> zeRXPk-k8T}*wS4l2X#B%y^|?k;hihCZx$~jG%U@TlGhD=|DgKYJ;ebbFH>i9b#*=U z^}P}S?Bn9%PPw^h;DNHP-}dar%g>WvVjksWs(pE%^{W5%uPpeSV!4ib{L&=Q;wgwC zSa-hI%L`G{Zck4Ie2dU3)kxkQ7cX66PWY?V;n=Yh^k=kzDf&M-mz^inD@8&CiI^}v zd^u1Y(!p&PB2*DpJ-}nd4`4dadea!k{h568ixVtSP{l_`)fr+ zm!#10_1C4gM8)P->w z@$m6Q0hrpbefv7~RCXwaY+XGJp|}YFxeEyQ?LF^@nbT49>|EQPQ8iw4ywh5+n8n1# zCUkzF;<;6?Ubw(W$SwGE!Obc9&fS0ib2$@Jl@_1x5>!s&y++l-@UI=w5Bwf@E**w= zDkinY%*;&De^HtGNE%HA$F-EIg#{ltF@xiOfySgmKab8yZs@0)hn`%3yC5X6%BKw} zs_CFnx2r7tc&n!*ou5iHX`s!apscQ~Wx$y0!J7=oS=#3L$67xB zXV}5mSFNh?(Uv^?kP|VfGXB^?=u~BPm7Iw)2TfjnW-9x+F91}#Gs z7eoP}J`N6s0P3TS&e-z=gjJCwq&ir%dBw*srFHMVp1&;+A$dbq88i~BcoF)OlKZTt zeS!J==VbprF(KX6kVRP*n7C_a-1vt&qN-;S{KM=*Am<(9gCF$g|?zW#lw zO)cx`GiSg7o-@s4iq=6GLw>Y(Bw8k~x8&<;>B?j!#} z{E=Q*7^9^1U#maP{%7@vbf-Hgy@W^?s-w(J6lxnwC#OGwAV7|F>=&^lt!!-QK%N0p zrTINS$>szL*`K_uv)9A?U_j*XDmNXqN2CR9vbxdr@@EZzkgo}^QY%qeLXiT z{gy|mW%(P+ew4I?#3qQO9auiz?js2|ne4BB4=-zGvqm!B5~rszsXKDCXExR3U=#a#Ki1ypN=(M*ZF7ftDmzjkp$_S{teNAk}$1g(P2qNGdL!FCh z+Z4G~-zAI+swlTTIi6V9q+f;XtobKBP*T1Bcc%5@C#{4Lt_o?)n{laqsaW~FH3#Fk7}TIz|ISO%`=P{$G;!5 zgY@<}>O_Nbg-bs@ra)xlBsf8tQT?;WKMHll&eKaxjYh$-gWn?u@SEI-?PEwPETOw= z^jsKb6o4%Ea-y{Vk0JdZUq7tFXCQSqDvC{(V{vRCwwQ^H&3^Em+%ABbL;x~Uk$t;+ z?r*sVI&a!PI-S&q$i&_#F7D_fur823S|O7CTuS~DSe+>~%T{A3`u^j`y#gnUP58C;g&$rl<8O2VJfM5?#9akA|3V`oSYpXj_`t}_ z{EXBRG+9VVC7c;S8^W*L+2YI)9m2lsyzZeyuR9qT!BHx6jNsl*gm+)VwVZT2rmS$} zQVpKhN8^hrsn0Y$`#2ms3zq^KYH+c$v%CK!{e1b)?z6;qf`N*vFMSC}T!uynARA#| zlT~~(D!gAoEr_T?GWDW%yFDf@H&kmP#s_C)+g&u=HMhpXhX!L3fSV0+a#uS2fdncl zDM^3%^!CjgO4~muA}UG?qh~MLEI>g%KR!N9yWK(b;c zt$Ttdru;C594lT1=m+sc^NWgXN;==Y>jRd0#d{Im6;Tx9H^O$!hladC1>GCz>|j5@ z1Az;imLgdcQ2*nPz!GbLAj2efLSh3ylOq5`oz(;A_CsfP_##>K&anQ zShCg7t3&*?%g09eG4}j4k=oX}tF-=WPt0j&9+Kee0pE3iT1oS;MuU|*O8j*~mZ+-NrCZj+GLh11D zZt!e#zW}ylr8*PXKGt z27vRsj)571^yc<rf7XlA>eUq1nCb100#oB zN&x2v^&Z;zVBq%A*ufPteE=AutBC3Av&L7Mnwl=506OMPd6^)|`RY|ocXxMfjIa3F z*sx>J$*n9f(IEmBYVg|*A)#PQL*#E>ynLB(cG-D#8T3-F;O@C!FE{~06~$w+<^9%G z8g=a0v1r8pKq+Ec;iEJQ^9Gb6?a?*Xm!aMsRKbft^zOJPd;j_?3=Lix;L%yghwGr_ zJ~1Czjo%nrEl}#*L$oOVzkm8SxJ*>|K~%N1pr~jAI$#Wfu^<*=Fmh_|kVLyPf%!Vi z2~Z&Xm=SHN?@|yeSUo)AA78uR_@iKCA(yec$b8c^L&I~**`EyMZ9bH_5%mKlBp-_= z4EPT0DaKz<=qAcBfm8h(T2lKT=HRFIHlYFYnFq)uA7}fuJ&zCKgFoclwJnFO=lGCC zA#MMP2MTEjx(JC4cb#j{i)+`fFJ}#Q8x?__i>tmxk2@Rs3!tjjixvysxPVo73JfG& zg@B2K3Bn-q5l#R+ZwN~((PsilsK)e82u$d-2e|q(gJj+ zC{hTpoXC)U>Qw&w_Uvxwn%+~~Ezh^y?JO+0-GD8D4{Zps8xpM^=v3LptsY5||98Ts zQD?-jhewf^!XW<;I+|(6nCG+uuK|IFi0TXbiT^XIo}=eHe$OJ z%ubL=?o4ZI>+b+41mUA0Yr*YCd0tVuZgXKI6d4kUJh6B~7}XhfS)W6i`8ns1DT%pd5QD*8BPSfgv^cjD&&erY82cw{@%0hF(JZOGkm0E3kjf z%dSp9W2m0Bi$A<1J1c60LC-rNJz;=d_j(vz1Bs~o8&auijI)I6qhgxP@kI@#6Cg= zx@!zkY2ZVb!`7sR?51oELtbEwrzZxOAfAUci%_;C=^UE~Ho0f$i>W{KA<)T)$$d8z z15uw~dWJ{C2CX?2fM?AM@0p#-X=F_YK-sR4=}1Yzy>h7h^a?y4AOz56k&`__k&^NB zsrt~!hz2CjI5bB7RV(BZc+TW{fB+pkaiSmIwT7NYkTzmMEP%uV>?#=~fnXE+75*)m zEuh7+lbD$!KX_(-^7S~#R4WYaEN5<_f{CPkW4G&5R4RY4P)|2na;R*>?x0uuFem)X9sP`GE z^98;0Zb(93Lapt$K9}6>rS3B-iv~ZiG43Gy-NdEaHYC0}oRj(k1LTpiAjvP5fD7a!F^GFaO&j5}|kR_Jnc~W1#)bzUgqEtg&|&#c#}wEP-k$ z68!${x6u_`N7?I6E~BCH6$8%2+^8+2S^6-=;6Q$UekJ55qu~wcp%T z>1EB_+Upb3>0^%UXU${zz1OJ-(ewK>8c%&nmNh85XIq@^rBK|aN;^^)S7+;k6^TyzM6ttN;+xN^S+Q#gZXZJR%b6e2HnUMkeGZrE@``AjsD zAU0+nPNm6M$~nGyyy{igjCVN)MoHv`MK463jtQb##ec~K$QAi+jm@9CCnQ|`J!k9pio0BXNkhyO zT;RJQ{=wt~r4wpQ$j2;&*9A^35XPMvs9S;2JLu)7H0Pd*?-LWmTyiUNLol&TF|xA4 zJ-)NAua5|}ps0rf2mTE%V%bw!oUwrxzY?4T7M7H}{CT{zfRdT8sIX@c(d`g?ix5M! z?R}l-F!Jtod!FaUjpBh!yp*L)W(do-CtT?{f5&E8qbo)}PfAt#`h;yBo8Eu;knmnJ z0T5H+OQTjQoA9+=UF(SO7Umc1FP|zb{!4p{1JAW<*FtaH;IUJ@i^dtl)2YRIZ;GJc z1aqfip6o<}CeoP>F=P?#9heMIvWAnYRXtAP-UXR6mE|9DVO=NrT&8P_UmKgf%G9&0 zCyQ71dVPSCAJ3tNhE;LSnJdR7!gp5+O~p<@#SOoOF?XrF<`$r2S4}IH)`h{Y)!v3Z_DHbJlD1$1yzVPQvSmx+jqq9IcsRKlZwaH^Mf z@&Uqa(8x3JXO$h47+OL>40?I5Ibg^j!epUdq3p-qoj>?lldf+23wZfGB81$;kUej- zf-i)7?~r{u8#r8`M9um}>l>g1Bdi-lfQ(CM@w(Fm2ytu@Sb`=SqeeBr=ma-3+w6b` z@s$Le!sLn81({rz@6r>m9$^9y$wdo{6T`s?Cxw@FbpZth(v&4o3zYn$yRATtsR1B^ zoE2T^E{Ls*LTUkLlh+YTxTr=nLw}58jLw@ubAGSLX(~WfDCBmb5Q72MKd~17O`%xA zqu^ds$!c8}Er7o5Xktco;Y-)DL-SBTz>Ks6uSf)hLW4Y@dx>6tv(dc(kdp*X29Z$q zP06y7f>@UcZ6C?}pl9I@#{Hrw!ZiKwA&c0@9td{CocjzVH$f#HyI#deUifh3kj{^7 zE%FD7)Hj|R^KwpPDu4K;@1x_s{LYxIxSgVUH&k(gTGHAX)?-`5FP_;7PH~<4m*cA^ z{gJTAb90gxvK+KsfS%w5kpkpMrSSRSgaYmROm}HqFK#}Cz-mhBjj2I;eBbdvw{YL+ zxVR9AT|NT!ZAg3uln%T#XRB2+_k|lIf@s`Y{EWFn|BE8=d%R--11!}*+mg0>#p@dd z*f!sI7&o1PXi?h^g8jA;AKk3^8ekj2g9oEMR8_SPLEGohKF-9bi}nB~xi#V4E8= zo4d`O5<`qCN+>cL@LwXp26--)n}-TzQYKgHP0YMsQf9)cX z76!wJMnv)t_xg(-fSphO7+Qfz_7oa4m}KkAjSnNqj2R|E@KRuG9=q&|!)6?GT-KK# z4~4k4fT+hip_)K(@F!=Hf*go&Icb{V5Zqp*@3Da{lStH`NAXFz`GcLseU3+t#(C0M z00xePq5>Ir5s|d`$r_kn&tcENCyhVrVg}*6Htv7C8d^cVHZgILK+j&2l>j+-Wi7i7T8Wzd774H-uNzPs!mwmpGJH3%U z`u4}(-azQj%c8*K2dAZ3M~tB#&Y7Uc^q}k6^Zd|X-EPC%b8{Dp$6|{gm)PKJyrGdv z*4pOwJKnjiM~?6*JQ_dT>!gs7hK)^;M=wI}8(KCDZ^Z^#W^G6WF`F`*xOt$*#bYeb zDf!-*?(DB)%YIB9Kjdw;%-5IO_-4ql$)q6+#4`}H(`avN#&J~n5Pt6VNZWewfX@BY zL^faFcM5ZJ_npT41g%gP|36~>O0;KDRtg+iHzhQh9UMu|+t;<*BDo|->`1avPT=3O zcj@zKv=l^gND)AugeRf(0Mb>3j;!~~m(Hicw{SRRcbi>eJ~^av8o(hr0aD)yxIO-H zFue&!MR01$<6IS8G`^h?X{y#QRnxcKecx43O>}18%(?~)Sk!%)Viv+wi449U7d0@BN~iV zm$+n!sm$j`vsR;Fi(Z!45x(qCb(3ejaexfgK8^z=ojTI%q9AQaXdh`B39L((U!9va z(9Zb}z4ne7@UK7Q9cXoNGsuoW*lCn=5UIVG>d-3hJFuqoZS3c%KG(7t#8>r>j4VfO z%d42nZd-9HNhLt;e8`qVn#JNEmM+xvwQZp{3cVzv<}PW+HJqb+kC0s@zan7R0G$IcR~+?0YBAnNFm@qevu}Jk+}t)@V}PG^es(` zJP}^`HW3uV6_mUmOt`OjCV@F|KyD1**agJ6l!5ylS@j?`_3vdheM{IvGI4o0Inecr zUJs4|!J=UOPiy4=Su+3g9}kjn6J~_`v~Ag)-df`4;el`xVwfE!*gMMFOC{Z4fKIY(1G3h%%0Av`Xum{8W#$06As1EH8M8#$UiwA$Hp5Q-t zJsoicV2rZ(5C%t1t!!>yk)54QHYZ?Y`Y0MB9(Y5RAPfg!1RT+H1kt{9$#$CP*)U-2 zH!=#rUWAZ_OVGV-d-)tTT3g#CKn6&}7+SO4kiq~h1>QZ50TGEA%-H5hRRhcv?}l?s z5b(ihV`GC&JQemqLR;R-J=_|Mh80d)WZt1ALs_6y=gb#?5?WbX8`u=r;4|BR7;fmt z`-(8mRcO9ob=gc}5s;?*d2+zDOiD)PTvnDCFeYiV$bW=ek}Jcckc0!_B;gcM>N*OK zE-%^%6~v$7m8{a(Vh7m$Pq;w5qKhB~HwfE<@XIy;dV4jo--!bWXOA$t6kzmK#vvQz z`-l|VXR6h>n0F_bZnBWjgg@~At%3w58uaND)I@^NZ=55rADDOyFMkxk*j(tZ68IZS z0faviD65cid=b`$rR1v^87r}#(Dk!Xpclv)z-K`i|TwOGKFyo)U)C=*yRBBqb$r1Ove(hlhtB z_$_QB6E{vRVup>KHc<7ejK0&ZSfP|VnIbYc{@`ivsS7xbU{V%vR6;EhgKj`RD--o| ztV@g7-)r$RgcKD`2Foy!h$Q<=gte4l(&13Ju8TM4=*`LLss6sc3z(!`yu3t%*#T{; z6O_V350h3s-cy((#m(+tVCGSbhEz78oq?P_ya&2mF|ja;AJVNW+w7pAu!U2yfh?C; z(;_?RD6i2DH60zD3)(`0@Tq?OtksrhA?XnT9FiUHW5QAX;AnCpF|Goq9uWOt(amSr zIvjR{va5q8R_zADW%km{!x?mHR*xFGHhm)_5sX*&Cg%=zY5YP>s0HT(?EZJHvXP^tBF?+lzMGdoB z4I$R8pD{A?D$gYO8N>5j1NZ8mAZ2_b`5hwrZZa!eZ&Zfs&v|dF;x0k`q$gV`@~zOL zR(_sz0W1Iuh*fi%ra>+8XRL7ZPfrc6B8!`y5rN_c_TGHhOyCMZsrbw3^W?0yj*jCq z=Ku#?92eshi8tV(*S9Uj5|Bg?P2inmzl%U+)F4#{%0~j6@N;ba*y%yDy|^MqPNF6o z8lFKfSSX@a!79f&WQW9$ls|axjYOW}I9({Dki9^`>V;=v57(S={~=-FTV{DL z`^V$kEukJ?tKy?b%=i#5QWR0Aff=L)xw#o{6`hBN2Wi&eA;>B!#}C047rX4e;ky^A zOnd$C$;Y0OP6hQ3^o~qV?%j7?BM3&KgN9ffQW99QY2aH3g(dYV3K`a(JguQ`hhs^D z8$eKU=rNv0n*Z;`hH3mCR40w$2MBDof>d!W5DijSVE0B-uYEW#`-z&+%2rceTm;G1 zLY+XZrv=~<|JE%aBz9O%p*afGC0$F`1}9R;`XtfM0y z;AAYiutKPWX#>jr782P7Khz)A*H^;|)4}%66ji%HL`1|Ecg$e@0H#)9cn<}k0o-M1 zcolR&>(G+CuVZ>YO2c*uu3WoT8^?Qv0a$bK^b`RMV?X^3kSq91cIczhQd89+jj00{ z5WkSw7KVHhxY`9_GYW3Biv@DjKKBa@D7h+%A5vcV7aq{{^?mxJiKP+-mTUa`cYcHl z-J9~h2kkMO6YU*-xc6z9ncB#;(F7ur05w>1ON$T&-vK-@VPHDZ+a$MdzdpM_%MX$Z z6ejkBy8Fl9Uclgm)2xLsaR{U35kSXQ92e$r-30pThXR*KI~GMkH*pI6Ml1_7ToLNz f;JyFtA18i$p1RGQOhYA7_|G0qy logs/log_0 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 diff --git a/research/mi_poison_2022/scripts/train_demo_multigpu.sh b/research/mi_poison_2022/scripts/train_demo_multigpu.sh new file mode 100644 index 00000000..7d8d81fb --- /dev/null +++ b/research/mi_poison_2022/scripts/train_demo_multigpu.sh @@ -0,0 +1,32 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 0 --logdir exp/cifar10 &> logs/log_0 & +CUDA_VISIBLE_DEVICES='1' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1 & +CUDA_VISIBLE_DEVICES='2' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2 & +CUDA_VISIBLE_DEVICES='3' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3 & +CUDA_VISIBLE_DEVICES='4' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 & +CUDA_VISIBLE_DEVICES='5' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 & +CUDA_VISIBLE_DEVICES='6' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 & +CUDA_VISIBLE_DEVICES='7' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 & +wait; +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8 & +CUDA_VISIBLE_DEVICES='1' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9 & +CUDA_VISIBLE_DEVICES='2' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10 & +CUDA_VISIBLE_DEVICES='3' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11 & +CUDA_VISIBLE_DEVICES='4' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 & +CUDA_VISIBLE_DEVICES='5' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 & +CUDA_VISIBLE_DEVICES='6' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 & +CUDA_VISIBLE_DEVICES='7' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 & +wait; diff --git a/research/mi_poison_2022/train_poison.py b/research/mi_poison_2022/train_poison.py new file mode 100644 index 00000000..ae7b788f --- /dev/null +++ b/research/mi_poison_2022/train_poison.py @@ -0,0 +1,198 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import json + +import numpy as np +import tensorflow as tf # For data augmentation. +import tensorflow_datasets as tfds +from absl import app, flags + +from objax.util import EasyDict + +# from mi_lira_2021 +from dataset import DataSet +from train import augment, MemModule, network + +FLAGS = flags.FLAGS + + +def get_data(seed): + """ + This is the function to generate subsets of the data for training models. + + First, we get the training dataset either from the numpy cache + or otherwise we load it from tensorflow datasets. + + Then, we compute the subset. This works in one of two ways. + + 1. If we have a seed, then we just randomly choose examples based on + a prng with that seed, keeping FLAGS.pkeep fraction of the data. + + 2. Otherwise, if we have an experiment ID, then we do something fancier. + If we run each experiment independently then even after a lot of trials + there will still probably be some examples that were always included + or always excluded. So instead, with experiment IDs, we guarantee that + after FLAGS.num_experiments are done, each example is seen exactly half + of the time in train, and half of the time not in train. + + """ + DATA_DIR = os.path.join(os.environ['HOME'], 'TFDS') + + if os.path.exists(os.path.join(FLAGS.logdir, "x_train.npy")): + inputs = np.load(os.path.join(FLAGS.logdir, "x_train.npy")) + labels = np.load(os.path.join(FLAGS.logdir, "y_train.npy")) + else: + print("First time, creating dataset") + data = tfds.as_numpy(tfds.load(name=FLAGS.dataset, batch_size=-1, data_dir=DATA_DIR)) + inputs = data['train']['image'] + labels = data['train']['label'] + + inputs = (inputs/127.5)-1 + np.save(os.path.join(FLAGS.logdir, "x_train.npy"), inputs) + np.save(os.path.join(FLAGS.logdir, "y_train.npy"), labels) + + nclass = np.max(labels)+1 + + np.random.seed(seed) + if FLAGS.num_experiments is not None: + np.random.seed(0) + keep = np.random.uniform(0, 1, size=(FLAGS.num_experiments, len(inputs))) + order = keep.argsort(0) + keep = order < int(FLAGS.pkeep * FLAGS.num_experiments) + keep = np.array(keep[FLAGS.expid], dtype=bool) + else: + keep = np.random.uniform(0, 1, size=len(inputs)) <= FLAGS.pkeep + + xs = inputs[keep] + ys = labels[keep] + + if FLAGS.num_poison_targets > 0: + + # select some points as targets + np.random.seed(FLAGS.poison_pos_seed) + poison_pos = np.random.choice(len(inputs), size=FLAGS.num_poison_targets, replace=False) + + # create mislabeled poisons for the targeted points and replicate each + # poison `FLAGS.poison_reps` times + y_noise = np.mod(labels[poison_pos] + np.random.randint(low=1, high=nclass, size=FLAGS.num_poison_targets), nclass) + ypoison = np.repeat(y_noise, FLAGS.poison_reps) + xpoison = np.repeat(inputs[poison_pos], FLAGS.poison_reps, axis=0) + xs = np.concatenate((xs, xpoison), axis=0) + ys = np.concatenate((ys, ypoison), axis=0) + + if not os.path.exists(os.path.join(FLAGS.logdir, "poison_pos.npy")): + np.save(os.path.join(FLAGS.logdir, "poison_pos.npy"), poison_pos) + + if FLAGS.augment == 'weak': + aug = lambda x: augment(x, 4) + elif FLAGS.augment == 'mirror': + aug = lambda x: augment(x, 0) + elif FLAGS.augment == 'none': + aug = lambda x: augment(x, 0, mirror=False) + else: + raise + + print(xs.shape, ys.shape) + train = DataSet.from_arrays(xs, ys, + augment_fn=aug) + test = DataSet.from_tfds(tfds.load(name=FLAGS.dataset, split='test', data_dir=DATA_DIR), xs.shape[1:]) + train = train.cache().shuffle(len(xs)).repeat().parse().augment().batch(FLAGS.batch) + train = train.nchw().one_hot(nclass).prefetch(FLAGS.batch) + test = test.cache().parse().batch(FLAGS.batch).nchw().prefetch(FLAGS.batch) + + return train, test, xs, ys, keep, nclass + + +def main(argv): + del argv + tf.config.experimental.set_visible_devices([], "GPU") + + seed = FLAGS.seed + if seed is None: + import time + seed = np.random.randint(0, 1000000000) + seed ^= int(time.time()) + + args = EasyDict(arch=FLAGS.arch, + lr=FLAGS.lr, + batch=FLAGS.batch, + weight_decay=FLAGS.weight_decay, + augment=FLAGS.augment, + seed=seed) + + if FLAGS.expid is not None: + logdir = "experiment-%d_%d" % (FLAGS.expid, FLAGS.num_experiments) + else: + logdir = "experiment-"+str(seed) + logdir = os.path.join(FLAGS.logdir, logdir) + + if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz" % FLAGS.epochs)): + print(f"run {FLAGS.expid} already completed.") + return + else: + if os.path.exists(logdir): + print(f"deleting run {FLAGS.expid} that did not complete.") + shutil.rmtree(logdir) + + print(f"starting run {FLAGS.expid}.") + if not os.path.exists(logdir): + os.makedirs(logdir) + + train, test, xs, ys, keep, nclass = get_data(seed) + + # Define the network and train_it + tm = MemModule(network(FLAGS.arch), nclass=nclass, + mnist=FLAGS.dataset == 'mnist', + epochs=FLAGS.epochs, + expid=FLAGS.expid, + num_experiments=FLAGS.num_experiments, + pkeep=FLAGS.pkeep, + save_steps=FLAGS.save_steps, + **args + ) + + r = {} + r.update(tm.params) + + open(os.path.join(logdir, 'hparams.json'), "w").write(json.dumps(tm.params)) + np.save(os.path.join(logdir,'keep.npy'), keep) + + tm.train(FLAGS.epochs, len(xs), train, test, logdir, + save_steps=FLAGS.save_steps) + + +if __name__ == '__main__': + flags.DEFINE_string('arch', 'cnn32-3-mean', 'Model architecture.') + flags.DEFINE_float('lr', 0.1, 'Learning rate.') + flags.DEFINE_string('dataset', 'cifar10', 'Dataset.') + flags.DEFINE_float('weight_decay', 0.0005, 'Weight decay ratio.') + flags.DEFINE_integer('batch', 256, 'Batch size') + flags.DEFINE_integer('epochs', 100, 'Training duration in number of epochs.') + flags.DEFINE_string('logdir', 'experiments', 'Directory where to save checkpoints and tensorboard data.') + flags.DEFINE_integer('seed', None, 'Training seed.') + flags.DEFINE_float('pkeep', .5, 'Probability to keep examples.') + flags.DEFINE_integer('expid', None, 'Experiment ID') + flags.DEFINE_integer('num_experiments', None, 'Number of experiments') + flags.DEFINE_string('augment', 'weak', 'Strong or weak augmentation') + flags.DEFINE_integer('eval_steps', 1, 'how often to get eval accuracy.') + flags.DEFINE_integer('save_steps', 10, 'how often to get save model.') + + flags.DEFINE_integer('num_poison_targets', 250, 'Number of points to target ' + 'with the poisoning attack.') + flags.DEFINE_integer('poison_reps', 8, 'Number of times to repeat each poison.') + flags.DEFINE_integer('poison_pos_seed', 0, '') + app.run(main) From b029b6ab005e93aa4fa8af122db437a9de6ed0e3 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:46:39 +0000 Subject: [PATCH 03/10] fix plot output --- research/mi_poison_2022/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md index 071c209b..dd35fe04 100644 --- a/research/mi_poison_2022/README.md +++ b/research/mi_poison_2022/README.md @@ -88,11 +88,11 @@ which should give (something like) the following output ![Log-log ROC Curve for all attacks](fprtpr.png "Log-log ROC Curve") ``` -Attack No poison +Attack No poison (LiRA) AUC 0.7025, Accuracy 0.6258, TPR@0.1%FPR of 0.0544 Attack No poison (Global threshold) AUC 0.6191, Accuracy 0.6173, TPR@0.1%FPR of 0.0012 -Attack With poison +Attack With poison (LiRA) AUC 0.9943, Accuracy 0.9653, TPR@0.1%FPR of 0.4945 Attack With poison (Global threshold) AUC 0.9922, Accuracy 0.9603, TPR@0.1%FPR of 0.3930 From 46a6e48e03f53a45676ca0654c56ac3a303edda9 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:47:50 +0000 Subject: [PATCH 04/10] fix names --- research/mi_poison_2022/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md index dd35fe04..670981fa 100644 --- a/research/mi_poison_2022/README.md +++ b/research/mi_poison_2022/README.md @@ -111,7 +111,7 @@ You can cite this paper with ``` @article{tramer2022truth, title={Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets}, - author={Tramer, Florian and Shokri, Reza and Joaquin, Ayrton San and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, + author={Tramer, Florian and Shokri, Reza and San Joaquin, Ayrton and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, journal={arXiv preprint arXiv:2204.00032}, year={2022} } From 59c03b5ab0dee3556e630f37e7fdec9fbb308b9c Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:53:24 +0000 Subject: [PATCH 05/10] some comments --- research/mi_poison_2022/plot_poison.py | 6 ++++++ research/mi_poison_2022/train_poison.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/research/mi_poison_2022/plot_poison.py b/research/mi_poison_2022/plot_poison.py index ce7bb787..bb156506 100644 --- a/research/mi_poison_2022/plot_poison.py +++ b/research/mi_poison_2022/plot_poison.py @@ -62,24 +62,29 @@ def do_plot_all(fn, keep, scores, legend='', metric='auc', sweep_fn=sweep, **plo def fig_fpr_tpr(poison_mask, scores, keep): plt.figure(figsize=(4, 3)) + + # evaluate LiRA on the points that were not targeted by poisoning do_plot_all(functools.partial(generate_ours, fix_variance=True), keep[:, ~poison_mask], scores[:, ~poison_mask], "No poison (LiRA)\n", metric='auc', ) + # evaluate the global-threshold attack on the points that were not targeted by poisoning do_plot_all(generate_global, keep[:, ~poison_mask], scores[:, ~poison_mask], "No poison (Global threshold)\n", metric='auc', ls="--", c=plt.gca().lines[-1].get_color() ) + # evaluate LiRA on the points that were targeted by poisoning do_plot_all(functools.partial(generate_ours, fix_variance=True), keep[:, poison_mask], scores[:, poison_mask], "With poison (LiRA)\n", metric='auc', ) + # evaluate the global-threshold attack on the points that were targeted by poisoning do_plot_all(generate_global, keep[:, poison_mask], scores[:, poison_mask], "With poison (Global threshold)\n", @@ -98,6 +103,7 @@ def fig_fpr_tpr(poison_mask, scores, keep): plt.savefig("/tmp/fprtpr.png") plt.show() + if __name__ == '__main__': logdir = "exp/cifar10/" scores, keep = load_data(logdir) diff --git a/research/mi_poison_2022/train_poison.py b/research/mi_poison_2022/train_poison.py index ae7b788f..d7a57cab 100644 --- a/research/mi_poison_2022/train_poison.py +++ b/research/mi_poison_2022/train_poison.py @@ -49,6 +49,19 @@ def get_data(seed): after FLAGS.num_experiments are done, each example is seen exactly half of the time in train, and half of the time not in train. + Finally, we add some poisons. The same poisoned samples are added for + each randomly generated training set. + We first select FLAGS.num_poison_targets victim points that will be targeted + by the poisoning attack. For each of these victim points, the attacker will + insert FLAGS.poison_reps mislabeled replicas of the point into the training + set. + + For CIFAR-10, we recommend that: + + `FLAGS.num_poison_targets * FLAGS.poison_reps < 5000` + + Otherwise, the poisons might introduce too much label noise and the model's + accuracy (and the attack's success rate) will be degraded. """ DATA_DIR = os.path.join(os.environ['HOME'], 'TFDS') From 680ee66a01ead8f9ae21369b0f3d48c2288317d7 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:34:19 +0000 Subject: [PATCH 06/10] clean up some LiRA code --- research/mi_lira_2021/README.md | 6 +++--- research/mi_lira_2021/inference.py | 30 +++++++----------------------- research/mi_lira_2021/plot.py | 5 +++-- research/mi_lira_2021/train.py | 5 ++--- 4 files changed, 15 insertions(+), 31 deletions(-) diff --git a/research/mi_lira_2021/README.md b/research/mi_lira_2021/README.md index 7cb30e17..65eddfe2 100644 --- a/research/mi_lira_2021/README.md +++ b/research/mi_lira_2021/README.md @@ -2,9 +2,9 @@ This directory contains code to reproduce our paper: -**"Membership Inference Attacks From First Principles"** -https://arxiv.org/abs/2112.03570 -by Nicholas Carlini, Steve Chien, Milad Nasr, Shuang Song, Andreas Terzis, and Florian Tramer. +**"Membership Inference Attacks From First Principles"**
+https://arxiv.org/abs/2112.03570
+by Nicholas Carlini, Steve Chien, Milad Nasr, Shuang Song, Andreas Terzis, and Florian Tramèr. ### INSTALLING diff --git a/research/mi_lira_2021/inference.py b/research/mi_lira_2021/inference.py index 11ad696a..7fd3f814 100644 --- a/research/mi_lira_2021/inference.py +++ b/research/mi_lira_2021/inference.py @@ -12,32 +12,19 @@ # See the License for the specific language governing permissions and # limitations under the License. -import functools -import os -from typing import Callable import json - +import os import re -import jax -import jax.numpy as jn -import numpy as np -import tensorflow as tf # For data augmentation. -import tensorflow_datasets as tfds -from absl import app, flags -from tqdm import tqdm, trange -import pickle -from functools import partial +import numpy as np import objax -from objax.jaxboard import SummaryWriter, Summary -from objax.util import EasyDict -from objax.zoo import convnet, wide_resnet - -from dataset import DataSet +import tensorflow as tf # For data augmentation. +from absl import app +from absl import flags -from train import MemModule, network +from train import MemModule +from train import network -from collections import defaultdict FLAGS = flags.FLAGS @@ -142,9 +129,6 @@ def features(model, xbatch, ybatch): flags.DEFINE_string('dataset', 'cifar10', 'Dataset.') flags.DEFINE_string('logdir', 'experiments/', 'Directory where to save checkpoints and tensorboard data.') flags.DEFINE_string('regex', '.*experiment.*', 'keep files when matching') - flags.DEFINE_bool('random_labels', False, 'use random labels.') flags.DEFINE_integer('dataset_size', 50000, 'size of dataset.') flags.DEFINE_integer('from_epoch', None, 'which epoch to load from.') - flags.DEFINE_integer('seed_mod', None, 'keep mod seed.') - flags.DEFINE_integer('modulus', 8, 'modulus.') app.run(main) diff --git a/research/mi_lira_2021/plot.py b/research/mi_lira_2021/plot.py index 435125ce..422b18e4 100644 --- a/research/mi_lira_2021/plot.py +++ b/research/mi_lira_2021/plot.py @@ -220,5 +220,6 @@ def fig_fpr_tpr(): plt.show() -load_data("exp/cifar10/") -fig_fpr_tpr() +if __name__ == '__main__': + load_data("exp/cifar10/") + fig_fpr_tpr() diff --git a/research/mi_lira_2021/train.py b/research/mi_lira_2021/train.py index 19ff0e3c..dd0b901d 100644 --- a/research/mi_lira_2021/train.py +++ b/research/mi_lira_2021/train.py @@ -24,12 +24,11 @@ import tensorflow as tf # For data augmentation. import tensorflow_datasets as tfds from absl import app, flags -from tqdm import tqdm, trange import objax from objax.jaxboard import SummaryWriter, Summary from objax.util import EasyDict -from objax.zoo import convnet, wide_resnet, dnnet +from objax.zoo import convnet, wide_resnet from dataset import DataSet @@ -269,7 +268,7 @@ def main(argv): logdir = "experiment-"+str(seed) logdir = os.path.join(FLAGS.logdir, logdir) - if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz"%10)): + if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz"%FLAGS.epochs)): print(f"run {FLAGS.expid} already completed.") return else: From b924d0aba4730094794dfc4a46b3f0d0bd111777 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:38:19 +0000 Subject: [PATCH 07/10] Add code to reproduce Truth Serum --- research/mi_poison_2022/README.md | 118 +++++++++++ research/mi_poison_2022/fprtpr.png | Bin 0 -> 32936 bytes research/mi_poison_2022/logs/.keep | 0 research/mi_poison_2022/plot_poison.py | 107 ++++++++++ research/mi_poison_2022/scripts/train_demo.sh | 30 +++ .../scripts/train_demo_multigpu.sh | 32 +++ research/mi_poison_2022/train_poison.py | 198 ++++++++++++++++++ 7 files changed, 485 insertions(+) create mode 100644 research/mi_poison_2022/README.md create mode 100644 research/mi_poison_2022/fprtpr.png create mode 100644 research/mi_poison_2022/logs/.keep create mode 100644 research/mi_poison_2022/plot_poison.py create mode 100644 research/mi_poison_2022/scripts/train_demo.sh create mode 100644 research/mi_poison_2022/scripts/train_demo_multigpu.sh create mode 100644 research/mi_poison_2022/train_poison.py diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md new file mode 100644 index 00000000..071c209b --- /dev/null +++ b/research/mi_poison_2022/README.md @@ -0,0 +1,118 @@ +## Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets + +This directory contains code to reproduce results from the paper: + +**"Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets"**
+https://arxiv.org/abs/2204.00032
+by Florian Tramèr, Reza Shokri, Ayrton San Joaquin, Hoang Le, Matthew Jagielski, Sanghyun Hong and Nicholas Carlini + +### INSTALLING + +The experiments in this directory are built on top of the [LiRA +membership inference attack](../mi_lira_2021). + +After following the [installation instructions](../mi_lira_2021#installing) +for LiRa, make sure the attack code is on your `PYTHONPATH`: + +```bash +export PYTHONPATH="${PYTHONPATH}:../mi_lira_2021" +``` + + +### RUNNING THE CODE + +#### 1. Train the models + +The first step in our attack is to train shadow models, with some data points +targeted by a poisoning attack. You can train 16 shadow models +with the command + +> bash scripts/train_demo.sh + +or if you have multiple GPUs on your machine and want to train these models +in parallel, then modify and run + +> bash scripts/train_demo_multigpu.sh + +This will train 16 CIFAR-10 wide ResNet models to ~91% accuracy each, with +250 points targeted for poisoning. For each of these 250 targeted points, the +attacker adds 8 mislabeled poisoned copies of the point into the training set. +The training run will output a bunch of files under the directory exp/cifar10 with structure: + +``` +exp/cifar10/ +- xtrain.npy +- ytain.npy +- poison_pos.npy +- experiment_N_of_16 +-- hparams.json +-- keep.npy +-- ckpt/ +--- 0000000100.npz +-- tb/ +``` + +The following flags control the poisoning attack: +- `num_poison_targets (default=250)`. The number of targeted points. +- `poison_reps (default=8)`. The number of replicas per poison. +- `poison_pos_seed (default=0)`. The random seed to use to choose the target points. + +We recommend that `num_poison_targets * poison_reps < 5000` on CIFAR-10, as +otherwise the poisons introduce too much label noise and the model's +accuracy (and the attack's success rate) will be degraded. + +#### 2. Perform inference and compute scores + +Exactly as for LiRA, we then evaluate the models on the entire CIFAR-10 dataset, +and generate logit-scaled membership inference scores. +See [here](../mi_lira_2021#2-perform-inference) +and [here](../mi_lira_2021#3-compute-membership-inference-scores) +for details. + +```bash +python3 -m inference --logdir=exp/cifar10/ +python3 -m score exp/cifar10/ +``` + +### PLOTTING THE RESULTS + +Finally we can generate pretty pictures, and run the plotting code + +```bash +python3 plot_poison.py +``` + +which should give (something like) the following output + + +![Log-log ROC Curve for all attacks](fprtpr.png "Log-log ROC Curve") + +``` +Attack No poison + AUC 0.7025, Accuracy 0.6258, TPR@0.1%FPR of 0.0544 +Attack No poison (Global threshold) + AUC 0.6191, Accuracy 0.6173, TPR@0.1%FPR of 0.0012 +Attack With poison + AUC 0.9943, Accuracy 0.9653, TPR@0.1%FPR of 0.4945 +Attack With poison (Global threshold) + AUC 0.9922, Accuracy 0.9603, TPR@0.1%FPR of 0.3930 +``` + +where the baselines are LiRA and a simple global threshold on the +membership scores, both without poisoning. +With poisoning, both LiRA and the global threshold attack are boosted +significantly. Note that because we only train a few models, we use +the fixed variance variant of LiRA. + +### Citation + +You can cite this paper with + +``` +@article{tramer2022truth, + title={Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets}, + author={Tramer, Florian and Shokri, Reza and Joaquin, Ayrton San and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, + journal={arXiv preprint arXiv:2204.00032}, + year={2022} +} +``` diff --git a/research/mi_poison_2022/fprtpr.png b/research/mi_poison_2022/fprtpr.png new file mode 100644 index 0000000000000000000000000000000000000000..a870cb96e91c44f568ee83c02ef636a49d94fcfa GIT binary patch literal 32936 zcmafbbySsU)bF87x;q4(X)5F6(%>Qx2z*r)1swzeB?*4J z$s>O5urEe%5gd0FV^0Kv&&D_w4n-XCI@~#gABrZkSd-r7*~~ znuE27_0R|3?Nv5Un$$F|6qv<$&n+xezFgaX)vo=}um%rLlde1}rUBjV!E$)gR_F6# zNfI2Qq=~<#lKYoE!=aLeTeJEJBiat6s~h)0hep`)WS-M!lpp>|)JZ2qtN z!w!TuH9W!#0(z7ED~)UstI5~m8RDMo2+NDJ12LZ;bZ0+4$C7g!^b8CrR1PJ-dWA_y zm}pXGsI46m6N9_`#)wBLC`j7N+q>8LP1ZU!@+I-ZiO47^`v(TlU7p_DCn?TB$ZF*x9k|uD-jgs-eL)I5;>kH1uL?dwanzQNPN7 z7;zGLHO+WF>lmb?qeFAq_NmmF$$MmCB0_~Vw4k71?y0=IyyWvMIz-RcCrA5r-F=IT zl-6&kECNsUb8gmmU7b7gJZz5hH8nEA#>2xaE-sdpmoM_o#hz~_Le+aE{Dm%P!c|!Y z@gz^wY&k1i)5mhVO*HX0KGrW%TgECY)N!2Df88nkzFkcloZWw)%O{W8kS4 z^_p0OcHhJx_PgDFvY;@L2v58*W$4+S^ZVZPhYmf0jpbb*>q|>*%qxnnf8hQ5bLJ~DqrN5zj zODa>~X_a50n6v*~#ZhTi6P{>5(rb|I7aF1O2N8p>IYsOe?4bDskOKcMhL#L`O|<**DXc zbAK5_(8l@PsT0NS4ERd|PcuC;(-)3IYH4j9T3A@O7}sih|)3(wDIirTBR)|n{t3od6Mt7H)Mu@rcMZf}kl{T0HS(8Oq{ zkB2B8Gs+3Q-Lri^OR+K_^=eI;e7U#azP$AJ3Ib&&vdyi2y?^|Y#QevsKe6mEMD(7S z2HdT_HPn_l5?CW0cpglIDk>_t$6`uuC80CD7k}+SA{UmM!Ww^u}iR@ZaBKn!ua!+Io8EP&ddZC}!Ase1(e%HxAad znrg&R!*?Wm+)B2tF*tj!CD*=JnUh+2BJ5m8u>1q3O1coT+VM<8F-JRLdM+2DD2^B@ zzayJtP^HgyV+f@>WaTp-y2am=lA^b=vdR#!LVYHgJ8qNp<+w%zE9vmi{m=RN!VfxMT4d(Ui<&;D${$(=6y6%Xlgozm_btl% zTiaWV`DIsECN@|%E@NY3%l=$_mmP>IR1-2+FtI3$UgD;7(ivN+x!HcTgpEQFhpBVZ z?q?kXRZ1-uje%B_R4rQv%4OQOfTJ&bTQ-}1SeIc|ARJmlT=Ta*DueguYcUp642#?Qw z`t_#xS}F3v`h6^V-^GeFKi|TXYRdAQu@$^?C#<32HV+Tao=bxh!COq7NQ0^np;E`a z?30yjzdc+fI>q4Qwcps>VZ%MNXo!DURxVY%=S>dRRzU+K{ zGDjv@_uMexk2xg=XXjrQa+=hLAd&NRkNw99ygWSN(Awr~@bQ;Zv7P0enI?+oFJ!sK zXecDd*HQzb#&j$MvYyW&Bxv6_6(5;T!eXNKVKxC zd)-rwAc{?pR9NTm&u~(0!1(iVUg|RcoTJKPbEtQ9l>t#4s;X9qqVNI(kHWGy6hl4y zvHmY1jd^`E2M2*l&B~v0U3)@`-cSGCCr(f-wH%(ORM&lpy!^?2OycT3s=gv$ZYkxL z)XU$=23iTXLx>`!gX*sKyM#x%n3+Rxm}(+0lbRFCem!107nihL;!i2P)k5ud&Y5vJ zj~X_s&=WMCJ^a=&rf_Shj%;H($8gC*n0R?G#g6%8o?NiS!chI_Z7}YGUmpHPj_lHR zX(E2zC+6|iKkRpi^s6%qOe2r%?}ZBep~L5=eW3KOe3as%O%4Qu=a$Ub0YY9|FXuj2 z_!47s+&j^fq=zgBM1}*G)oSqLjBk%8Pdtd^>M!^3{skO~W1|FApVl9sElTY8bV>=< zGZ$5xpyD&}78Imj?v2*o*VfBUvfz?EoM1&?yRSrSfsGKD*BL@x`$6^7o_(H}4+_x1 z$go(D3_0R(!=D11&XJI>?8oFUXeFkt?kKXCq-YP(@R@E4Io|SfesD$_%Sw8N$d9F@ zmr&FG$-&lr!~ZcW^Oia8C^uUkvDvBq8%tTAxF>=kp>g?I=A3<-Z{_NZWnl`{_iZ>A z-;IxtPq(;PK+o*&?=PCDiMt;;3uPR_6G3u<;Z1!<%13L+)`P242qpu)zE` zWA^L1{l&QsnSNQ>Zp4zuDaNGKWUva)-=(*rA9|4QVeN)4((!}v_jhY)>pQUOe zg5g98N=khT3(d=A1k+ZXyoy@5*ff79F3s34<`<*Q^gF13-L%|eUw*oC9X!TOYiJNt zSWG16E+tLbQQO^f`p4A1Be98L`)vP3*Jp3%8uCv;$&teq5gD^}+=w1oGaSslCQ%uE zM!VBLM>gXbf<;HpFI$LcWVep0TxY9!e%IctTs06@_`OYGIetAKc+kEbqeaFRVZqs# zS?=}yU1s=BW{TqLB_7MxR2;YoF4k;&fWS-+v#As_E$H zR!lu#@(2aLP22v5NEGMAVM46`<($6Q%o?6%ZX?sL`=^6Kuo@IC^O)^so!;%j`w_#v z3O=6Wd(IempWD~^Sp<9AJ*Vn_+mj&RhJv5k=W?eiYZ4nO3GtbJ8IfUpd$)gU1vhPX^%V&-%Vo*1zrL?U5#nCA4 zjfN-M6W#b!Cu#q#&+B^T@#Fak6}5??acU*U`ns}5qMjuWyZ8vq_t53lQvGl+{VDK~ ztM1pRz<#2?%6q<$6ja)7P4`~+TbmM7R*lQ=r>4@gsFoPAYATDM9MDv#ua@jqbXj#m zbvWQlxxT(YbKn2{VlC{Y5UZou;{C2SY0OH`NVr)Y?8yYrvhRu?|_GI=|s_%ik3u743txhDLlnUY;8VXR@K?_il< z?xVv_^kuH!i>~vMM37o>{ER_qFXO9gg!Ux6&hFe-Hlmm}ygVp(I(Z6;OrGc2oN61G z@=>b2;t9#hG70a@e`WSy&1dUy0+vIMbF(D?=LAGNihsG*n4!o_ETW`zB>1od@5 z#*0)!-6Pfm4cDUBT3qRq@1gLL)31t}wbIjlOESm<2&2VIv<%iQyWCn#!iGQYE`HqP zs5+lUuq3wI7aF?RlK@UHI0=#qZjc<|;3GF%VO7-DP~l;3O@`YhbGH5wB7MoC8Y`LD zuM`|c%-)vWUzPp%ouks0$Au!pHhpJeXv2LfM-Hn+As?}-E2qe*HzR? zV$|}6HGfx~ah3<8?sE=26TO3wH4GHy`PKN!c%env_+?t!+gQX5nF9gvlusi(o>@jY~LM~ti2QWUl{@d znzi*w=e7Uq5YG?PI;59rCG* zRg2{`?fq7b>+>3v{dDofmTq&!@@`*W8owW|UiXh0;XqBF=-c~nma(^7GUG2tHEZ-`B9bTlFAMP34>9sO3p~;nbP9P&Ag9wJ2`ht*gbw<(E^S=RCC6Xgb)eet+ z(CE%5K@Zz&Tv`P0@a@bX5!THc&7$FN>^JYMi8Mw3*j5QNOMou|i^;iFr)M18XlQ6+ z5)#cisvjBtd)c>u_^9@Qi=6DS?F+zEf7bE($bTx<=ch2a&?-R#$fFVO zr&eRhY17Nz%S04RHFyAusvX`5w&1is+b#bkP#=kc`H{MB*o1Mkb32E8)o2&vE3(K; z-H9zI?iq3;H4~V7v)#~0$ROF%=DAZmQ|B|FZQ#W&p{PB<&e=kiZgbn?tFZ`+Mb`_J zf7>0UZ3(yk`B7QdzhjoH@p@>RJJj*)a8FoznKAkk2?}7|H$E@!z8-&T%TQHOf%C@b z6Xz{V5i(@?LqRB5hho*4t7Vm1HypCG=s9)Q6iB=fqq0iMVg9Zmw4&iT`Np5~&yN#Q z>~QN70iEGm#~;EHV;HC?sVDUhTOMaf`HO%k^l`$|g;yw8OYfF}4)ehL5}j$nh=P#| z0q)q^$s_k=p3X&m$}1^+cyVSPua6cYyMS09)(*%sk=fMeFpFqB2{CEHiP0~rt~Kj; zvE2AbURfCnVQFu_V50cg5SB~1W;jvkz-yiGVf)Rso#oc_*Hb)w=!mRQ+;46-sm14~ zlnD1+t4TWxfF^%`z5RlFs^QIHje5IVVuRnGL*_eEVJUzY$mV;=Um3En2vRP7@$}&3 z!>XtoD>-|(S7}-QuGq0My^W%^sraV@KlGF@ijO&!#MLCa4m5NAGv9YAYKzp~IT3MC0HfU3W4ic+9rNEXhY}tVphN(Xm%im=`bI*|FbS)xoFt zjDEj{pM><^6HW?Xp4f=PeJg}#PFCY*M(KbkJW4*qx7WLi*(Ss1+=bhqbRPVJ0`0 zgt#~}7Z)x80fCpbC22mqOoZ9NXFY^(D&2WS1!@v}#zKhctejHLq4$(}xaYT3BF{-h15?cvjDgkuuLI29oQ z%uQy>77|Ynz^iq?#){9XxSwn+>ti8nUC#odl)m+qcD9e`MGr*Ho_Wtk zI8z>&P)hFcNFS`>2eTH{cAaP2#=Gq_=oq?PmAK!%iJXg7lu(S%s>5j<%IATTlmGWZjD;xrkwm?1?^r@8dxdqm_2vKe~rZ zrd>MqJV!Wdd*Z|COcSRDC&E9`6qR`C`9kH0ylOuGHmgsll;st!^>S{}E~D)y!#< zur)jFw*}TShS620R}FU-f)nEz0^0ZcKkIbp#=UalNzSmmm{)LVr^8AH$Zv$!QQ9#^ zCy-xPmk8pLoUU%Fh;dUACPq?|^Wv98nX8^Dnd>-l&uxK^^mKIfzQ^`7vgMkbRZ)ri z*CjY@FEb)Pk?h)YN(n3eITb+O3f|N84)<}BHygQJKoAcuWu83>LP@p?Av-`K}VvA1X6(a=GzHd|XHGhcUi z(5NjD<_In>DJcNyUAQ@R&|1&HU{oxua789LW8Kk@`7!&}hJiKT57w{pOaYB7jM;3e zON(ntjOL%cH&zv~Bp>fEn13W2_tqnlY!u}RXKD3LE;ddV9(DTn=Sb+;+Z0R;qPutR znsx`}RPY&ydwfCnJ571lETGsV8cx*to_Db3+KJjH^_TPbxyZbZ(({Lbu(lP~9{Wuj zd~MQkyi4#V!}9p$GqEfuYbe_Y74NiJZ3EW_Fd(BZxdlOHgGax|L$O)ceI@7O!gKa> zEwTIhs-taGMSJ2tEBmut{)Lai!e-M$9hBi3AM&4`vAdxEIpCq z$8$xbuya$yVkjyqZVMt-Z08kB0vJ>lYt1}_|&%qH~RmE-I`RqfvoR*&M z7zfWVP;3R(ZDtlhLJ+enE}ddbpBe4;RAuu?R(nm)8ig$gm@MVesv2ES-H94N2cT5H zDKARppb{!S&%q5QOB|yN0}p762_oh9P*}yqjqUvs0Up6D2^#|xEig~VCHnhI_;L69 z{)uo?$*D93ld;YEjTJ>a>4lU8w_A63-M>w*7@W52Az&N6uHp=$+rd`kD;whAMsV=* zMgU#nPJfl6Sc`bFJx;xSP04Q_Ub>x~Ow5vuZCGm>Qc_}kDA{GA3R zY!DRy0umuY&0BXWz_(+d-&=mgLvM@bPl|uTLjW~)un_PKbKAM>l8fqm?x6L?K|eVe znWdv+w8isF@|#L=4!JtbBr8}m&7Gs%@pYmgu))I{-3(MajSg`U$=+^oLg8J&($RnUwxOZ#o&WFrva-eUY;8Sx zJ#59+V;086FHhWs`LK?b*>o;6n$nI|Va~CZ_T&>wu^m$Z%+N2A9DV#MrWb=P^pc!N zZ1id(C^-s;LFm%&VkTcY2^^ViE-b)3GkBxOMwFgwzhN)OH_E+s(&n+m)9U@EjOnTGCe7h|v*%@E-={z-l0h`0Yp~z2l)D zQ%OlltKTV?`8z+15Lu=sx1%qE8U_NDo}dk|q#hGfWQQFtWd%~dpo#hRQG@X6>5Id? zQaoPr6`px>$@`o=usD;>+vy|;Xl3G>hl^L?hK`Po99&%1DU45dXG)Y_;8h+tYax6A z(Y(N?eF3s4IB#+QmD<|cw7?z<3AvYjgUMz%LGjr$v48*mxqlgmUf|O#yG25ZGvfRk!*Z|$S~M8UI#1W zGS~m^AUzIHSA`A5X z!(rBVjXqpOJ)=$7euYb{ahi50>%MxbzP_^fgx$P{5JxZv2Zw;XAO4LShkNtfC*PMz zv$C>OR8<`hC+YyJ0`vL;UFZG#_dO%2w*>9SsVZzhaH;p$RI;_VN4la=Y41u(O40=I zSNamOIA%#)vyv7F8b3>Z$knuV{{eZ}_Zu6|&IlhhSBd?OFyGfEQEbYLU#gg;1O|Gs z5thH!DnopJt_}{yQf8j)d!^|Qf5=D7R2lMr>hK|4SXlVB)-3~NHu}>iQ#vts1VA|a zE?-AS&eu6Pg3b#(nN8_QNkrgD-hiebSypyS#BDV$I2Zx0XJ1bb0lA6gm+<9lJmQ%Z_@owW|C-22M3O(&(qWO z_x9^9E-p4d6e%OCeP$+g0;5a<*lF)2&%lx%YOov27}gg~nhYfheV<< zvTF5KdPnGG`OG_`#?tx2hBr4h6J?a3MIQb8>sf8|2KCWYuA;20tmmu74|4MIU!@JS zwUdqg^Bc>FV%&kGLSz2o7eJt6k*;8OzC>kvY-qghRSLBY=2TyH6&~!VF=`d zumi@ng6yuWaczU2GsPNBgikP%r2Nk;As;aP;Y08KVjV;u9=oi(#X+u+V%5~ctr1dt zB}d&4v$JTC{r3zEUd^3VSMyL&Qw#6DlOP3eci1taH8UyJ`KS#PpkP@J`!jCNdcnC~ zDgYYyp#3jV&3K%rZT&1QmV0lDY9#Rj~_n{4-b2`C$hCoOky(cJnEVa z+;6n(MddN752h}eUM=q8lV3V^K@z`-a7nUPYIVPBY6l|9UJ{&t{05{-%4^oJq{xON z6G=K6i?mg^IU;cr!$9|Gy=vLwVWM8SUhhRa`MH*-dE%FcZ278bmpg1tSvsi~>^MqdOC{eDi4k^15K zyfaeRqs3`c8Bfj~D64(1nD@9H8fUEZ=acErvhlS;%JZr`>`Ots#NFmGVJhm=iiQ|> zrJsa>kfEcio5`ODHn#F9`j1pULcc)hleYB zur2wK2cHDmFpI8Jv6P{Xr42aBAiVU>-ltp~>*e& z38@Yb4{!cG-hmPbZgzfAbvca#BDsKVJA`#R&2q7cvXvq|>_o0>PMTXvAFLsqH3m#@ z0!!UYDveofDL26eJz{AgGZm>C`p2ZKtZe(}Q4y4-w<9qr(P!X^bApYlXl?436|sLL z&cRbR@b^}x&c&rajb(ifF;Jb7im3{FA8~fPJo>;Woe*rm0N7c3xOdU)!e6|wUhfGR zuMe!;EDU6FH4^OBNl6kAXM8TLqWHwzINqlEL{D)11NQ8>(F~ogUMx{8PG}!VJ*N;s zI3m|TVo79>JhagC`_damh>}3L9)!AO(owE=cX}O>l93S=cyY!oC`kOU*#+Pm%4D7l zAs$|=W==z!I@7?$-MXX<-I($;)CywV`8E@WUL!SXT`8VJb3?QG-EiqjjL({*{dMV^ z2!yT_Z@p77J;Q2Z8jao8&CS`-lH!$E;NMk-b=Cm4zkO`sM`~-{`wP9$_Y)bU`k)Q( znblVmF^8{znN@e`K1WTQ#dVI$n9<$r{h&ws41>UhfgiuWO=_o=9{2^Im}`)Zmc;<@=~J_=IP2k!^} z4pSowe4G>gx}TGz(QXAK7`R(a9QsvKyND|m6tV9e*4W;t7Y=@z%Z0F9W}@BN+Oix@ zp#Pthb^BZ4N&?8VU1C?*pdW0&The77GmtV>b|Ge>L*_ahv=AR=X5^HVFsrMpi<@Nv z{t!U`K*OZI+4j%Ez6xjZst72xLv2}oiwnW3t{U2|gU9160e*)lDNodTH|qu8Phz8n zARr6`8~pzL`x&?GaOdXd9aG8Y9q`Y$vL$_Qzv8z@*c?s_8?;6ea45WgzQpnixURh2 z=CY~s&osnrF2sF3F7yF(H85E$H7HFASD56m6IRuFEAKpDCW^5ttO$42HM4rp*?)HM z@dNnJtsNa`h?CRPhiz~5{1@|mncxxdD0pPu-337GX*K18-%HgqL)X_W-Nopu!dfeF zocI57s2}}6F>9J^`oufmm1cl_G@tiDj4}rWao99JJanX~66ZS**Dy0X+u!MXEb+klc1xi?n8khLm4t8am-Ph# zB3^0aY`U>nNwlJmdp>^b&=z%iu;L`)6O$4_)aRAZ{hssQJzusP;&$ri-T>cw!>~ZA zNK2z2L`JG$@c*ob5gu+tGWJbQ)@qBo@;l&%JB|H$$x?my_YIJ3x)>Hxm(6u_LU|uH z%W{xGkJhU}#)|IYsT7x$Z5~g_3o#wWjti0p@A2uXRK>@N~t zUtPHArq}6zpuwJ}$nnc~QFrYZ2ayN}OB{`%KQgM)4;|hV98))ZR|Au;pKV}~ti?*Q z0THV|R~J8t>FGaL?@l3;2zn27K(tJceY51D(ctCvoHo!Ub1QV|7NN)P4BNNeT_m=8 z5gIDTou1}IXu?bTb0S;R6y{tB`r8>VZ zvd>Lz=1i-lTmG)i5lSbYucVm^pNB=Jo+Cj{dP~!G#pLeom2kzT=HgE}>^Lk4Tgs#4 zlkdrT=iX59o#H6m`r1LvD|$S$`gS0tnM|J&l(woV>A{gmN9^mq^pWq?WkdaI5Ax)A zC}yKW^s1^_Fp-+X4SESOpo(Ii+v+)#w8Bn$sT8$M$kC-~i2a*!tjx+#LV=11y3VIm zlIX6mf^-@SA0Gk07bxjYZz-rF2JIV@Gc$3(3aVdjdO&OS#)$q0?OhmPwT|oCjIoId z%r*IRKme8`mND*Qs3Mt3clUE=%EUcclVY5z^Kt@-zOce9Wk6-F*X))LjZhV=QPs@I zzh`_2FQ+XS=cr!-QMN3!L z=IrR1q>+iqm&@O4Zom4p$(Yfpsi}eF2#AWhHM{9RriD{ZiI{PZi^ZYdNI=UloFBA8b&9}M6)Ugv-7hjwoqn8I3Uyo7(Yd zy`rylt?>IPG|W`y=gb7aGAcGUEH{_oZBx^JHR{)$9Z|;(ld(_?ym|oD_Gzf|=PSo? z&VoP!s?ZT&jpkP{DQ!59_($|_$Uc17VTPqr>FPZBTk{>tR?AvYSbMv~@%7n1rPA_J zYez>MsJc*m!*}c4eSC)byZ?ovpkb=ek6ATZRhHZ+P(jBY)zV|7)-HjW;pMGSI^C}} zZ6j%JZqAVK-n%Q4ehw?n5|Br$_ddmviwF-dugBgTXKQP#3jOb+VMd$1xr)QnQ$NT8HmzJ^vcw8BzYyj9-^dL|A&$psPME(Lo#|)kaN(0g(h#It3&y}S1^Yi<&c6~KW zw|Q`Y4`Y&%naM0Bma&|hlY{T(<|gTX7I%gx9I~=1&)da*9)?AD2*IZ9qutpuq#hG+ zJR$lPKYh;&6%Fl;{RB#9{`g(g&?mqn<3==>`PF_$} zhNZ8s4_SD~r`o`Nq7rr@1C3z1&O(Ou)-AN8x5m%^eoc3GcMlE=L!;m|8R(p{AI}JZ zVvUBF%$35sb?X)ka}Qh;=shNam6f$;Z7pxv2Zq1&HDCpxvXPOISTJ7oZfnw+Q^`|1 zH(6N~YPDaXVctk;S0)E>3rG?KkW-c^XlY}Mi&?*~bu*S}=E!Ml6E{u)g8SU=-Cv@X zzW46dd^-s5(7$3^Tg4DPv6TD*jx(&~<>l*tjyC`7Rn;N)_Pl}k-Mjq|3GTi;qeTQG zASQCNr5KgA%OW9H%7vKN0}tqH?@E(1GU*Sx_rkZIt-i#INoMMVIv$!&(-Yk|M%L6Hk7FXvEDRxWyeeT4|F&@Hou)O1ur!qCy? zu%Q2$O9i0^l#fsE{3&;)@`k?*#xj%PI-6d=-m7nD@GB#BCnqQWbMdoV5T4%tBY@Mx z56X-Xajr`S=nlni*t*=q@jQ4+ASga!C*Z8^uSJ zwOv=Tgq@N7&eqnhs89zT+7cZF-tWiegKs%gwMWb;GDP!RhDMvSgLI}xFB)Ef)vKMErfgmwX9?K4U9)D!Qyr}4-`7HeIBAQATh zQrr+$Pm!Ye~Lvjo<(mgXXBmgjb zkUI+$mu*w&JVHCWuS%cOoy75RnV>5p(LVv5STK@o{5jk}1cS`3^jxi={Om^~fBNRn zZn>oF92|{WB0ILh!=s~nkVpprt8HWy)qQo22i8!EZ;*lldTK-9?;oF`CSgSOhje$# zfKf#lB8w={iu&B)V{^FPD?Iu9>Yw=cgM&Bj8*%~XyJf9`7b1YTH}>{MPh4S}AqA3e zi**)Zx!I|yq}Dbzw0~%mON)xa-~u-}IIy7!A!AiY5p-+I7WVSy*Rgb?W|vGo-Ytk7 zxIcXwRCb{=Q3s^vczfd6;W`QhpJ|j^_ay~f01JVHVdm!M6FRanPW$Z-GuBgm^H}b3 zlfHbJ@|QuMF-iZDbuH+E7vk%7xj$i1!NvO_?E7f)buYy45h_`Yre2WHD2 zlr&?(#Eg%}{jI2Lt1zrHN2ctNIxwgxmay@3E}&S%z>LW;5d{;4BEJir0H2F{{#TYQ z6@T6oR%E~9FUMas@dTT$fv&Dp{LJVWUthlG`cl-?RW%C01=#%Ll5su z>+Koi#36@0%AM`=nc5B`Vf+d+L@O?B(T<}g>4bdi6}DwDr?334XOiE(eG_ut`@V47 zst?`P&JGw;%X?359-itZ#z?1!GATB=3U#LN&!e#uep4BYsQG=VSC3EI)q~?urNgZSmv|^n^u;XoU)_Ta+&|x0 zkx(+B&T9JBffy*<;mNM<*3QmN85zb0xsvyw+EVbFe*(O%_mG{0w0Jw68B)V?Z>a9$ z(q&XMGIQ~VYH_4xA(XXqG{Um#kpnV4tz|zh!Q|Kn50@e?QTz_R%0W6S@?l}iO^6UO zZAS*%cPqc_oWb2nSG9k>Uv<<&dZcZ@k~3CWQf=1pidtXT_m?gB^qSiqAdW*NM=)i2 z9>If2`ki2FYHD(DbN55JL0Xjc%P!Rx-3-7mv6yz_u0j3Hhr>IdG$3t$6qJdJj?$GD zaF^lGRAYN>e*UzTK`fes$ZB)kp!K|ixsxWO{W)rg?kPv(2RRaRU0vvWm}&bjgmY6l z0q_kC1TaCNh)7STypz_|4&`6qS_b@SgF173XmW@iKqaV9)?w>00d0cEh20xbTgw*^ z5O4_CPu|#=8ngzv?x4%0+*|^E{do%C1ig#deD>-PG%V|p6gHhPm(r3FD{ztykB^-J zD(b0`l7_yvmL9nF#&0m@$#CMxfBQ`sIPL$~PlqG7^76qDaewlp599zK0nxCND`-IP zjMfW|j*1FyX%U5uv$3@`uvML+4BJTh{M*g)%F6#*r~O19@VY%Ix2asIahH?I{qby3 z!eHB|AHgVS?!UHF5k1od@|h$42?+^I;Y_z~DgU^wlk*^~Yt;2)Gk%d$d?delP)|tNxg_mjkJTudv zbXDF0IB$J#zDmq}9np9IH22#~N$i|AxQ+UopJ0*-0H^5;>&$%wH~^1$1VXH7+mi_D zp^uG^9{jnu5LMgYsAMCOLni9&nHgE+dhFli_c1et2M=0@fNmXIp+C<-z$hID4L5jd z2k@@^g9mk!{t5~TJqru3jDLqt*#DQyvSn9s!R_tsr4qDOcxUJT@g=umz2B*mX;(m- zS#frmN#Z9ISP$q>^}q!U&dXy2gf#d1YhgJBGg)kMibaBTmn~)+S0Y}ZU1-V|)OrQf zQq{`ye~};=Sne;>fV~ool)_$%hryLY$G+aj?@SVnOp_EQf+KYm>g^*VBi@r-x56!< zr~T=^zL2Rn&|ef35wWk;4MdE?zz71ZL}m^E3O{+7w>J<)clEpbnwTP1(Z$6j3gW4d zyWWcB?@HR4dsZ(N;=s;RA8`n z)W>ao{hADC3sPRaQi30eGUBQASOpawc}Zg2h~mP+&>OTOabkDxrjCx@FRZM*pDP`B zADek=(c(FqzT2CB{{pOSZ52R=Q^ICeU?oZJ?38SS<0TYI04~8(#MwChJ1{X356|Ab zun8v(qk(!6M=R)jG^{aE1b)<^yvyLc+qC_EjzT z)VI-LxYSei=1H(%(<5C=r8ue(fM<7n-yndj(B68GI{>!a`qmaYWI8@|`s)5@AfOUJ zq_S&-78Tv}JNq7eFANjuha-aDzgJtqF2we zBq(1QsrOjFQ*|}xBf7`Ip?vh56qPn>ET(X!-5@X8Tw9x2nBw(>J-Af`ot>T9q9x@E zJ#}Qv(jDl3OWa<8`w*P+#Tt#ZLOAoBbydDTRQjc(NPIA>0w17MS| z>n=xSG+7!B81w(SeHqB74K_k1>;;7hl2zskNzphfZzgdEgG5v}RCpVZ!Ly?;7{?i1 zN9D~?M4@rl3!GT1aSbHcTp4=-4^>DXXdfh-qOeLzc4o%j)7JQTaI$~yBzZUxkVbY4 zhgmHxEfEBfx>NcbF_ju)Y~-+Ez>d{<%n6C1dSf1{DsEXL^YN{Qnd#|o?)y)*^wf$_ z!uOJZYuykc1BWp(85ggF4FmCwgFD9Q{s}kLyj7YWcCxN%ujGP`L2zy;Q3;!d%;+e< zIVr!BU@Z>Y=C+QG79-D}5EBBGseq!*5GXhDWE}hEQ#d6HRX!eM5CP{miPU#6LHt0l za_T|uLHDOO2rqWEN9K9J`z-+}HJLa1VFtG+xZ+M6(ILsl=Ybq zBX08F>b}vgDqixQZ(IIk4U0Z@5Ot?pc&v&YQ^c-O9~Q#d_vKmJMdv8Spg$;pM_rTs zYm6H)0eV^n`P_LlInWB~DgjFb>@iro;m|8vo+g*GLp$#r=ayGuW<*Da1dm=GZB;=5 zN?>3hvR#Pb0~af_vn$$r&bSlFRySQe86>>k7$cu*=tE(5{R2bG&hTQz6NZ}u93xh zOamn|{M|b#I(m9R?>#+W*S9a=+_HfCy6nlf4`|TC6+mo4sLBn4JDd+F5e(*ZZY{%- zin>vPB%sd-r27p?5lb7JFvv~_cK5?;Bku_|43gX4zJ1&M=!@brfB$~4I2jli)Ds%u zVNMP{Qcg}z!gvh$tOX8#61{m7d9!c;e3%4uQBda)-rYg-
)pw`qkHAS?v_{1+? zGX@;qt9dR{Q(MsX09ge~HFNGGms?5}7%t4Vz=F2UPxs>|%VtbLi5`LZ#$gTz6@v&u zQ}A9azyAtR8thkbQyOH-s^mV>o3hi_ghH(H_;KwxoqI#6tuQO3*z5!43n!9;>3a_< zJv{Wl;2_Ap^eL}jt79{xBcS?Oz%wAFit>tzWT8*^zZa(52sYyT0!@^bf^$uU$EaAHXZAX9-1%YpyQ-gwDyt z7Eo{UPIvT%47i)WayZ6=P{>)xmedWKuJ9)@;1ht$&I^aD|wlNSs z7${L+cYnj7p&|8LmYX+kg6b`}Mo@mnTUU&Vv&;smfr1{Zez{aE4`1d#0n%JtT*be% zgv*4T7c9Yw9~a%>0?wXs*M|=QYARxLC|=&dp?uu=nXj+ln8Nd4Bex*<6RE-~2Vn`& zWKaN{f;m2(c?T<<-=fx|3*g8r9^=rw=U3DpK74?)-Expofln&v;bFIPbw%n)$QY)0 zT_sMnav}r7m*0a3>~SzRr$ds>zskIx->)^$tX0*ix|cY|rokH~0oo@|nuY z$)Um?z=DQ|j0w7k&;krZ;^%tjMZSQGgBD0)rOfqN$-lS}0yrCy58B!I>DRzZWb6Pa zEYfMx)g=XuE_ubx);1jGT}gOC7)Sszifauk3{t`Ab2H6=7zcs8(F7dBPO;j0pDZ9K zh!Q)PQfY07gNuvAxoK%>>$|(raK=;Qk2C_Q`9vwO*faA~2HLzaVp5o~wW}Ped}D2? z7T?eS`QbE9-TJ2oD}G}awN4#Fr5f2tZ420oCB$3)z*6`s(S447A2bJHrhJeZW9G1+ zAiYaUN(u(Tj)rLW-hTlKQo!Zu;&}6rWMs?lscg|i$4QVQLgV8#iezN>{c_>sE5@&J zn5l#uh(Nc<$OAA2isdFSjbRYqkRKR;L>AFJ`yE}?I|g|ziJQlNL3MV0vSkU$G048c z&>Grwfcn_jZX*Bx{oD!x*K9<_)3ui8mHZvqd3lKtVzSzq%)#foIsY`O())>16-STh zLuwPcdsRCP=RZ1rx*x9JtLPSg{`~p(l_oqyQfg}0|57-Rr(qCF@BBp}71g`pz7A<3 zV9WAML||rFgZ820?4#%V|Ccy!uXTHSJb99`N7n**DlmBd6WD$SI6790A75TxdVF6J zB)d$negYP%&eR4hb_mB%9Dk~!DDwx^3HCH9MJ!U+-2h?j$xA~sGqcLX5@u9XR3aJ( zCw_CVe>hLM0EHgby!FakN96Q*z|!%nyY+K_2Ob(0PY6*cOaYA=xs3lACapHGOc zF^Py!fc*%iW}HPQl7PMsEYNz_k0Nk4&|W$w70YekG&I2He=q}{#|5bdk&Hq86)zqQ6$(>NvAY2JgNi0#$CzcprYniX&ZIKp*1Hb_{>^J$@IIc5V)Jjnf!+%KrE zz5$OK2Fhy}nC0viUqnPjDOr-sm9TPQ26R8yO~FvOpJ|*amZ8U_Jbo(LDIje8gu6J33{~5h-t2yG+*L`BaF1>=Ux!;U}Lzg-&|wQ`ZZf1LWtn@4{Z@?G#DBN%W>ialFfq5-6Lm!`TD8TGzf8 z%21}Ffn>^%BC{yTkf@EK%#|@pBy-9bkz|MxO2!Ih4k?Kw6h#?RWJoeph!FaIPy2lD z`|j@@uJ=3FIoCe54eP(wdhX}Gf7A1v*{;pSyaalvwk_&+4i--wHE+#4WP*1U?i$_0 ztHl5#qrB-bm|unN&8~y5RrhCs|pf;T$% zbY6Ko=B>;Nbng^mOGNS_O~jvL)J|OJ0#3yaGKha={bJi*`5o$vlck^9-%|Y_8a_JR zq8Yy%n)&gsI9($U@Nsd3Q?mAB%$zbCEnVFG&HtO^&jP1TEcgU{!&uRM&r#5Ad9@g; z09m-qe7Ay*S$*HWs097TXfN=t@?D6Es)ce~_-fHGrp%eqqSxeh;sz7A8l+3(VtMBD zT{W5v2ZJM5F;9~*78=`l72h*K&O`$c9K2-E1+){&P)d3_9l!yia|JjW`sfjn+!6{2 zCH}>Y{FW=L#Z3YbwIFqZ{y^hq2=|Foepr7Rp2ABRX2Z8a!F8t;7OoT#5dnJ{geIHA z_TcG>-(M64I!cD|nG{_I7%|RNeR3rd43sD<;N0?&U|}}nug3w4e?P69#dHoGeLQ3z zpv{CmSOu04EjDx+^9u_r)~sQ`wgBC&x~XYdXEI-_iu_R0RG0IU`uDtR*ItN;iJ|XN zmXTRmQBl#*pUpaoMsowo1*U6tJv~+e?V@YLj7)^7974&fHJyMp zO2QAmo9^5wcIx}Yr@uHkIV<7J#%}3!vaA!>tsp8KWFmuug9QFYTLqO}{Ok&lY5R?h z?Ook8ckTL!v4!Y$83jy=>{wvrkZ4~}S63%2jGCe0DoP4!Vjd;eU3OIG1D6ep3qLLF zRCK09ED+AbT}u~}&;!F!gEBMPOF*+k`mlEE$s0aTIJ4g#kIA#cNGE_t@*TfKZm68_ zn8FpYN1^k(PnSoxDTlr9B}dfv*EaQ~{Z^5?Aq>0BwW9UTK0bDJ2c2b3n}OFeqfSvV zO3R^=YIrYXIx#s($UOqcqq?$(*XX2}K6Dp*`|L-_e@4$%qLa&U@d21ySXg*!?#FTR zZ2<$}rcPn3cgb4By~0jejp9l`RXih^u=Qel<3yPFF(a&ZID8zghVs6`iv=@v8n9zD z@LMS7Zx`vj+xx!0|L=GkUQFS0X#GVde^W%E=kl?V2JQ6s+4(b}rn~n77_x_6KU*I$AIzQ7Ng{dtRK;VP- z(>?QA^2ziI9_5Fx3J3FJ>biXSGLe{Ezi*iXrJGzb40+6*J}lO>nVVh4-zsVi#oe*| z)@_3u#JGI9hN|k=`T?gZh}QG5TM}RoEi_>#007|CzE$B~w{CtQE6~Kmis&7!av(c1WZnO4*0|$no z!D9fGSqVBW2(eo7##~vf&))4{zd2+&0Jfq?|Iv^>`*s)XP$>FDRF5vmc}%d4%u#qF zL($n{(?o5pX~azrDuZEd>R*ZCQBs_3{I4 z$WS9tTK((S&Oh$|oDRulcq6B$qAmR7aOBl@ahOi=2I_!7uE#x4g=~9+8%xx~HoD^y zw1EL+uyh}3i-4Iv+PPvj1doqHow4(gmn517L9k=C)gdeybLQfLg6B{;R^Z;?#_0gx ze4Fws_sHW!6I0U_n>TN^%Rw$&nb$X7GHp&OZrytA5|2_0a@#o2ajfLzd^T#1g()pB zFRaWP1&($lZgfmik}jiwkd#z3+Td8E8^r?MjzM27KiIBOEZvy!B3M&<$K#Gx1V!~? zIsnz`P!iC#`rtRZu<&pVD=YAva%PWA#`F?YFMQKesj^^uo05|j(A6`ecKKx$06KU` znDC6Srma9LVMe&@%>rlw+afO8BhydeqJvoRV8Op|gpKaj7 z2i+I7#Mg2_ynfSg``hS*`}THr1XRH*W{IREIjQ{zx3{KHg`Q((KI1iONbSMC6fgb7 zaH!OM88xjXWYd~(hkr@a{=U9PD?AzV+X85{-h8^9n;Qu`AxT%$Z5&J4m~92j^EZTJ z=S#V`7vl)6x*L9aiz^GBEBQVJ}iONYhsb#-qjFLDj{?b`=d-Ugz?x|c*hjM<`4{IB3T3D-Ped)3dM z)58;*H3|K~;GP{N#l?iyrYyTkoZ{tAtd_mbN0EF%!afoD05ehYomf~0gxi2bonA> zzE!ohgTp2OQy9rE01shnPk<9!)@=u>MOkk7r@M1I zpl+6l0-UPn2VjA_Hpi^^qPS@xt~kSQXE6Ip4h{`;@bR_4m*Nu>WAKG5(Q{{waH!Y8 ze^zH~;cIvzA#*S(Y>P36k=Fd`J@PlSb-pVXt-VFN;^9#C$00{!+ui5M_-SPq=S7dU z6_-W!Az&Z4QBsZ_QW>{zpM&^CT$~vbQM=5H+R*ChWI=A?-50NK1=l;dAfZc8X%Xejdx#>d5bt9a~ek=iX4h zgi8vl?u8mC^?+|PQCz{+ggvBM{LK%5p8|q{v{;n}%Wt*X11tsNq+_9KZJpN858(+A ztg81fA%bD!Q`u!=vNnX7ixy>{sziS^j!4k9wVE9eB@wWJ4852hp#+nlQ~`d0ccDYG zflUt4mn;CkPVS#2aWcK9u$PPcoXlOhZryUAOc>a$h{t8{x^uiQG4v)bC(;4`#aV_t zizjx|>oCyE%NJMu%t#D5M6*kLH6Y@YvX^h{;2EK$rKKNWh(EFBmf;RK7#n=Oq^JE$ zL6$I6*hVOeBz2x`?1bGU`vr0-bE1<(AF9ANz+i%Ty^X<$v3OkJQvM_Qoo>wn> zeRXPk-k8T}*wS4l2X#B%y^|?k;hihCZx$~jG%U@TlGhD=|DgKYJ;ebbFH>i9b#*=U z^}P}S?Bn9%PPw^h;DNHP-}dar%g>WvVjksWs(pE%^{W5%uPpeSV!4ib{L&=Q;wgwC zSa-hI%L`G{Zck4Ie2dU3)kxkQ7cX66PWY?V;n=Yh^k=kzDf&M-mz^inD@8&CiI^}v zd^u1Y(!p&PB2*DpJ-}nd4`4dadea!k{h568ixVtSP{l_`)fr+ zm!#10_1C4gM8)P->w z@$m6Q0hrpbefv7~RCXwaY+XGJp|}YFxeEyQ?LF^@nbT49>|EQPQ8iw4ywh5+n8n1# zCUkzF;<;6?Ubw(W$SwGE!Obc9&fS0ib2$@Jl@_1x5>!s&y++l-@UI=w5Bwf@E**w= zDkinY%*;&De^HtGNE%HA$F-EIg#{ltF@xiOfySgmKab8yZs@0)hn`%3yC5X6%BKw} zs_CFnx2r7tc&n!*ou5iHX`s!apscQ~Wx$y0!J7=oS=#3L$67xB zXV}5mSFNh?(Uv^?kP|VfGXB^?=u~BPm7Iw)2TfjnW-9x+F91}#Gs z7eoP}J`N6s0P3TS&e-z=gjJCwq&ir%dBw*srFHMVp1&;+A$dbq88i~BcoF)OlKZTt zeS!J==VbprF(KX6kVRP*n7C_a-1vt&qN-;S{KM=*Am<(9gCF$g|?zW#lw zO)cx`GiSg7o-@s4iq=6GLw>Y(Bw8k~x8&<;>B?j!#} z{E=Q*7^9^1U#maP{%7@vbf-Hgy@W^?s-w(J6lxnwC#OGwAV7|F>=&^lt!!-QK%N0p zrTINS$>szL*`K_uv)9A?U_j*XDmNXqN2CR9vbxdr@@EZzkgo}^QY%qeLXiT z{gy|mW%(P+ew4I?#3qQO9auiz?js2|ne4BB4=-zGvqm!B5~rszsXKDCXExR3U=#a#Ki1ypN=(M*ZF7ftDmzjkp$_S{teNAk}$1g(P2qNGdL!FCh z+Z4G~-zAI+swlTTIi6V9q+f;XtobKBP*T1Bcc%5@C#{4Lt_o?)n{laqsaW~FH3#Fk7}TIz|ISO%`=P{$G;!5 zgY@<}>O_Nbg-bs@ra)xlBsf8tQT?;WKMHll&eKaxjYh$-gWn?u@SEI-?PEwPETOw= z^jsKb6o4%Ea-y{Vk0JdZUq7tFXCQSqDvC{(V{vRCwwQ^H&3^Em+%ABbL;x~Uk$t;+ z?r*sVI&a!PI-S&q$i&_#F7D_fur823S|O7CTuS~DSe+>~%T{A3`u^j`y#gnUP58C;g&$rl<8O2VJfM5?#9akA|3V`oSYpXj_`t}_ z{EXBRG+9VVC7c;S8^W*L+2YI)9m2lsyzZeyuR9qT!BHx6jNsl*gm+)VwVZT2rmS$} zQVpKhN8^hrsn0Y$`#2ms3zq^KYH+c$v%CK!{e1b)?z6;qf`N*vFMSC}T!uynARA#| zlT~~(D!gAoEr_T?GWDW%yFDf@H&kmP#s_C)+g&u=HMhpXhX!L3fSV0+a#uS2fdncl zDM^3%^!CjgO4~muA}UG?qh~MLEI>g%KR!N9yWK(b;c zt$Ttdru;C594lT1=m+sc^NWgXN;==Y>jRd0#d{Im6;Tx9H^O$!hladC1>GCz>|j5@ z1Az;imLgdcQ2*nPz!GbLAj2efLSh3ylOq5`oz(;A_CsfP_##>K&anQ zShCg7t3&*?%g09eG4}j4k=oX}tF-=WPt0j&9+Kee0pE3iT1oS;MuU|*O8j*~mZ+-NrCZj+GLh11D zZt!e#zW}ylr8*PXKGt z27vRsj)571^yc<rf7XlA>eUq1nCb100#oB zN&x2v^&Z;zVBq%A*ufPteE=AutBC3Av&L7Mnwl=506OMPd6^)|`RY|ocXxMfjIa3F z*sx>J$*n9f(IEmBYVg|*A)#PQL*#E>ynLB(cG-D#8T3-F;O@C!FE{~06~$w+<^9%G z8g=a0v1r8pKq+Ec;iEJQ^9Gb6?a?*Xm!aMsRKbft^zOJPd;j_?3=Lix;L%yghwGr_ zJ~1Czjo%nrEl}#*L$oOVzkm8SxJ*>|K~%N1pr~jAI$#Wfu^<*=Fmh_|kVLyPf%!Vi z2~Z&Xm=SHN?@|yeSUo)AA78uR_@iKCA(yec$b8c^L&I~**`EyMZ9bH_5%mKlBp-_= z4EPT0DaKz<=qAcBfm8h(T2lKT=HRFIHlYFYnFq)uA7}fuJ&zCKgFoclwJnFO=lGCC zA#MMP2MTEjx(JC4cb#j{i)+`fFJ}#Q8x?__i>tmxk2@Rs3!tjjixvysxPVo73JfG& zg@B2K3Bn-q5l#R+ZwN~((PsilsK)e82u$d-2e|q(gJj+ zC{hTpoXC)U>Qw&w_Uvxwn%+~~Ezh^y?JO+0-GD8D4{Zps8xpM^=v3LptsY5||98Ts zQD?-jhewf^!XW<;I+|(6nCG+uuK|IFi0TXbiT^XIo}=eHe$OJ z%ubL=?o4ZI>+b+41mUA0Yr*YCd0tVuZgXKI6d4kUJh6B~7}XhfS)W6i`8ns1DT%pd5QD*8BPSfgv^cjD&&erY82cw{@%0hF(JZOGkm0E3kjf z%dSp9W2m0Bi$A<1J1c60LC-rNJz;=d_j(vz1Bs~o8&auijI)I6qhgxP@kI@#6Cg= zx@!zkY2ZVb!`7sR?51oELtbEwrzZxOAfAUci%_;C=^UE~Ho0f$i>W{KA<)T)$$d8z z15uw~dWJ{C2CX?2fM?AM@0p#-X=F_YK-sR4=}1Yzy>h7h^a?y4AOz56k&`__k&^NB zsrt~!hz2CjI5bB7RV(BZc+TW{fB+pkaiSmIwT7NYkTzmMEP%uV>?#=~fnXE+75*)m zEuh7+lbD$!KX_(-^7S~#R4WYaEN5<_f{CPkW4G&5R4RY4P)|2na;R*>?x0uuFem)X9sP`GE z^98;0Zb(93Lapt$K9}6>rS3B-iv~ZiG43Gy-NdEaHYC0}oRj(k1LTpiAjvP5fD7a!F^GFaO&j5}|kR_Jnc~W1#)bzUgqEtg&|&#c#}wEP-k$ z68!${x6u_`N7?I6E~BCH6$8%2+^8+2S^6-=;6Q$UekJ55qu~wcp%T z>1EB_+Upb3>0^%UXU${zz1OJ-(ewK>8c%&nmNh85XIq@^rBK|aN;^^)S7+;k6^TyzM6ttN;+xN^S+Q#gZXZJR%b6e2HnUMkeGZrE@``AjsD zAU0+nPNm6M$~nGyyy{igjCVN)MoHv`MK463jtQb##ec~K$QAi+jm@9CCnQ|`J!k9pio0BXNkhyO zT;RJQ{=wt~r4wpQ$j2;&*9A^35XPMvs9S;2JLu)7H0Pd*?-LWmTyiUNLol&TF|xA4 zJ-)NAua5|}ps0rf2mTE%V%bw!oUwrxzY?4T7M7H}{CT{zfRdT8sIX@c(d`g?ix5M! z?R}l-F!Jtod!FaUjpBh!yp*L)W(do-CtT?{f5&E8qbo)}PfAt#`h;yBo8Eu;knmnJ z0T5H+OQTjQoA9+=UF(SO7Umc1FP|zb{!4p{1JAW<*FtaH;IUJ@i^dtl)2YRIZ;GJc z1aqfip6o<}CeoP>F=P?#9heMIvWAnYRXtAP-UXR6mE|9DVO=NrT&8P_UmKgf%G9&0 zCyQ71dVPSCAJ3tNhE;LSnJdR7!gp5+O~p<@#SOoOF?XrF<`$r2S4}IH)`h{Y)!v3Z_DHbJlD1$1yzVPQvSmx+jqq9IcsRKlZwaH^Mf z@&Uqa(8x3JXO$h47+OL>40?I5Ibg^j!epUdq3p-qoj>?lldf+23wZfGB81$;kUej- zf-i)7?~r{u8#r8`M9um}>l>g1Bdi-lfQ(CM@w(Fm2ytu@Sb`=SqeeBr=ma-3+w6b` z@s$Le!sLn81({rz@6r>m9$^9y$wdo{6T`s?Cxw@FbpZth(v&4o3zYn$yRATtsR1B^ zoE2T^E{Ls*LTUkLlh+YTxTr=nLw}58jLw@ubAGSLX(~WfDCBmb5Q72MKd~17O`%xA zqu^ds$!c8}Er7o5Xktco;Y-)DL-SBTz>Ks6uSf)hLW4Y@dx>6tv(dc(kdp*X29Z$q zP06y7f>@UcZ6C?}pl9I@#{Hrw!ZiKwA&c0@9td{CocjzVH$f#HyI#deUifh3kj{^7 zE%FD7)Hj|R^KwpPDu4K;@1x_s{LYxIxSgVUH&k(gTGHAX)?-`5FP_;7PH~<4m*cA^ z{gJTAb90gxvK+KsfS%w5kpkpMrSSRSgaYmROm}HqFK#}Cz-mhBjj2I;eBbdvw{YL+ zxVR9AT|NT!ZAg3uln%T#XRB2+_k|lIf@s`Y{EWFn|BE8=d%R--11!}*+mg0>#p@dd z*f!sI7&o1PXi?h^g8jA;AKk3^8ekj2g9oEMR8_SPLEGohKF-9bi}nB~xi#V4E8= zo4d`O5<`qCN+>cL@LwXp26--)n}-TzQYKgHP0YMsQf9)cX z76!wJMnv)t_xg(-fSphO7+Qfz_7oa4m}KkAjSnNqj2R|E@KRuG9=q&|!)6?GT-KK# z4~4k4fT+hip_)K(@F!=Hf*go&Icb{V5Zqp*@3Da{lStH`NAXFz`GcLseU3+t#(C0M z00xePq5>Ir5s|d`$r_kn&tcENCyhVrVg}*6Htv7C8d^cVHZgILK+j&2l>j+-Wi7i7T8Wzd774H-uNzPs!mwmpGJH3%U z`u4}(-azQj%c8*K2dAZ3M~tB#&Y7Uc^q}k6^Zd|X-EPC%b8{Dp$6|{gm)PKJyrGdv z*4pOwJKnjiM~?6*JQ_dT>!gs7hK)^;M=wI}8(KCDZ^Z^#W^G6WF`F`*xOt$*#bYeb zDf!-*?(DB)%YIB9Kjdw;%-5IO_-4ql$)q6+#4`}H(`avN#&J~n5Pt6VNZWewfX@BY zL^faFcM5ZJ_npT41g%gP|36~>O0;KDRtg+iHzhQh9UMu|+t;<*BDo|->`1avPT=3O zcj@zKv=l^gND)AugeRf(0Mb>3j;!~~m(Hicw{SRRcbi>eJ~^av8o(hr0aD)yxIO-H zFue&!MR01$<6IS8G`^h?X{y#QRnxcKecx43O>}18%(?~)Sk!%)Viv+wi449U7d0@BN~iV zm$+n!sm$j`vsR;Fi(Z!45x(qCb(3ejaexfgK8^z=ojTI%q9AQaXdh`B39L((U!9va z(9Zb}z4ne7@UK7Q9cXoNGsuoW*lCn=5UIVG>d-3hJFuqoZS3c%KG(7t#8>r>j4VfO z%d42nZd-9HNhLt;e8`qVn#JNEmM+xvwQZp{3cVzv<}PW+HJqb+kC0s@zan7R0G$IcR~+?0YBAnNFm@qevu}Jk+}t)@V}PG^es(` zJP}^`HW3uV6_mUmOt`OjCV@F|KyD1**agJ6l!5ylS@j?`_3vdheM{IvGI4o0Inecr zUJs4|!J=UOPiy4=Su+3g9}kjn6J~_`v~Ag)-df`4;el`xVwfE!*gMMFOC{Z4fKIY(1G3h%%0Av`Xum{8W#$06As1EH8M8#$UiwA$Hp5Q-t zJsoicV2rZ(5C%t1t!!>yk)54QHYZ?Y`Y0MB9(Y5RAPfg!1RT+H1kt{9$#$CP*)U-2 zH!=#rUWAZ_OVGV-d-)tTT3g#CKn6&}7+SO4kiq~h1>QZ50TGEA%-H5hRRhcv?}l?s z5b(ihV`GC&JQemqLR;R-J=_|Mh80d)WZt1ALs_6y=gb#?5?WbX8`u=r;4|BR7;fmt z`-(8mRcO9ob=gc}5s;?*d2+zDOiD)PTvnDCFeYiV$bW=ek}Jcckc0!_B;gcM>N*OK zE-%^%6~v$7m8{a(Vh7m$Pq;w5qKhB~HwfE<@XIy;dV4jo--!bWXOA$t6kzmK#vvQz z`-l|VXR6h>n0F_bZnBWjgg@~At%3w58uaND)I@^NZ=55rADDOyFMkxk*j(tZ68IZS z0faviD65cid=b`$rR1v^87r}#(Dk!Xpclv)z-K`i|TwOGKFyo)U)C=*yRBBqb$r1Ove(hlhtB z_$_QB6E{vRVup>KHc<7ejK0&ZSfP|VnIbYc{@`ivsS7xbU{V%vR6;EhgKj`RD--o| ztV@g7-)r$RgcKD`2Foy!h$Q<=gte4l(&13Ju8TM4=*`LLss6sc3z(!`yu3t%*#T{; z6O_V350h3s-cy((#m(+tVCGSbhEz78oq?P_ya&2mF|ja;AJVNW+w7pAu!U2yfh?C; z(;_?RD6i2DH60zD3)(`0@Tq?OtksrhA?XnT9FiUHW5QAX;AnCpF|Goq9uWOt(amSr zIvjR{va5q8R_zADW%km{!x?mHR*xFGHhm)_5sX*&Cg%=zY5YP>s0HT(?EZJHvXP^tBF?+lzMGdoB z4I$R8pD{A?D$gYO8N>5j1NZ8mAZ2_b`5hwrZZa!eZ&Zfs&v|dF;x0k`q$gV`@~zOL zR(_sz0W1Iuh*fi%ra>+8XRL7ZPfrc6B8!`y5rN_c_TGHhOyCMZsrbw3^W?0yj*jCq z=Ku#?92eshi8tV(*S9Uj5|Bg?P2inmzl%U+)F4#{%0~j6@N;ba*y%yDy|^MqPNF6o z8lFKfSSX@a!79f&WQW9$ls|axjYOW}I9({Dki9^`>V;=v57(S={~=-FTV{DL z`^V$kEukJ?tKy?b%=i#5QWR0Aff=L)xw#o{6`hBN2Wi&eA;>B!#}C047rX4e;ky^A zOnd$C$;Y0OP6hQ3^o~qV?%j7?BM3&KgN9ffQW99QY2aH3g(dYV3K`a(JguQ`hhs^D z8$eKU=rNv0n*Z;`hH3mCR40w$2MBDof>d!W5DijSVE0B-uYEW#`-z&+%2rceTm;G1 zLY+XZrv=~<|JE%aBz9O%p*afGC0$F`1}9R;`XtfM0y z;AAYiutKPWX#>jr782P7Khz)A*H^;|)4}%66ji%HL`1|Ecg$e@0H#)9cn<}k0o-M1 zcolR&>(G+CuVZ>YO2c*uu3WoT8^?Qv0a$bK^b`RMV?X^3kSq91cIczhQd89+jj00{ z5WkSw7KVHhxY`9_GYW3Biv@DjKKBa@D7h+%A5vcV7aq{{^?mxJiKP+-mTUa`cYcHl z-J9~h2kkMO6YU*-xc6z9ncB#;(F7ur05w>1ON$T&-vK-@VPHDZ+a$MdzdpM_%MX$Z z6ejkBy8Fl9Uclgm)2xLsaR{U35kSXQ92e$r-30pThXR*KI~GMkH*pI6Ml1_7ToLNz f;JyFtA18i$p1RGQOhYA7_|G0qy logs/log_0 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 diff --git a/research/mi_poison_2022/scripts/train_demo_multigpu.sh b/research/mi_poison_2022/scripts/train_demo_multigpu.sh new file mode 100644 index 00000000..7d8d81fb --- /dev/null +++ b/research/mi_poison_2022/scripts/train_demo_multigpu.sh @@ -0,0 +1,32 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 0 --logdir exp/cifar10 &> logs/log_0 & +CUDA_VISIBLE_DEVICES='1' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 1 --logdir exp/cifar10 &> logs/log_1 & +CUDA_VISIBLE_DEVICES='2' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 2 --logdir exp/cifar10 &> logs/log_2 & +CUDA_VISIBLE_DEVICES='3' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 3 --logdir exp/cifar10 &> logs/log_3 & +CUDA_VISIBLE_DEVICES='4' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 4 --logdir exp/cifar10 &> logs/log_4 & +CUDA_VISIBLE_DEVICES='5' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 5 --logdir exp/cifar10 &> logs/log_5 & +CUDA_VISIBLE_DEVICES='6' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 6 --logdir exp/cifar10 &> logs/log_6 & +CUDA_VISIBLE_DEVICES='7' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 7 --logdir exp/cifar10 &> logs/log_7 & +wait; +CUDA_VISIBLE_DEVICES='0' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 8 --logdir exp/cifar10 &> logs/log_8 & +CUDA_VISIBLE_DEVICES='1' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 9 --logdir exp/cifar10 &> logs/log_9 & +CUDA_VISIBLE_DEVICES='2' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 10 --logdir exp/cifar10 &> logs/log_10 & +CUDA_VISIBLE_DEVICES='3' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 11 --logdir exp/cifar10 &> logs/log_11 & +CUDA_VISIBLE_DEVICES='4' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 12 --logdir exp/cifar10 &> logs/log_12 & +CUDA_VISIBLE_DEVICES='5' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 13 --logdir exp/cifar10 &> logs/log_13 & +CUDA_VISIBLE_DEVICES='6' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 14 --logdir exp/cifar10 &> logs/log_14 & +CUDA_VISIBLE_DEVICES='7' python3 -u train_poison.py --dataset=cifar10 --epochs=100 --save_steps=20 --arch wrn28-2 --num_experiments 16 --expid 15 --logdir exp/cifar10 &> logs/log_15 & +wait; diff --git a/research/mi_poison_2022/train_poison.py b/research/mi_poison_2022/train_poison.py new file mode 100644 index 00000000..ae7b788f --- /dev/null +++ b/research/mi_poison_2022/train_poison.py @@ -0,0 +1,198 @@ +# Copyright 2021 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import os +import shutil +import json + +import numpy as np +import tensorflow as tf # For data augmentation. +import tensorflow_datasets as tfds +from absl import app, flags + +from objax.util import EasyDict + +# from mi_lira_2021 +from dataset import DataSet +from train import augment, MemModule, network + +FLAGS = flags.FLAGS + + +def get_data(seed): + """ + This is the function to generate subsets of the data for training models. + + First, we get the training dataset either from the numpy cache + or otherwise we load it from tensorflow datasets. + + Then, we compute the subset. This works in one of two ways. + + 1. If we have a seed, then we just randomly choose examples based on + a prng with that seed, keeping FLAGS.pkeep fraction of the data. + + 2. Otherwise, if we have an experiment ID, then we do something fancier. + If we run each experiment independently then even after a lot of trials + there will still probably be some examples that were always included + or always excluded. So instead, with experiment IDs, we guarantee that + after FLAGS.num_experiments are done, each example is seen exactly half + of the time in train, and half of the time not in train. + + """ + DATA_DIR = os.path.join(os.environ['HOME'], 'TFDS') + + if os.path.exists(os.path.join(FLAGS.logdir, "x_train.npy")): + inputs = np.load(os.path.join(FLAGS.logdir, "x_train.npy")) + labels = np.load(os.path.join(FLAGS.logdir, "y_train.npy")) + else: + print("First time, creating dataset") + data = tfds.as_numpy(tfds.load(name=FLAGS.dataset, batch_size=-1, data_dir=DATA_DIR)) + inputs = data['train']['image'] + labels = data['train']['label'] + + inputs = (inputs/127.5)-1 + np.save(os.path.join(FLAGS.logdir, "x_train.npy"), inputs) + np.save(os.path.join(FLAGS.logdir, "y_train.npy"), labels) + + nclass = np.max(labels)+1 + + np.random.seed(seed) + if FLAGS.num_experiments is not None: + np.random.seed(0) + keep = np.random.uniform(0, 1, size=(FLAGS.num_experiments, len(inputs))) + order = keep.argsort(0) + keep = order < int(FLAGS.pkeep * FLAGS.num_experiments) + keep = np.array(keep[FLAGS.expid], dtype=bool) + else: + keep = np.random.uniform(0, 1, size=len(inputs)) <= FLAGS.pkeep + + xs = inputs[keep] + ys = labels[keep] + + if FLAGS.num_poison_targets > 0: + + # select some points as targets + np.random.seed(FLAGS.poison_pos_seed) + poison_pos = np.random.choice(len(inputs), size=FLAGS.num_poison_targets, replace=False) + + # create mislabeled poisons for the targeted points and replicate each + # poison `FLAGS.poison_reps` times + y_noise = np.mod(labels[poison_pos] + np.random.randint(low=1, high=nclass, size=FLAGS.num_poison_targets), nclass) + ypoison = np.repeat(y_noise, FLAGS.poison_reps) + xpoison = np.repeat(inputs[poison_pos], FLAGS.poison_reps, axis=0) + xs = np.concatenate((xs, xpoison), axis=0) + ys = np.concatenate((ys, ypoison), axis=0) + + if not os.path.exists(os.path.join(FLAGS.logdir, "poison_pos.npy")): + np.save(os.path.join(FLAGS.logdir, "poison_pos.npy"), poison_pos) + + if FLAGS.augment == 'weak': + aug = lambda x: augment(x, 4) + elif FLAGS.augment == 'mirror': + aug = lambda x: augment(x, 0) + elif FLAGS.augment == 'none': + aug = lambda x: augment(x, 0, mirror=False) + else: + raise + + print(xs.shape, ys.shape) + train = DataSet.from_arrays(xs, ys, + augment_fn=aug) + test = DataSet.from_tfds(tfds.load(name=FLAGS.dataset, split='test', data_dir=DATA_DIR), xs.shape[1:]) + train = train.cache().shuffle(len(xs)).repeat().parse().augment().batch(FLAGS.batch) + train = train.nchw().one_hot(nclass).prefetch(FLAGS.batch) + test = test.cache().parse().batch(FLAGS.batch).nchw().prefetch(FLAGS.batch) + + return train, test, xs, ys, keep, nclass + + +def main(argv): + del argv + tf.config.experimental.set_visible_devices([], "GPU") + + seed = FLAGS.seed + if seed is None: + import time + seed = np.random.randint(0, 1000000000) + seed ^= int(time.time()) + + args = EasyDict(arch=FLAGS.arch, + lr=FLAGS.lr, + batch=FLAGS.batch, + weight_decay=FLAGS.weight_decay, + augment=FLAGS.augment, + seed=seed) + + if FLAGS.expid is not None: + logdir = "experiment-%d_%d" % (FLAGS.expid, FLAGS.num_experiments) + else: + logdir = "experiment-"+str(seed) + logdir = os.path.join(FLAGS.logdir, logdir) + + if os.path.exists(os.path.join(logdir, "ckpt", "%010d.npz" % FLAGS.epochs)): + print(f"run {FLAGS.expid} already completed.") + return + else: + if os.path.exists(logdir): + print(f"deleting run {FLAGS.expid} that did not complete.") + shutil.rmtree(logdir) + + print(f"starting run {FLAGS.expid}.") + if not os.path.exists(logdir): + os.makedirs(logdir) + + train, test, xs, ys, keep, nclass = get_data(seed) + + # Define the network and train_it + tm = MemModule(network(FLAGS.arch), nclass=nclass, + mnist=FLAGS.dataset == 'mnist', + epochs=FLAGS.epochs, + expid=FLAGS.expid, + num_experiments=FLAGS.num_experiments, + pkeep=FLAGS.pkeep, + save_steps=FLAGS.save_steps, + **args + ) + + r = {} + r.update(tm.params) + + open(os.path.join(logdir, 'hparams.json'), "w").write(json.dumps(tm.params)) + np.save(os.path.join(logdir,'keep.npy'), keep) + + tm.train(FLAGS.epochs, len(xs), train, test, logdir, + save_steps=FLAGS.save_steps) + + +if __name__ == '__main__': + flags.DEFINE_string('arch', 'cnn32-3-mean', 'Model architecture.') + flags.DEFINE_float('lr', 0.1, 'Learning rate.') + flags.DEFINE_string('dataset', 'cifar10', 'Dataset.') + flags.DEFINE_float('weight_decay', 0.0005, 'Weight decay ratio.') + flags.DEFINE_integer('batch', 256, 'Batch size') + flags.DEFINE_integer('epochs', 100, 'Training duration in number of epochs.') + flags.DEFINE_string('logdir', 'experiments', 'Directory where to save checkpoints and tensorboard data.') + flags.DEFINE_integer('seed', None, 'Training seed.') + flags.DEFINE_float('pkeep', .5, 'Probability to keep examples.') + flags.DEFINE_integer('expid', None, 'Experiment ID') + flags.DEFINE_integer('num_experiments', None, 'Number of experiments') + flags.DEFINE_string('augment', 'weak', 'Strong or weak augmentation') + flags.DEFINE_integer('eval_steps', 1, 'how often to get eval accuracy.') + flags.DEFINE_integer('save_steps', 10, 'how often to get save model.') + + flags.DEFINE_integer('num_poison_targets', 250, 'Number of points to target ' + 'with the poisoning attack.') + flags.DEFINE_integer('poison_reps', 8, 'Number of times to repeat each poison.') + flags.DEFINE_integer('poison_pos_seed', 0, '') + app.run(main) From 182d5e629549da19da3f45cadfe3cf2c71680401 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:46:39 +0000 Subject: [PATCH 08/10] fix plot output --- research/mi_poison_2022/README.md | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md index 071c209b..dd35fe04 100644 --- a/research/mi_poison_2022/README.md +++ b/research/mi_poison_2022/README.md @@ -88,11 +88,11 @@ which should give (something like) the following output ![Log-log ROC Curve for all attacks](fprtpr.png "Log-log ROC Curve") ``` -Attack No poison +Attack No poison (LiRA) AUC 0.7025, Accuracy 0.6258, TPR@0.1%FPR of 0.0544 Attack No poison (Global threshold) AUC 0.6191, Accuracy 0.6173, TPR@0.1%FPR of 0.0012 -Attack With poison +Attack With poison (LiRA) AUC 0.9943, Accuracy 0.9653, TPR@0.1%FPR of 0.4945 Attack With poison (Global threshold) AUC 0.9922, Accuracy 0.9603, TPR@0.1%FPR of 0.3930 From d87f13c7996d8500d390a06e3123f7d172cfa0a8 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:47:50 +0000 Subject: [PATCH 09/10] fix names --- research/mi_poison_2022/README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/research/mi_poison_2022/README.md b/research/mi_poison_2022/README.md index dd35fe04..670981fa 100644 --- a/research/mi_poison_2022/README.md +++ b/research/mi_poison_2022/README.md @@ -111,7 +111,7 @@ You can cite this paper with ``` @article{tramer2022truth, title={Truth Serum: Poisoning Machine Learning Models to Reveal Their Secrets}, - author={Tramer, Florian and Shokri, Reza and Joaquin, Ayrton San and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, + author={Tramer, Florian and Shokri, Reza and San Joaquin, Ayrton and Le, Hoang and Jagielski, Matthew and Hong, Sanghyun and Carlini, Nicholas}, journal={arXiv preprint arXiv:2204.00032}, year={2022} } From 7fd9637d0f124478accc3c51a783b8710b9d9337 Mon Sep 17 00:00:00 2001 From: Florian Tramer Date: Thu, 28 Apr 2022 16:53:24 +0000 Subject: [PATCH 10/10] some comments --- research/mi_poison_2022/plot_poison.py | 6 ++++++ research/mi_poison_2022/train_poison.py | 13 +++++++++++++ 2 files changed, 19 insertions(+) diff --git a/research/mi_poison_2022/plot_poison.py b/research/mi_poison_2022/plot_poison.py index ce7bb787..bb156506 100644 --- a/research/mi_poison_2022/plot_poison.py +++ b/research/mi_poison_2022/plot_poison.py @@ -62,24 +62,29 @@ def do_plot_all(fn, keep, scores, legend='', metric='auc', sweep_fn=sweep, **plo def fig_fpr_tpr(poison_mask, scores, keep): plt.figure(figsize=(4, 3)) + + # evaluate LiRA on the points that were not targeted by poisoning do_plot_all(functools.partial(generate_ours, fix_variance=True), keep[:, ~poison_mask], scores[:, ~poison_mask], "No poison (LiRA)\n", metric='auc', ) + # evaluate the global-threshold attack on the points that were not targeted by poisoning do_plot_all(generate_global, keep[:, ~poison_mask], scores[:, ~poison_mask], "No poison (Global threshold)\n", metric='auc', ls="--", c=plt.gca().lines[-1].get_color() ) + # evaluate LiRA on the points that were targeted by poisoning do_plot_all(functools.partial(generate_ours, fix_variance=True), keep[:, poison_mask], scores[:, poison_mask], "With poison (LiRA)\n", metric='auc', ) + # evaluate the global-threshold attack on the points that were targeted by poisoning do_plot_all(generate_global, keep[:, poison_mask], scores[:, poison_mask], "With poison (Global threshold)\n", @@ -98,6 +103,7 @@ def fig_fpr_tpr(poison_mask, scores, keep): plt.savefig("/tmp/fprtpr.png") plt.show() + if __name__ == '__main__': logdir = "exp/cifar10/" scores, keep = load_data(logdir) diff --git a/research/mi_poison_2022/train_poison.py b/research/mi_poison_2022/train_poison.py index ae7b788f..d7a57cab 100644 --- a/research/mi_poison_2022/train_poison.py +++ b/research/mi_poison_2022/train_poison.py @@ -49,6 +49,19 @@ def get_data(seed): after FLAGS.num_experiments are done, each example is seen exactly half of the time in train, and half of the time not in train. + Finally, we add some poisons. The same poisoned samples are added for + each randomly generated training set. + We first select FLAGS.num_poison_targets victim points that will be targeted + by the poisoning attack. For each of these victim points, the attacker will + insert FLAGS.poison_reps mislabeled replicas of the point into the training + set. + + For CIFAR-10, we recommend that: + + `FLAGS.num_poison_targets * FLAGS.poison_reps < 5000` + + Otherwise, the poisons might introduce too much label noise and the model's + accuracy (and the attack's success rate) will be degraded. """ DATA_DIR = os.path.join(os.environ['HOME'], 'TFDS')