Skip to content

Commit

Permalink
update result table based on new tdc submission results
Browse files Browse the repository at this point in the history
  • Loading branch information
Blazej Banaszewski committed Aug 2, 2024
1 parent 0259dc5 commit c259bd4
Show file tree
Hide file tree
Showing 2 changed files with 46 additions and 75 deletions.
46 changes: 23 additions & 23 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -47,33 +47,33 @@ The model has been evaluated on 22 benchmarks from the ADMET group of [Therapeut
|----------------------|----------|------------|-----------------|----------------|----------|---------------|----------|
| **Name** | **Size** | **Metric** | **SoTA Result** | **Result** | **Rank** | **Result** | **Rank** |
| **Absorption** | | | | | | | |
| Caco2 Wang | 906 | MAE | 0.276 ± 0.005 | 0.310 ± 0.010 | 6 | 0.324 ± 0.012 | 7 |
| Bioavailability Ma | 640 | AUROC | 0.748 ± 0.033 | 0.654 ± 0.028 | 7 | 0.699 ± 0.008 | 6 |
| Lipophilicity AZ | 4,200 | MAE | 0.467 ± 0.006 | 0.469 ± 0.009 | 3 | 0.455 ± 0.001 | 1 |
| Solubility AqSolDB | 9,982 | MAE | 0.761 ± 0.025 | 0.792 ± 0.005 | 5 | 0.750 ± 0.012 | 1 |
| HIA Hou | 578 | AUROC | 0.989 ± 0.001 | 0.963 ± 0.019 | 7 | 0.994 ± 0.003 | 1 |
| Pgp Broccatelli | 1,212 | AUROC | 0.938 ± 0.002 | 0.915 ± 0.005 | 7 | 0.994 ± 0.002 | 1 |
| Caco2 Wang | 906 | MAE | 0.276 ± 0.005 | 0.310 ± 0.010 | 6 | 0.350 ± 0.018 | 7 |
| Bioavailability Ma | 640 | AUROC | 0.748 ± 0.033 | 0.654 ± 0.028 | 7 | 0.689 ± 0.020 | 5 |
| Lipophilicity AZ | 4,200 | MAE | 0.467 ± 0.006 | 0.469 ± 0.009 | 3 | 0.456 ± 0.008 | 1 |
| Solubility AqSolDB | 9,982 | MAE | 0.761 ± 0.025 | 0.792 ± 0.005 | 5 | 0.741 ± 0.013 | 1 |
| HIA Hou | 578 | AUROC | 0.989 ± 0.001 | 0.963 ± 0.019 | 7 | 0.993 ± 0.005 | 1 |
| Pgp Broccatelli | 1,212 | AUROC | 0.938 ± 0.002 | 0.915 ± 0.005 | 7 | 0.942 ± 0.002 | 1 |
| **Distribution** | | | | | | | |
| BBB Martins | 1,975 | AUROC | 0.916 ± 0.001 | 0.903 ± 0.005 | 7 | 0.923 ± 0.002 | 1 |
| PPBR AZ | 1,797 | MAE | 7.526 ± 0.106 | 8.073 ± 0.335 | 6 | 7.807 ± 0.188 | 4 |
| VDss Lombardo | 1,130 | Spearman | 0.713 ± 0.007 | 0.654 ± 0.031 | 3 | 0.570 ± 0.015 | 7 |
| BBB Martins | 1,975 | AUROC | 0.916 ± 0.001 | 0.903 ± 0.005 | 7 | 0.924 ± 0.003 | 1 |
| PPBR AZ | 1,797 | MAE | 7.526 ± 0.106 | 8.073 ± 0.335 | 6 | 7.696 ± 0.125 | 4 |
| VDss Lombardo | 1,130 | Spearman | 0.713 ± 0.007 | 0.654 ± 0.031 | 3 | 0.535 ± 0.027 | 7 |
| **Metabolism** | | | | | | | |
| CYP2C9 Veith | 12,092 | AUPRC | 0.859 ± 0.001 | 0.801 ± 0.003 | 5 | 0.819 ± 0.001 | 4 |
| CYP2D6 Veith | 13,130 | AUPRC | 0.790 ± 0.001 | 0.682 ± 0.008 | 6 | 0.718 ± 0.003 | 5 |
| CYP3A4 Veith | 12,328 | AUPRC | 0.916 ± 0.000 | 0.867 ± 0.003 | 7 | 0.878 ± 0.001 | 5 |
| CYP2C9 Substrate | 666 | AUPRC | 0.441 ± 0.033 | 0.446 ± 0.062 | 2 | 0.481 ± 0.013 | 1 |
| CYP2D6 Substrate | 664 | AUPRC | 0.736 ± 0.024 | 0.699 ± 0.018 | 7 | 0.726 ± 0.006 | 2 |
| CYP3A4 Substrate | 667 | AUROC | 0.662 ± 0.031 | 0.670 ± 0.018 | 1 | 0.644 ± 0.006 | 6 |
| CYP2C9 Veith | 12,092 | AUPRC | 0.859 ± 0.001 | 0.801 ± 0.003 | 5 | 0.823 ± 0.006 | 4 |
| CYP2D6 Veith | 13,130 | AUPRC | 0.790 ± 0.001 | 0.682 ± 0.008 | 6 | 0.719 ± 0.004 | 5 |
| CYP3A4 Veith | 12,328 | AUPRC | 0.916 ± 0.000 | 0.867 ± 0.003 | 7 | 0.877 ± 0.001 | 4 |
| CYP2C9 Substrate | 666 | AUPRC | 0.441 ± 0.033 | 0.446 ± 0.062 | 2 | 0.474 ± 0.025 | 1 |
| CYP2D6 Substrate | 664 | AUPRC | 0.736 ± 0.024 | 0.699 ± 0.018 | 7 | 0.695 ± 0.032 | 6 |
| CYP3A4 Substrate | 667 | AUROC | 0.662 ± 0.031 | 0.670 ± 0.018 | 1 | 0.663 ± 0.008 | 2 |
| **Excretion** | | | | | | | |
| Half Life Obach | 667 | Spearman | 0.562 ± 0.008 | 0.549 ± 0.024 | 4 | 0.493 ± 0.002 | 7 |
| Clearance Hepatocyte | 1,102 | Spearman | 0.498 ± 0.009 | 0.381 ± 0.038 | 7 | 0.448 ± 0.006 | 4 |
| Clearance Microsome | 1,020 | Spearman | 0.630 ± 0.010 | 0.607 ± 0.027 | 6 | 0.652 ± 0.007 | 1 |
| Half Life Obach | 667 | Spearman | 0.562 ± 0.008 | 0.549 ± 0.024 | 4 | 0.495 ± 0.042 | 6 |
| Clearance Hepatocyte | 1,102 | Spearman | 0.498 ± 0.009 | 0.381 ± 0.038 | 7 | 0.446 ± 0.029 | 3 |
| Clearance Microsome | 1,020 | Spearman | 0.630 ± 0.010 | 0.607 ± 0.027 | 6 | 0.628 ± 0.005 | 2 |
| **Toxicity** | | | | | | | |
| LD50 Zhu | 7,385 | MAE | 0.552 ± 0.009 | 0.823 ± 0.019 | 7 | 0.588 ± 0.010 | 3 |
| hERG | 648 | AUROC | 0.880 ± 0.002 | 0.813 ± 0.009 | 7 | 0.849 ± 0.007 | 6 |
| Ames | 7,255 | AUROC | 0.871 ± 0.002 | 0.883 ± 0.005 | 1 | 0.856 ± 0.001 | 5 |
| DILI | 475 | AUROC | 0.925 ± 0.005 | 0.577 ± 0.021 | 7 | 0.944 ± 0.007 | 1 |
| | | | | **Mean Rank:** | 5.2 | | 3.4 |
| LD50 Zhu | 7,385 | MAE | 0.552 ± 0.009 | 0.823 ± 0.019 | 7 | 0.585 ± 0.005 | 2 |
| hERG | 648 | AUROC | 0.880 ± 0.002 | 0.813 ± 0.009 | 7 | 0.846 ± 0.016 | 4 |
| Ames | 7,255 | AUROC | 0.871 ± 0.002 | 0.883 ± 0.005 | 1 | 0.849 ± 0.004 | 5 |
| DILI | 475 | AUROC | 0.925 ± 0.005 | 0.577 ± 0.021 | 7 | 0.956 ± 0.006 | 1 |
| | | | | **Mean Rank:** | 5.2 | | 3.3 |

## License

Expand Down
75 changes: 23 additions & 52 deletions tdc_leaderboard_submission.py
View file
Original file line number Diff line number Diff line change
Expand Up @@ -143,57 +143,29 @@ def __getitem__(self, idx):
REPETITIONS = 5
ENSEMBLE_SIZE = 5
RESULTS_FILE_PATH = 'results_best_val.pkl'
DEFAULT_HEAD_HPARAMS = {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003}
MODE = 'best_val'
SWEEP_RESULTS = {
'best_val': {
'caco2_wang': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'hia_hou': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'pgp_broccatelli': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0003},
'bioavailability_ma': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003},
'lipophilicity_astrazeneca': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'solubility_aqsoldb': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0005},
'bbb_martins': {'hidden_dim': 2048, 'depth': 3, 'combine': True, 'lr': 0.0001},
'ppbr_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'vdss_lombardo': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
'cyp2d6_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp3a4_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2c9_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2d6_substrate_carbonmangels': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp3a4_substrate_carbonmangels': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2c9_substrate_carbonmangels': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0005},
'half_life_obach': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0003},
'clearance_microsome_az': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0005},
'clearance_hepatocyte_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'herg': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003},
'ames': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'dili': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0005},
'ld50_zhu': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
},
'best_test': {
'caco2_wang': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003},
'hia_hou': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0001},
'pgp_broccatelli': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0001},
'bioavailability_ma': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0001},
'lipophilicity_astrazeneca': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'solubility_aqsoldb': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
'bbb_martins': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0001},
'ppbr_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'vdss_lombardo': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0003},
'cyp2d6_veith': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'cyp3a4_veith': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0005},
'cyp2c9_veith': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'cyp2d6_substrate_carbonmangels': {'hidden_dim': 2048, 'depth': 3, 'combine': True, 'lr': 0.0003},
'cyp3a4_substrate_carbonmangels': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
'cyp2c9_substrate_carbonmangels': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0001},
'half_life_obach': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0005},
'clearance_microsome_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'clearance_hepatocyte_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'herg': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'ames': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'dili': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0005},
'ld50_zhu': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0003},
},
'caco2_wang': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'hia_hou': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'pgp_broccatelli': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0003},
'bioavailability_ma': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003},
'lipophilicity_astrazeneca': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'solubility_aqsoldb': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0005},
'bbb_martins': {'hidden_dim': 2048, 'depth': 3, 'combine': True, 'lr': 0.0001},
'ppbr_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0003},
'vdss_lombardo': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
'cyp2d6_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp3a4_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2c9_veith': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2d6_substrate_carbonmangels': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp3a4_substrate_carbonmangels': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'cyp2c9_substrate_carbonmangels': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0005},
'half_life_obach': {'hidden_dim': 1024, 'depth': 3, 'combine': True, 'lr': 0.0003},
'clearance_microsome_az': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0005},
'clearance_hepatocyte_az': {'hidden_dim': 2048, 'depth': 4, 'combine': True, 'lr': 0.0005},
'herg': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0003},
'ames': {'hidden_dim': 512, 'depth': 3, 'combine': True, 'lr': 0.0001},
'dili': {'hidden_dim': 512, 'depth': 4, 'combine': True, 'lr': 0.0005},
'ld50_zhu': {'hidden_dim': 1024, 'depth': 4, 'combine': True, 'lr': 0.0001},
}

if os.path.exists(RESULTS_FILE_PATH):
Expand Down Expand Up @@ -236,9 +208,8 @@ def __getitem__(self, idx):
val_loader = DataLoader(AdmetDataset(mols_valid), batch_size=128, shuffle=False)
train_loader = DataLoader(AdmetDataset(mols_train), batch_size=32, shuffle=True)

hparams = SWEEP_RESULTS[MODE][dataset_name]
hparams = SWEEP_RESULTS[dataset_name]
model, optimiser, lr_scheduler, loss_fn = model_factory(**hparams, task=task)
# model, optimiser, lr_scheduler, loss_fn = model_factory(**DEFAULT_HEAD_HPARAMS, task=task)

best_epoch = {"model": None, "result": None}

Expand Down

0 comments on commit c259bd4

Please sign in to comment.