Skip to content

Commit

Permalink
Merge pull request #580 from adamdecaf/server-more-download-flags
Browse files Browse the repository at this point in the history 
cmd/server: add more configuration for disabling default lists
  • Loading branch information
@adamdecaf
adamdecaf authored Dec 13, 2024
2 parents 5a87eee + 017a192 commit d985d75
Show file tree
Hide file tree
Showing 3 changed files with 53 additions and 33 deletions.
3 changes: 3 additions & 0 deletions README.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -220,6 +220,9 @@ PONG
| `OFAC_DOWNLOAD_TEMPLATE` | HTTP address for downloading raw OFAC files. | `https://www.treasury.gov/ofac/downloads/%s` |
| `DPL_DOWNLOAD_TEMPLATE` | HTTP address for downloading the DPL. | `https://www.bis.doc.gov/dpl/%s` |
| `EU_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading EU Consolidated Screening List | Subresource of `webgate.ec.europa.eu` |
| `WITH_OFAC_LIST` | Download and parse the US OFAC List | Default: `true` |
| `WITH_US_DPL_LIST` | Download and parse the US Denied Persons List (DPL) | Default: `true` |
| `WITH_US_CSL_SANCTIONS_LIST` | Download and parse the US Consolidated Screening List | Default: `true` |
| `WITH_EU_SCREENING_LIST` | Download and parse the EU Consolidated Screening List | Default: `true` |
| `WITH_UK_CSL_SANCTIONS_LIST` | Download and parse the UK CSL Sanctions List on startup. | Default: `true` |
| `UK_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading UK Consolidated Screening List | Subresource of `www.gov.uk` |
Expand Down
80 changes: 47 additions & 33 deletions cmd/server/download.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -263,26 +263,35 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) {
RefreshedAt: lastRefresh(initialDir),
}

var err error
lastDataRefreshFailure.WithLabelValues("SDNs").Set(float64(time.Now().Unix()))

results, err := ofacRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("SDNs").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("OFAC: %v", err))
var ofacResults *ofac.Results
withOFACList := cmp.Or(os.Getenv("WITH_OFAC_LIST"), "true")
if strx.Yes(withOFACList) {
ofacResults, err = ofacRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("SDNs").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("OFAC: %v", err))
}
}
if results == nil {
results = &ofac.Results{}
if ofacResults == nil {
ofacResults = &ofac.Results{}
}

sdns := precomputeSDNs(results.SDNs, results.Addresses, s.pipe)
adds := precomputeAddresses(results.Addresses)
alts := precomputeAlts(results.AlternateIdentities, s.pipe)
sdnComments := results.SDNComments
sdns := precomputeSDNs(ofacResults.SDNs, ofacResults.Addresses, s.pipe)
adds := precomputeAddresses(ofacResults.Addresses)
alts := precomputeAlts(ofacResults.AlternateIdentities, s.pipe)
sdnComments := ofacResults.SDNComments

deniedPersons, err := dplRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("DPs").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("DPL: %v", err))
var deniedPersons []*dpl.DPL
withDPLList := cmp.Or(os.Getenv("WITH_US_DPL_LIST"), "true")
if strx.Yes(withDPLList) {
deniedPersons, err = dplRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("DPs").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("DPL: %v", err))
}
}
dps := precomputeDPs(deniedPersons, s.pipe)

Expand Down Expand Up @@ -323,25 +332,30 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) {
}

// csl records from US downloaded here
consolidatedLists, err := cslRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("CSL").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("CSL: %v", err))
}
if consolidatedLists == nil {
consolidatedLists = new(csl.CSL)
}
els := precomputeCSLEntities[csl.EL](consolidatedLists.ELs, s.pipe)
meus := precomputeCSLEntities[csl.MEU](consolidatedLists.MEUs, s.pipe)
ssis := precomputeCSLEntities[csl.SSI](consolidatedLists.SSIs, s.pipe)
uvls := precomputeCSLEntities[csl.UVL](consolidatedLists.UVLs, s.pipe)
isns := precomputeCSLEntities[csl.ISN](consolidatedLists.ISNs, s.pipe)
fses := precomputeCSLEntities[csl.FSE](consolidatedLists.FSEs, s.pipe)
plcs := precomputeCSLEntities[csl.PLC](consolidatedLists.PLCs, s.pipe)
caps := precomputeCSLEntities[csl.CAP](consolidatedLists.CAPs, s.pipe)
dtcs := precomputeCSLEntities[csl.DTC](consolidatedLists.DTCs, s.pipe)
cmics := precomputeCSLEntities[csl.CMIC](consolidatedLists.CMICs, s.pipe)
ns_mbss := precomputeCSLEntities[csl.NS_MBS](consolidatedLists.NS_MBSs, s.pipe)
var usConsolidatedLists *csl.CSL
withUSConsolidatedLists := cmp.Or(os.Getenv("WITH_US_CSL_SANCTIONS_LIST"), "true")
if strx.Yes(withUSConsolidatedLists) {
usConsolidatedLists, err = cslRecords(s.logger, initialDir)
if err != nil {
lastDataRefreshFailure.WithLabelValues("CSL").Set(float64(time.Now().Unix()))
stats.Errors = append(stats.Errors, fmt.Errorf("US CSL: %v", err))
}
}
if usConsolidatedLists == nil {
usConsolidatedLists = new(csl.CSL)
}

els := precomputeCSLEntities[csl.EL](usConsolidatedLists.ELs, s.pipe)
meus := precomputeCSLEntities[csl.MEU](usConsolidatedLists.MEUs, s.pipe)
ssis := precomputeCSLEntities[csl.SSI](usConsolidatedLists.SSIs, s.pipe)
uvls := precomputeCSLEntities[csl.UVL](usConsolidatedLists.UVLs, s.pipe)
isns := precomputeCSLEntities[csl.ISN](usConsolidatedLists.ISNs, s.pipe)
fses := precomputeCSLEntities[csl.FSE](usConsolidatedLists.FSEs, s.pipe)
plcs := precomputeCSLEntities[csl.PLC](usConsolidatedLists.PLCs, s.pipe)
caps := precomputeCSLEntities[csl.CAP](usConsolidatedLists.CAPs, s.pipe)
dtcs := precomputeCSLEntities[csl.DTC](usConsolidatedLists.DTCs, s.pipe)
cmics := precomputeCSLEntities[csl.CMIC](usConsolidatedLists.CMICs, s.pipe)
ns_mbss := precomputeCSLEntities[csl.NS_MBS](usConsolidatedLists.NS_MBSs, s.pipe)

// OFAC
stats.SDNs = len(sdns)
Expand Down
3 changes: 3 additions & 0 deletions docs/usage-configuration.md
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,9 @@ menubar: docs-menu
| `OFAC_DOWNLOAD_TEMPLATE` | HTTP address for downloading raw OFAC files. | `https://www.treasury.gov/ofac/downloads/%s` |
| `DPL_DOWNLOAD_TEMPLATE` | HTTP address for downloading the DPL. | `https://www.bis.doc.gov/dpl/%s` |
| `EU_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading EU Consolidated Screening List | Subresource of `webgate.ec.europa.eu` |
| `WITH_OFAC_LIST` | Download and parse the US OFAC List | Default: `true` |
| `WITH_US_DPL_LIST` | Download and parse the US Denied Persons List (DPL) | Default: `true` |
| `WITH_US_CSL_SANCTIONS_LIST` | Download and parse the US Consolidated Screening List | Default: `true` |
| `WITH_EU_SCREENING_LIST` | Download and parse the EU Consolidated Screening List | Default: `true` |
| `WITH_UK_CSL_SANCTIONS_LIST` | Download and parse the UK CSL Sanctions List on startup. | Default: `true` |
| `UK_CSL_DOWNLOAD_URL` | Use an alternate URL for downloading UK Consolidated Screening List | Subresource of `www.gov.uk` |
Expand Down

0 comments on commit d985d75

Please sign in to comment.