Merge pull request #528 from adamdecaf/search-crypto-addresses

feat: add /crypto address and extract digital currency addresses from OFAC

cmd/server/download.go

@@ -281,6 +281,7 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) {
 	sdns := precomputeSDNs(results.SDNs, results.Addresses, s.pipe)
 	adds := precomputeAddresses(results.Addresses)
 	alts := precomputeAlts(results.AlternateIdentities, s.pipe)
+	sdnComments := results.SDNComments
 
 	deniedPersons, err := dplRecords(s.logger, initialDir)
 	if err != nil {
@@ -390,6 +391,7 @@ func (s *searcher) refreshData(initialDir string) (*DownloadStats, error) {
 	s.SDNs = sdns
 	s.Addresses = adds
 	s.Alts = alts
+	s.SDNComments = sdnComments
 	// BIS
 	s.DPs = dps
 	// CSL

cmd/server/search.go

@@ -36,9 +36,10 @@ var (
 // This data comes from various US and EU Federal agencies
 type searcher struct {
 	// OFAC
-	SDNs      []*SDN
-	Addresses []*Address
-	Alts      []*Alt
+	SDNs        []*SDN
+	Addresses   []*Address
+	Alts        []*Alt
+	SDNComments []*ofac.SDNComments
 
 	// BIS
 	DPs []*DP
@@ -410,6 +411,10 @@ func (s *searcher) debugSDN(entityID string) *SDN {
 	s.RLock()
 	defer s.RUnlock()
 
+	return s.findSDNWithoutLock(entityID)
+}
+
+func (s *searcher) findSDNWithoutLock(entityID string) *SDN {
 	for i := range s.SDNs {
 		if s.SDNs[i].EntityID == entityID {
 			return s.SDNs[i]

cmd/server/search_crypto.go

@@ -0,0 +1,74 @@
+// Copyright 2022 The Moov Authors
+// Use of this source code is governed by an Apache License
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"net/http"
+	"strings"
+
+	moovhttp "github.com/moov-io/base/http"
+	"github.com/moov-io/base/log"
+	"github.com/moov-io/watchman/pkg/ofac"
+)
+
+type cryptoAddressSearchResult struct {
+	OFAC []SDNWithDigitalCurrencyAddress `json:"ofac"`
+}
+
+func searchByCryptoAddress(logger log.Logger, searcher *searcher) http.HandlerFunc {
+	return func(w http.ResponseWriter, r *http.Request) {
+		cryptoAddress := strings.TrimSpace(r.URL.Query().Get("address"))
+		cryptoName := strings.TrimSpace(r.URL.Query().Get("name"))
+		if cryptoAddress == "" {
+			moovhttp.Problem(w, errNoSearchParams)
+			return
+		}
+
+		limit := extractSearchLimit(r)
+
+		// Find SDNs with a crypto address that exactly matches
+		resp := cryptoAddressSearchResult{
+			OFAC: searcher.FindSDNCryptoAddresses(limit, cryptoName, cryptoAddress),
+		}
+
+		w.Header().Set("Content-Type", "application/json; charset=utf-8")
+		w.WriteHeader(http.StatusOK)
+		json.NewEncoder(w).Encode(resp)
+	}
+}
+
+type SDNWithDigitalCurrencyAddress struct {
+	SDN *ofac.SDN `json:"sdn"`
+
+	DigitalCurrencyAddresses []ofac.DigitalCurrencyAddress `json:"digitalCurrencyAddresses"`
+}
+
+func (s *searcher) FindSDNCryptoAddresses(limit int, name, needle string) []SDNWithDigitalCurrencyAddress {
+	s.RLock()
+	defer s.RUnlock()
+
+	var out []SDNWithDigitalCurrencyAddress
+	for i := range s.SDNComments {
+		addresses := s.SDNComments[i].DigitalCurrencyAddresses
+		for j := range addresses {
+			// Skip addresses of a different coin
+			if name != "" && addresses[j].Currency != name {
+				continue
+			}
+			if addresses[j].Address == needle {
+				// Find SDN
+				sdn := s.findSDNWithoutLock(s.SDNComments[i].EntityID)
+				if sdn != nil {
+					out = append(out, SDNWithDigitalCurrencyAddress{
+						SDN:                      sdn.SDN,
+						DigitalCurrencyAddresses: addresses,
+					})
+				}
+			}
+		}
+	}
+	return out
+}

cmd/server/search_crypto_test.go

@@ -0,0 +1,92 @@
+// Copyright 2022 The Moov Authors
+// Use of this source code is governed by an Apache License
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"encoding/json"
+	"fmt"
+	"net/http"
+	"net/http/httptest"
+	"path/filepath"
+	"testing"
+
+	"github.com/moov-io/base/log"
+	"github.com/moov-io/watchman/pkg/ofac"
+
+	"github.com/gorilla/mux"
+	"github.com/stretchr/testify/require"
+)
+
+var (
+	cryptoSearcher = newSearcher(log.NewNopLogger(), noLogPipeliner, 1)
+)
+
+func init() {
+	// Set SDN Comments
+	ofacResults, err := ofac.Read(filepath.Join("..", "..", "test", "testdata", "sdn_comments.csv"))
+	if err != nil {
+		panic(fmt.Sprintf("ERROR reading sdn_comments.csv: %v", err))
+	}
+
+	cryptoSearcher.SDNComments = ofacResults.SDNComments
+	cryptoSearcher.SDNs = precomputeSDNs([]*ofac.SDN{
+		{
+			EntityID: "39796", // matches TestSearchCrypto
+			SDNName:  "Person A",
+			SDNType:  "individual",
+			Title:    "Guy or Girl doing crypto stuff",
+		},
+	}, nil, noLogPipeliner)
+}
+
+func TestSearchCryptoSetup(t *testing.T) {
+	require.Len(t, cryptoSearcher.SDNComments, 13)
+	require.Len(t, cryptoSearcher.SDNs, 1)
+}
+
+type expectedCryptoAddressSearchResult struct {
+	OFAC []SDNWithDigitalCurrencyAddress `json:"ofac"`
+}
+
+func TestSearchCrypto(t *testing.T) {
+	router := mux.NewRouter()
+	addSearchRoutes(log.NewNopLogger(), router, cryptoSearcher)
+
+	w := httptest.NewRecorder()
+	req := httptest.NewRequest("GET", "/crypto?address=0x242654336ca2205714071898f67E254EB49ACdCe", nil)
+	router.ServeHTTP(w, req)
+	w.Flush()
+	require.Equal(t, http.StatusOK, w.Code)
+
+	var response expectedCryptoAddressSearchResult
+	err := json.NewDecoder(w.Body).Decode(&response)
+	require.NoError(t, err)
+
+	require.Len(t, response.OFAC, 1)
+	require.Equal(t, "39796", response.OFAC[0].SDN.EntityID)
+
+	// Now with cryptocurrency name specified
+	req = httptest.NewRequest("GET", "/crypto?name=ETH&address=0x242654336ca2205714071898f67E254EB49ACdCe", nil)
+	router.ServeHTTP(w, req)
+	w.Flush()
+	require.Equal(t, http.StatusOK, w.Code)
+
+	err = json.NewDecoder(w.Body).Decode(&response)
+	require.NoError(t, err)
+
+	require.Len(t, response.OFAC, 1)
+	require.Equal(t, "39796", response.OFAC[0].SDN.EntityID)
+
+	// With wrong cryptocurrency name
+	req = httptest.NewRequest("GET", "/crypto?name=QRR&address=0x242654336ca2205714071898f67E254EB49ACdCe", nil)
+	router.ServeHTTP(w, req)
+	w.Flush()
+	require.Equal(t, http.StatusOK, w.Code)
+
+	err = json.NewDecoder(w.Body).Decode(&response)
+	require.NoError(t, err)
+
+	require.Len(t, response.OFAC, 0)
+}

cmd/server/search_handlers.go

@@ -33,6 +33,7 @@ var (
 
 // TODO: modify existing search endpoint with additional eu info and add an eu only endpoint
 func addSearchRoutes(logger log.Logger, r *mux.Router, searcher *searcher) {
+	r.Methods("GET").Path("/crypto").HandlerFunc(searchByCryptoAddress(logger, searcher))
 	r.Methods("GET").Path("/search").HandlerFunc(search(logger, searcher))
 	r.Methods("GET").Path("/search/us-csl").HandlerFunc(searchUSCSL(logger, searcher))
 	r.Methods("GET").Path("/search/eu-csl").HandlerFunc(searchEUCSL(logger, searcher))

pkg/ofac/ofac.go

@@ -69,4 +69,15 @@ type SDNComments struct {
 	EntityID string `json:"entityID"`
 	// RemarksExtended is remarks extended on a Specially Designated National
 	RemarksExtended string `json:"remarksExtended"`
+	// DigitalCurrencyAddresses are wallet addresses for digital currencies
+	DigitalCurrencyAddresses []DigitalCurrencyAddress `json:"digitalCurrencyAddresses"`
+}
+
+type DigitalCurrencyAddress struct {
+	// Currency is the name of the digital currency.
+	// Examples: XBT (Bitcoin), ETH (Ethereum)
+	Currency string
+
+	// Address is a digital wallet address
+	Address string
 }

pkg/ofac/reader.go

@@ -233,8 +233,9 @@ func csvSDNCommentsFile(path string) (*Results, error) {
 		}
 		line = replaceNull(line)
 		out = append(out, &SDNComments{
-			EntityID:        line[0],
-			RemarksExtended: line[1],
+			EntityID:                 line[0],
+			RemarksExtended:          line[1],
+			DigitalCurrencyAddresses: readDigitalCurrencyAddresses(line[1]),
 		})
 	}
 	return &Results{SDNComments: out}, nil
@@ -262,3 +263,59 @@ func splitPrograms(in string) []string {
 	norm := cleanPrgmsList(in)
 	return strings.Split(norm, "; ")
 }
+
+var (
+	digitalCurrencies = []string{
+		"XBT",  // Bitcoin
+		"ETH",  // Ethereum
+		"XMR",  // Monero
+		"LTC",  // Litecoin
+		"ZEC",  // ZCash
+		"DASH", // Dash
+		"BTG",  // Bitcoin Gold
+		"ETC",  // Ethereum Classic
+		"BSV",  // Bitcoin Satoshi Vision
+		"BCH",  // Bitcoin Cash
+		"XVG",  // Verge
+		"USDC", // USD Coin
+		"USDT", // USD Tether
+		"XRP",  // Ripple
+		"TRX",  // Tron
+		"ARB",  // Arbitrum
+		"BSC",  // Binance Smart Chain
+	}
+)
+
+func readDigitalCurrencyAddresses(remarks string) []DigitalCurrencyAddress {
+	var out []DigitalCurrencyAddress
+
+	// The format is semicolon delineated, but "Digital Currency Address" is sometimes truncated badly
+	//
+	//   alt. Digital Currency Address - XBT 12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax;
+	//
+	parts := strings.Split(remarks, ";")
+	for i := range parts {
+		// Check if the currency is in the remark
+		var addressIndex int
+		for j := range digitalCurrencies {
+			idx := strings.Index(parts[i], fmt.Sprintf(" %s ", digitalCurrencies[j]))
+			if idx > -1 {
+				addressIndex = idx
+				break
+			}
+		}
+		if addressIndex > 0 {
+			fields := strings.Fields(parts[i][addressIndex:])
+			if len(fields) < 2 {
+				break // bad parsing
+			}
+			out = append(out, DigitalCurrencyAddress{
+				Currency: fields[0],
+				Address:  fields[1],
+			})
+			continue
+		}
+	}
+
+	return out
+}

pkg/ofac/reader_test.go

@@ -9,6 +9,8 @@ import (
 	"path/filepath"
 	"reflect"
 	"testing"
+
+	"github.com/stretchr/testify/require"
 )
 
 // TestOFAC__read validates reading an OFAC Address CSV File
@@ -113,3 +115,26 @@ func TestSDNComments(t *testing.T) {
 		}
 	}
 }
+
+func TestSDNComments_CryptoCurrencies(t *testing.T) {
+	fd, err := os.CreateTemp("", "sdn-comments")
+	require.NoError(t, err)
+
+	_, err = fd.WriteString(`42496," alt. Digital Currency Address - XBT 12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax; alt. Digital Currency Address - XBT 1J378PbmTKn2sEw6NBrSWVfjZLBZW3DZem; alt. Digital Currency Address - XBT 18aqbRhHupgvC9K8qEqD78phmTQQWs7B5d; alt. Digital Currency Address - XBT 16ti2EXaae5izfkUZ1Zc59HMcsdnHpP5QJ; Secondary sanctions risk: North Korea Sanctions Regulations, sections 510.201 and 510.210; Transactions Prohibited For Persons Owned or Controlled By U.S. Financial Institutions: North Korea Sanctions Regulations section 510.214; Passport E59165201 (China) expires 01 Sep 2025; Identification Number 371326198812157611 (China); a.k.a. 'WAKEMEUPUPUP'; a.k.a. 'FAST4RELEASE'; Linked To: LAZARUS GROUP."`)
+	require.NoError(t, err)
+
+	sdn, err := csvSDNCommentsFile(fd.Name())
+	require.NoError(t, err)
+	require.Len(t, sdn.SDNComments, 1)
+
+	addresses := sdn.SDNComments[0].DigitalCurrencyAddresses
+	require.Len(t, addresses, 4)
+
+	expected := []DigitalCurrencyAddress{
+		{Currency: "XBT", Address: "12jVCWW1ZhTLA5yVnroEJswqKwsfiZKsax"},
+		{Currency: "XBT", Address: "1J378PbmTKn2sEw6NBrSWVfjZLBZW3DZem"},
+		{Currency: "XBT", Address: "18aqbRhHupgvC9K8qEqD78phmTQQWs7B5d"},
+		{Currency: "XBT", Address: "16ti2EXaae5izfkUZ1Zc59HMcsdnHpP5QJ"},
+	}
+	require.ElementsMatch(t, expected, addresses)
+}