Merge pull request #530 from adamdecaf/basic-v2-search-model

[WIP] feat: add generalized entity model for v2 search

.dockerignore

@@ -0,0 +1 @@
+libpostal/

.github/workflows/go.yml

@@ -91,17 +91,3 @@ jobs:
     - name: Docker Build
       if: runner.os == 'Linux'
       run: make docker-hub
-
-    - name: Build batchsearch
-      if: runner.os == 'Linux'
-      run: make build-batchsearch
-
-    # - name: Integration Test
-    #   if: runner.os == 'Linux'
-    #   run: make test-integration
-
-    - name: Test Cleanup
-      if: runner.os == 'Linux' && always()
-      run: |
-        docker compose logs
-        make clean-integration

.github/workflows/release.yml

@@ -151,9 +151,6 @@ jobs:
     - name: Docker Static
       run: make docker-static
 
-    - name: Docker watchmantest
-      run: make docker-watchmantest
-
     - name: Docker Push
       run: |+
           echo "$DOCKER_PASSWORD" | docker login -u "$DOCKER_USERNAME" --password-stdin

.gitignore

@@ -28,6 +28,8 @@ openapi-generator*jar
 
 *.db
 
+/libpostal/
+
 webui/build/
 webui/node_modules/
 

Dockerfile

@@ -1,25 +1,79 @@
-FROM golang:alpine as backend
+# Backend build stage
+FROM golang:1.23-bookworm as backend
 ARG VERSION
 WORKDIR /src
-COPY . /src
+
+# Install system dependencies first
+RUN apt-get update && apt-get install -y \
+    curl \
+    autoconf \
+    automake \
+    libtool \
+    pkg-config \
+    git
+
+# Clone and build libpostal (rarely changes)
+RUN git clone https://github.com/openvenues/libpostal.git /src/libpostal
+WORKDIR /src/libpostal
+RUN ./bootstrap.sh && \
+    ./configure && \
+    make -j$(shell nproc) && \
+    make install && \
+    ldconfig
+
+# Download libpostal data (rarely changes)
+RUN libpostal_data download all /usr/local/share/libpostal
+
+# Copy go.mod and go.sum first to cache dependencies
+COPY go.mod go.sum /src/
 RUN go mod download
-RUN CGO_ENABLED=0 go build -ldflags "-X github.com/moov-io/watchman.Version=${VERSION}" -o ./bin/server /src/cmd/server
 
-FROM node:21-alpine as frontend
+# Now copy the rest of the source code (frequently changes)
+COPY . /src/
+WORKDIR /src
+RUN VERSION=${VERSION} GOTAGS="-tags libpostal" make build-server
+
+# Frontend build stage
+FROM node:22-bookworm as frontend
 ARG VERSION
-COPY webui/ /watchman/
 WORKDIR /watchman/
+
+# Copy package files first to cache dependencies
+COPY webui/package*.json webui/
+WORKDIR /watchman/webui/
 RUN npm install --legacy-peer-deps
+
+# Copy and build frontend source (frequently changes)
+COPY webui/ ./
 RUN npm run build
 
-FROM alpine:latest
+# Final stage
+FROM debian:bookworm
 LABEL maintainer="Moov <[email protected]>"
+
+# Install runtime dependencies
+RUN apt-get update && \
+    apt-get install -y \
+    libssl3 \
+    ca-certificates \
+    && rm -rf /var/lib/apt/lists/*
+
+# Create necessary directories and copy libpostal files
+RUN mkdir -p /usr/local/share/libpostal
+COPY --from=backend /usr/local/lib/libpostal.so* /usr/local/lib/
+COPY --from=backend /usr/local/lib/pkgconfig/libpostal.pc /usr/local/lib/pkgconfig/
+COPY --from=backend /usr/local/share/libpostal/ /usr/local/share/libpostal/
+RUN ldconfig
+
+# Copy application files
 COPY --from=backend /src/bin/server /bin/server
-COPY --from=frontend /watchman/build/ /watchman/
-ENV WEB_ROOT=/watchman/
+COPY --from=frontend /watchman/webui/build/ /watchman/
 
+# Set environment variables
+ENV WEB_ROOT=/watchman/
+ENV LD_LIBRARY_PATH=/usr/local/lib
+ENV LIBPOSTAL_DATA_DIR=/usr/local/share/libpostal
 
 EXPOSE 8084
 EXPOSE 9094
-
 ENTRYPOINT ["/bin/server"]

Dockerfile-openshift

@@ -1,28 +1,103 @@
-FROM quay.io/fedora/fedora:40-x86_64 as builder
+# Stage 1: Install dependencies with root privileges
+FROM registry.access.redhat.com/ubi9/go-toolset as builder-deps
+USER root
+
+# Install system dependencies first - rarely changes
+RUN dnf install -y --allowerasing --setopt=tsflags=nodocs \
+    curl \
+    autoconf \
+    automake \
+    libtool \
+    pkgconfig \
+    gcc \
+    gcc-c++ \
+    make \
+    git \
+    && dnf clean all
+
+# Install libpostal with models - rarely changes
+RUN git clone https://github.com/openvenues/libpostal && \
+    cd libpostal && \
+    ./bootstrap.sh && \
+    ./configure --prefix=/usr/local && \
+    make -j4 && \
+    make install && \
+    mkdir -p /usr/local/share/libpostal
+
+# Download libpostal data - separate step for better caching
+RUN cd libpostal/src && \
+    PATH=$PATH:/usr/local/bin ./libpostal_data download all /usr/local/share/libpostal
+
+# Set permissions - should be last in this stage
+RUN chown -R 1001:0 /usr/local && \
+    chmod -R g=u /usr/local
+
+# Stage 2: Build the application
+FROM registry.access.redhat.com/ubi9/go-toolset AS builder
 ARG VERSION
-RUN yum install -y git golang make npm wget glibc
 WORKDIR /opt/app-root/src/
-COPY . .
+
+# Copy only the necessary files from builder-deps
+COPY --from=builder-deps /usr/local /usr/local
+COPY --from=builder-deps /usr/lib64 /usr/lib64
+
+# Set environment variables for build
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+ENV LD_LIBRARY_PATH=/usr/local/lib
+
+# Copy go.mod and go.sum first to cache dependencies
+COPY go.mod go.sum ./
 RUN go mod download
-RUN VERSION=${VERSION} make build-server
 
+# Copy source files
+COPY . .
+
+# Create bin directory and set permissions BEFORE building
+USER root
+RUN mkdir -p bin && \
+    chown -R 1001:0 . && \
+    chmod -R g=u .
+
+USER 1001
+RUN VERSION=${VERSION} GOTAGS="-tags libpostal" make build-server
+
+# Stage 3: Frontend build
 FROM node:21-bookworm as frontend
-COPY webui/ /watchman/
 WORKDIR /watchman/
+
+# Copy package files first to cache dependencies
+COPY webui/package*.json ./
 RUN npm install --legacy-peer-deps
-RUN npm run build
 
-FROM quay.io/fedora/fedora:40-x86_64
-RUN yum install -y glibc
+# Copy frontend source and build - frequently changes
+COPY webui/ ./
+RUN npm run build
 
+# Stage 4: Final stage
+FROM registry.access.redhat.com/ubi9/ubi-minimal:9.5-1733767867
 ARG VERSION=unknown
 LABEL maintainer="Moov <[email protected]>"
 LABEL name="watchman"
 LABEL version=$VERSION
 
-COPY --from=builder /opt/app-root/src/bin/server /bin/server
+# Install runtime dependencies
+USER root
+RUN microdnf install -y \
+    libstdc++ \
+    && microdnf clean all
+
+# Copy libpostal files and setup
+COPY --from=builder-deps /usr/local /usr/local
+ENV LD_LIBRARY_PATH=/usr/local/lib
 
+# Copy application files
+COPY --from=builder /opt/app-root/src/bin/server /bin/server
 COPY --from=frontend /watchman/build/ /watchman/
 ENV WEB_ROOT=/watchman/
 
+# Set final permissions and switch to non-root user
+RUN chown -R 1001:0 /bin/server /watchman && \
+    chmod -R g=u /bin/server /watchman
+USER 1001
+
 ENTRYPOINT ["/bin/server"]

README.md

@@ -259,20 +259,11 @@ By design, Watchman  **does not persist** (save) any data about the search queri
 
 ### Go library
 
-This project uses [Go Modules](https://go.dev/blog/using-go-modules) and Go v1.18 or newer. See [Golang's install instructions](https://golang.org/doc/install) for help setting up Go. You can download the source code and we offer [tagged and released versions](https://github.com/moov-io/watchman/releases/latest) as well. We highly recommend you use a tagged release for production.
-
-```
-$ [email protected]:moov-io/watchman.git
-
-# Pull down into the Go Module cache
-$ go get -u github.com/moov-io/watchman
-
-$ go doc github.com/moov-io/watchman/client Search
-```
+Watchman offers [several packages for usage as libraries](https://pkg.go.dev/github.com/moov-io/watchman/pkg).
 
 ### In-browser Watchman search
 
-Using our [in-browser utility](https://oss.moov.io/watchman/), you can instantly perform advanced Watchman searches. Simply fill search fields and generate a detailed report that includes match percentage, alternative names, effective/expiration dates, IDs, addresses, and other useful information. This tool is particularly useful for completing quick searches with the aid of a intuitive interface.
+Using the [WebUI](https://moov-io.github.io/watchman/webui/), you can instantly perform advanced OFAC Watchman searches. Simply fill search fields and generate a detailed report that includes match percentage, alternative names, effective/expiration dates, IDs, addresses, and other useful information. This tool is particularly useful for completing quick searches with the aid of a intuitive interface.
 
 ## Reporting blocks to OFAC
 
@@ -309,7 +300,7 @@ Note: 32-bit platforms have known issues and are not supported.
 
 Yes please! Please review our [Contributing guide](CONTRIBUTING.md) and [Code of Conduct](https://github.com/moov-io/ach/blob/master/CODE_OF_CONDUCT.md) to get started! Checkout our [issues for first time contributors](https://github.com/moov-io/watchman/contribute) for something to help out with.
 
-This project uses [Go Modules](https://go.dev/blog/using-go-modules) and Go v1.18 or newer. See [Golang's install instructions](https://golang.org/doc/install) for help setting up Go. You can download the source code and we offer [tagged and released versions](https://github.com/moov-io/watchman/releases/latest) as well. We highly recommend you use a tagged release for production.
+Run `make install` to setup [gopostal](https://github.com/openvenues/gopostal) / [libpostal](https://github.com/openvenues/libpostal) for Watchman.
 
 ### Releasing