Migrate owned did nuts to SQL (#3392)

* migrate did:nuts documents to SQL

* add e2e-test setup

* test migration hack is safe for production network

* fix broken tests

* reduce complexity

* dont migrate documents past deactivation

* fix merge conflict

* generalize methods

* add tests

* e2e test; add SQL db checks

* e2e test; add more SQL db checks

* pr feedback + documentation

* revert e2e-test

* add e2e-test

.github/workflows/e2e-tests.yaml

@@ -75,7 +75,7 @@ jobs:
       - name: Run E2E tests
         run: |
             cd e2e-tests && \
-            find . -type f -name "docker-compose.yml" | xargs -I{} sed -i 's~nutsfoundation/nuts-node:master~ghcr.io/nuts-foundation/nuts-node-ci:${{ env.SHA }}~g' {} && \
+            find . -type f -name "docker-compose*.yml" | xargs -I{} sed -i 's~nutsfoundation/nuts-node:master~ghcr.io/nuts-foundation/nuts-node-ci:${{ env.SHA }}~g' {} && \
             find . -type f -name "run-test.sh" | xargs -I{} sed -i 's/docker-compose exec/docker-compose exec -T/g' {} && \
             ./run-tests.sh
 

docs/pages/integrating/version-incompatibilities.rst

@@ -17,3 +17,11 @@ There are basically two options.
 Do not use the VDR V1 and VDR V2 API at the same time. This will lead to unexpected behavior.
 Once you use the VDR V2 API, you cannot go back to the VDR V1 API. The VDR V1 API has also been marked as deprecated.
 
+Nodes running v6 with ``nuts`` configured as one of the ``vdr.did_methods`` will migrate all owned ``did:nuts`` DID documents to the new SQL storage.
+This migration includes all historic document updates as published upto a potential deactivation of the document.
+For DIDs with a document conflict this is different than the resolved version of the document, which contains a merge of all conflicting document updates.
+To prevent the state of the resolver and the SQL storage to be in conflict, all DID document conflicts must be resolved before upgrading to v6.
+See ``/status/diagnostics`` if you own any DIDs with a document conflict. If so, use ``/internal/vdr/v1/did/conflicted`` to find the DIDs with a conflict.
+
+The document migration will run on every restart of the node, meaning that any updates made using the VDR V1 API will be migrated on the next restart.
+When switching from the VDR V1 API to the V2 API, the node must be restarted first to migrate any recent changes.

docs/pages/release_notes.rst

@@ -17,6 +17,7 @@ Breaking changes
   When migrating from v5, change the owner of the data directory on the host to that of the container's user. (``chown -R 18081:18081 /path/to/host/data-dir``)
 - Docker image tags have been changed: previously version tags had were prefixed with ``v`` (e.g., ``v5.0.0``), this prefix has been dropped to better adhere to industry standards.
 - The VDR v1 ``createDID`` (``POST /internal/vdr/v1/did``) no longer supports the ``controller`` and ``selfControl`` fields. All did:nuts documents are now self controlled. All existing documents will be migrated to self controlled at startup.
+- Managed ``did:nuts`` DIDs are migrated to the new SQL storage. Unresolved DID document conflicts may contain an incorrect state after migrating to v6. See ``/status/diagnostics`` if you own any DIDs with a document conflict; use ``/internal/vdr/v1/did/conflicted`` to find the specific DIDs.
 - Removed legacy API authentication tokens.
 
 ============

e2e-tests/migration/docker-compose-post-migration.yml

@@ -0,0 +1,32 @@
+services:
+  nodeA:
+    image: "${IMAGE_NODE_A:-nutsfoundation/nuts-node:master}"
+    container_name: nodeA
+    user: &usr "$USER:$USER"
+    ports:
+      - "18081:8081"
+    environment:
+      NUTS_URL: "http://nodeA:8080"
+    volumes:
+      - "./nuts-v6.yaml:/nuts/config/nuts.yaml"
+      - "./nodeA/data:/nuts/data"
+      - "../tls-certs/nodeA-certificate.pem:/nuts/config/certificate-and-key.pem:ro"
+      - "../tls-certs/truststore.pem:/nuts/config/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often
+  nodeB:
+    image: nutsfoundation/nuts-node:v5.4.11 # must be v5.4.11+ for bugfixes. sync with docker-compose-post-migration.yml
+    container_name: nodeB
+    user: *usr
+    ports:
+      - "28081:1323"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+      NUTS_NETWORK_BOOTSTRAPNODES: "nodeA:5555"
+    volumes:
+      - "./nuts-v5.yaml:/opt/nuts/nuts.yaml"
+      - "./nodeB/data:/opt/nuts/data"
+      - "../tls-certs/nodeB-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often

e2e-tests/migration/docker-compose-pre-migration.yml

@@ -0,0 +1,32 @@
+services:
+  nodeA:
+    image: &im nutsfoundation/nuts-node:v5.4.11 # must be v5.4.11+ for bugfixes. sync with docker-compose-post-migration.yml
+    container_name: nodeA
+    user: &usr "$USER:$USER"
+    ports: # use v6 ports to minimize changes
+      - "18081:1323"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+    volumes:
+      - "./nuts-v5.yaml:/opt/nuts/nuts.yaml"
+      - "./nodeA/data:/opt/nuts/data"
+      - "../tls-certs/nodeA-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often
+  nodeB:
+    image: *im
+    container_name: nodeB
+    user: *usr
+    ports:
+      - "28081:1323"
+    environment:
+      NUTS_CONFIGFILE: /opt/nuts/nuts.yaml
+      NUTS_NETWORK_BOOTSTRAPNODES: "nodeA:5555"
+    volumes:
+      - "./nuts-v5.yaml:/opt/nuts/nuts.yaml"
+      - "./nodeB/data:/opt/nuts/data"
+      - "../tls-certs/nodeB-certificate.pem:/opt/nuts/certificate-and-key.pem:ro"
+      - "../tls-certs/truststore.pem:/opt/nuts/truststore.pem:ro"
+    healthcheck:
+      interval: 1s # Make test run quicker by checking health status more often

e2e-tests/migration/main_test.go

@@ -0,0 +1,120 @@
+//go:build e2e_tests
+
+/*
+ * Copyright (C) 2023 Nuts community
+ *
+ * This program is free software: you can redistribute it and/or modify
+ * it under the terms of the GNU General Public License as published by
+ * the Free Software Foundation, either version 3 of the License, or
+ * (at your option) any later version.
+ *
+ * This program is distributed in the hope that it will be useful,
+ * but WITHOUT ANY WARRANTY; without even the implied warranty of
+ * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the
+ * GNU General Public License for more details.
+ *
+ * You should have received a copy of the GNU General Public License
+ * along with this program.  If not, see <https://www.gnu.org/licenses/>.
+ *
+ */
+
+package migration
+
+import (
+	"encoding/json"
+	did "github.com/nuts-foundation/go-did/did"
+	"github.com/stretchr/testify/assert"
+	"os"
+	"testing"
+	"time"
+
+	"github.com/nuts-foundation/nuts-node/storage"
+	"github.com/nuts-foundation/nuts-node/storage/orm"
+	"github.com/nuts-foundation/nuts-node/vdr/didsubject"
+	"github.com/stretchr/testify/require"
+)
+
+func Test_Migrations(t *testing.T) {
+	db := storage.NewTestStorageEngineInDir(t, "./nodeA/data").GetSQLDatabase()
+
+	DIDs, err := didsubject.NewDIDManager(db).All()
+	require.NoError(t, err)
+	require.Len(t, DIDs, 4)
+
+	t.Run("vendor", func(t *testing.T) {
+		// versions for did:nuts:
+		// - LC0: init -> no controller because vendor
+		// - LC4: add service1
+		// - LC4: add service2, conflicts with above
+		// - LC8: add verification method, solves conflict
+		// no updates during migration
+		//
+		// total 4 versions in SQL; latest has 2 services and 2 VMs
+		id := did.MustParseDID(os.Getenv("VENDOR_DID"))
+		var doc orm.DidDocument
+		err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error
+		require.NoError(t, err)
+
+		assert.Equal(t, 3, doc.Version)
+		assert.Len(t, doc.Services, 2)
+		assert.Len(t, doc.VerificationMethods, 2)
+	})
+	t.Run("org1", func(t *testing.T) {
+		// versions for did:nuts:
+		// - LC1: init -> has controller
+		// - LC5: add service2
+		// - LC6: add service1, conflicts with above
+		// migration removes controller (solves document conflict)
+		//
+		// total 4 versions in SQL; latest one has no controller, 2 services, and 1 VM
+		id := did.MustParseDID(os.Getenv("ORG1_DID"))
+		var doc orm.DidDocument
+		err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error
+		require.NoError(t, err)
+
+		assert.Equal(t, 3, doc.Version)
+		assert.Len(t, doc.Services, 2)
+		assert.Len(t, doc.VerificationMethods, 1)
+		didDoc := new(did.Document)
+		require.NoError(t, json.Unmarshal([]byte(doc.Raw), didDoc))
+		assert.Empty(t, didDoc.Controller)
+	})
+	t.Run("org2", func(t *testing.T) {
+		// versions for did:nuts:
+		// - LC2: init -> has controller
+		// - LC5: deactivate
+		// - LC6: service2, conflicts with above
+		// deactivated, so no updates during migration;
+		//
+		// total 2 versions in SQL, migration stopped at LC5; no controller, 0 service, 0 VM
+		id := did.MustParseDID(os.Getenv("ORG2_DID"))
+		var doc orm.DidDocument
+		err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error
+		require.NoError(t, err)
+
+		assert.Equal(t, 1, doc.Version)
+		assert.Len(t, doc.Services, 0)
+		assert.Len(t, doc.VerificationMethods, 0)
+	})
+	t.Run("org3", func(t *testing.T) {
+		// versions for did:nuts:
+		// - LC3: init -> has controller
+		// - LC7: add service1
+		// - LC7: add verification method, conflicts with above
+		// - LC9: add service2, solves conflict
+		// migration removes controller, total 5 versions in SQL
+		//
+		// total 5 versions in SQL; no controller, 2 services, 2 VMs
+		id := did.MustParseDID(os.Getenv("ORG3_DID"))
+		var doc orm.DidDocument
+		err = db.Preload("DID").Preload("Services").Preload("VerificationMethods").Where("did = ? AND updated_at <= ?", id.String(), time.Now()).Order("version desc").First(&doc).Error
+		require.NoError(t, err)
+
+		assert.Equal(t, 4, doc.Version)
+		assert.Len(t, doc.Services, 2)
+		assert.Len(t, doc.VerificationMethods, 2)
+		didDoc := new(did.Document)
+		require.NoError(t, json.Unmarshal([]byte(doc.Raw), didDoc))
+		assert.Empty(t, didDoc.Controller)
+	})
+}

e2e-tests/migration/nuts-v5.yaml

@@ -0,0 +1,12 @@
+datadir: /opt/nuts/data
+verbosity: debug
+strictmode: false
+tls:
+  truststorefile: "/opt/nuts/truststore.pem"
+  certfile: "/opt/nuts/certificate-and-key.pem"
+  certkeyfile: "/opt/nuts/certificate-and-key.pem"
+auth:
+  contractvalidators:
+    - dummy
+goldenhammer:
+  enabled: false

e2e-tests/migration/nuts-v6.yaml

@@ -0,0 +1,15 @@
+verbosity: debug
+strictmode: false
+#url: http://nodeA:1323 # set in as env variable
+tls:
+  truststorefile: "/nuts/config/truststore.pem"
+  certfile: "/nuts/config/certificate-and-key.pem"
+  certkeyfile: "/nuts/config/certificate-and-key.pem"
+auth:
+  contractvalidators:
+    - dummy
+goldenhammer:
+  enabled: false
+http:
+  internal:
+    address: :8081