Fixup some inserts data to make compatible with what is in production.

CHANGELOG.rst

@@ -14,6 +14,16 @@ Change Log
 * Update: Added tests to verify the presence and non-empty content of `<pre>` elements inside static section
 
 
+8.4.5
+=====
+* 2024-11-18/dmichaels
+* Fixed up some inserts data to match 4dn/data/staging.
+* Updated dcicutils to latest version (8.16.4).
+* Updated dcicsnovault to latest version (11.23.0).
+* Added comments to base.ini on how to use the open sourch mitmproxy utility to observe OpenSearch
+  traffic easily when running locally for development; via the elasticsearch.server.actual_port property.
+
+
 8.4.4
 =====
 

Makefile

@@ -112,7 +112,9 @@ kibana-stop:
 kill:  # kills back-end processes associated with the application. Use with care.
 	pkill -f postgres &
 	pkill -f elasticsearch &
+	-pkill -f opensearch &
 	pkill -f moto_server &
+	-pkill -f nginx &
 
 clean-python:
 	@echo -n "Are you sure? This will wipe all libraries installed on this virtualenv [y/N] " && read ans && [ $${ans:-N} = y ]

base.ini

@@ -9,6 +9,14 @@ file_wfout_bucket = encoded-4dn-files
 # this really shouldn't be used with development, only uncomment if your sure
 system_bucket = elasticbeanstalk-encoded-4dn-system
 elasticsearch.server = 127.0.0.1:9200
+# The elasticsearch.server.actual_port property is useful (only) for running a localhost ElasticSearch
+# proxy in order to observe traffic (requests/responses) between portal and ElasticSearch with a tool like
+# mitmweb; e.g. setting elasticsearch.server.actual_port to 9201 and elasticsearch.server to localhost:9200
+# will case ElasticSearch to actually run on port 9201 but will cause portal to talk to it via port 9200,
+# and then we can run mitmweb --mode reverse:http://localhost:9201 -p 9200 --web-port 8081 which will
+# allow us to browse to http://localhost:8081 locally to observe all of the ElasticSearch traffic;
+# and note to install mitmweb do: pip install mitmproxy
+# elasticsearch.server.actual_port = 9201
 ontology_path = %(here)s/ontology.json
 aws_ip_ranges_path = %(here)s/aws-ip-ranges.json
 #this is to reroute downloads to something other than aws s3

pyproject.toml

@@ -49,8 +49,8 @@ colorama = "0.3.3"
 # we get odd 'pyo3_runtime.PanicException: Python API call failed' error on import
 # of cryptography.hazmat.bindings._rust in cryptography package. 2023-04-21.
 # cryptography = "39.0.2"
-dcicsnovault = "^11.22.0"
-dcicutils = "^8.16.1"
+dcicsnovault = "^11.23.0"
+dcicutils = "^8.16.4"
 elasticsearch = "7.13.4"
 elasticsearch-dsl = "^7.0.0"  # TODO: port code from cgap-portal to get rid of uses
 execnet = "1.4.1"

src/encoded/tests/data/inserts/award.json

@@ -40,7 +40,7 @@
         "status": "current",
         "title": "EXPLORING HOW THE GENOME FOLDS THROUGH PROXIMITY LIGATION AND SEQUENCING",
         "url": "https://projectreporter.nih.gov/project_info_details.cfm?aid=8146738&icde=30734626",
-        "uuid": "b0b9c607-f8b4-4f02-93f4-9895b461334c",
+        "uuid": "36a06537-7831-494d-b10d-3e9fea931021",
         "project": "External"
     },
     {

src/encoded/tests/data/inserts/bio_feature.json

@@ -5,7 +5,7 @@
         "feature_type": "0231da78-1adc-4b8a-89b9-4f91908412a3",
         "uuid": "a8ab8bca-4840-41b7-88d7-04cc19a54657",
         "genome_location": ["d1115d5e-40aa-43bc-b81c-32c70c9afb01"],
-        "relevant_genes": ["3f3496f7-31bf-429f-8da7-7f1fdf840dcc"],
+        "relevant_genes": ["594e3125-a9cb-4ffa-bc2b-17870c1690f0"],
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
@@ -17,7 +17,7 @@
         "description": "A protein feature",
         "feature_type": "91f427e6-5246-4992-8123-b4f8fa9eef01",
         "uuid": "5a5b6f55-0b54-441c-86bf-292d41e443a0",
-        "relevant_genes": ["d5ee3bf3-63b0-4032-b133-314173b3cc4d"],
+        "relevant_genes": ["a093769c-d596-4a26-8916-06ae491575ba"],
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",

src/encoded/tests/data/inserts/biosource.json

@@ -4,24 +4,24 @@
         "description": "GM12878 test cells 1",
         "biosource_type": "immortalized cell line",
         "individual":"4DNINOOOAAQ1",
-        "cell_line": "530036bc-8535-4448-903e-854af460b24c",
+        "cell_line": "b9668b9a-be39-47de-8eab-5bf1b0854417",
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "status": "in review by lab",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "331111bc-8535-4448-903e-854af460a254",
         "description": "GM12878 test cells 2",
         "biosource_type": "immortalized cell line",
         "individual":"4DNINOOOAAQ1",
-        "cell_line": "530036bc-8535-4448-903e-854af460b24c",
+        "cell_line": "b9668b9a-be39-47de-8eab-5bf1b0854417",
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "status": "in review by lab",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "331111bc-8535-4448-903e-854ab460b254",
@@ -32,18 +32,18 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "status": "in review by lab",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "331111bc-8535-2248-903e-854af460b254",
         "description": "test modified cells",
         "biosource_type": "immortalized cell line",
         "individual":"4DNINOOOAAQ1",
-        "cell_line": "530036bc-8535-4448-903e-854af460b24c",
+        "cell_line": "b9668b9a-be39-47de-8eab-5bf1b0854417",
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e",
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f",
         "status": "in review by lab",
         "modifications": ["431106bc-8535-4448-903e-854af460b265"]
     },
@@ -56,7 +56,7 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "status": "in review by lab",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "111116bc-8535-4448-903e-854af460b254",
@@ -76,17 +76,18 @@
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "0f011b1e-b772-4f2a-8c24-cc55de28a994",
+        "accession": "4DNSRVWZS4XO",
         "description": "Homo sapiens GM12878 immortalized cell line",
         "biosource_type": "immortalized cell line",
         "individual":"4DNINOOOAAQ1",
         "award": "1U01CA200059-01",
         "lab": "dcic-testing-lab",
         "submitted_by": "[email protected]",
-        "biosource_vendor": "b31106bc-8535-4448-903e-854af460b21e"
+        "biosource_vendor": "11f94a17-51ed-4a0f-93b1-1cac2fd2844f"
     },
     {
         "uuid": "c9165aa4-2ab5-428d-b5a7-db86dbb2d815",

src/encoded/tests/data/inserts/enzyme.json

@@ -1,7 +1,7 @@
 [
     {
         "name": "HindIII",
-        "aliases": ["dcic:hindIII_neb"],
+        "aliases": ["4dn-dcic-lab:hindIII_neb", "dcic:hindIII_neb"],
         "enzyme_source": "new-england-biolabs",
         "catalog_number":"R0104",
         "recognition_sequence": "AAGCTT",
@@ -11,7 +11,7 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "url": "https://www.neb.com/products/r0104-hindiii",
-        "uuid": "4d44d12c-b37c-478b-9bc7-d843cba4bb91"
+        "uuid": "2ace2f00-c913-4bac-a6ef-0b316bc2108e"
     },
     {
         "name": "DNaseI",
@@ -27,7 +27,10 @@
     },
     {
         "name": "MboI",
-        "aliases":["dcic:mboI_neb"],
+        "aliases": [
+            "dcic:mboI_neb",
+            "4dn-dcic-lab:mboI_neb"
+        ],
         "enzyme_source": "new-england-biolabs",
         "catalog_number":"R0147",
         "recognition_sequence": "GATC",
@@ -37,7 +40,7 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "url": "https://www.neb.com/products/r0147-mboi",
-        "uuid": "37750c1c-fce5-4ec5-baa7-ce7858f898ae"
+        "uuid": "d8da42d0-bc20-40e5-bacf-c263b8603e03"
     },
     {
         "name": "MNase",
@@ -53,7 +56,7 @@
     },
     {
         "name": "DpnII",
-        "aliases":["dcic:dpnII_neb"],
+        "aliases":["4dn-dcic-lab:dpnII_neb", "dcic:dpnII_neb"],
         "enzyme_source": "new-england-biolabs",
         "catalog_number":"R0543",
         "recognition_sequence": "GATC",
@@ -63,21 +66,21 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "url": "https://www.neb.com/products/r0543-dpnii",
-        "uuid": "557d7469-d1f6-4200-8ed4-c40374383dd3"
+        "uuid": "356a57a1-1f1d-463d-a972-27742f79a6a5"
     },
     {
         "name": "NcoI_MspI_BspHI",
-        "aliases":["dcic:3nzNMB_neb"],
+        "aliases":["dcic:3nzNMB_neb", "4dn-dcic-lab:3nzNMB_neb"],
         "enzyme_source": "new-england-biolabs",
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "description": "Mixed Enzymes",
-        "uuid": "6b88eb35-92e7-46fd-87b8-a779cec40c31"
+        "uuid": "f57bbab4-acb9-4951-9099-a57b015b80c6"
     },
     {
         "name": "NcoI",
-        "aliases":["dcic:ncoI_neb"],
+        "aliases":["dcic:ncoI_neb", "4dn-dcic-lab:ncoI_neb"],
         "enzyme_source": "new-england-biolabs",
         "catalog_number":"R0193",
         "recognition_sequence": "CCATGG",
@@ -87,11 +90,11 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "url": "https://www.neb.com/products/r0193-ncoi",
-        "uuid": "aa5ecd3d-69bf-478c-acbc-82f88b785f7c"
+        "uuid": "9ff58460-3e4c-4bb1-820d-81bc1127ac53"
     },
     {
         "name": "MspI",
-        "aliases":["dcic:mspI_neb"],
+        "aliases":["dcic:mspI_neb", "4dn-dcic-lab:mspI_neb"],
         "enzyme_source": "new-england-biolabs",
         "catalog_number":"R0106",
         "recognition_sequence": "CCGG",
@@ -101,15 +104,18 @@
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
         "url": "https://www.neb.com/products/r0106-mspi",
-        "uuid": "1ff66969-52bc-4226-a056-53b9b0541fe2"
+        "uuid": "bd56fbbe-9124-4431-8529-2918dc487385"
     },
     {
         "name": "Tn5 Transposase",
-        "aliases":["dcic:tn5transposase"],
+        "aliases": [
+            "dcic:transposase_illumina",
+            "4dn-dcic-lab:transposase_illumina"
+        ],
         "enzyme_source": "dcic:illumina",
         "award": "1U01CA200059-01",
         "lab": "4dn-dcic-lab",
         "submitted_by": "[email protected]",
-        "uuid": "1ff66969-52bc-4226-a056-111110541fe2"
+        "uuid": "cac89b2d-6985-4b14-af7e-0808a5fd103b"
     }
 ]

src/encoded/tests/data/inserts/experiment_atacseq.json

@@ -2,7 +2,7 @@
     {
         "accession":"4DNEX167AAA1",
         "biosample":"a9163992-7486-4edd-b21d-f492884bff63",
-        "experiment_type": "c17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "5b35245d-5777-4983-b7e8-8dffd9ab83ab",
         "enzyme_lot_number": "123456",
         "enzyme_incubation_time": 30,
         "incubation_temperature": 55,

src/encoded/tests/data/inserts/experiment_hi_c.json

@@ -3,7 +3,7 @@
         "accession":"4DNEXO67APU1",
         "biosample":"231111bc-8535-4448-903e-854af460b254",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -44,7 +44,7 @@
         "accession":"4DNEXO67APT1",
         "biosample":"231111bc-8535-4448-903e-854af460b254",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -82,7 +82,7 @@
         "accession":"4DNEXO67APV1",
         "biosample": "231111bc-8535-4448-903e-854af460b242",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "digestion_enzyme": "DNaseI",
         "digestion_time": 30,
         "digestion_temperature": 37,
@@ -108,7 +108,7 @@
         "accession":"4DNEXO67APY1",
         "biosample":"231111bc-8535-4448-903e-854af460ba4d",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -138,7 +138,7 @@
         "accession":"4DNEXO67APZ1",
         "biosample":"231111bc-8535-4448-903e-854af460b254",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -168,7 +168,7 @@
         "accession":"4DNEXO67APZ7",
         "biosample":"111111bc-1111-4448-903e-854af460b123",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -193,7 +193,7 @@
         "accession":"4DNEXO67APZ8",
         "biosample":"111112bc-1111-4448-903e-854af460b123",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,
@@ -218,7 +218,7 @@
         "accession":"4DNEXO67APZ9",
         "biosample":"111112bc-1111-4448-903e-854af460b123",
         "fragmentation_method": "chemical",
-        "experiment_type": "b17b17cd-544a-3a3b-abc3-17a544544b3c",
+        "experiment_type": "832b7989-69ab-4bb6-97b9-195d5dd965ea",
         "enzyme_lot_number": "123456",
         "average_fragment_size": 100,
         "digestion_time": 30,