From d1426bb99721f46dbee39e63df83d440ff2fbe2f Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Fri, 22 Mar 2024 08:04:40 -0400 Subject: [PATCH 01/22] scaffolding --- dlx_rest/static/js/modals/export.js | 59 +++++++++++++++++++++++++++++ dlx_rest/static/js/search/search.js | 10 ++++- 2 files changed, 68 insertions(+), 1 deletion(-) create mode 100644 dlx_rest/static/js/modals/export.js diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js new file mode 100644 index 00000000..6e8d035e --- /dev/null +++ b/dlx_rest/static/js/modals/export.js @@ -0,0 +1,59 @@ +import { Jmarc } from "../jmarc.mjs" + +export let exportmodal = { + props: ["links"], + template: `
+ + + +
`, + data: function() { + return { + collection: null, + recordId: null, + showModal: false, + record: null, + showSpinner: false + } + }, + methods: { + show: async function() { + Jmarc.apiUrl = this.api_prefix + if (this.collection && this.recordId) { + this.showSpinner = true + Jmarc.get(this.collection, this.recordId).then(jmarc => { + // The normal way to set things like this is to use a reactive data property + // But we can also just set the innerText to what we get here, which preserves + // the newline characters + let previewText = document.getElementById("preview-text") + previewText.innerText = jmarc.toStr() + }).then( () => {this.showSpinner = false}) + } else { + this.showModal = false + } + this.showModal = true + } + } +} \ No newline at end of file diff --git a/dlx_rest/static/js/search/search.js b/dlx_rest/static/js/search/search.js index adbfff59..b81538e0 100644 --- a/dlx_rest/static/js/search/search.js +++ b/dlx_rest/static/js/search/search.js @@ -3,6 +3,7 @@ import { countcomponent } from "./count.js"; import basket from "../api/basket.js"; import user from "../api/user.js"; import { Jmarc } from "../jmarc.mjs"; +import { exportmodal } from "../modals/export.js"; export let searchcomponent = { // onclick="addRemoveBasket("add","{{record['id']}}","{{coll}}","{{prefix}}")" @@ -155,6 +156,7 @@ export let searchcomponent = { None Send Selected to Basket (limit: 100) Speech Review +
@@ -215,6 +217,7 @@ export let searchcomponent = {
  • Next
  • +
    `, data: function () { let myParams = this.search_url.split("?")[1]; @@ -812,10 +815,15 @@ export let searchcomponent = { let toggleButton = document.getElementById("preview-toggle-" + recordId); toggleButton.className = "fas fa-file preview-toggle"; toggleButton.title = "preview record"; + }, + showExportModal() { + console.log(this.links.format) + this.$refs.exportmodal.show() } }, components: { 'sortcomponent': sortcomponent, - 'countcomponent': countcomponent + 'countcomponent': countcomponent, + 'exportmodal': exportmodal } } \ No newline at end of file From c1656c02392bb3f96838850cec2c682ad970de20 Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Fri, 22 Mar 2024 09:58:45 -0400 Subject: [PATCH 02/22] add format selector --- dlx_rest/static/js/modals/export.js | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index 6e8d035e..a0981023 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -19,6 +19,14 @@ export let exportmodal = { Loading...
    +
    + Select format + +
    `, data: function() { return { - collection: null, - recordId: null, showModal: false, - record: null, - showSpinner: false + showSpinner: false, + selectedExportUrl: this["links"], + results: [] } }, methods: { show: async function() { - Jmarc.apiUrl = this.api_prefix - if (this.collection && this.recordId) { - this.showSpinner = true - Jmarc.get(this.collection, this.recordId).then(jmarc => { - // The normal way to set things like this is to use a reactive data property - // But we can also just set the innerText to what we get here, which preserves - // the newline characters - let previewText = document.getElementById("preview-text") - previewText.innerText = jmarc.toStr() - }).then( () => {this.showSpinner = false}) - } else { - this.showModal = false - } this.showModal = true + }, + setFormat(format) { + let exportUrl = this["links"][format.toUpperCase()] + let previewUrl = this["links"]["brief"].replace(/\&limit=\d{1,3}/, "&limit=5") + fetch(previewUrl).then(response => response.json().then( jsonData => { + console.log(jsonData) + this.results = jsonData.data + })) } } } \ No newline at end of file From 4d8a2230187cdb5c084796801261809ce771e8a1 Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Fri, 5 Apr 2024 09:05:43 -0400 Subject: [PATCH 04/22] use links object and enable download --- dlx_rest/static/js/modals/export.js | 45 ++++++++++++++++++++++------- dlx_rest/static/js/search/search.js | 6 ++-- 2 files changed, 37 insertions(+), 14 deletions(-) diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index 9730fac9..bc63ce3d 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -1,7 +1,11 @@ import { Jmarc } from "../jmarc.mjs" export let exportmodal = { - props: ["links"], + props: { + links: { + type: Object + } + }, template: `
    -
    - Select format - -
    +
    @@ -49,27 +57,32 @@ export let exportmodal = { return { showModal: false, showSpinner: false, - selectedExportUrl: null, - results: [] + selectedFormat: 'mrk', + selectedFields: null, + selectedExportUrl: null } }, methods: { show: async function() { this.showModal = true + this.setFormat('mrk') }, setFormat(format) { + this.selectedFormat = format this.selectedExportUrl = this.links.format[format.toUpperCase()] - this.submitExport(format) }, - setOutputFields(fields) { - if (this.selectedExportUrl) { - this.selectedExportUrl = `${this.selectedExportUrl}&of=${encodeURIComponent(fields)}` - } + setOutputFields(e) { + this.selectedFields = e.target.value + let url = new URL(this.selectedExportUrl) + let search = new URLSearchParams(url.search) + search.set("of", e.target.value) + url.search = search + this.selectedExportUrl = url }, - submitExport(format) { + submitExport() { fetch(this.selectedExportUrl).then( response => { response.blob().then( blob => { - this.download(blob, `export.${format}`) + this.download(blob, `export.${this.selectedFormat}`) }) }) }, From 5a08dd22910c06b2141e1fd87c300ed908f01ced Mon Sep 17 00:00:00 2001 From: jbukhari Date: Wed, 5 Jun 2024 11:26:00 -0400 Subject: [PATCH 07/22] Jbukhari/patch/aaronhelton/issue779 (#1439) * disable non functional inputs * create download tag inside template --- dlx_rest/static/js/modals/export.js | 24 ++++++++++++++---------- 1 file changed, 14 insertions(+), 10 deletions(-) diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index ad66854a..48cf9a82 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -26,11 +26,7 @@ export let exportmodal = { @@ -79,7 +83,7 @@ export let exportmodal = { url.search = search this.selectedExportUrl = url }, - submitExport() { + /* submitExport() { fetch(this.selectedExportUrl).then( response => { response.blob().then( blob => { this.download(blob, `export.${this.selectedFormat}`) @@ -96,6 +100,6 @@ export let exportmodal = { a.click() document.body.removeChild(a) window.URL.revokeObjectURL(url) - } + } */ } } \ No newline at end of file From fb519f2713dc44fea1f8cce9f87e292c83f00bce Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Wed, 19 Jun 2024 15:07:40 -0400 Subject: [PATCH 08/22] enable CSV in export --- dlx_rest/api/__init__.py | 7 ++++++- dlx_rest/static/js/modals/export.js | 4 ++-- requirements.txt | 2 +- 3 files changed, 9 insertions(+), 4 deletions(-) diff --git a/dlx_rest/api/__init__.py b/dlx_rest/api/__init__.py index 17637bc7..13be56c2 100644 --- a/dlx_rest/api/__init__.py +++ b/dlx_rest/api/__init__.py @@ -193,7 +193,7 @@ class RecordsList(Resource): args.add_argument( 'format', type=str, - choices=['json', 'xml', 'mrk', 'mrc', 'brief', 'brief_speech'], + choices=['json', 'xml', 'mrk', 'mrc', 'csv', 'tsv', 'brief', 'brief_speech'], help='Formats the list as a batch of records in the specified format' ) args.add_argument( @@ -301,6 +301,10 @@ def get(self, collection): return Response(recordset.to_xml(), mimetype='text/xml') elif fmt == 'mrk': return Response(recordset.to_mrk(), mimetype='text/plain') + elif fmt == 'csv': + return Response(recordset.to_csv(), mimetype='text/csv') + elif fmt == 'tsv': + return Response(recordset.to_csv(), mimetype='text/tab-separated-values') elif fmt == 'brief': schema_name='api.brieflist' make_brief = brief_bib if recordset.record_class == Bib else brief_auth @@ -345,6 +349,7 @@ def get(self, collection): 'list': URL('api_records_list', collection=collection, start=start, limit=limit, search=search, sort=sort_by, direction=args.direction, subtype=args.subtype).to_str(), 'XML': URL('api_records_list', collection=collection, start=start, limit=limit, search=search, format='xml', sort=sort_by, direction=args.direction, subtype=args.subtype).to_str(), 'MRK': URL('api_records_list', collection=collection, start=start, limit=limit, search=search, format='mrk', sort=sort_by, direction=args.direction, subtype=args.subtype).to_str(), + 'CSV': URL('api_records_list', collection=collection, start=start, limit=limit, search=search, format='csv', sort=sort_by, direction=args.direction, subtype=args.subtype).to_str(), }, 'sort': { 'updated': URL('api_records_list', collection=collection, start=start, limit=limit, search=search, format=fmt, sort='updated', direction=new_direction, subtype=args.subtype).to_str() diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index 48cf9a82..ddcbe5b9 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -36,8 +36,8 @@ export let exportmodal = {
    - - + +

    Output Fields: diff --git a/requirements.txt b/requirements.txt index bb6ee420..c3ed4ffa 100644 --- a/requirements.txt +++ b/requirements.txt @@ -10,7 +10,7 @@ cfn-flip==1.3.0 charset-normalizer==3.3.2 click==8.1.7 cryptography==42.0.7 -dlx @ git+https://github.com/dag-hammarskjold-library/dlx@v1.4.3 +dlx @ git+https://github.com/dag-hammarskjold-library/dlx@v1.4.4 dnspython==2.6.1 email-validator==1.1.3 exceptiongroup==1.2.1 From dd09719181f26258e6d52c0ed6ebbadceba5f6cc Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Thu, 20 Jun 2024 09:33:59 -0400 Subject: [PATCH 09/22] set output fields --- dlx_rest/api/__init__.py | 14 ++++++++++++++ dlx_rest/static/js/modals/export.js | 2 +- 2 files changed, 15 insertions(+), 1 deletion(-) diff --git a/dlx_rest/api/__init__.py b/dlx_rest/api/__init__.py index 13be56c2..91a51dce 100644 --- a/dlx_rest/api/__init__.py +++ b/dlx_rest/api/__init__.py @@ -206,6 +206,11 @@ class RecordsList(Resource): type=str, choices=['default', 'speech', 'vote', 'all', ''] ) + args.add_argument( + 'of', + type=str, + help='Comma separated list of fields you want returned (e.g., for export)' + ) @ns.doc(description='Return a list of MARC Bibliographic or Authority Records') @ns.expect(args) @@ -267,6 +272,15 @@ def get(self, collection): # make sure logical fields are available for sorting tags += (list(DlxConfig.bib_logical_fields.keys()) + list(DlxConfig.auth_logical_fields.keys())) project = dict.fromkeys(tags, True) + elif fmt in ['mrk', 'xml', 'csv']: + project = None + output_fields = args.get("of") + if output_fields is not None: + tags = output_fields.split(',') + # make sure logical fields are available for sorting + tags += (list(DlxConfig.bib_logical_fields.keys()) + list(DlxConfig.auth_logical_fields.keys())) + project = dict.fromkeys(tags, True) + elif fmt: project = None else: diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index ddcbe5b9..af6fbe57 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -41,7 +41,7 @@ export let exportmodal = {
    Output Fields: - +
    - Output Fields: - + Output Fields: + From c6b45cd958deb0f493a91d3da9205581441c1595 Mon Sep 17 00:00:00 2001 From: Aaron Helton Date: Fri, 26 Jul 2024 15:55:06 -0400 Subject: [PATCH 19/22] add export "list type" in search args and try to use it in export.js --- dlx_rest/api/__init__.py | 22 ++++++++++++++-------- dlx_rest/static/js/modals/export.js | 8 ++++++++ 2 files changed, 22 insertions(+), 8 deletions(-) diff --git a/dlx_rest/api/__init__.py b/dlx_rest/api/__init__.py index e9a1659c..7547de4d 100644 --- a/dlx_rest/api/__init__.py +++ b/dlx_rest/api/__init__.py @@ -218,6 +218,13 @@ class RecordsList(Resource): help='Toggle the search type between Atas and Community', default='community' ) + args.add_argument( + 'listtype', + type=str, + choices=['default','export'], + help='Choose whether this is a record export or not', + default='default' + ) @ns.doc(description='Return a list of MARC Bibliographic or Authority Records') @ns.expect(args) @@ -276,12 +283,17 @@ def get(self, collection): # limit limit = int(args.limit or 100) + + # List type + list_type = args.listtype or 'default' + print(list_type) # format fmt = args['format'] or None - if fmt != 'brief_speech' and limit > 1000: - abort(404, 'Maximum limit is 1000') + if limit > 1000: + if fmt != brief_speech and list_type != 'export': + abort(404, 'Maximum limit is 1000') if fmt == 'brief': tags = ['191', '245', '269', '520', '596', '700', '710', '711', '791', '989', '991', '992'] if collection == 'bibs' \ @@ -448,12 +460,6 @@ def post(self, collection): else: abort(500, 'POST request failed for unknown reasons') -# Records Export -@ns.route('/marc//records/export') -@ns.param('collection', "bibs" or "auths") -class RecordsExport(Resource): - def post(self): - pass # Records list count @ns.route('/marc//records/count') diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index 0bed148e..28285728 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -74,12 +74,20 @@ export let exportmodal = { setFormat(format) { this.selectedFormat = format this.selectedExportUrl = this.links.format[format.toUpperCase()] + let url = new URL(this.selectedExportUrl) + let search = new URLSearchParams(url.search) + search.set("limit", 10000) + search.set("listtype", "export") + url.search = search + this.selectedExportUrl = url }, setOutputFields(e) { this.selectedFields = e.target.value let url = new URL(this.selectedExportUrl) let search = new URLSearchParams(url.search) search.set("fields", e.target.value) + search.set("limit", 10000) + search.set("listtype", "export") url.search = search this.selectedExportUrl = url }, From 8f2283a9411ddd6eca13369b4e54116fb02be69e Mon Sep 17 00:00:00 2001 From: jbukhari Date: Fri, 2 Aug 2024 13:30:16 -0400 Subject: [PATCH 20/22] Use API pagination to build export file (#1501) * Bump zipp from 3.8.0 to 3.19.1 (#1476) Bumps [zipp](https://github.com/jaraco/zipp) from 3.8.0 to 3.19.1. - [Release notes](https://github.com/jaraco/zipp/releases) - [Changelog](https://github.com/jaraco/zipp/blob/main/NEWS.rst) - [Commits](https://github.com/jaraco/zipp/compare/v3.8.0...v3.19.1) --- updated-dependencies: - dependency-name: zipp dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * NOT xref; $ regex search; CSV serialization (#1450) * update dlx * enable csv and tsv formatted records list api response * update dlx and urllib3 * Enable subtype=all search and count (#1465) * enable subtype=all search and count * enable subtype=all from record editor and highlight all bibs collections (#1467) * enable subtype=all from record editor and highlight all bibs collections * Update count component to link to subtype=all searches for auths --------- Co-authored-by: Aaron Helton * delete 999 on auth clone (#1468) * Update python-tests.yml (#1473) Run tests on PRs against all branches * browse list bugfixes (#1475) * Build auth cache on startup (#1480) * build auth cache on startup * Update dlx * update dlx * Update workflow triggers (#1481) * Bump sentry-sdk from 1.25.1 to 2.8.0 (#1482) Bumps [sentry-sdk](https://github.com/getsentry/sentry-python) from 1.25.1 to 2.8.0. - [Release notes](https://github.com/getsentry/sentry-python/releases) - [Changelog](https://github.com/getsentry/sentry-python/blob/master/CHANGELOG.md) - [Commits](https://github.com/getsentry/sentry-python/compare/1.25.1...2.8.0) --- updated-dependencies: - dependency-name: sentry-sdk dependency-type: direct:production ... Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> * make a new read-only record view (#1483) * make a new read-only record view * add edit link and control fields * downgrade Vue 2 because the latest release breaks advanced search (#1485) * Bump version to 2.10.12 (#1494) Fixes #1486 * Add subtype filtering to bibs search pages (#1497) * build export file by paginating api * adjust xml buffer * spinner and layout tweaks * reload window on modal close to cancel ongoing page fetching * show api page traversal status * xml node list syntax bugfix --------- Signed-off-by: dependabot[bot] Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Aaron Helton --- .github/workflows/python-tests.yml | 2 +- dlx_rest/api/__init__.py | 30 +- dlx_rest/api/utils.py | 2 - dlx_rest/config.py | 2 +- dlx_rest/routes.py | 4 +- dlx_rest/static/js/modals/export.js | 103 +- dlx_rest/static/js/readonly_record.js | 29 + dlx_rest/static/js/search/sort.js | 24 +- dlx_rest/static/js/vue/vue2.js | 12020 +++++++++++++++++++++++- dlx_rest/templates/record.html | 15 +- requirements.txt | 4 +- 11 files changed, 12168 insertions(+), 67 deletions(-) create mode 100644 dlx_rest/static/js/readonly_record.js diff --git a/.github/workflows/python-tests.yml b/.github/workflows/python-tests.yml index 82c1aa3b..baf40c02 100644 --- a/.github/workflows/python-tests.yml +++ b/.github/workflows/python-tests.yml @@ -5,7 +5,7 @@ name: Python package on: pull_request: - types: [ opened ] + types: [ opened, synchronize, reponend ] env: DLX_REST_TESTING: True diff --git a/dlx_rest/api/__init__.py b/dlx_rest/api/__init__.py index 7547de4d..9a443b26 100644 --- a/dlx_rest/api/__init__.py +++ b/dlx_rest/api/__init__.py @@ -31,13 +31,13 @@ from dlx_rest.api.utils import ClassDispatch, URL, ApiResponse, Schemas, abort, brief_bib, brief_auth, brief_speech, item_locked, has_permission # Init -authorizations = { - 'basic': { - 'type': 'basic' - } -} -api = Api(app, doc='/api/', authorizations=authorizations) +# build the auth cache in a non blocking thread +def build_cache(): Auth.build_cache() + +threading.Thread(target=build_cache, args=[]).start() + +api = Api(app, doc='/api/', authorizations={'basic': {'type': 'basic'}}) ns = api.namespace('api', description='DLX MARC REST API') # Set up the login manager for the API @@ -218,13 +218,6 @@ class RecordsList(Resource): help='Toggle the search type between Atas and Community', default='community' ) - args.add_argument( - 'listtype', - type=str, - choices=['default','export'], - help='Choose whether this is a record export or not', - default='default' - ) @ns.doc(description='Return a list of MARC Bibliographic or Authority Records') @ns.expect(args) @@ -283,17 +276,12 @@ def get(self, collection): # limit limit = int(args.limit or 100) - - # List type - list_type = args.listtype or 'default' - print(list_type) # format fmt = args['format'] or None - if limit > 1000: - if fmt != brief_speech and list_type != 'export': - abort(404, 'Maximum limit is 1000') + if fmt != 'brief_speech' and limit > 1000: + abort(404, 'Maximum limit is 1000') if fmt == 'brief': tags = ['191', '245', '269', '520', '596', '700', '710', '711', '791', '989', '991', '992'] if collection == 'bibs' \ @@ -304,7 +292,7 @@ def get(self, collection): project = dict.fromkeys(tags, True) elif fmt == 'brief_speech': tags = ['269', '596', '700', '710', '711', '791', '991', '992'] - + # make sure logical fields are available for sorting tags += (list(DlxConfig.bib_logical_fields.keys()) + list(DlxConfig.auth_logical_fields.keys())) project = dict.fromkeys(tags, True) diff --git a/dlx_rest/api/utils.py b/dlx_rest/api/utils.py index 060598d8..0e28cd33 100644 --- a/dlx_rest/api/utils.py +++ b/dlx_rest/api/utils.py @@ -211,7 +211,6 @@ def brief_bib(record): # Should be truncated in display with hover showing the rest f520 = [record.get_value('520', 'a')] - print(f520) return { '_id': record.id, @@ -323,7 +322,6 @@ def has_permission(user, action, record, collection): bool_list.append("F") else: bool_list.append("F") - #print("boolean list:",bool_list) if "F" in bool_list: return False else: diff --git a/dlx_rest/config.py b/dlx_rest/config.py index 771b07fa..1844229e 100644 --- a/dlx_rest/config.py +++ b/dlx_rest/config.py @@ -6,7 +6,7 @@ class Config(object): DEBUG = False TESTING = False - VERSION = "v2.10.10" # Set this in each new milestone/release. + VERSION = "v2.10.12" # Set this in each new milestone/release. bucket = 'undl-files' diff --git a/dlx_rest/routes.py b/dlx_rest/routes.py index 3cd39422..34fd92ee 100644 --- a/dlx_rest/routes.py +++ b/dlx_rest/routes.py @@ -575,10 +575,8 @@ def review_speeches(): @app.route('/records//', methods=['GET']) @login_required def get_record_by_id(coll,id): - # register the permission, but don't require it yet, TBI - #register_permission('updateRecord') this_prefix = url_for('doc', _external=True) - return render_template('record.html', coll=coll, record_id=id, prefix=this_prefix) + return render_template('record.html', collection=coll, record_id=id, api_prefix=this_prefix) @app.route('/records//new') @login_required diff --git a/dlx_rest/static/js/modals/export.js b/dlx_rest/static/js/modals/export.js index 28285728..bdc5b12b 100644 --- a/dlx_rest/static/js/modals/export.js +++ b/dlx_rest/static/js/modals/export.js @@ -11,19 +11,13 @@ export let exportmodal = {