Skip to content

Commit

Permalink
Revert "rebase"
Browse files Browse the repository at this point in the history
This reverts commit 5bbfb5f.
  • Loading branch information
ErykKul committed Oct 5, 2023
1 parent 5bbfb5f commit cceb20c
Show file tree
Hide file tree
Showing 32 changed files with 11,264 additions and 15 deletions.
90 changes: 90 additions & 0 deletions .github/scripts/build_previewers.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
import os, yaml, subprocess, shutil
from checksumdir import dirhash


def _check_metadata_extension(source_dir):
"""Check if the previewer metadata file has the correct extension"""

# List all the files in the previewers directory
files = os.listdir(".")

if "previewer-meta.yaml" in files:
return "previewer-meta.yaml"
elif "previewer-meta.yml" in files:
return "previewer-meta.yml"
else:
raise FileNotFoundError(
f"No previewer metadata file found for source directory {source_dir}"
)


def build_react_app(source_dir, TARGET_DIR):
"""Builds the react app and copies the files to the target directory
Args:
source_dir (str): Source directory of the react app.
TARGET_DIR (str): Target directory to copy the files to.
REPO_DIR (str): Directory of the repository.
"""

# Move to the source directory to read metadata and instructions
os.chdir(source_dir)

# Check if a previewer metadata file exists
metadata_file = _check_metadata_extension(source_dir)

# Read the previewers metadata
metadata = yaml.safe_load(open(metadata_file))

# Create path map to deploy the necessary files
extension_paths = {
"js": os.path.join(TARGET_DIR, "js"),
"css": os.path.join(TARGET_DIR, "css"),
"html": TARGET_DIR,
}

# Get the hash of the source directory
source_hash = dirhash(metadata["checkdir"], "sha256")

if metadata.get("checksum") == source_hash:
print(f"No changes detected for {metadata['name']}")
return

print(f"Changes detected for {metadata['name']} - Building previewer")

for command in metadata["build"]:
# Run the build commands
subprocess.call(command, shell=True)

for file in metadata["files"]:
# Copy the files to the target directory
fname = os.path.basename(file)
extension = fname.split(".")[-1]
shutil.copy(file, os.path.join(extension_paths[extension], fname))

# Update checksum in metadata file
metadata["checksum"] = source_hash
yaml.safe_dump(metadata, open(metadata_file, "w"), sort_keys=False)

print("Successfully built previewer - Checksum updated")


if __name__ == "__main__":
# Build the react previewers
BASE_DIR = "./previewers/react-source/"
REPO_DIR = os.getcwd()
TARGET_DIR = os.path.join(REPO_DIR, "previewers", "betatest")

# Get all the react previewers
react_previewers = [
os.path.join(BASE_DIR, path)
for path in os.listdir(BASE_DIR)
if not path.startswith(".") and path != "README.md"
]

for source_dir in react_previewers:
# Build the react app
build_react_app(source_dir, TARGET_DIR)

# Return to the repository directory
os.chdir(REPO_DIR)
32 changes: 32 additions & 0 deletions .github/workflows/build_react_previewers.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,32 @@
name: Build React previewers

on: [push]

jobs:
build_react_previewers:
runs-on: ubuntu-latest
steps:
- name: Checkout
uses: actions/checkout@v2
- name: Set up Python 3.10
uses: actions/setup-python@v2
with:
python-version: "3.10"
- name: Install dependencies
run: python3 -m pip install pyyaml checksumdir
- name: Set up npm
uses: actions/setup-node@v3
- name: Run script and build apps
run: python3 ./.github/scripts/build_previewers.py
- name: Push built previewers
run: |
if [[ `git status --porcelain` ]]; then
git add --all
git config --global user.name 'Previewer Builder'
git config --global user.email '[email protected]'
git commit -am "Previewer update"
git push
else
echo "Nothing changed!"
fi
39 changes: 39 additions & 0 deletions .gitignore
Original file line number Diff line number Diff line change
@@ -1,2 +1,41 @@
/.project
/.settings
.DS_STORE
node_modules
dist
scripts/flow/*/.flowconfig
.flowconfig
*~
*.pyc
.grunt
_SpecRunner.html
__benchmarks__
build/
remote-repo/
coverage/
.module-cache
fixtures/dom/public/react-dom.js
fixtures/dom/public/react.js
test/the-files-to-test.generated.js
*.log*
chrome-user-data
*.sublime-project
*.sublime-workspace
.idea
*.iml
.vscode
*.swp
*.swo

packages/react-devtools-core/dist
packages/react-devtools-extensions/chrome/build
packages/react-devtools-extensions/chrome/*.crx
packages/react-devtools-extensions/chrome/*.pem
packages/react-devtools-extensions/firefox/build
packages/react-devtools-extensions/firefox/*.xpi
packages/react-devtools-extensions/firefox/*.pem
packages/react-devtools-extensions/shared/build
packages/react-devtools-extensions/.tempUserDataDir
packages/react-devtools-inline/dist
packages/react-devtools-shell/dist
packages/react-devtools-timeline/dist
93 changes: 86 additions & 7 deletions 5.2curlcommands.md
Original file line number Diff line number Diff line change
Expand Up @@ -696,6 +696,54 @@ curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin
}'
```

### H5Web Previewer for HDF5 and NetCDF files:

```bash
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools -d \
'{
"displayName":"H5Web",
"description":"Explore and visualize HDF5 files",
"toolName":"HDF5Preview",
"scope":"file",
"types":["preview"],
"toolUrl":"https://gdcc.github.io/dataverse-previewers/previewers/betatest/HDF5Preview.html",
"toolParameters": {
"queryParameters":[
{"fileid":"{fileId}"},
{"siteUrl":"{siteUrl}"},
{"apiToken":"{apiToken}"},
{"datasetid":"{datasetId}"},
{"datasetversion":"{datasetVersion}"},
{"locale":"{localeCode}"}
]
},
"contentType":"application/x-hdf5"
}'
```

```bash
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools -d \
'{
"displayName":"H5Web",
"description":"Explore and visualize HDF5 files",
"toolName":"HDF5Preview",
"scope":"file",
"types":["preview"],
"toolUrl":"https://gdcc.github.io/dataverse-previewers/previewers/betatest/HDF5Preview.html",
"toolParameters": {
"queryParameters":[
{"fileid":"{fileId}"},
{"siteUrl":"{siteUrl}"},
{"apiToken":"{apiToken}"},
{"datasetid":"{datasetId}"},
{"datasetversion":"{datasetVersion}"},
{"locale":"{localeCode}"}
]
},
"contentType":"application/netcdf"
}'
```

### Markdown Previewer (beta)

```bash
Expand All @@ -721,17 +769,48 @@ curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin
}'
```

### RO-Crate Previewer (beta)
### ESRI Shape Previewer (beta)

This previewer includes a hard-coded file size limit of a zipped shp file of 20 MB (larger zips will not be loaded). If you want to change this limit you can change the value in "previewers/betatest/js/mapshp.js", but then you have to host the customised previewer yourself (e.g. via github pages). Instructions on how to build external tools yourself can be found here: https://guides.dataverse.org/en/latest/api/external-tools.html.

```bash
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools -d \
'{
"displayName":"View Map",
"description":"View a map of the file.",
"toolName":"mapShpPreviewer",
"scope":"file",
"types":["preview"],
"toolUrl":"gdcc.github.io/dataverse-previewers/previewers/betatest/MapShpPreview.html",
"toolParameters": {
"queryParameters":[
{"fileid":"{fileId}"},
{"siteUrl":"{siteUrl}"},
{"key":"{apiToken}"},
{"datasetid":"{datasetId}"},
{"datasetversion":"{datasetVersion}"},
{"locale":"{localeCode}"}
]
},
"contentType":"application/zipped-shapefile"
}'
```

### GeoTIFF Previewer (beta)

Please note that the Geotiff Previewer tries to display ALL image/tiff files in a map (because there is no own mimetype for Geotiffs at the moment). Therefore please only use it if you want to use Geotiffs only. As soon as there is an own mimetype for Geotiffs, this will be updated.

Limits are also defined for previewing raster files. Besides the file size limit (20 MB), a column and row limit (50,000) and a loading timeout (30 seconds) are defined. In case of an unsupported GeoTIFF (e.g. no projection specified), an error message is displayed after 30 seconds stating that the tiff image cannot be loaded. These limits can be adjusted as needed in previewers/betatest/js/mapraster.js, although this adjusted version must then also be hosted by yourself (e.g. via github pages). Instructions on how to build external tools yourself can be found here: https://guides.dataverse.org/en/latest/api/external-tools.html.

```bash
curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin/externalTools -d \
'{
"displayName":"Show RO-Crate",
"description":"View the RO-Crate metadata file.",
"toolName":"rocratePreviewer",
"displayName":"View Map",
"description":"View a map of the file.",
"toolName":"mapShpPreviewer",
"scope":"file",
"types":["preview"],
"toolUrl":"https://gdcc.github.io/dataverse-previewers/previewers/betatest/ROCratePreview.html",
"toolUrl":"gdcc.github.io/dataverse-previewers/previewers/betatest/MapRasterPreview.html",
"toolParameters": {
"queryParameters":[
{"fileid":"{fileId}"},
Expand All @@ -742,6 +821,6 @@ curl -X POST -H 'Content-type: application/json' http://localhost:8080/api/admin
{"locale":"{localeCode}"}
]
},
"contentType":"application/ld+json; profile=\"http://www.w3.org/ns/json-ld#flattened http://www.w3.org/ns/json-ld#compacted https://w3id.org/ro/crate\""
"contentType":"image/tiff"
}'
```
```
4 changes: 3 additions & 1 deletion README.md
Original file line number Diff line number Diff line change
Expand Up @@ -74,7 +74,9 @@ The Spreadsheet Previewer was contributed by [anncie-pcss](https://github.com/an

[Max Planck Digital Library](https://github.com/MPDL) contributed the ZIP Previewer.

[erykkul](https://github.com/erykkul) contributed the Markdown (MD) Previewer and the RO-Crate previewer.
[erykkul](https://github.com/erykkul) contributed the Markdown (MD) Previewer.

[Jan Range](https://github.com/JR-1991) contributed the H5Web Previewer.

## How can I help?

Expand Down
19 changes: 19 additions & 0 deletions previewers/betatest/HDF5Preview.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,19 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="utf-8" />
<meta name="viewport" content="width=device-width, initial-scale=1" />

<title>H5Web</title>

<meta name="description" content="Web-based HDF5 file viewer." />
<script src="https://polyfill.io/v3/polyfill.min.js?features=globalThis"></script>
<script type="module" crossorigin src="./js/hdf5.js"></script>
<link rel="stylesheet" href="./css/hdf5.css">
</head>
<body>
<noscript>You need to enable JavaScript to run this app.</noscript>
<div id="root"></div>

</body>
</html>
49 changes: 49 additions & 0 deletions previewers/betatest/MapRasterPreview.html
Original file line number Diff line number Diff line change
@@ -0,0 +1,49 @@
<html>
<head>
<meta charset="utf-8">
<title class="mapPreviewText">Map Preview</title>
<script src="https://ajax.googleapis.com/ajax/libs/jquery/3.5.1/jquery.min.js"></script>
<script type="text/javascript" src="js/xss.js"></script>
<script src="lib/jquery.i18n.js"></script>
<script src="lib/jquery.i18n.messagestore.js"></script>
<script src="lib/jquery.i18n.language.js"></script>
<script type="text/javascript" src="js/retriever.js"></script>
<!-- Latest compiled and minified CSS -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap.min.css"
integrity="sha384-HSMxcRTRxnN+Bdg0JdbxYKrThecOKuH5zCYotlSAcp1+c8xmyTe9GYg1l9a69psu" crossorigin="anonymous">
<!-- Optional theme -->
<link rel="stylesheet" href="https://stackpath.bootstrapcdn.com/bootstrap/3.4.1/css/bootstrap-theme.min.css"
integrity="sha384-6pzBo3FDv/PJ8r2KRkGHifhEocL+1X2rVCTTkUfGk7/0pbek5mMa1upzvWbrUbOZ" crossorigin="anonymous">
<!-- Leaflet -->
<link rel="stylesheet" href="https://unpkg.com/[email protected]/dist/leaflet.css"
integrity="sha512-xodZBNTC5n17Xt2atTPuE1HxjVMSvLVW9ocqUKLsCC5CXdbqCmblAshOMAS6/keqq/sMZMZ19scR4PsZChSR7A==" crossorigin="anonymous"/>
<script src="https://unpkg.com/[email protected]/dist/leaflet.js"
integrity="sha512-XQoYMqMTK8LvdxXYG3nZ448hOEQiglfqkJs1NOQV44cWnUrBc8PkAOcXy20w0vlaXaVUearIOBhiXZ5V3ynxwA==" crossorigin="anonymous"></script>
<link type="text/css" rel="stylesheet" href="css/preview.css" />
<!--drawing raster-->
<script src="https://unpkg.com/georaster"
integrity="sha512-EPpN3KleOAW9ST4LCK50R9Fu/UWlB92OSsKiCLUOwX0rpDw7PwdWwzLNerdMxlbLZC+cnxybJMS3jl2I1EdpVQ==" crossorigin="anonymous"></script>
<script src="https://unpkg.com/georaster-layer-for-leaflet"
integrity="sha512-bNJ+bvpXepatbIMeaida52kR2AT5lr6a9dTfb/OB0c/3O2KzUrdUi3Gtri0zvJUbfKgmG+wN3GKTkBi/6OkVVA==" crossorigin="anonymous"></script>
<!-- spinner-->
<script src="https://cdnjs.cloudflare.com/ajax/libs/spin.js/2.3.2/spin.js"
integrity="sha512-C7tgVIfPE0ivBKcs2WstAh5y7Njir2odGBjnuIa64SmVzZIoTb8kRrNursRzEv4bNcesPywtVAXqH1GmqRBmpg==" crossorigin="anonymous"></script>
</head>

<body class="container">
<main>
<img id='logo' alt='Site Logo'>
<h1 class="page-title mapPreviewText">Map Preview</h1>
<div class="alert alert-warning" id="file_error" hidden>
<span class="glyphicon glyphicon-warning-sign"></span>&nbsp;<strong>Drawing Error</strong>
</div>
<div class='preview-container'>
<div class='preview-header'></div>
<div class='preview'>
<div id="map" style="width: 800px; height: 500px;"></div>
</div>
</div>
</main>
<script type="text/javascript" src="js/mapraster.js"></script>
</body>
</html>
Loading

0 comments on commit cceb20c

Please sign in to comment.