TDMS file renderer

.github/workflows/test-build.yml

@@ -0,0 +1,63 @@
+name: mfr_ci_testing
+
+on: [push, pull_request, workflow_dispatch]
+
+jobs:
+
+  build:
+    runs-on: ubuntu-20.04
+    env:
+      GHA_DISTRO: ubuntu-20.04
+    if: "!contains(github.event.head_commit.message, 'skip ci')"
+    strategy:
+      matrix:
+        python-version: [3.6]
+    steps:
+      - name: Git checkout
+        uses: actions/checkout@v2
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v2
+        with:
+          python-version: ${{ matrix.python-version }}
+      - name: Cache Build Requirements
+        id: pip-cache-step
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+      - name: install dependencies
+        if: steps.pip-cache-step.outputs.cache-hit != 'true'
+        run: |
+          python -m pip install --upgrade pip
+          pip install -r dev-requirements.txt
+
+  runtests:
+    name: Run unit tests
+    needs: build
+    runs-on: ubuntu-20.04
+    env:
+      GHA_DISTRO: ubuntu-20.04
+    steps:
+      - uses: actions/checkout@v2
+      - name: Set up Python 3.6
+        uses: actions/setup-python@v2
+        with:
+          python-version: 3.6
+      - name: Cache pip
+        uses: actions/cache@v2
+        with:
+          path: ${{ env.pythonLocation }}
+          key: ${{ env.GHA_DISTRO }}-${{ env.pythonLocation }}-${{ hashFiles('requirements.txt', 'dev-requirements.txt') }}
+      - name: run syntax checks
+        run: |
+          flake8 .
+      - name: build plugins
+        run: |
+          python setup.py develop
+      - name: run unit tests
+        run: |
+          py.test --cov-report term-missing --cov mfr tests
+      - name: Upload coverage data to coveralls.io
+        run: coveralls --service=github
+        env:
+          GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}

AUTHORS.rst

@@ -34,3 +34,4 @@ Contributors
 - Longze Chen `@cslzchen <https://github.com/cslzchen>`_
 - Jonathon Love `@jonathon-love <https://github.com/jonathon-love>`_
 - Josh Bird `@birdbrained <https://github.com/birdbrained>`_
+- Connor Bailey `@cbbcbail <https://github.com/cbbcbail>`_

README.md

@@ -1,10 +1,8 @@
 # MFR (Modular File Renderer)
 
-`master` Build Status: [![Build Status](https://travis-ci.org/CenterForOpenScience/modular-file-renderer.svg?branch=master)](https://travis-ci.org/CenterForOpenScience/modular-file-renderer)
+`master` Build Status: ![Build Status](https://github.com/CenterForOpenScience/modular-file-renderer/actions/workflows/test-build.yml/badge.svg?branch=master)[![Coverage Status](https://coveralls.io/repos/github/CenterForOpenScience/modular-file-renderer/badge.svg?branch=master)](https://coveralls.io/github/CenterForOpenScience/modular-file-renderer?branch=master)
 
-`develop` Build Status: [![Build Status](https://travis-ci.org/CenterForOpenScience/modular-file-renderer.svg?branch=develop)](https://travis-ci.org/CenterForOpenScience/modular-file-renderer)
-
-[![Coverage Status](https://coveralls.io/repos/github/CenterForOpenScience/modular-file-renderer/badge.svg)](https://coveralls.io/github/CenterForOpenScience/modular-file-renderer)
+`develop` Build Status: ![Build Status](https://github.com/CenterForOpenScience/modular-file-renderer/actions/workflows/test-build.yml/badge.svg?branch=develop)[![Coverage Status](https://coveralls.io/repos/github/CenterForOpenScience/modular-file-renderer/badge.svg?branch=develop)](https://coveralls.io/github/CenterForOpenScience/modular-file-renderer?branch=develop)
 
 A Python package for rendering files to HTML via an embeddable iframe.
 

configuringDevEnv.md

@@ -0,0 +1,94 @@
+# Configuring the Development Environment for OSF:
+
+Instructions for MacOS Big Sur compiled from [**README-docker-compose.md**](https://github.com/CenterForOpenScience/osf.io/blob/develop/README-docker-compose.md), correspondence with Fitz Elliot, and the [documentation](https://cosdev.readthedocs.io/en/latest/osf/setup.html): modified to fix errors.
+
+- Clone the repository:
+  - ```$ git clone [osf.io]```
+  - ```$ cd osf.io```
+
+- Install the Docker Client
+
+- Grant Docker minimum resources of 1 CPU, 8GB memory, 2GB swap, and 32GB disk image size
+
+- Alias the loopback interface:
+  - ```$ export libdir='/Library/LaunchDaemons'```
+  - ```$ export file='com.runlevel1.lo0.192.168.168.167.plist'```
+  - ```$ sudo cp $file $libdir```
+  - ```$ sudo chmod 0644 $libdir/$file```
+  - ```$ sudo chown root:wheel $libdir/$file```
+  - ```$ sudo launchctl load $libdir/$file```
+
+- Configure application:
+  - ```$ cp ./website/settings/local-dist.py ./website/settings/local.py```
+  - ```$ cp ./api/base/settings/local-dist.py ./api/base/settings/local.py```
+  - ```$ cp ./docker-compose-dist.override.yml ./docker-compose.override.yml```
+  - ```$ cp ./tasks/local-dist.py ./tasks/local.py```
+
+- Start Application:
+  - ```$ docker compose up requirements mfr_requirements wb_requirements```
+  - ```$ docker compose up -d elasticsearch postgres mongo rabbitmq```
+  - ```$ rm -Rf ./node_modules```
+  - ```$ docker compose up -d assets```
+  - ```$ docker compose up -d admin_assets```
+  - ```$ docker-compose up -d mfr wb fakecas sharejs```
+  - ```$ docker-compose run --rm web python3 manage.py migrate```
+  - ```$ docker-compose up -d worker web api admin preprints registries ember\_osf\_web```
+
+- *Note: once it has been setup, you can [quickstart](https://github.com/CenterForOpenScience/osf.io/blob/develop/README-docker-compose.md\#quickstart-running-all-osf-services-in-the-background) to quickly launch in the future.*
+- Access the OSF at **http://localhost:5000**.
+- Click sign up to create an account.
+- Access the 'web' containers logs to view the confirmation email for new accounts:
+- ```$ docker compose logs -f --tail 1000 web```
+- Copy the confirmation link and paste into the browser to confirm the account then log in using the email address used to create the account. This allows test files, projects, and folders to be created and accessed through the normal methods.
+
+
+## Instructions for building and testing the modified MFR container:
+
+- Go to the **modular-file-renderer** directory
+- ```$ docker image build -t mfr:local .```
+- Go to the **osf.io** directory
+- Edit the **docker-compose.yml** file. Under **mfr** and **mfr_requirements**, change the image to **image: mfr:local**
+- ```$ docker compose up -d --force-recreate mfr_requirements```
+- ```$ docker compose up -d --force-recreate mfr```
+- Restart the Docker application
+
+
+### In order to verify that the version of MFR you are running is the correct, most up to date version:
+
+- Modify the version number in **modular-file-renderer/mfr/version.py** to indicate that changes have been made.
+- Check the version of the container at **http://localhost:7778/status**
+- If the status version number matches the modified version number, you know that the modified version of the MFR container has been successfully built and deployed by Docker.
+
+
+## Debugging Problems
+
+- Running out of memory in Docker can be solved by doing a system prune of the containers and reinstalling them. Allocating more memory in Docker is a good idea.
+- ```$ docker system prune -a```
+
+
+# Configuring the development environment for MFR:
+Instructions for MacOS Big Sur compiled from []**CONTRIBUTING.rst**](https://github.com/CenterForOpenScience/modular-file-renderer/blob/develop/CONTRIBUTING.rst), [**README.md**](https://github.com/CenterForOpenScience/modular-file-renderer/blob/develop/README.md), the [installation documentation](https://modular-file-renderer.readthedocs.io/en/latest/install.html\#install) and the [contribution documentation](https://modular-file-renderer.readthedocs.io/en/latest/contributing.html) and modified to fix errors.
+
+- Clone the repository:
+  - ```$ git clone [modular-file-renderer]```
+  - ```$ cd modular-file-renderer```
+- Create the virtual environment:
+  - ```$ pip install virtualenv virtualenvwrapper```
+  - ```$ brew update```
+  - ```$ brew install python3 r pspp unoconv pyenv```
+
+- Add the following initialization lines to either the **~/.bashrc** or **~/.zprofile** files for either bash or zsh shells respectively:
+```
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+```
+
+- ```$ source ~/.[bashrc or zprofile]```
+- ```$ pyenv virtualenv 3.6.4 mfr```
+- ```$ pip install setuptools==37.0.0```
+- ```$ pip install invoke==0.13.0```
+- Modify **requirements.txt** to update the version number for **reportlab==3.4.0** to **reportlab==3.5.56** due to an incompatibility with the old version.
+- ```$ invoke install -d```
+
+- In order to run the development unit testing procedure:
+  - ```$ invoke test```

dev-requirements.txt

@@ -5,12 +5,12 @@
 
 beautifulsoup4
 colorlog==2.5.0
+coveralls
 flake8==3.0.4
 ipdb
 mccabe
 pydevd==0.0.6
 pyflakes
 pytest==2.8.2
 pytest-cov==2.2.0
-python-coveralls==2.9.1
 pyzmq==14.4.1

mfr/extensions/tdms/__init__.py

@@ -0,0 +1 @@
+from .render import TdmsRenderer  # noqa

mfr/extensions/tdms/render.py

@@ -0,0 +1,134 @@
+import os
+import base64
+from io import BytesIO
+
+import nptdms
+import pandas as pd
+import matplotlib.pyplot as plt
+import numpy as np
+import datetime
+from mako.lookup import TemplateLookup
+
+from mfr.core import extension
+from mfr.core import utils
+
+class TdmsRenderer(extension.BaseRenderer):
+
+    TEMPLATE = TemplateLookup(
+        directories=[
+            os.path.join(os.path.dirname(__file__), 'templates')
+        ]).get_template('viewer.mako')
+
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.metrics.add('nptdms_version', nptdms.version.__version__)
+
+    def render(self):
+        """Render a tdms file to html."""
+        maxTableLength = 100  # maximum rows to include in data table
+        minDigitLength = 4  # minumum digits before scientific notation is used
+        formatString = "{:." + str(minDigitLength) + "g}"
+
+        fig, ax = plt.subplots()  # create matplotlib figure object
+        ax.grid(True, alpha=0.1)  # specify matplotlib grid properties
+
+        # empty data structures to be filled with file data in loops
+        channelNames = []
+        lineCollections = []
+        properties = ""
+        data = pd.DataFrame()
+
+        tdms_file = nptdms.TdmsFile.open(self.file_path, raw_timestamps=True)
+        fileMetadata = TdmsRenderer.formatMetadata(tdms_file.properties.items())
+
+        # Parse group data and metadata and generate html
+        for group in tdms_file.groups():
+            groupClass = group.name.replace(" ", "")
+            buttonId = groupClass + "Button"
+            showHide = "showHide(\'" + groupClass + "\', \'" + buttonId + "\')"
+            rowTag = "<tr class=\'group\' onclick=\"" + showHide + "\">"
+            buttonTag = "<td class=\'button\' id=\'" + buttonId + "\'>+</td>"
+            properties += rowTag + "<td>Group " + group.name + "</td>"
+            properties += buttonTag + "</tr>"
+
+            # Parse channel data and metadata and generate html
+            for channel in group.channels():
+                channelClass = channel.name.replace(" ", "")
+                buttonId = channel.name.replace(" ", "") + "Button"
+                showHide = "showHide(\'" + channelClass + "\', \'" + buttonId + "\')"
+                rowTag = "<tr class=\'channel " + groupClass + "\' onclick=\"" + showHide + "\">"
+                buttonTag = "<td class=\'button\' id=\'" + buttonId + "\'>+</td>"
+                properties += rowTag + "<td>Channel " + channel.name + "</td>" + buttonTag + "</tr>"
+                channelLength = len(channel)
+
+                # Parse dictionary of properties:
+                for property, value in channel.properties.items():
+                    rowTag = "<tr class=\'property " + channelClass + " " + groupClass + "Child\'>"
+                    leftCol = rowTag + "<td>" + property + "</td>"
+                    if utils.isfloat(value):  # reformat float values
+                        value = formatString.format(float(value))
+                    rightCol = "<td>" + str(value) + "</td></tr>"
+                    properties += leftCol + rightCol
+
+                # Access numpy array of data for channel:
+                if (channelLength > 1):  # Only access channels with datasets > 1
+                    # Plotting on a time axis
+                    if (channel.properties['wf_start_time'] and channel.properties['wf_increment']):
+                        start = channel.properties['wf_start_time'].as_datetime()
+                        start = (start - datetime.datetime(1904, 1, 1)).total_seconds()
+                        increment = channel.properties['wf_increment']
+                        stop = start + increment * channelLength
+                        timeAxis = np.linspace(start, stop, channelLength)
+                        line = ax.plot(timeAxis, channel, linewidth=2, label=channel.name)
+                        plt.xticks(rotation=45)
+                        plt.xlabel("Time (s)")
+                    else:
+                        line = ax.plot(channel, linewidth=2, label=channel.name)
+
+                    lineCollections.append(line)
+                    channelNames.append(channel.name)
+                    data[channel.name] = channel[:maxTableLength]
+
+        ax.legend(channelNames, bbox_to_anchor=(1, 1))
+        plotFile = BytesIO()  # create byte sream object
+        fig.savefig(plotFile, format='png', bbox_inches='tight')  # export plot to png in byte stream
+        encoded = base64.b64encode(plotFile.getvalue()).decode('utf-8')  # encode base 64 byte stream data
+        plot = '\'data:image/png;base64,{}\''.format(encoded)  # format encoded data as string
+        table = data.to_html()  # export pandas DataFrame to HTML
+
+        return self.TEMPLATE.render(base=self.assets_url, fileMetadata=fileMetadata, properties=properties, plot=plot, table=table)
+
+    def formatMetadata(items):
+        # Parse property value pairs in file level metadata and generate html
+
+        minDigitLength = 4  # minumum digits before scientific notation is used
+        formatString = "{:." + str(minDigitLength) + "G}"
+        fileMetadata = ""
+
+        for property, value in items:
+            value = str(value)
+            if value.find("\n") > 1:
+                value = value.split("\n")
+                fileMetadata += "<li>" + value[0] + "</li>"
+                fileMetadata += "<ul>"
+                for v in value[1:]:
+                    v = v.replace("\\n", " ").replace("\"", "").split("=")
+                    v[0] = "".join(v[0].split(".")[1:])
+                    v[1] = v[1].strip()
+                    if utils.isfloat(v[1]):  # reformat float values
+                        v[1] = formatString.format(float(v[1]))
+                    fileMetadata += "<li>" + "= ".join(v) + "</li>"
+                fileMetadata += "</ul>"
+            else:
+                fileMetadata += "<li>" + str(property) + ": " + value + "</li>"
+        fileMetadata += "</ul>"
+
+        return fileMetadata
+
+    @property
+    def file_required(self):
+        return True
+
+    @property
+    def cache_result(self):
+        return True