From 63856a34a6049d3f0f786850dbe63ec9cabf2877 Mon Sep 17 00:00:00 2001 From: Emma Turetsky <turetske@gmail.com> Date: Tue, 16 Apr 2024 16:26:32 -0500 Subject: [PATCH] Added packaging workflows and setup and slightly better README --- .github/workflows/pypi-publish.yml | 0 .github/workflows/testpypi-publish.yml | 91 ++++++++++++++++++++++++++ README.md | 44 ++++++++++++- pyproject.toml | 8 +++ setup.cfg | 11 ++++ setup.py | 43 ++++++++++++ 6 files changed, 195 insertions(+), 2 deletions(-) create mode 100644 .github/workflows/pypi-publish.yml create mode 100644 .github/workflows/testpypi-publish.yml create mode 100644 pyproject.toml create mode 100644 setup.cfg create mode 100644 setup.py diff --git a/.github/workflows/pypi-publish.yml b/.github/workflows/pypi-publish.yml new file mode 100644 index 0000000..e69de29 diff --git a/.github/workflows/testpypi-publish.yml b/.github/workflows/testpypi-publish.yml new file mode 100644 index 0000000..74fcd7c --- /dev/null +++ b/.github/workflows/testpypi-publish.yml @@ -0,0 +1,91 @@ +name: Publish PelicanFS distribution to TestPyPI +on: pre-release + +jobs: + build: + name: Build distribution + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - name: Set up Python + uses: actions/setup-python@v4 + with: + python-version: "3.x" + - name: Install pypa/build + run: >- + python 3 -m + pip install + build + --user + - name: Build a binary wheel and a source tarball + run: python3 -m build + - name: Store the distribution packages + uses: actions/upload-artiface@v3 + with: + name: python-package-distributions + path: dist/ + + publish-to-test-pypi: + name: >- + Publish Python Distribution to TestPyPI + needs: + - build + runs-on: ubuntu-latest + environment: + name: testpypi + url: https://test.pypi.org/p/pelicanfs + permissions: + id-token: write + steps: + - name: Download all the dists + uses: actions/downaload-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Publish distribution to TestPyPI + uses: pypa/gh-action-pypi-publish@release/v1 + with: + respository-url: https://test.pypi.org/legacy + + github-release: + name: >- + Sign the Python distribution with Sigstore + and upload them to GitHub Release + needs: + - publish-to-test-pypi + runs-on: ubuntu-latest + + permissions: + contents: write # IMPORTANT: mandatory for making GitHub Releases + id-token: write # IMPORTANT: mandatory for sigstore + + steps: + - name: Download all the dists + uses: actions/download-artifact@v3 + with: + name: python-package-distributions + path: dist/ + - name: Sign the dists with Sigstore + uses: sigstore/gh-action-sigstore-python@v1.2.3 + with: + inputs: >- + ./dist/*.tar.gz + ./dist/*.whl + - name: Create GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + run: >- + gh release create + '${{ github.ref_name }}' + --repo '${{ github.repository }}' + --notes "" + - name: Upload artifact signatures to GitHub Release + env: + GITHUB_TOKEN: ${{ github.token }} + # Upload to GitHub Release using the `gh` CLI. + # `dist/` contains the built packages, and the + # sigstore-produced signatures and certificates. + run: >- + gh release upload + '${{ github.ref_name }}' dist/** + --repo '${{ github.repository }}' \ No newline at end of file diff --git a/README.md b/README.md index a21b3dd..d82a589 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,42 @@ -# pelicanfs -An ffspec implementation that uses the pelican client +# PelicanFS + +## Overview + +PelicanFS is a file system interface (fsspec) for the Pelican Platform. For more information about the Pelican Platform, please visit the [Pelican Platform](https://pelicanplatform.org) and the [Pelican Platform Github](https://github.com/PelicanPlatform/pelican) pages. For more information about fsspec, visit the [filesystem-spec](https://filesystem-spec.readthedocs.io/en/latest/index.html) page. + + +## Limitations + +PelicanFS is built on top of the http fsspec implementation. As such, any functionality that isn’t available in the http implementation is also *not* available in PelicanFS. + +### Installation + +To install pelican, run:```pip install pelicanfs```### Using PelicanFS + +To use pelicanfs, first create a `PelicanFileSystem` and provide it with the url for the director of your data federation. As an example using the OSDF director + +```python +from pelicanfs.core import PelicanFileSystem + +pelfs = PelicanFileSystem("https://osdf-director.osg-htc.org/") +``` + +From there, use `pelfs` as you would an http fsspec using a namespace path as the url path. For example: + +```python +hello_world = pelfs.cat('/ospool/uc-shared/public/OSG-Staff/validation/test.txt') +print(hello_world) +``` + +### Getting an FSMap + +Sometimes various systems that interact with an fsspec want a key-value mapper rather than a url. To do that, call the `PelicanMap` function with the namespace path and a `PelicanFileSystem` object rather than using the fsspec `get_mapper` call. For example + +```python +from pelicanfs.core import PelicanFileSystem, PelicanMap + +pelfs = PelicanFileSystem(“some-director-url”) +file1 = PelicanMap(“namespace/file/1”, pelfs=pelfs) +file2 = PelicanMap(“namespace/file/2”, pelfs=pelfs) +ds = xarray.open_mfdataset([file1,file2], engine='zarr') +``` \ No newline at end of file diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..2b7baad --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,8 @@ +[tool.pytest.ini_options] +addopts = [ + "--import-mode=importlib", +] + +[build-system] +requires = ["setuptools>=61.0", "wheel"] +build-backend = "setuptools.build_meta" \ No newline at end of file diff --git a/setup.cfg b/setup.cfg new file mode 100644 index 0000000..5d0c905 --- /dev/null +++ b/setup.cfg @@ -0,0 +1,11 @@ +[metadata] +license_files = LICENSE + +[options] +packages = find: +package_dir = + =src + +[options.packages.find] +where = src +include = pelicanfs* \ No newline at end of file diff --git a/setup.py b/setup.py new file mode 100644 index 0000000..c2f2f06 --- /dev/null +++ b/setup.py @@ -0,0 +1,43 @@ +from setuptools import setup, find_packages + +packages = find_packages() +print(packages) + +setup( + name="pelicanfs", + version="0.0.1", + description="An FSSpec Implementation using the Pelican System", + url = "https://github.com/PelicanPlatform/pelicanfs", + classifiers=[ + "Development Status :: 2 - Pre-Alpha", + "Intended Audience :: Science/Research", + "Programming Language :: Python :: 3", + "Programming Language :: Python :: 3.7", + "Programming Language :: Python :: 3.8", + "Programming Language :: Python :: 3.9", + "Programming Language :: Python :: 3.10", + "Programming Language :: Python :: 3 :: Only", + "License :: OSI Approved :: Apache Software License", + ], + keywords="pelican, fsspec", + packages=find_packages( + where='src', + include=['pelicanfs*'], + ), + package_dir={"": "src"}, + python_requires=">=3.7, <4", + install_requires=["aiohttp==3.9.4", + "aiosignal==1.3.1", + "async-timeout==4.0.3", + "attrs==23.2.0", + "frozenlist==1.4.1", + "fsspec==2024.3.1", + "idna==3.7", + "multidict==6.0.5", + "yarl==1.9.4"], + project_urls={ + "Source": "https://github.com/PelicanPlatform/pelicanfs", + "Pelican Source": "https://github.com/PelicanPlatform/pelican", + "Bug Reports":"https://github.com/PelicanPlatform/pelicanfs/issues" + }, +) \ No newline at end of file