From 1d2f75eabafbff70e1a96e9b186280d7f7ba3911 Mon Sep 17 00:00:00 2001 From: mj-will Date: Fri, 10 Nov 2023 17:56:53 +0000 Subject: [PATCH 01/14] feat: add basic package --- src/demo_sampler_bilby/__init__.py | 11 +++++ src/demo_sampler_bilby/plugin.py | 69 ++++++++++++++++++++++++++++++ 2 files changed, 80 insertions(+) create mode 100644 src/demo_sampler_bilby/__init__.py create mode 100644 src/demo_sampler_bilby/plugin.py diff --git a/src/demo_sampler_bilby/__init__.py b/src/demo_sampler_bilby/__init__.py new file mode 100644 index 0000000..521b09d --- /dev/null +++ b/src/demo_sampler_bilby/__init__.py @@ -0,0 +1,11 @@ +"""An example of how to implement a sampler plugin in for bilby. + +This package provides the 'demo_sampler' sampler. +""" +from importlib.metadata import PackageNotFoundError, version + +try: + __version__ = version(__name__) +except PackageNotFoundError: + # package is not installed + __version__ = "unknown" diff --git a/src/demo_sampler_bilby/plugin.py b/src/demo_sampler_bilby/plugin.py new file mode 100644 index 0000000..4a6eae8 --- /dev/null +++ b/src/demo_sampler_bilby/plugin.py @@ -0,0 +1,69 @@ +"""Example plugin for using a sampler in bilby. + +Here we demonstrate the how to implement the class. +""" +import bilby +from bilby.core.sampler.base_sampler import MCMCSampler, NestedSampler +import numpy as np + + +class DemoSampler(NestedSampler): + """Bilby wrapper for your sampler. + + This class should inherit from :code:`MCMCSampler` or :code:`NestedSampler` + """ + + @property + def external_sampler_name(self) -> str: + """The name of package that provides the sampler.""" + # In this template we do not require any external codes, so we just + # use bilby. You should change this. + return "bilby" + + @property + def default_kwargs(self) -> dict: + """Dictionary of default keyword arguments. + + Any arguments not included here will be removed before calling the + sampler. + """ + return dict( + ninitial=100, + ) + + def run_sampler(self) -> dict: + """Run the sampler. + + This method should run the sampler and update the result object. + It should also return the result object. + """ + + # The code below shows how you can call different method. + # Replace this code with calls to your sampler + + # Keyword arguments are stored in self.kwargs + prior_samples = np.array( + list(self.priors.sample(self.kwargs["ninitial"]).values()), + ).T + # We can evaluate the log-prior + logp = self.log_prior(prior_samples) + # And similarly for the log-likelihood + logl = np.empty(len(prior_samples)) + for i, sample in enumerate(prior_samples): + logl[i] = self.log_likelihood(sample) + + # Generate posterior samples + logw = logl.copy() - logl.max() + keep = logw > np.log(np.random.rand(len(logw))) + posterior_samples = prior_samples[keep] + + # The result object is created automatically + # So we just have to populate the different methods + # Add the posterior samples to the result object + # This should be a numpy array of shape (# samples x # parameters) + self.result.samples = posterior_samples + # We can also add the log-evidence + self.result.ln_evidence = np.mean(logl) + + # Must return the result object + return self.result From 8222187770200938c8f7c9c231802718691cc04b Mon Sep 17 00:00:00 2001 From: mj-will Date: Fri, 10 Nov 2023 17:57:05 +0000 Subject: [PATCH 02/14] build: add pyproject.toml --- pyproject.toml | 35 +++++++++++++++++++++++++++++++++++ 1 file changed, 35 insertions(+) create mode 100644 pyproject.toml diff --git a/pyproject.toml b/pyproject.toml new file mode 100644 index 0000000..9d40086 --- /dev/null +++ b/pyproject.toml @@ -0,0 +1,35 @@ +[build-system] +requires = ["setuptools>=45", "setuptools-scm[toml]>=6.2"] +build-backend = "setuptools.build_meta" + +[project] +name = "demo_sampler_bilby" +authors = [ + {name = "your name", email = "your@email.com"}, +] +description = "" +readme = "README.md" +requires-python = ">=3.9" +license = {text = "MIT"} +classifiers = [ + "Programming Language :: Python :: 3", +] +dependencies = [ + "bilby", + "numpy", +] + +dynamic = ["version"] + +[project.optional-dependencies] +dev = [ + "pytest", +] + +[tool.setuptools_scm] + +[tool.black] +line-length = 79 + +[project.entry-points."bilby.samplers"] +demo_sampler = "demo_sampler_bilby.plugin:DemoSampler" From 114c441588ff26f8cdd2f62a907a93d3913474e0 Mon Sep 17 00:00:00 2001 From: mj-will Date: Fri, 10 Nov 2023 17:57:16 +0000 Subject: [PATCH 03/14] test: add a basic integration test --- tests/test_bilby_integration.py | 48 +++++++++++++++++++++++++++++++++ 1 file changed, 48 insertions(+) create mode 100644 tests/test_bilby_integration.py diff --git a/tests/test_bilby_integration.py b/tests/test_bilby_integration.py new file mode 100644 index 0000000..d8abd85 --- /dev/null +++ b/tests/test_bilby_integration.py @@ -0,0 +1,48 @@ +import bilby +import numpy as np +import pytest + + +def model(x, m, c): + return m * x + c + + +@pytest.fixture() +def bilby_likelihood(): + bilby.core.utils.random.seed(42) + rng = bilby.core.utils.random.rng + x = np.linspace(0, 1, 11) + injection_parameters = dict(m=0.5, c=0.2) + sigma = 0.1 + y = model(x, **injection_parameters) + rng.normal(0., sigma, len(x)) + likelihood = bilby.likelihood.GaussianLikelihood( + x, y, model, sigma + ) + return likelihood + + +@pytest.fixture() +def bilby_priors(): + priors = bilby.core.prior.PriorDict() + priors["m"] = bilby.core.prior.Uniform(0, 5, boundary="periodic") + priors["c"] = bilby.core.prior.Uniform(-2, 2, boundary="reflective") + return priors + + +@pytest.fixture() +def sampler_kwargs(): + return dict( + nlive=100, + ) + + +def test_run_sampler(bilby_likelihood, bilby_priors, tmp_path, sampler_kwargs): + + outdir = tmp_path / "test_run_sampler" + + bilby.run_sampler( + likelihood=bilby_likelihood, + priors=bilby_priors, + sampler="demo_sampler", + outdir=outdir, + ) \ No newline at end of file From a6409ea60bc4d65c1b6eef1491025ccab34c94eb Mon Sep 17 00:00:00 2001 From: mj-will Date: Fri, 10 Nov 2023 17:57:35 +0000 Subject: [PATCH 04/14] style: apply black --- tests/test_bilby_integration.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/tests/test_bilby_integration.py b/tests/test_bilby_integration.py index d8abd85..b8c79eb 100644 --- a/tests/test_bilby_integration.py +++ b/tests/test_bilby_integration.py @@ -14,10 +14,8 @@ def bilby_likelihood(): x = np.linspace(0, 1, 11) injection_parameters = dict(m=0.5, c=0.2) sigma = 0.1 - y = model(x, **injection_parameters) + rng.normal(0., sigma, len(x)) - likelihood = bilby.likelihood.GaussianLikelihood( - x, y, model, sigma - ) + y = model(x, **injection_parameters) + rng.normal(0.0, sigma, len(x)) + likelihood = bilby.likelihood.GaussianLikelihood(x, y, model, sigma) return likelihood @@ -37,7 +35,6 @@ def sampler_kwargs(): def test_run_sampler(bilby_likelihood, bilby_priors, tmp_path, sampler_kwargs): - outdir = tmp_path / "test_run_sampler" bilby.run_sampler( @@ -45,4 +42,4 @@ def test_run_sampler(bilby_likelihood, bilby_priors, tmp_path, sampler_kwargs): priors=bilby_priors, sampler="demo_sampler", outdir=outdir, - ) \ No newline at end of file + ) From 5d815681527ed4499f16f166ce54796e04e35100 Mon Sep 17 00:00:00 2001 From: mj-will Date: Fri, 10 Nov 2023 18:02:08 +0000 Subject: [PATCH 05/14] fix: fix typo in kwargs --- tests/test_bilby_integration.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_bilby_integration.py b/tests/test_bilby_integration.py index b8c79eb..5186b2a 100644 --- a/tests/test_bilby_integration.py +++ b/tests/test_bilby_integration.py @@ -30,7 +30,7 @@ def bilby_priors(): @pytest.fixture() def sampler_kwargs(): return dict( - nlive=100, + ninitial=100, ) From e8aa733362d7d19f7e08818707144cc16a94c9ff Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 15:34:07 +0000 Subject: [PATCH 06/14] add more result attributes --- src/demo_sampler_bilby/plugin.py | 20 ++++++++++++++------ 1 file changed, 14 insertions(+), 6 deletions(-) diff --git a/src/demo_sampler_bilby/plugin.py b/src/demo_sampler_bilby/plugin.py index 4a6eae8..949f73f 100644 --- a/src/demo_sampler_bilby/plugin.py +++ b/src/demo_sampler_bilby/plugin.py @@ -38,19 +38,19 @@ def run_sampler(self) -> dict: It should also return the result object. """ - # The code below shows how you can call different method. + # The code below shows how you can call different methods. # Replace this code with calls to your sampler # Keyword arguments are stored in self.kwargs prior_samples = np.array( list(self.priors.sample(self.kwargs["ninitial"]).values()), ).T - # We can evaluate the log-prior - logp = self.log_prior(prior_samples) - # And similarly for the log-likelihood + # We can evaluate the log-prior and log-likelihood logl = np.empty(len(prior_samples)) + logp = np.empty(len(prior_samples)) for i, sample in enumerate(prior_samples): logl[i] = self.log_likelihood(sample) + logp[i] = self.log_prior(sample) # Generate posterior samples logw = logl.copy() - logl.max() @@ -62,8 +62,16 @@ def run_sampler(self) -> dict: # Add the posterior samples to the result object # This should be a numpy array of shape (# samples x # parameters) self.result.samples = posterior_samples - # We can also add the log-evidence - self.result.ln_evidence = np.mean(logl) + # We can also store the log-likelihood and log-prior values for each + # posterior sample + self.result.log_likelihood_evaluations = logl[keep] + self.result.log_prior_evaluations = logp[keep] + # If it is a nested sampler, we can add the nested samples + self.result.nested_samples = prior_samples + # We can also add the log-evidence and the error + # These can be NaNs for samplers that no not estimate the evidence + self.result.log_evidence = np.mean(logl) + self.result.log_evidence_err = np.std(logl) # Must return the result object return self.result From 5f1920f0ee9164e9aa4c151ddbe5430539eca987 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 15:34:24 +0000 Subject: [PATCH 07/14] add more comments --- tests/test_bilby_integration.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/tests/test_bilby_integration.py b/tests/test_bilby_integration.py index 5186b2a..e6c5739 100644 --- a/tests/test_bilby_integration.py +++ b/tests/test_bilby_integration.py @@ -29,6 +29,7 @@ def bilby_priors(): @pytest.fixture() def sampler_kwargs(): + # Any keyword arguments that need to be set of you want to test return dict( ninitial=100, ) @@ -40,6 +41,6 @@ def test_run_sampler(bilby_likelihood, bilby_priors, tmp_path, sampler_kwargs): bilby.run_sampler( likelihood=bilby_likelihood, priors=bilby_priors, - sampler="demo_sampler", + sampler="demo_sampler", # This should match the name of the sampler outdir=outdir, ) From 2a3e86cb2d3a9b06b480c92e25b5353bcb19fd02 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 15:34:39 +0000 Subject: [PATCH 08/14] change dev to test --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index 9d40086..ca5410d 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -22,7 +22,7 @@ dependencies = [ dynamic = ["version"] [project.optional-dependencies] -dev = [ +test = [ "pytest", ] From b1d474d35ddf9ee983b63e3d8d134eabbeff95ff Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:16:07 +0000 Subject: [PATCH 09/14] ci: add basic CI for running unit tests --- .github/workflow/tests.yaml | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) create mode 100644 .github/workflow/tests.yaml diff --git a/.github/workflow/tests.yaml b/.github/workflow/tests.yaml new file mode 100644 index 0000000..35480a1 --- /dev/null +++ b/.github/workflow/tests.yaml @@ -0,0 +1,36 @@ +name: Unit tests + +on: + push: + branches: [ main ] + pull_request: + branches: [ main ] + +concurrency: + group: ${{ github.workflow }}-${{ github.ref }} + cancel-in-progress: true + +jobs: + unit-tests: + name: Unit tests - Python ${{ matrix.python-version }} + + strategy: + fail-fast: false + matrix: + python-version: ["3.9", "3.10", "3.11"] + runs-on: ubuntu-latest + + steps: + - uses: actions/checkout@v4 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + cache: 'pip' + - name: Install dependencies + run: | + python -m pip install --upgrade pip + pip install .[test] + - name: Test with pytest + run: | + python -m pytest From c53724ce28213d2258a92a77b290b1f7598c449f Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:16:21 +0000 Subject: [PATCH 10/14] docs: add initial documentation --- README.md | 72 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 72 insertions(+) diff --git a/README.md b/README.md index 46223b0..0e3162b 100644 --- a/README.md +++ b/README.md @@ -1,2 +1,74 @@ # sampler-template + Template for sampler plugins in bilby + +There are three main components to a sampler plugin for bilby: + +- The sampler class +- The `pyproject.toml` +- The test suite + + +## The sampler class + +This is the interface between the external sampling code and bilby, in this +template repo is it located in `demo_sampler_bilby/src/plugin.py`. + +The class should inherit from one of: + +- `bilby.core.sampler.Sampler` +- `bilby.core.sampler.MCMCSampler` +- `bilby.core.sampler.NestedSampler` + +The sampler is run in the `run_sampler` method + + +## pyproject.toml + +Various fields in the `pyproject.toml` need to be set, these are: + +* `name`: this is the name Python package, we recommend using `_bilby` +* `author`: this should include any authors +* `dependencies`: any dependencies should be included here, this must include `bilby` +* `[project.entry-points."bilby.samplers"]` see below + + +### Adding the entry points + +This section of the `pyproject.toml` makes the sampler 'visible' within bilby. + +``` +[project.entry-points."bilby.samplers"] +demo_sampler = "demo_sampler_bilby.plugin:DemoSampler" +``` + +The name of the sampler within bilby is determined based on the name used here +(e.g. `demo_sampler` in this case). + +The string should points to the file and, after the colon, the sampler class. + + +## Tests + +The plugin should include a test suite. + +`tests/test_bilby_integration.py` includes a standard test that +all samplers should pass but other tests can be included in here. The file should +be updated name matches the name of the sampler provided by the plugin and any keyword +arguments should be set. + + +### Continuous Integration + +The tests are run automatically via GitHub Actions. These are configured in +`.github/workflow/test.yaml`. You may wish to change the version of Python +being tested or included additional configuration, e.g. for more complex installation +processes. + + +## Plugin in an existing package + +It is also possible to include the plugin in an existing package rather +than creating a separate plugin package. In this case, you need to define +the sampler class somewhere within the existing package and then add an entry +point. From 4263fbd5748fdab15a7b467945e592554c24b425 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:18:11 +0000 Subject: [PATCH 11/14] fix typo in directory name --- .github/{workflow => workflows}/tests.yaml | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/{workflow => workflows}/tests.yaml (100%) diff --git a/.github/workflow/tests.yaml b/.github/workflows/tests.yaml similarity index 100% rename from .github/workflow/tests.yaml rename to .github/workflows/tests.yaml From 995f4b44a4df34cf4064b6818c3b767389b6ea10 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:24:13 +0000 Subject: [PATCH 12/14] ci: try to fix ci --- .github/workflows/tests.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 35480a1..5f8e567 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -11,7 +11,7 @@ concurrency: cancel-in-progress: true jobs: - unit-tests: + unittests: name: Unit tests - Python ${{ matrix.python-version }} strategy: From 5c13efbfa50f71edff768f6ddc46d55c31b2dc65 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:26:50 +0000 Subject: [PATCH 13/14] change .yaml to .yml --- .github/workflows/{tests.yaml => tests.yml} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename .github/workflows/{tests.yaml => tests.yml} (100%) diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yml similarity index 100% rename from .github/workflows/tests.yaml rename to .github/workflows/tests.yml From 9acf5c93af73cce3eccea35a91cc869cdaa99708 Mon Sep 17 00:00:00 2001 From: mj-will Date: Thu, 11 Jan 2024 16:37:47 +0000 Subject: [PATCH 14/14] docs: fix path --- README.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/README.md b/README.md index 0e3162b..df3d32a 100644 --- a/README.md +++ b/README.md @@ -12,7 +12,7 @@ There are three main components to a sampler plugin for bilby: ## The sampler class This is the interface between the external sampling code and bilby, in this -template repo is it located in `demo_sampler_bilby/src/plugin.py`. +template repo is it located in `src/demo_sampler_bilby/plugin.py`. The class should inherit from one of: