Merge pull request #11 from legend-exp/dev

Slow Control database interface

docs/source/conf.py

@@ -50,9 +50,9 @@
 intersphinx_mapping = {
     "python": ("https://docs.python.org/3", None),
     "numpy": ("http://docs.scipy.org/doc/numpy", None),
-    "scipy": ("http://docs.scipy.org/doc/scipy/reference", None),
     "pandas": ("https://pandas.pydata.org/docs", None),
-    "matplotlib": ("http://matplotlib.org/stable", None),
+    "matplotlib": ("https://matplotlib.org/stable", None),
+    "sqlalchemy": ("https://docs.sqlalchemy.org", None),
 }  # add new intersphinx mappings here
 
 # sphinx-autodoc

docs/source/tutorial.rst

@@ -138,4 +138,75 @@ corresponding to a certain DAQ channel:
      'channel': 3,
      ...
 
-For further details, have a look at the documentation of :meth:`.AttrsDict.map`.
+For further details, have a look at the documentation for :meth:`.AttrsDict.map`.
+
+Slow Control interface
+----------------------
+
+A number of parameters related to the LEGEND hardware configuration and status
+are recorded in the Slow Control database. The latter, PostgreSQL database
+resides on the ``legend-sc.lngs.infn.it`` host, part of the LNGS network.
+
+Connecting to the database from within the LEGEND LNGS environment does not
+require any special configuration:
+
+.. code:: python
+
+   >>> from legendmeta import LegendSlowControlDB
+   >>> scdb = LegendSlowControlDB()
+   >>> scdb.connect(password="···")
+
+.. note::
+
+   The database password (for the ``scuser`` user) is confidential and may be
+   found on the LEGEND internal wiki pages.
+
+.. tip::
+
+   Alternatively to giving the password to ``connect()``, it can be stored
+   in the ``$LEGEND_SCDB_PW`` shell variable (in e.g. ``.bashrc``):
+
+   .. code-block:: bash
+      :caption: ``~/.bashrc``
+
+      export LEGEND_SCDB_PW="···"
+
+More :meth:`.LegendSlowControlDB.connect` keyword-arguments are available to
+customize hostname and port through which the database can be contacted (in
+case of e.g. custom port forwarding).
+
+:meth:`.LegendSlowControlDB.dataframe` can be used to execute an SQL query and
+return a :class:`pandas.DataFrame`. The following selects three rows from the
+``slot``, ``channel`` and ``vmon`` columns in the ``diode_snap`` table:
+
+.. code:: python
+
+   >>> scdb.dataframe("SELECT slot, channel, vmon FROM diode_snap LIMIT 3")
+      slot  channel    vmon
+   0     3        6  4300.0
+   1     9        2  2250.0
+   2    10        3  3699.9
+
+It's even possible to get an entire table as a dataframe:
+
+.. code:: python
+
+   >>> scdb.dataframe("diode_conf")
+         confid  crate  slot  channel    vset  iset  rup  rdown  trip  vmax pwkill pwon                    tstamp
+   0         15      0     0        0  4000.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+   1         15      0     0        1  4300.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+   2         15      0     0        2  4200.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+   ...
+
+Executing queries natively through an `SQLAlchemy
+<ihttps://www.sqlalchemy.org>`_ :class:`~sqlalchemy.orm.Session` is also
+possible:
+
+.. code:: python
+
+   >>> import sqlalchemy as sql
+   >>> from legendmeta.slowcontrol import DiodeSnap
+   >>> session = scdb.make_session()
+   >>> result = session.execute(sql.select(DiodeSnap.channel, DiodeSnap.imon).limit(3))
+   >>> result.all()
+   [(2, 0.0007), (1, 0.0001), (5, 5e-05)]

setup.cfg

@@ -32,6 +32,8 @@ classifiers =
 packages = find:
 install_requires =
     GitPython
+    pandas
+    sqlalchemy>=2.*
 python_requires = >=3.9
 include_package_data = True
 package_dir =

src/legendmeta/__init__.py

@@ -18,5 +18,6 @@
 from legendmeta._version import version as __version__
 from legendmeta.core import LegendMetadata
 from legendmeta.jsondb import JsonDB
+from legendmeta.slowcontrol import LegendSlowControlDB
 
-__all__ = ["__version__", "LegendMetadata", "JsonDB"]
+__all__ = ["__version__", "LegendMetadata", "LegendSlowControlDB", "JsonDB"]

src/legendmeta/core.py

@@ -38,7 +38,7 @@ class LegendMetadata(JsonDB):
     path
         path to legend-metadata repository. If not existing, will attempt a
         git-clone through SSH. If ``None``, legend-metadata will be cloned
-        in a temporary directory (see :func:`gettempdir`).
+        in a temporary directory (see :func:`tempfile.gettempdir`).
     """
 
     def __init__(self, path: str = None) -> None:
@@ -81,9 +81,11 @@ def checkout(self, git_ref: str) -> None:
         """Select legend-metadata version."""
         try:
             self._repo.git.checkout(git_ref)
+            self._repo.git.submodule("update", "--init")
         except GitCommandError:
             self._repo.remote().pull()
             self._repo.git.checkout(git_ref)
+            self._repo.git.submodule("update", "--init")
 
     def reset(self) -> None:
         """Checkout legend-metadata to default Git ref."""

src/legendmeta/jsondb.py

@@ -162,7 +162,7 @@ class JsonDB:
 
     The database is represented on disk by a collection of JSON files
     arbitrarily scattered in a filesystem. Subdirectories are also
-    :class:`JsonDB` objects. In memory, the database is represented as an
+    :class:`.JsonDB` objects. In memory, the database is represented as an
     :class:`AttrsDict`.
 
     Note
@@ -185,7 +185,7 @@ class JsonDB:
     """
 
     def __init__(self, path: str | Path, lazy: bool = False) -> None:
-        """Construct a :class:`JsonDB` object.
+        """Construct a :class:`.JsonDB` object.
 
         Parameters
         ----------
@@ -230,11 +230,11 @@ def on(
         Parameters
         ----------
         timestamp
-            a :class:`datetime` object or a string matching the pattern
-            ``YYYYmmddTHHMMSSZ``.
+            a :class:`~datetime.datetime` object or a string matching the
+            pattern ``YYYYmmddTHHMMSSZ``.
         pattern
             query by filename pattern.
-        system: {'all', 'phy', 'cal', 'lar', ...}
+        system: 'all', 'phy', 'cal', 'lar', ...
             query only a data taking "system".
         """
         # get the files from the jsonl

src/legendmeta/slowcontrol.py

@@ -0,0 +1,225 @@
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass
+from datetime import datetime
+
+import pandas
+import sqlalchemy as db
+from sqlalchemy.orm import DeclarativeBase, Mapped
+
+
+class LegendSlowControlDB:
+    """A class for interacting with the LEGEND Slow Control database.
+
+    The LEGEND Slow Control system relies on a `PostgreSQL
+    <https://www.postgresql.org/docs/current/index.html>`_ database living on
+    ``legend-sc.lngs.infn.it``. The aim of the :class:`LegendSlowControlDB`
+    class is to simplify access to the database from Python.
+    """
+
+    def __init__(self) -> None:
+        self.connection: db.engine.base.Connection = None
+
+    def connect(
+        self, host: str = "localhost", port: int = 5432, password: str | None = None
+    ) -> None:
+        """Establish a connection to the database.
+
+        Authentication is attempted with the read-only user ``scuser`` on a
+        database named ``scdb``.
+
+        Parameters
+        ----------
+        host
+            database host. Can be a hostname (``localhost``,
+            ``legend-sc.lngs.infn.it``, etc.) or an IP address.
+        port
+            port through which the database should be contacted.
+        password
+            password for user ``scuser`` of the ``scdb`` database. May be found
+            on LEGEND's internal documentation (e.g. the Wiki web pages). If
+            ``None``, uses the value of the ``$LEGEND_SCDB_PW`` shell variable.
+
+        Examples
+        --------
+        If the Slow Control database connection is forwarded to a local machine
+        (port 6942) (through e.g. an SSH tunnel), use:
+
+        >>> scdb = LegendSlowControlDB()
+        >>> scdb.connect("localhost", port=6942, password="···")
+
+        Alternatively to giving the password to ``connect()``, it can be stored
+        in the ``$LEGEND_SCDB_PW`` shell variable (in e.g. ``.bashrc``):
+
+        .. code-block:: bash
+           :caption: ``~/.bashrc``
+
+           export LEGEND_SCDB_PW="···"
+
+        Then:
+
+        >>> scdb.connect("localhost", port=6942)
+        """
+        if password is None:
+            password = os.getenv("LEGEND_SCDB_PW")
+
+        if password is None:
+            raise ValueError("must supply the database password")
+
+        if self.connection is not None and not self.connection.closed:
+            self.disconnect()
+
+        self.connection = db.create_engine(
+            f"postgresql://scuser:{password}@{host}:{port}/scdb"
+        ).connect()
+
+    def disconnect(self) -> None:
+        """Disconnect from the database."""
+        self.connection.close()
+
+    def make_session(self) -> db.orm.Session:
+        """Open and return a new  :class:`~sqlalchemy.orm.Session` object for executing database operations.
+
+        Examples
+        --------
+        >>> import sqlalchemy as sql
+        >>> from legendmeta.slowcontrol import DiodeSnap
+        >>> session = scdb.make_session()
+        >>> result = session.execute(sql.select(DiodeSnap.channel, DiodeSnap.imon).limit(3))
+        >>> result.all()
+        [(2, 0.0007), (1, 0.0001), (5, 5e-05)]
+
+        See Also
+        --------
+        `SQLAlchemy documentation <https://www.sqlalchemy.org/>`_
+        """
+        return db.orm.Session(self.connection)
+
+    def dataframe(self, expr: str | db.sql.Select) -> pandas.DataFrame:
+        """Query the database and return a dataframe holding the result.
+
+        Parameters
+        ----------
+        expr
+            SQL table name, select SQL command text or SQLAlchemy selectable
+            object.
+
+        Examples
+        --------
+        SQL select syntax text or table name:
+
+        >>> scdb.dataframe("SELECT channel, vmon FROM diode_snap LIMIT 3")
+           channel    vmon
+        0        2  2250.0
+        1        1  3899.4
+        2        5  1120.2
+
+        >>> scdb.dataframe("diode_conf")
+              confid  crate  slot  channel    vset  iset  rup  rdown  trip  vmax pwkill pwon                    tstamp
+        0         15      0     0        0  4000.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+        1         15      0     0        1  4300.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+        2         15      0     0        2  4200.0   6.0   10      5  10.0  6000   KILL  Dis 2022-10-07 13:49:56+00:00
+        ...
+
+        :class:`sqlalchemy.sql.Select` object:
+
+        >>> import sqlalchemy as sql
+        >>> from legendmeta.slowcontrol import DiodeSnap
+        >>> scdb.dataframe(sql.select(DiodeSnap.channel, DiodeSnap.vmon).limit(3))
+           channel    vmon
+        0        2  2250.0
+        1        1  3899.4
+        2        5  1120.2
+
+        See Also
+        --------
+        pandas.read_sql
+        """
+        try:
+            return pandas.read_sql(expr, self.connection)
+        except db.exc.ObjectNotExecutableError:
+            return pandas.read_sql(db.text(expr), self.connection)
+        # TODO: automatically rollback if failed transaction
+        # except db.exc.ProgrammingError as e:
+        #     self.connection.rollback()
+        #     raise e
+
+    def status(self, channel: dict, at: str | datetime, system: str = "ged") -> dict:
+        """Query information about a channel.
+
+        >>> channel = lmeta.hardware.configuration.channelmaps.on(ts).B00089B
+        >>> scdb.status(channel, at=ts)
+        """
+        raise NotImplementedError
+        # df = self.dataframe(...tables...).sort_values(by=["tstamp"])
+        # return df.loc(df.tstamp <= at).iloc(-1)
+
+
+class Base(DeclarativeBase):
+    pass
+
+
+@dataclass
+class DiodeSnap(Base):
+    """Monitored parameters of HPGe detectors."""
+
+    __tablename__ = "diode_snap"
+
+    crate: Mapped[int]
+    slot: Mapped[int]
+    channel: Mapped[int]
+    vmon: Mapped[float]
+    imon: Mapped[float]
+    status: Mapped[int]
+    almask: Mapped[int]
+    tstamp: Mapped[datetime] = db.orm.mapped_column(primary_key=True)
+
+
+@dataclass
+class DiodeConf(Base):
+    """Configuration parameters of HPGe detectors."""
+
+    __tablename__ = "diode_conf"
+
+    confid: Mapped[int]
+    crate: Mapped[int]
+    slot: Mapped[int]
+    channel: Mapped[int]
+    vset: Mapped[float]
+    iset: Mapped[float]
+    rup: Mapped[int]
+    rdown: Mapped[int]
+    trip: Mapped[float]
+    vmax: Mapped[int]
+    pwkill: Mapped[str]
+    pwon: Mapped[str]
+    tstamp: Mapped[datetime] = db.orm.mapped_column(primary_key=True)
+
+
+@dataclass
+class SiPMSnap(Base):
+    """Monitored parameters of SiPMs."""
+
+    __tablename__ = "sipm_snap"
+
+    board: Mapped[int]
+    channel: Mapped[int]
+    vmon: Mapped[float]
+    imon: Mapped[float]
+    status: Mapped[int]
+    almask: Mapped[int]
+    tstamp: Mapped[datetime] = db.orm.mapped_column(primary_key=True)
+
+
+class SiPMConf(Base):
+    """Configuration parameters of SiPMs."""
+
+    __tablename__ = "sipm_conf"
+
+    confid: Mapped[int]
+    board: Mapped[int]
+    channel: Mapped[int]
+    vset: Mapped[float]
+    iset: Mapped[float]
+    tstamp: Mapped[datetime] = db.orm.mapped_column(primary_key=True)