Data Security Changes

azure-pipelines.yml

@@ -7,18 +7,12 @@ resources:
       endpoint: github
 
 jobs:
-  - template: collection-2021-1.2.yml@templates
+  - template: 2024-2.3-py310.yml@templates
     parameters:
       beamline_acronym: SMI
-  - template: nsls2-collection-2021-2.2.yml@templates
+  - template: 2024-2.3-py311.yml@templates
     parameters:
       beamline_acronym: SMI
-  - template: nsls2-collection-2021-2.2-py39.yml@templates
+  - template: 2024-2.3-py312-tiled.yml@templates
     parameters:
-      beamline_acronym: SMI
-  - template: nsls2-collection-2021-3.0-py37.yml@templates
-    parameters:
-      beamline_acronym: SMI
-  - template: nsls2-collection-2021-3.0-py39.yml@templates
-    parameters:
-      beamline_acronym: SMI
+      beamline_acronym: SMI

smi_config.csv

@@ -322,4 +322,5 @@
 320,2.79932,13.000806,0.000234,9.049794,Mon Sep 16 14:20:49 2024
 321,1.99938,13.000806,0.000234,9.049794,Tue Sep 17 01:37:27 2024
 322,2.999305,13.000806,0.000234,9.34986,Sat Sep 28 10:11:25 2024
-323,1.398485,13.000805999999999,0.000234,9.34986,Sun Sep 29 12:18:37 2024
+323,1.398485,13.000806,0.000234,9.34986,Sun Sep 29 12:18:37 2024
+324,1.7991599999999999,13.000805999999999,0.000234,9.34986,Thu Oct 24 10:56:21 2024

startup/00-base.py

@@ -1,154 +1,62 @@
 print(f"Loading {__file__}")
-
-###############################################################################
-# TODO: remove this block once https://github.com/bluesky/ophyd/pull/959 is
-# merged/released.
 from datetime import datetime
 from ophyd.signal import EpicsSignalBase, EpicsSignal, DEFAULT_CONNECTION_TIMEOUT
+import redis
+from redis_json_dict import RedisJSONDict
 
+import nslsii
+import redis
+import os
 
-def print_now():
-    return datetime.strftime(datetime.now(), "%Y-%m-%d %H:%M:%S.%f")
+import time
+from redis_json_dict import RedisJSONDict
+from tiled.client import from_profile
+from databroker import Broker
 
+# Configure a Tiled writing client
+tiled_writing_client = from_profile("nsls2", api_key=os.environ["TILED_BLUESKY_WRITING_API_KEY_SMI"])["smi"]["raw"]
 
-def wait_for_connection_base(self, timeout=DEFAULT_CONNECTION_TIMEOUT):
-    """Wait for the underlying signals to initialize or connect"""
-    if timeout is DEFAULT_CONNECTION_TIMEOUT:
-        timeout = self.connection_timeout
-    # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...')
-    start = time.time()
-    try:
-        self._ensure_connected(self._read_pv, timeout=timeout)
-        # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.')
-    except TimeoutError:
-        if self._destroyed:
-            raise DestroyedError("Signal has been destroyed")
-        raise
+class TiledInserter:
+    def insert(self, name, doc):
+        ATTEMPTS = 20
+        error = None
+        for _ in range(ATTEMPTS):
+            try:
+                tiled_writing_client.post_document(name, doc)
+            except Exception as exc:
+                print("Document saving failure:", repr(exc))
+                error = exc
+            else:
+                break
+            time.sleep(2)
+        else:
+            # Out of attempts
+            raise error
 
+tiled_inserter = TiledInserter()
 
-def wait_for_connection(self, timeout=DEFAULT_CONNECTION_TIMEOUT):
-    """Wait for the underlying signals to initialize or connect"""
-    if timeout is DEFAULT_CONNECTION_TIMEOUT:
-        timeout = self.connection_timeout
-    # print(f'{print_now()}: waiting for {self.name} to connect within {timeout:.4f} s...')
-    start = time.time()
-    self._ensure_connected(self._read_pv, self._write_pv, timeout=timeout)
-    # print(f'{print_now()}: waited for {self.name} to connect for {time.time() - start:.4f} s.')
+# The function below initializes RE and subscribes tiled_inserter to it
+nslsii.configure_base(get_ipython().user_ns,
+               tiled_inserter,
+               bec_derivative=True, 
+               publish_documents_with_kafka=True)
 
+print("\nInitializing Tiled reading client...\nMake sure you check for duo push.")
+tiled_reading_client = from_profile("nsls2", username=None)["smi"]["raw"]
 
-EpicsSignalBase.wait_for_connection = wait_for_connection_base
-EpicsSignal.wait_for_connection = wait_for_connection
-###############################################################################
+db = Broker(tiled_reading_client)
 
-from ophyd.signal import EpicsSignalBase
+# set plot properties for 4k monitors
+plt.rcParams['figure.dpi']=200
 
-EpicsSignalBase.set_defaults(timeout=10, connection_timeout=10)
+# Set the metadata dictionary
+RE.md = RedisJSONDict(redis.Redis("info.smi.nsls2.bnl.gov"), prefix="swaxs-")
 
-import warnings
-import nslsii
-from databroker import Broker
-from ophyd import Signal
+# Setup the path to the secure assets folder for the current proposal
+assets_path = f"/nsls2/data/smi/proposals/{RE.md['cycle']}/{RE.md['data_session']}/assets/"
 
-nslsii.configure_base(
-    get_ipython().user_ns, "smi", bec_derivative=True, publish_documents_with_kafka=True
-)
 # Disable printing scan info
 bec.disable_baseline()
 
 # Populating oLog entries with scans, comment out to disable
 nslsii.configure_olog(get_ipython().user_ns, subscribe=True)
-
-# from bluesky.utils import PersistentDict
-from pathlib import Path
-
-import appdirs
-
-
-try:
-    from bluesky.utils import PersistentDict
-except ImportError:
-    import msgpack
-    import msgpack_numpy
-    import zict
-
-    class PersistentDict(zict.Func):
-        """
-        A MutableMapping which syncs it contents to disk.
-        The contents are stored as msgpack-serialized files, with one file per item
-        in the mapping.
-        Note that when an item is *mutated* it is not immediately synced:
-        >>> d['sample'] = {"color": "red"}  # immediately synced
-        >>> d['sample']['shape'] = 'bar'  # not immediately synced
-        but that the full contents are synced to disk when the PersistentDict
-        instance is garbage collected.
-        """
-
-        def __init__(self, directory):
-            self._directory = directory
-            self._file = zict.File(directory)
-            self._cache = {}
-            super().__init__(self._dump, self._load, self._file)
-            self.reload()
-
-            # Similar to flush() or _do_update(), but without reference to self
-            # to avoid circular reference preventing collection.
-            # NOTE: This still doesn't guarantee call on delete or gc.collect()!
-            #       Explicitly call flush() if immediate write to disk required.
-            def finalize(zfile, cache, dump):
-                zfile.update((k, dump(v)) for k, v in cache.items())
-
-            import weakref
-
-            self._finalizer = weakref.finalize(
-                self, finalize, self._file, self._cache, PersistentDict._dump
-            )
-
-        @property
-        def directory(self):
-            return self._directory
-
-        def __setitem__(self, key, value):
-            self._cache[key] = value
-            super().__setitem__(key, value)
-
-        def __getitem__(self, key):
-            return self._cache[key]
-
-        def __delitem__(self, key):
-            del self._cache[key]
-            super().__delitem__(key)
-
-        def __repr__(self):
-            return f"<{self.__class__.__name__} {dict(self)!r}>"
-
-        @staticmethod
-        def _dump(obj):
-            "Encode as msgpack using numpy-aware encoder."
-            # See https://github.com/msgpack/msgpack-python#string-and-binary-type
-            # for more on use_bin_type.
-            return msgpack.packb(obj, default=msgpack_numpy.encode, use_bin_type=True)
-
-        @staticmethod
-        def _load(file):
-            return msgpack.unpackb(file, object_hook=msgpack_numpy.decode, raw=False)
-
-        def flush(self):
-            """Force a write of the current state to disk"""
-            for k, v in self.items():
-                super().__setitem__(k, v)
-
-        def reload(self):
-            """Force a reload from disk, overwriting current cache"""
-            self._cache = dict(super().items())
-
-
-runengine_metadata_dir = appdirs.user_data_dir(appname="bluesky") / Path(
-    "runengine-metadata"
-)
-print('starting RE.md link')
-# PersistentDict will create the directory if it does not exist
-RE.md = PersistentDict(runengine_metadata_dir)
-print('finished RE.md link')
-RE.md["beamline_name"] = "SMI"
-RE.md["facility"] = "NSLS-II"
-RE.md["optional_comments"] = ""  # Any comment can be added if needed

startup/09-beamstop.py

@@ -97,7 +97,11 @@ def beamstop_save():
 
     # load the previous config file
     smi_config = pds.read_csv(SMI_CONFIG_FILENAME, index_col=0)
-    smi_config_update = smi_config.append(current_config_DF, ignore_index=True)
+    #smi_config_update = smi_config.append(current_config_DF, ignore_index=True)
+    #YZ changed here on Octa 24, 2024
+    smi_config_update = pds.concat([smi_config, current_config_DF ], ignore_index=True)  
+
+
 
     # save to file
     smi_config_update.to_csv(SMI_CONFIG_FILENAME)

startup/09-machine.py

@@ -119,7 +119,7 @@ class InsertionDevice(EpicsMotor):
     )
 
     def move(self, *args, **kwargs):
-        set_and_wait(self.brake, 1)
+        self.brake.set(1).wait() # changed from set_and_wait Oct 2024 - Eliot
         return super().move(*args, **kwargs)
 
 

startup/10-hexapods.py

@@ -43,8 +43,11 @@ class SMARACT(Device):
     x = Cpt(EpicsMotor, "0}Mtr", labels=["piezo"])
     y = Cpt(EpicsMotor, "3}Mtr", labels=["piezo"])
     z = Cpt(EpicsMotor, "6}Mtr", labels=["piezo"])
-    th = Cpt(EpicsMotor, "4}Mtr", labels=["piezo"])
-    ch = Cpt(EpicsMotor, "1}Mtr", labels=["piezo"])
+    # swapping Th and ch as of Oct 2024 when old th motor seems to fail it's sensor
+    #th = Cpt(EpicsMotor, "4}Mtr", labels=["piezo"])
+    #ch = Cpt(EpicsMotor, "1}Mtr", labels=["piezo"])
+    ch = Cpt(EpicsMotor, "4}Mtr", labels=["piezo"])
+    th = Cpt(EpicsMotor, "1}Mtr", labels=["piezo"])
 
 
 class BDMStage(Device):

startup/11-energy.py

@@ -229,13 +229,13 @@ def set(self, position):
 
 bragg.read_attrs = ["user_readback"]
 
-new_ivu_gap = EpicsMotor("SR:C12-ID:G1{IVU:1-Ax:Gap}-Mtr", name="new_ivu_gap")
+#new_ivu_gap = EpicsMotor("SR:C12-ID:G1{IVU:1-Ax:Gap}-Mtr", name="new_ivu_gap")
 
 
-def move_E(E, gap_off=0):
-    energy.move(E)
-    if gap_off != 0:
-        cur_gap = ivugap.user_readback.value
-        ivugap.move(cur_gap + gap_off)
+# def move_E(E, gap_off=0):
+#     energy.move(E)
+#     if gap_off != 0:
+#         cur_gap = ivugap.user_readback.value
+#         ivugap.move(cur_gap + gap_off)
 
 dcm_theta = EpicsMotor("XF:12ID:m65", name="dcm_theta")