Fixed pandapower and constrained setuptools dependencies version

setup.py

@@ -30,7 +30,7 @@
         'ifcfg',
         'ninja',
         'testresources',
-        'c104~=1.16.0',
+        'c104@git+https://github.com/Fraunhofer-FIT-DIEN/iec104-python.git@main',
         'python-dateutil',
         'docker',
         'ipaddress==1.0.23',
@@ -41,9 +41,8 @@
         'numba==0.57.1',
         'numpy>=1.21.5',
         'packaging==20.3',
-        #'pandapower==2.13.2', # Not yet released but contains critical bug fix
-        'pandapower @ git+https://github.com/e2nIEE/pandapower.git',
-        'powerowl @ git+https://github.com/fkie-cad/powerowl.git',
+        'pandapower==2.14.1', # Contains critical bug fix
+        'powerowl @ git+https://github.com/Dosclic98/powerowl.git',
         'pandas==1.3.4',
         'psutil==5.7.0',
         'pydot',
@@ -63,7 +62,7 @@
         'pyzmq==20.0.0',
         'scapy==2.4.4',
         'scipy>=1.11.2',
-        'setuptools>=65.5.1',
+        'setuptools>=65.5.1,<=70.0.0',
         'shapely',
         'sqlalchemy==1.3.16',
         'tabulate',

wattson/apps/interface/util/messages.py

@@ -18,6 +18,7 @@
 from wattson.iec104.common.config import SUPPORTED_ASDU_TYPES
 from wattson.iec104.interface.types import MsgDirection, TypeID, COT, IEC_PARAMETER_SINGLE_VALUE
 from wattson.iec104.interface.apdus import I_FORMAT, APDU
+from wattson.iec104.interface.types import IECValue
 
 COA = Union[int, str]
 IOA = Union[int, str]
@@ -318,7 +319,7 @@ class ProcessInfoMonitoring(IECMsg):
     # send with empty val_map and cot == 10 to mark end of packets about that typeID
     # during general interro
     coa: int
-    val_map: Dict[IOA, Union[bool, int, float, Tuple]]
+    val_map: Dict[IOA, IECValue]
     ts_map: Dict[int, int]
     type_ID: Union[int, TypeID]
     cot: int
@@ -353,7 +354,7 @@ class ProcessInfoControl(IECMsg):
     # COT necessary if the MTU sends a set-command not communicated over a subscriber
     coa: COA
     type_ID: int
-    val_map: Dict[IOA, Union[bool, int, float]]
+    val_map: Dict[IOA, IECValue]
     reference_nr: str = UNSET_REFERENCE_NR
     max_tries: int = DEFAULT_MAX_TRIES
     queue_on_collision: bool = False
@@ -539,7 +540,7 @@ class PeriodicUpdate(ProcessInfoMonitoring):
     def __init__(
             self,
             coa: int,
-            val_map: Dict[IOA, Union[bool, int, float, Tuple]],
+            val_map: Dict[IOA, IECValue],
             ts_map: Dict[int, int],
             type_ID: Union[int, TypeID],
             reference_nr: str,

wattson/iec104/common/config.py

@@ -4,7 +4,7 @@
 SERVER_DEFAULT_PORT = 2404
 SERVER_TICK_RATE_MS = 2000
 # interval in seconds used to send periodic updates (COT=1) from server to client
-SERVER_UPDATE_PERIOD_S = 10
+SERVER_UPDATE_PERIOD_S = 40     # Originally 10
 SERVER_UPDATE_PERIOD_MS = SERVER_UPDATE_PERIOD_S * 1000
 
 CLIENT_COMMAND_TIMEOUT_MS = 2000
@@ -27,12 +27,13 @@
                        | {100, 102, 103, 113}
 # short floating point = IEE float
 
+# IEC 104 connection parameters (in # of APDUs and in seconds)
 APCI_PARAMETERS = {
     'k': 12,
-    'w': 8,
-    't0': 10,
+    'w': 8,         # Originally 8
+    't0': 10,       # Originally 10
     't1': 15,
-    't2': 10,
+    't2': 340,      # Originally 100
     't3': 20,
 }
 

wattson/iec104/implementations/c104/client/client.py

@@ -70,7 +70,7 @@ def __init__(self, org: int = 0, **kwargs):
 
         self._org = org
         self._servers = {}
-        self._client = c104.add_client(
+        self._client = c104.Client(
             tick_rate_ms=CLIENT_TICKRATE_MS,
             command_timeout_ms=CLIENT_COMMAND_TIMEOUT_MS
         )
@@ -220,9 +220,7 @@ def get_wattson_connection_state(self, coa: int) -> ConnectionState:
             c104.ConnectionState.CLOSED_AWAIT_OPEN: ConnectionState.CLOSED,
             c104.ConnectionState.CLOSED_AWAIT_RECONNECT: ConnectionState.CLOSED,
             c104.ConnectionState.OPEN: ConnectionState.INTERRO_DONE,
-            c104.ConnectionState.OPEN_AWAIT_CLOCK_SYNC: ConnectionState.OPEN,
             c104.ConnectionState.OPEN_AWAIT_CLOSED: ConnectionState.OPEN,
-            c104.ConnectionState.OPEN_AWAIT_INTERROGATION: ConnectionState.INTERRO_STARTED,
             c104.ConnectionState.OPEN_MUTED: ConnectionState.OPEN
         }
         connection_state = self.get_connection_state(coa)
@@ -242,6 +240,13 @@ def connect_server(self, server: Union[str, int, dict]):
                 port=server["port"],
                 init=self.c104_init
             )
+            # IEC104 Connection parameters
+            server["connection"].protocol_parameters.send_window_size = APCI_PARAMETERS["k"]
+            server["connection"].protocol_parameters.receive_window_size = APCI_PARAMETERS["w"]
+            server["connection"].protocol_parameters.connection_timeout = APCI_PARAMETERS["t0"] * 1000
+            server["connection"].protocol_parameters.message_timeout = APCI_PARAMETERS["t1"] * 1000
+            server["connection"].protocol_parameters.confirm_interval = APCI_PARAMETERS["t2"] * 1000
+            server["connection"].protocol_parameters.keep_alive_interval = APCI_PARAMETERS["t3"] * 1000
 
             self.logger.debug(f"Setting Connection Callbacks for {server['coa']}")
             server["connection"].on_receive_raw(callable=self._on_receive_raw_callback_wrapper)
@@ -481,15 +486,15 @@ def update_datapoint(self, coa: int, ioa: int, value) -> bool:
             server["datapoints"][str(ioa)].value = value
             return True
 
-    def _on_receive_datapoint(self, point: c104.Point, previous_state: dict,
+    def _on_receive_datapoint(self, point: c104.Point, previous_info: c104.Information,
                               message: c104.IncomingMessage) -> c104.ResponseState:
         # TODO: What happens during on_receive on client-side if return False?!
         with self._cb_lock:
             try:
                 if self.callbacks['on_receive_datapoint'] is not None:
                     p = C104Point(point)
-                    previous_state = C104Point.parse_to_previous_point(previous_state, point)
-                    success = self.callbacks['on_receive_datapoint'](p, previous_state, message)
+                    previous_info = C104Point.parse_to_previous_point(previous_info, point)
+                    success = self.callbacks['on_receive_datapoint'](p, previous_info, message)
                     return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE
                 return c104.ResponseState.NONE
             except Exception as e:

wattson/iec104/implementations/c104/common/c104_point.py

@@ -20,34 +20,34 @@ def __init__(self, c104_point: c104.Point):
                          c104_point.related_io_address,
                          fcs_quality_descriptor,
                          c104_point.value,
-                         c104_point.reported_at_ms,
-                         c104_point.updated_at_ms
+                         c104_point.processed_at.microsecond / 1000,
+                         0 if c104_point.recorded_at is None else c104_point.recorded_at.microsecond / 1000
                          )
 
     @staticmethod
-    def parse_to_previous_point(previous_state: dict, cur_point: c104.Point) -> 'C104Point':
+    def parse_to_previous_point(previous_state: c104.Information, cur_point: c104.Point) -> 'C104Point':
         new_v = cur_point.value
         p = C104Point(cur_point)
         p.c104_point = None
-        p._value = previous_state['value']
+        p._value = previous_state.value
         s = f"{p} {previous_state} {cur_point.value} {new_v} {type(new_v)} {type(cur_point.value)} {cur_point.value == new_v} {type(new_v) == type(cur_point.value)}"
         #s = str(p) + str(previous_state) + str(new_v) + str(cur_point.value)
         if new_v != float(cur_point.value) and not (isnan(new_v) and isnan(cur_point.value)):
             raise RuntimeError(f"Bad value translation {s}")
         if not (new_v == cur_point.value or (isnan(new_v) and isnan(cur_point.value))):
             raise RuntimeError("2")
-        elif not (p.value == previous_state['value'] or (isnan(p.value) and isnan(previous_state['value']))):
+        elif not (p.value == previous_state.value or (isnan(p.value) and isnan(previous_state.value))):
             raise RuntimeError("3")
 
-        p.updated_at_ms = previous_state['updatedAt_ms']
-        p.quality = C104Point._translate_c104_quality({previous_state['quality']})
+        p.updated_at_ms = 0 if previous_state.recorded_at is None else previous_state.recorded_at.microsecond / 1000
+        p.quality = C104Point._translate_c104_quality({previous_state.quality})
         return p
 
     def read(self) -> bool:
         res = self.c104_point.read()
         self._value = TypeID.convert_val_by_type(self.type, self.c104_point.value)
-        self.reported_at_ms = self.c104_point.reported_at_ms
-        self.updated_at_ms = self.c104_point.updated_at_ms
+        self.reported_at_ms = self.c104_point.processed_at.microsecond / 1000
+        self.updated_at_ms = 0 if self.c104_point.recorded_at is None else self.c104_point.recorded_at.microsecond / 1000
         self.report_ms = self.c104_point.report_ms
         self.quality = self._translate_c104_quality(self.c104_point.quality)
         return res
@@ -67,8 +67,8 @@ def transmit(self, cause: int) -> bool:
         res = self.c104_point.transmit(cot)
 
         # not entirely sure if these values change in control-direction
-        self.reported_at_ms = self.c104_point.reported_at_ms
-        self.updated_at_ms = self.c104_point.updated_at_ms
+        self.reported_at_ms = self.c104_point.processed_at.microsecond / 1000
+        self.updated_at_ms = 0 if self.c104_point.recorded_at is None else self.c104_point.recorded_at.microsecond / 1000
         self.report_ms = self.c104_point.report_ms
         self.quality = self._translate_c104_quality(self.c104_point.quality)
         return res
@@ -82,8 +82,10 @@ def _translate_c104_quality(c104_qualilty: c104.Quality) -> QualityByte:
             QualityBit.SUBSTITUTED: c104.Quality.Substituted,
             QualityBit.BLOCKED: c104.Quality.Blocked,
             QualityBit.ELAPSED_TIME_INVALID: c104.Quality.ElapsedTimeInvalid,
-            QualityBit.RESERVED: c104.Quality.Reserved,
             QualityBit.OVERFLOW: c104.Quality.Overflow
         }
-        fcs_bits = {fcs_bit for (fcs_bit, c104_bit) in c104_map.items() if c104_bit in c104_qualilty}
-        return QualityByte(fcs_bits)
+        if c104_qualilty is None:
+            return QualityByte(None)
+        else:
+            fcs_bits = {fcs_bit for (fcs_bit, c104_bit) in c104_map.items() if c104_bit in c104_qualilty}
+            return QualityByte(fcs_bits)

wattson/iec104/implementations/c104/server/server.py

@@ -13,6 +13,7 @@
 from wattson.iec104.interface.types import COT
 from wattson.iec104.interface.server import IECServerInterface
 from wattson.iec104.implementations.c104 import C104Point, build_apdu_from_c104_bytes
+from wattson.iec104.common.config import APCI_PARAMETERS 
 
 if TYPE_CHECKING:
     from wattson.hosts.rtu.rtu import RTU
@@ -31,7 +32,15 @@ def __init__(self, rtu: 'RTU', ip: str, **kwargs):
         if self.periodic_updates_start < 0:
             self.periodic_updates_start = self.periodic_updates_start % self.periodic_updates_ms
 
-        self.server = c104.add_server(ip=ip, port=port, tick_rate_ms=tick_rate_ms)
+        self.server = c104.Server(ip=ip, port=port, tick_rate_ms=tick_rate_ms)
+        # Set w, k, t0, t1, t2, t3 parameters
+        self.server.protocol_parameters.send_window_size = APCI_PARAMETERS["k"]
+        self.server.protocol_parameters.receive_window_size = APCI_PARAMETERS["w"]
+        self.server.protocol_parameters.connection_timeout = APCI_PARAMETERS["t0"] * 1000
+        self.server.protocol_parameters.message_timeout = APCI_PARAMETERS["t1"] * 1000
+        self.server.protocol_parameters.confirm_interval = APCI_PARAMETERS["t2"] * 1000
+        self.server.protocol_parameters.keep_alive_interval = APCI_PARAMETERS["t3"] * 1000
+
         self.station = self.server.add_station(common_address=rtu.coa)
 
         self._periodic_update_points_queue = []
@@ -165,21 +174,21 @@ def _on_unexpected_message(self, server: c104.Server, message: c104.IncomingMess
                                cause: c104.Umc) -> None:
         self.callbacks["on_unexpected_msg"](server, message, cause)
 
-    def _on_receive(self, point: c104.Point, previous_state: dict,
+    def _on_receive(self, point: c104.Point, previous_info: c104.Information,
                     message: c104.IncomingMessage) -> c104.ResponseState:
-        prev_point = C104Point.parse_to_previous_point(previous_state, point)
+        prev_point = C104Point.parse_to_previous_point(previous_info, point)
         success = self.callbacks["on_receive"](C104Point(point), prev_point, message)
         return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE
 
-    def _on_setpoint_command(self, point: c104.Point, previous_state: dict,
+    def _on_setpoint_command(self, point: c104.Point, previous_info: c104.Information,
                              message: c104.IncomingMessage) -> c104.ResponseState:
-        prev_point = C104Point.parse_to_previous_point(previous_state, point)
+        prev_point = C104Point.parse_to_previous_point(previous_info, point)
         success = self.callbacks["on_setpoint_command"](C104Point(point), prev_point, message)
         return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE
 
-    def _on_step_command(self, point: c104.Point, previous_state: dict,
+    def _on_step_command(self, point: c104.Point, previous_info: c104.Information,
                          message: c104.IncomingMessage) -> bool:
-        prev_point = C104Point.parse_to_previous_point(previous_state, point)
+        prev_point = C104Point.parse_to_previous_point(previous_info, point)
         return self.callbacks["on_step_command"](C104Point(point), prev_point, message)
 
     def _on_before_read(self, point: c104.Point) -> None:

wattson/iec104/interface/types/custom_iec_value.py

@@ -1,7 +1,9 @@
 from dataclasses import dataclass
 from typing import Union, Tuple
+import c104
 
-IEC_SINGLE_VALUE = Union[bool, float, int]
+#IEC_SINGLE_VALUE = Union[bool, float, int]
+IEC_SINGLE_VALUE = Union[None,bool,c104.Double,c104.Step,c104.Int7,c104.Int16,int,c104.Byte32,c104.NormalizedFloat,float,c104.EventState,c104.StartEvents,c104.OutputCircuits,c104.PackedSingle]
 IECValue = Union[IEC_SINGLE_VALUE, Tuple[IEC_SINGLE_VALUE, int]]
 IEC_PARAMETER_SINGLE_VALUE = float
 

wattson/iec104/interface/types/quality_bit.py

@@ -13,7 +13,6 @@ class QualityBit(IntEnum):
     SUBSTITUTED = 0x20
     BLOCKED = 0x10
     ELAPSED_TIME_INVALID = 0x08
-    RESERVED = 0x04
     OVERFLOW = 0x01
 
     # only indirectly used, not part of iec101

wattson/iec104/interface/types/quality_byte.py

@@ -11,7 +11,6 @@ def __init__(self, bits: Optional[Set['QualityBit']] = None):
         self._substituded = QualityBit.SUBSTITUTED in self.bits
         self._blocked = QualityBit.BLOCKED in self.bits
         self._elapsed_time_invalid = QualityBit.ELAPSED_TIME_INVALID in self.bits
-        self._reserved = QualityBit.RESERVED in self.bits
         self._overflow = QualityBit.OVERFLOW in self.bits
 
     def __str__(self):
@@ -44,10 +43,6 @@ def is_blocked(self):
     def has_invalid_elapsed_time(self):
         return self._elapsed_time_invalid
 
-    @property
-    def is_reserved(self):
-        return self._reserved
-
     @property
     def is_overflow(self):
         return self._overflow

wattson/powergrid/simulator/power_grid_simulator.py

@@ -23,6 +23,8 @@
 from wattson.powergrid.simulator.messages.power_grid_query_type import PowerGridQueryType
 from wattson.powergrid.simulator.threads.export_thread import ExportThread
 from wattson.services.wattson_python_service import WattsonPythonService
+from wattson.services.wattson_pcap_service import WattsonPcapService
+from wattson.services.configuration import ServiceConfiguration
 from wattson.cosimulation.simulators.physical.physical_simulator import PhysicalSimulator
 from wattson.powergrid.simulator.default_configurations.mtu_default_configuration import \
     MtuDefaultConfiguration
@@ -40,6 +42,7 @@
 from wattson.powergrid.simulator.threads.simulation_thread import SimulationThread
 from wattson.util.events.multi_event import MultiEvent
 from wattson.util.events.queue_event import QueueEvent
+from wattson.iec104.common.config import SERVER_UPDATE_PERIOD_MS
 
 
 class PowerGridSimulator(PhysicalSimulator):
@@ -277,6 +280,9 @@ def _configure_network_nodes(self):
                 mtu_configuration = MtuDefaultConfiguration()
                 from wattson.hosts.mtu import MtuDeployment
                 node.add_service(WattsonPythonService(MtuDeployment, mtu_configuration, node))
+                # Add pcap services by default for all interfaces of the MTU
+                for interface in node.get_interfaces():
+                    node.add_service(WattsonPcapService(interface=interface, service_configuration=ServiceConfiguration(), network_node=node))
                 rtu_map = node.config.get("rtu_map", {})
                 self._rtu_map[node.entity_id] = {rtu_id: {"coa": rtu_id, "ip": rtu_ip.split("/")[0]} for rtu_id, rtu_ip in rtu_map.items()}
                 self._required_sim_control_clients.add(node.entity_id)
@@ -304,7 +310,7 @@ def _fill_configuration_store(self):
         })
         self._configuration_store.register_configuration("do_periodic_updates", True)
         self._configuration_store.register_configuration("periodic_update_start", 0)
-        self._configuration_store.register_configuration("periodic_update_ms", 10000)
+        self._configuration_store.register_configuration("periodic_update_ms", SERVER_UPDATE_PERIOD_MS)
         self._configuration_store.register_configuration("allowed_mtu_ips", True)
         # General
         self._configuration_store.register_configuration("coas", lambda node, store: self._common_addresses)