diff --git a/setup.py b/setup.py index d1d6fc2..afef61c 100644 --- a/setup.py +++ b/setup.py @@ -30,7 +30,7 @@ 'ifcfg', 'ninja', 'testresources', - 'c104~=1.16.0', + 'c104@git+https://github.com/Fraunhofer-FIT-DIEN/iec104-python.git@main', 'python-dateutil', 'docker', 'ipaddress==1.0.23', @@ -41,9 +41,8 @@ 'numba==0.57.1', 'numpy>=1.21.5', 'packaging==20.3', - #'pandapower==2.13.2', # Not yet released but contains critical bug fix - 'pandapower @ git+https://github.com/e2nIEE/pandapower.git', - 'powerowl @ git+https://github.com/fkie-cad/powerowl.git', + 'pandapower==2.14.1', # Contains critical bug fix + 'powerowl @ git+https://github.com/Dosclic98/powerowl.git', 'pandas==1.3.4', 'psutil==5.7.0', 'pydot', @@ -63,7 +62,7 @@ 'pyzmq==20.0.0', 'scapy==2.4.4', 'scipy>=1.11.2', - 'setuptools>=65.5.1', + 'setuptools>=65.5.1,<=70.0.0', 'shapely', 'sqlalchemy==1.3.16', 'tabulate', diff --git a/wattson/apps/interface/util/messages.py b/wattson/apps/interface/util/messages.py index 0cde4ba..01f2e04 100644 --- a/wattson/apps/interface/util/messages.py +++ b/wattson/apps/interface/util/messages.py @@ -18,6 +18,7 @@ from wattson.iec104.common.config import SUPPORTED_ASDU_TYPES from wattson.iec104.interface.types import MsgDirection, TypeID, COT, IEC_PARAMETER_SINGLE_VALUE from wattson.iec104.interface.apdus import I_FORMAT, APDU +from wattson.iec104.interface.types import IECValue COA = Union[int, str] IOA = Union[int, str] @@ -318,7 +319,7 @@ class ProcessInfoMonitoring(IECMsg): # send with empty val_map and cot == 10 to mark end of packets about that typeID # during general interro coa: int - val_map: Dict[IOA, Union[bool, int, float, Tuple]] + val_map: Dict[IOA, IECValue] ts_map: Dict[int, int] type_ID: Union[int, TypeID] cot: int @@ -353,7 +354,7 @@ class ProcessInfoControl(IECMsg): # COT necessary if the MTU sends a set-command not communicated over a subscriber coa: COA type_ID: int - val_map: Dict[IOA, Union[bool, int, float]] + val_map: Dict[IOA, IECValue] reference_nr: str = UNSET_REFERENCE_NR max_tries: int = DEFAULT_MAX_TRIES queue_on_collision: bool = False @@ -539,7 +540,7 @@ class PeriodicUpdate(ProcessInfoMonitoring): def __init__( self, coa: int, - val_map: Dict[IOA, Union[bool, int, float, Tuple]], + val_map: Dict[IOA, IECValue], ts_map: Dict[int, int], type_ID: Union[int, TypeID], reference_nr: str, diff --git a/wattson/iec104/common/config.py b/wattson/iec104/common/config.py index dead3e0..5be858c 100644 --- a/wattson/iec104/common/config.py +++ b/wattson/iec104/common/config.py @@ -4,7 +4,7 @@ SERVER_DEFAULT_PORT = 2404 SERVER_TICK_RATE_MS = 2000 # interval in seconds used to send periodic updates (COT=1) from server to client -SERVER_UPDATE_PERIOD_S = 10 +SERVER_UPDATE_PERIOD_S = 40 # Originally 10 SERVER_UPDATE_PERIOD_MS = SERVER_UPDATE_PERIOD_S * 1000 CLIENT_COMMAND_TIMEOUT_MS = 2000 @@ -27,12 +27,13 @@ | {100, 102, 103, 113} # short floating point = IEE float +# IEC 104 connection parameters (in # of APDUs and in seconds) APCI_PARAMETERS = { 'k': 12, - 'w': 8, - 't0': 10, + 'w': 8, # Originally 8 + 't0': 10, # Originally 10 't1': 15, - 't2': 10, + 't2': 340, # Originally 100 't3': 20, } diff --git a/wattson/iec104/implementations/c104/client/client.py b/wattson/iec104/implementations/c104/client/client.py index 241becf..686218d 100644 --- a/wattson/iec104/implementations/c104/client/client.py +++ b/wattson/iec104/implementations/c104/client/client.py @@ -70,7 +70,7 @@ def __init__(self, org: int = 0, **kwargs): self._org = org self._servers = {} - self._client = c104.add_client( + self._client = c104.Client( tick_rate_ms=CLIENT_TICKRATE_MS, command_timeout_ms=CLIENT_COMMAND_TIMEOUT_MS ) @@ -220,9 +220,7 @@ def get_wattson_connection_state(self, coa: int) -> ConnectionState: c104.ConnectionState.CLOSED_AWAIT_OPEN: ConnectionState.CLOSED, c104.ConnectionState.CLOSED_AWAIT_RECONNECT: ConnectionState.CLOSED, c104.ConnectionState.OPEN: ConnectionState.INTERRO_DONE, - c104.ConnectionState.OPEN_AWAIT_CLOCK_SYNC: ConnectionState.OPEN, c104.ConnectionState.OPEN_AWAIT_CLOSED: ConnectionState.OPEN, - c104.ConnectionState.OPEN_AWAIT_INTERROGATION: ConnectionState.INTERRO_STARTED, c104.ConnectionState.OPEN_MUTED: ConnectionState.OPEN } connection_state = self.get_connection_state(coa) @@ -242,6 +240,13 @@ def connect_server(self, server: Union[str, int, dict]): port=server["port"], init=self.c104_init ) + # IEC104 Connection parameters + server["connection"].protocol_parameters.send_window_size = APCI_PARAMETERS["k"] + server["connection"].protocol_parameters.receive_window_size = APCI_PARAMETERS["w"] + server["connection"].protocol_parameters.connection_timeout = APCI_PARAMETERS["t0"] * 1000 + server["connection"].protocol_parameters.message_timeout = APCI_PARAMETERS["t1"] * 1000 + server["connection"].protocol_parameters.confirm_interval = APCI_PARAMETERS["t2"] * 1000 + server["connection"].protocol_parameters.keep_alive_interval = APCI_PARAMETERS["t3"] * 1000 self.logger.debug(f"Setting Connection Callbacks for {server['coa']}") server["connection"].on_receive_raw(callable=self._on_receive_raw_callback_wrapper) @@ -481,15 +486,15 @@ def update_datapoint(self, coa: int, ioa: int, value) -> bool: server["datapoints"][str(ioa)].value = value return True - def _on_receive_datapoint(self, point: c104.Point, previous_state: dict, + def _on_receive_datapoint(self, point: c104.Point, previous_info: c104.Information, message: c104.IncomingMessage) -> c104.ResponseState: # TODO: What happens during on_receive on client-side if return False?! with self._cb_lock: try: if self.callbacks['on_receive_datapoint'] is not None: p = C104Point(point) - previous_state = C104Point.parse_to_previous_point(previous_state, point) - success = self.callbacks['on_receive_datapoint'](p, previous_state, message) + previous_info = C104Point.parse_to_previous_point(previous_info, point) + success = self.callbacks['on_receive_datapoint'](p, previous_info, message) return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE return c104.ResponseState.NONE except Exception as e: diff --git a/wattson/iec104/implementations/c104/common/c104_point.py b/wattson/iec104/implementations/c104/common/c104_point.py index ee9776b..a2c1b00 100644 --- a/wattson/iec104/implementations/c104/common/c104_point.py +++ b/wattson/iec104/implementations/c104/common/c104_point.py @@ -20,34 +20,34 @@ def __init__(self, c104_point: c104.Point): c104_point.related_io_address, fcs_quality_descriptor, c104_point.value, - c104_point.reported_at_ms, - c104_point.updated_at_ms + c104_point.processed_at.microsecond / 1000, + 0 if c104_point.recorded_at is None else c104_point.recorded_at.microsecond / 1000 ) @staticmethod - def parse_to_previous_point(previous_state: dict, cur_point: c104.Point) -> 'C104Point': + def parse_to_previous_point(previous_state: c104.Information, cur_point: c104.Point) -> 'C104Point': new_v = cur_point.value p = C104Point(cur_point) p.c104_point = None - p._value = previous_state['value'] + p._value = previous_state.value s = f"{p} {previous_state} {cur_point.value} {new_v} {type(new_v)} {type(cur_point.value)} {cur_point.value == new_v} {type(new_v) == type(cur_point.value)}" #s = str(p) + str(previous_state) + str(new_v) + str(cur_point.value) if new_v != float(cur_point.value) and not (isnan(new_v) and isnan(cur_point.value)): raise RuntimeError(f"Bad value translation {s}") if not (new_v == cur_point.value or (isnan(new_v) and isnan(cur_point.value))): raise RuntimeError("2") - elif not (p.value == previous_state['value'] or (isnan(p.value) and isnan(previous_state['value']))): + elif not (p.value == previous_state.value or (isnan(p.value) and isnan(previous_state.value))): raise RuntimeError("3") - p.updated_at_ms = previous_state['updatedAt_ms'] - p.quality = C104Point._translate_c104_quality({previous_state['quality']}) + p.updated_at_ms = 0 if previous_state.recorded_at is None else previous_state.recorded_at.microsecond / 1000 + p.quality = C104Point._translate_c104_quality({previous_state.quality}) return p def read(self) -> bool: res = self.c104_point.read() self._value = TypeID.convert_val_by_type(self.type, self.c104_point.value) - self.reported_at_ms = self.c104_point.reported_at_ms - self.updated_at_ms = self.c104_point.updated_at_ms + self.reported_at_ms = self.c104_point.processed_at.microsecond / 1000 + self.updated_at_ms = 0 if self.c104_point.recorded_at is None else self.c104_point.recorded_at.microsecond / 1000 self.report_ms = self.c104_point.report_ms self.quality = self._translate_c104_quality(self.c104_point.quality) return res @@ -67,8 +67,8 @@ def transmit(self, cause: int) -> bool: res = self.c104_point.transmit(cot) # not entirely sure if these values change in control-direction - self.reported_at_ms = self.c104_point.reported_at_ms - self.updated_at_ms = self.c104_point.updated_at_ms + self.reported_at_ms = self.c104_point.processed_at.microsecond / 1000 + self.updated_at_ms = 0 if self.c104_point.recorded_at is None else self.c104_point.recorded_at.microsecond / 1000 self.report_ms = self.c104_point.report_ms self.quality = self._translate_c104_quality(self.c104_point.quality) return res @@ -82,8 +82,10 @@ def _translate_c104_quality(c104_qualilty: c104.Quality) -> QualityByte: QualityBit.SUBSTITUTED: c104.Quality.Substituted, QualityBit.BLOCKED: c104.Quality.Blocked, QualityBit.ELAPSED_TIME_INVALID: c104.Quality.ElapsedTimeInvalid, - QualityBit.RESERVED: c104.Quality.Reserved, QualityBit.OVERFLOW: c104.Quality.Overflow } - fcs_bits = {fcs_bit for (fcs_bit, c104_bit) in c104_map.items() if c104_bit in c104_qualilty} - return QualityByte(fcs_bits) + if c104_qualilty is None: + return QualityByte(None) + else: + fcs_bits = {fcs_bit for (fcs_bit, c104_bit) in c104_map.items() if c104_bit in c104_qualilty} + return QualityByte(fcs_bits) diff --git a/wattson/iec104/implementations/c104/server/server.py b/wattson/iec104/implementations/c104/server/server.py index faab184..ebcbbc4 100644 --- a/wattson/iec104/implementations/c104/server/server.py +++ b/wattson/iec104/implementations/c104/server/server.py @@ -13,6 +13,7 @@ from wattson.iec104.interface.types import COT from wattson.iec104.interface.server import IECServerInterface from wattson.iec104.implementations.c104 import C104Point, build_apdu_from_c104_bytes +from wattson.iec104.common.config import APCI_PARAMETERS if TYPE_CHECKING: from wattson.hosts.rtu.rtu import RTU @@ -31,7 +32,15 @@ def __init__(self, rtu: 'RTU', ip: str, **kwargs): if self.periodic_updates_start < 0: self.periodic_updates_start = self.periodic_updates_start % self.periodic_updates_ms - self.server = c104.add_server(ip=ip, port=port, tick_rate_ms=tick_rate_ms) + self.server = c104.Server(ip=ip, port=port, tick_rate_ms=tick_rate_ms) + # Set w, k, t0, t1, t2, t3 parameters + self.server.protocol_parameters.send_window_size = APCI_PARAMETERS["k"] + self.server.protocol_parameters.receive_window_size = APCI_PARAMETERS["w"] + self.server.protocol_parameters.connection_timeout = APCI_PARAMETERS["t0"] * 1000 + self.server.protocol_parameters.message_timeout = APCI_PARAMETERS["t1"] * 1000 + self.server.protocol_parameters.confirm_interval = APCI_PARAMETERS["t2"] * 1000 + self.server.protocol_parameters.keep_alive_interval = APCI_PARAMETERS["t3"] * 1000 + self.station = self.server.add_station(common_address=rtu.coa) self._periodic_update_points_queue = [] @@ -165,21 +174,21 @@ def _on_unexpected_message(self, server: c104.Server, message: c104.IncomingMess cause: c104.Umc) -> None: self.callbacks["on_unexpected_msg"](server, message, cause) - def _on_receive(self, point: c104.Point, previous_state: dict, + def _on_receive(self, point: c104.Point, previous_info: c104.Information, message: c104.IncomingMessage) -> c104.ResponseState: - prev_point = C104Point.parse_to_previous_point(previous_state, point) + prev_point = C104Point.parse_to_previous_point(previous_info, point) success = self.callbacks["on_receive"](C104Point(point), prev_point, message) return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE - def _on_setpoint_command(self, point: c104.Point, previous_state: dict, + def _on_setpoint_command(self, point: c104.Point, previous_info: c104.Information, message: c104.IncomingMessage) -> c104.ResponseState: - prev_point = C104Point.parse_to_previous_point(previous_state, point) + prev_point = C104Point.parse_to_previous_point(previous_info, point) success = self.callbacks["on_setpoint_command"](C104Point(point), prev_point, message) return c104.ResponseState.SUCCESS if success else c104.ResponseState.FAILURE - def _on_step_command(self, point: c104.Point, previous_state: dict, + def _on_step_command(self, point: c104.Point, previous_info: c104.Information, message: c104.IncomingMessage) -> bool: - prev_point = C104Point.parse_to_previous_point(previous_state, point) + prev_point = C104Point.parse_to_previous_point(previous_info, point) return self.callbacks["on_step_command"](C104Point(point), prev_point, message) def _on_before_read(self, point: c104.Point) -> None: diff --git a/wattson/iec104/interface/types/custom_iec_value.py b/wattson/iec104/interface/types/custom_iec_value.py index 7c68ee6..b946fac 100644 --- a/wattson/iec104/interface/types/custom_iec_value.py +++ b/wattson/iec104/interface/types/custom_iec_value.py @@ -1,7 +1,9 @@ from dataclasses import dataclass from typing import Union, Tuple +import c104 -IEC_SINGLE_VALUE = Union[bool, float, int] +#IEC_SINGLE_VALUE = Union[bool, float, int] +IEC_SINGLE_VALUE = Union[None,bool,c104.Double,c104.Step,c104.Int7,c104.Int16,int,c104.Byte32,c104.NormalizedFloat,float,c104.EventState,c104.StartEvents,c104.OutputCircuits,c104.PackedSingle] IECValue = Union[IEC_SINGLE_VALUE, Tuple[IEC_SINGLE_VALUE, int]] IEC_PARAMETER_SINGLE_VALUE = float diff --git a/wattson/iec104/interface/types/quality_bit.py b/wattson/iec104/interface/types/quality_bit.py index 7c2cb7f..7120d4b 100644 --- a/wattson/iec104/interface/types/quality_bit.py +++ b/wattson/iec104/interface/types/quality_bit.py @@ -13,7 +13,6 @@ class QualityBit(IntEnum): SUBSTITUTED = 0x20 BLOCKED = 0x10 ELAPSED_TIME_INVALID = 0x08 - RESERVED = 0x04 OVERFLOW = 0x01 # only indirectly used, not part of iec101 diff --git a/wattson/iec104/interface/types/quality_byte.py b/wattson/iec104/interface/types/quality_byte.py index 472c8d7..5089876 100644 --- a/wattson/iec104/interface/types/quality_byte.py +++ b/wattson/iec104/interface/types/quality_byte.py @@ -11,7 +11,6 @@ def __init__(self, bits: Optional[Set['QualityBit']] = None): self._substituded = QualityBit.SUBSTITUTED in self.bits self._blocked = QualityBit.BLOCKED in self.bits self._elapsed_time_invalid = QualityBit.ELAPSED_TIME_INVALID in self.bits - self._reserved = QualityBit.RESERVED in self.bits self._overflow = QualityBit.OVERFLOW in self.bits def __str__(self): @@ -44,10 +43,6 @@ def is_blocked(self): def has_invalid_elapsed_time(self): return self._elapsed_time_invalid - @property - def is_reserved(self): - return self._reserved - @property def is_overflow(self): return self._overflow diff --git a/wattson/powergrid/simulator/power_grid_simulator.py b/wattson/powergrid/simulator/power_grid_simulator.py index 2a46c80..d8c6d94 100644 --- a/wattson/powergrid/simulator/power_grid_simulator.py +++ b/wattson/powergrid/simulator/power_grid_simulator.py @@ -23,6 +23,8 @@ from wattson.powergrid.simulator.messages.power_grid_query_type import PowerGridQueryType from wattson.powergrid.simulator.threads.export_thread import ExportThread from wattson.services.wattson_python_service import WattsonPythonService +from wattson.services.wattson_pcap_service import WattsonPcapService +from wattson.services.configuration import ServiceConfiguration from wattson.cosimulation.simulators.physical.physical_simulator import PhysicalSimulator from wattson.powergrid.simulator.default_configurations.mtu_default_configuration import \ MtuDefaultConfiguration @@ -40,6 +42,7 @@ from wattson.powergrid.simulator.threads.simulation_thread import SimulationThread from wattson.util.events.multi_event import MultiEvent from wattson.util.events.queue_event import QueueEvent +from wattson.iec104.common.config import SERVER_UPDATE_PERIOD_MS class PowerGridSimulator(PhysicalSimulator): @@ -277,6 +280,9 @@ def _configure_network_nodes(self): mtu_configuration = MtuDefaultConfiguration() from wattson.hosts.mtu import MtuDeployment node.add_service(WattsonPythonService(MtuDeployment, mtu_configuration, node)) + # Add pcap services by default for all interfaces of the MTU + for interface in node.get_interfaces(): + node.add_service(WattsonPcapService(interface=interface, service_configuration=ServiceConfiguration(), network_node=node)) rtu_map = node.config.get("rtu_map", {}) self._rtu_map[node.entity_id] = {rtu_id: {"coa": rtu_id, "ip": rtu_ip.split("/")[0]} for rtu_id, rtu_ip in rtu_map.items()} self._required_sim_control_clients.add(node.entity_id) @@ -304,7 +310,7 @@ def _fill_configuration_store(self): }) self._configuration_store.register_configuration("do_periodic_updates", True) self._configuration_store.register_configuration("periodic_update_start", 0) - self._configuration_store.register_configuration("periodic_update_ms", 10000) + self._configuration_store.register_configuration("periodic_update_ms", SERVER_UPDATE_PERIOD_MS) self._configuration_store.register_configuration("allowed_mtu_ips", True) # General self._configuration_store.register_configuration("coas", lambda node, store: self._common_addresses)