diff --git a/src/teamster/core/achieve3k/sensors.py b/src/teamster/core/achieve3k/sensors.py index a67726e489..5af9915a46 100644 --- a/src/teamster/core/achieve3k/sensors.py +++ b/src/teamster/core/achieve3k/sensors.py @@ -15,7 +15,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -30,9 +30,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor( - context: SensorEvaluationContext, ssh_achieve3k: SSHConfigurableResource - ): + def _sensor(context: SensorEvaluationContext, ssh_achieve3k: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/adp/sensors.py b/src/teamster/core/adp/sensors.py index a07bd15dd1..09670898c5 100644 --- a/src/teamster/core/adp/sensors.py +++ b/src/teamster/core/adp/sensors.py @@ -14,7 +14,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -28,9 +28,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor( - context: SensorEvaluationContext, ssh_adp_workforce_now: SSHConfigurableResource - ): + def _sensor(context: SensorEvaluationContext, ssh_adp_workforce_now: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/clever/sensors.py b/src/teamster/core/clever/sensors.py index 646987160c..c926a7d49b 100644 --- a/src/teamster/core/clever/sensors.py +++ b/src/teamster/core/clever/sensors.py @@ -16,7 +16,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -31,9 +31,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor( - context: SensorEvaluationContext, ssh_clever_reports: SSHConfigurableResource - ): + def _sensor(context: SensorEvaluationContext, ssh_clever_reports: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/datagun/assets.py b/src/teamster/core/datagun/assets.py index b1ae9ace49..10385065b5 100644 --- a/src/teamster/core/datagun/assets.py +++ b/src/teamster/core/datagun/assets.py @@ -12,7 +12,7 @@ from pandas import DataFrame from sqlalchemy import literal_column, select, table, text -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.classes import CustomJSONEncoder @@ -74,7 +74,7 @@ def transform_data(data, file_suffix, file_encoding=None, file_format=None): def load_sftp( context: AssetExecutionContext, - ssh: SSHConfigurableResource, + ssh: SSHResource, data, file_name, destination_path, diff --git a/src/teamster/core/edplan/sensors.py b/src/teamster/core/edplan/sensors.py index c8aa6b332c..dc6cdc25f1 100644 --- a/src/teamster/core/edplan/sensors.py +++ b/src/teamster/core/edplan/sensors.py @@ -14,7 +14,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -28,7 +28,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor(context: SensorEvaluationContext, ssh_edplan: SSHConfigurableResource): + def _sensor(context: SensorEvaluationContext, ssh_edplan: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/iready/sensors.py b/src/teamster/core/iready/sensors.py index 586226ddfb..5a2dec7ad8 100644 --- a/src/teamster/core/iready/sensors.py +++ b/src/teamster/core/iready/sensors.py @@ -15,7 +15,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -30,7 +30,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor(context: SensorEvaluationContext, ssh_iready: SSHConfigurableResource): + def _sensor(context: SensorEvaluationContext, ssh_iready: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/powerschool/assets.py b/src/teamster/core/powerschool/assets.py index 1701dc108c..39018990fb 100644 --- a/src/teamster/core/powerschool/assets.py +++ b/src/teamster/core/powerschool/assets.py @@ -10,7 +10,7 @@ from sqlalchemy import literal_column, select, table, text from teamster.core.sqlalchemy.resources import OracleResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.classes import FiscalYearPartitionsDefinition @@ -31,7 +31,7 @@ def build_powerschool_table_asset( ) def _asset( context: AssetExecutionContext, - ssh_powerschool: SSHConfigurableResource, + ssh_powerschool: SSHResource, db_powerschool: OracleResource, ): now = pendulum.now() diff --git a/src/teamster/core/powerschool/schedules.py b/src/teamster/core/powerschool/schedules.py index 2698699b04..42ebdf71dc 100644 --- a/src/teamster/core/powerschool/schedules.py +++ b/src/teamster/core/powerschool/schedules.py @@ -12,7 +12,7 @@ from sqlalchemy import text from teamster.core.sqlalchemy.resources import OracleResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_last_modified_schedule( @@ -32,7 +32,7 @@ def build_last_modified_schedule( ) def _schedule( context: ScheduleEvaluationContext, - ssh_powerschool: SSHConfigurableResource, + ssh_powerschool: SSHResource, db_powerschool: OracleResource, ): ssh_tunnel = ssh_powerschool.get_tunnel(remote_port=1521, local_port=1521) diff --git a/src/teamster/core/powerschool/sensors.py b/src/teamster/core/powerschool/sensors.py index dbc500bf70..84c4423269 100644 --- a/src/teamster/core/powerschool/sensors.py +++ b/src/teamster/core/powerschool/sensors.py @@ -13,7 +13,7 @@ from sqlalchemy import text from teamster.core.sqlalchemy.resources import OracleResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.classes import FiscalYearPartitionsDefinition @@ -27,7 +27,7 @@ def build_partition_sensor( ) def _sensor( context: SensorEvaluationContext, - ssh_powerschool: SSHConfigurableResource, + ssh_powerschool: SSHResource, db_powerschool: OracleResource, ): now = pendulum.now(timezone).start_of("minute") diff --git a/src/teamster/core/renlearn/sensors.py b/src/teamster/core/renlearn/sensors.py index 8eadf7e537..ffc38d44d6 100644 --- a/src/teamster/core/renlearn/sensors.py +++ b/src/teamster/core/renlearn/sensors.py @@ -16,7 +16,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -34,9 +34,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor( - context: SensorEvaluationContext, ssh_renlearn: SSHConfigurableResource - ): + def _sensor(context: SensorEvaluationContext, ssh_renlearn: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/core/sftp/assets.py b/src/teamster/core/sftp/assets.py index d2a6ffed02..ff808329c4 100644 --- a/src/teamster/core/sftp/assets.py +++ b/src/teamster/core/sftp/assets.py @@ -17,7 +17,7 @@ from paramiko import SFTPClient from slugify import slugify -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.functions import regex_pattern_replace @@ -37,7 +37,7 @@ def listdir_attr_r(sftp_client: SFTPClient, remote_dir: str, files: list = []): return files -def match_sftp_files(ssh: SSHConfigurableResource, remote_dir, remote_file_regex): +def match_sftp_files(ssh: SSHResource, remote_dir, remote_file_regex): # list files remote filepath with ssh.get_connection() as conn: with conn.open_sftp() as sftp_client: @@ -105,7 +105,7 @@ def build_sftp_asset( def _asset(context: AssetExecutionContext): context.log.debug(requests.get(url="https://api.ipify.org").text) - ssh: SSHConfigurableResource = getattr(context.resources, ssh_resource_key) + ssh: SSHResource = getattr(context.resources, ssh_resource_key) # find matching file for partition remote_file_regex_composed = compose_regex( diff --git a/src/teamster/core/ssh/resources.py b/src/teamster/core/ssh/resources.py index b82fe6160f..a277cb5847 100644 --- a/src/teamster/core/ssh/resources.py +++ b/src/teamster/core/ssh/resources.py @@ -1,59 +1,21 @@ -from dagster import ConfigurableResource, InitResourceContext from dagster_ssh import SSHResource -from pydantic import PrivateAttr +from sshtunnel import SSHTunnelForwarder -class SSHConfigurableResource(ConfigurableResource): - remote_host: str - remote_port: str = "22" - username: str = None - password: str = None - key_file: str = None - key_string: str = None - timeout: int = 10 - keepalive_interval: int = 30 - compress: bool = True - no_host_key_check: bool = True - allow_host_key_change: bool = False - tunnel_remote_host: str = "localhost" +class SSHResource(SSHResource): + remote_port = 22 + tunnel_remote_host: str = None - _internal_resource: SSHResource = PrivateAttr() - - def setup_for_execution(self, context: InitResourceContext) -> None: - self._internal_resource = SSHResource( - remote_host=self.remote_host, - remote_port=int(self.remote_port), - username=self.username, - password=self.password, - key_file=self.key_file, - key_string=self.key_string, - timeout=self.timeout, - keepalive_interval=self.keepalive_interval, - compress=self.compress, - no_host_key_check=self.no_host_key_check, - allow_host_key_change=self.allow_host_key_change, - logger=self.get_resource_context().log, - ) - - def get_connection(self): - return self._internal_resource.get_connection() - - def get_tunnel(self, remote_port, remote_host=None, local_port=None): - if remote_host is None: + def get_tunnel( + self, remote_port, remote_host=None, local_port=None + ) -> SSHTunnelForwarder: + if remote_host is not None: + pass + elif self.tunnel_remote_host is not None: remote_host = self.tunnel_remote_host + else: + remote_host = "localhost" - return self._internal_resource.get_tunnel( + return super().get_tunnel( remote_port=remote_port, remote_host=remote_host, local_port=local_port ) - - def sftp_get(self, remote_filepath, local_filepath): - return self._internal_resource.sftp_get( - remote_filepath=remote_filepath, local_filepath=local_filepath - ) - - def sftp_put(self, remote_filepath, local_filepath, confirm=True): - return self._internal_resource.sftp_put( - remote_filepath=remote_filepath, - local_filepath=local_filepath, - confirm=confirm, - ) diff --git a/src/teamster/core/titan/sensors.py b/src/teamster/core/titan/sensors.py index 1476cfba2e..40664df5e0 100644 --- a/src/teamster/core/titan/sensors.py +++ b/src/teamster/core/titan/sensors.py @@ -14,7 +14,7 @@ from paramiko.ssh_exception import SSHException from teamster.core.sftp.assets import listdir_attr_r -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def build_sftp_sensor( @@ -29,7 +29,7 @@ def build_sftp_sensor( minimum_interval_seconds=minimum_interval_seconds, asset_selection=AssetSelection.assets(*asset_defs), ) - def _sensor(context: SensorEvaluationContext, ssh_titan: SSHConfigurableResource): + def _sensor(context: SensorEvaluationContext, ssh_titan: SSHResource): cursor: dict = json.loads(context.cursor or "{}") now = pendulum.now(tz=timezone) diff --git a/src/teamster/kippcamden/definitions.py b/src/teamster/kippcamden/definitions.py index e6a3aee1da..80cf3344cb 100644 --- a/src/teamster/kippcamden/definitions.py +++ b/src/teamster/kippcamden/definitions.py @@ -10,7 +10,7 @@ from teamster.core.deanslist.resources import DeansListResource from teamster.core.google.io.resources import gcs_io_manager from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from . import ( CODE_LOCATION, @@ -84,34 +84,34 @@ subdomain="kippnj", api_key_map="/etc/secret-volume/deanslist_api_key_map_yaml", ), - "ssh_couchdrop": SSHConfigurableResource( + "ssh_couchdrop": SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), ), - "ssh_cpn": SSHConfigurableResource( + "ssh_cpn": SSHResource( remote_host="sftp.careevolution.com", username=EnvVar("CPN_SFTP_USERNAME"), password=EnvVar("CPN_SFTP_PASSWORD"), ), - "ssh_edplan": SSHConfigurableResource( + "ssh_edplan": SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_PASSWORD"), ), - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="pskcna.kippnj.org", - remote_port=EnvVar("KIPPCAMDEN_PS_SSH_PORT"), + remote_port=EnvVar("KIPPCAMDEN_PS_SSH_PORT").get_value(), username=EnvVar("KIPPCAMDEN_PS_SSH_USERNAME"), password=EnvVar("KIPPCAMDEN_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("KIPPCAMDEN_PS_SSH_REMOTE_BIND_HOST"), ), - "ssh_pythonanywhere": SSHConfigurableResource( + "ssh_pythonanywhere": SSHResource( remote_host="ssh.pythonanywhere.com", username=EnvVar("PYTHONANYWHERE_SFTP_USERNAME"), password=EnvVar("PYTHONANYWHERE_SFTP_PASSWORD"), ), - "ssh_titan": SSHConfigurableResource( + "ssh_titan": SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPCAMDEN_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_TITAN_SFTP_PASSWORD"), diff --git a/src/teamster/kippmiami/definitions.py b/src/teamster/kippmiami/definitions.py index 1d8d355645..d28e87ec4b 100644 --- a/src/teamster/kippmiami/definitions.py +++ b/src/teamster/kippmiami/definitions.py @@ -10,7 +10,7 @@ from teamster.core.deanslist.resources import DeansListResource from teamster.core.google.io.resources import gcs_io_manager from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from . import ( CODE_LOCATION, @@ -84,29 +84,29 @@ subdomain="kippnj", api_key_map="/etc/secret-volume/deanslist_api_key_map_yaml", ), - "ssh_couchdrop": SSHConfigurableResource( + "ssh_couchdrop": SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), ), - "ssh_iready": SSHConfigurableResource( + "ssh_iready": SSHResource( remote_host="prod-sftp-1.aws.cainc.com", username=EnvVar("IREADY_SFTP_USERNAME"), password=EnvVar("IREADY_SFTP_PASSWORD"), ), - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="ps.kippmiami.org", - remote_port=EnvVar("KIPPMIAMI_PS_SSH_PORT"), + remote_port=EnvVar("KIPPMIAMI_PS_SSH_PORT").get_value(), username=EnvVar("KIPPMIAMI_PS_SSH_USERNAME"), password=EnvVar("KIPPMIAMI_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("KIPPMIAMI_PS_SSH_REMOTE_BIND_HOST"), ), - "ssh_pythonanywhere": SSHConfigurableResource( + "ssh_pythonanywhere": SSHResource( remote_host="ssh.pythonanywhere.com", username=EnvVar("PYTHONANYWHERE_SFTP_USERNAME"), password=EnvVar("PYTHONANYWHERE_SFTP_PASSWORD"), ), - "ssh_renlearn": SSHConfigurableResource( + "ssh_renlearn": SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPMIAMI_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPMIAMI_RENLEARN_SFTP_PASSWORD"), diff --git a/src/teamster/kippnewark/definitions.py b/src/teamster/kippnewark/definitions.py index f16c17de84..23449a8456 100644 --- a/src/teamster/kippnewark/definitions.py +++ b/src/teamster/kippnewark/definitions.py @@ -10,7 +10,7 @@ from teamster.core.deanslist.resources import DeansListResource from teamster.core.google.io.resources import gcs_io_manager from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from . import ( CODE_LOCATION, @@ -94,34 +94,34 @@ subdomain="kippnj", api_key_map="/etc/secret-volume/deanslist_api_key_map_yaml", ), - "ssh_couchdrop": SSHConfigurableResource( + "ssh_couchdrop": SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), ), - "ssh_edplan": SSHConfigurableResource( + "ssh_edplan": SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPNEWARK_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_EDPLAN_SFTP_PASSWORD"), ), - "ssh_iready": SSHConfigurableResource( + "ssh_iready": SSHResource( remote_host="prod-sftp-1.aws.cainc.com", username=EnvVar("IREADY_SFTP_USERNAME"), password=EnvVar("IREADY_SFTP_PASSWORD"), ), - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="psteam.kippnj.org", - remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT"), + remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT").get_value(), username=EnvVar("KIPPNEWARK_PS_SSH_USERNAME"), password=EnvVar("KIPPNEWARK_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("KIPPNEWARK_PS_SSH_REMOTE_BIND_HOST"), ), - "ssh_renlearn": SSHConfigurableResource( + "ssh_renlearn": SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPNJ_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPNJ_RENLEARN_SFTP_PASSWORD"), ), - "ssh_titan": SSHConfigurableResource( + "ssh_titan": SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPNEWARK_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_TITAN_SFTP_PASSWORD"), diff --git a/src/teamster/kipptaf/definitions.py b/src/teamster/kipptaf/definitions.py index 2b2f422a65..983e9d496f 100644 --- a/src/teamster/kipptaf/definitions.py +++ b/src/teamster/kipptaf/definitions.py @@ -22,7 +22,7 @@ from teamster.core.ldap.resources import LdapResource from teamster.core.schoolmint.grow.resources import SchoolMintGrowResource from teamster.core.smartrecruiters.resources import SmartRecruitersResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.jobs import asset_observation_job from teamster.kipptaf import ( CODE_LOCATION, @@ -164,91 +164,91 @@ "smartrecruiters": SmartRecruitersResource( smart_token=EnvVar("SMARTRECRUITERS_SMARTTOKEN") ), - "ssh_achieve3k": SSHConfigurableResource( + "ssh_achieve3k": SSHResource( remote_host="xfer.achieve3000.com", username=EnvVar("ACHIEVE3K_SFTP_USERNAME"), password=EnvVar("ACHIEVE3K_SFTP_PASSWORD"), ), - "ssh_adp_workforce_now": SSHConfigurableResource( + "ssh_adp_workforce_now": SSHResource( remote_host="sftp.kippnj.org", username=EnvVar("ADP_SFTP_USERNAME"), password=EnvVar("ADP_SFTP_PASSWORD"), ), - "ssh_blissbook": SSHConfigurableResource( + "ssh_blissbook": SSHResource( remote_host="sftp.blissbook.com", - remote_port="3022", + remote_port=3022, username=EnvVar("BLISSBOOK_SFTP_USERNAME"), password=EnvVar("BLISSBOOK_SFTP_PASSWORD"), ), - "ssh_clever": SSHConfigurableResource( + "ssh_clever": SSHResource( remote_host="sftp.clever.com", username=EnvVar("CLEVER_SFTP_USERNAME"), password=EnvVar("CLEVER_SFTP_PASSWORD"), ), - "ssh_clever_reports": SSHConfigurableResource( + "ssh_clever_reports": SSHResource( remote_host="reports-sftp.clever.com", username=EnvVar("CLEVER_REPORTS_SFTP_USERNAME"), password=EnvVar("CLEVER_REPORTS_SFTP_PASSWORD"), ), - "ssh_couchdrop": SSHConfigurableResource( + "ssh_couchdrop": SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), ), - "ssh_coupa": SSHConfigurableResource( + "ssh_coupa": SSHResource( remote_host="fileshare.coupahost.com", username=EnvVar("COUPA_SFTP_USERNAME"), password=EnvVar("COUPA_SFTP_PASSWORD"), ), - "ssh_deanslist": SSHConfigurableResource( + "ssh_deanslist": SSHResource( remote_host="sftp.deanslistsoftware.com", username=EnvVar("DEANSLIST_SFTP_USERNAME"), password=EnvVar("DEANSLIST_SFTP_PASSWORD"), ), - "ssh_egencia": SSHConfigurableResource( + "ssh_egencia": SSHResource( remote_host="eusftp.egencia.com", username=EnvVar("EGENCIA_SFTP_USERNAME"), key_file="/etc/secret-volume/id_rsa_egencia", ), - "ssh_illuminate": SSHConfigurableResource( + "ssh_illuminate": SSHResource( remote_host="sftp.illuminateed.com", username=EnvVar("ILLUMINATE_SFTP_USERNAME"), password=EnvVar("ILLUMINATE_SFTP_PASSWORD"), ), - "ssh_kipptaf": SSHConfigurableResource( + "ssh_kipptaf": SSHResource( remote_host="sftp.kippnj.org", username=EnvVar("KTAF_SFTP_USERNAME"), password=EnvVar("KTAF_SFTP_PASSWORD"), ), - "ssh_idauto": SSHConfigurableResource( + "ssh_idauto": SSHResource( remote_host="sftp.kippnj.org", username=EnvVar("KTAF_SFTP_USERNAME"), password=EnvVar("KTAF_SFTP_PASSWORD"), ), - "ssh_littlesis": SSHConfigurableResource( + "ssh_littlesis": SSHResource( remote_host="upload.littlesis.app", username=EnvVar("LITTLESIS_SFTP_USERNAME"), password=EnvVar("LITTLESIS_SFTP_PASSWORD"), ), - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="teamacademy.clgpstest.com", - remote_port=EnvVar("STAGING_PS_SSH_PORT"), + remote_port=EnvVar("STAGING_PS_SSH_PORT").get_value(), username=EnvVar("STAGING_PS_SSH_USERNAME"), password=EnvVar("STAGING_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("STAGING_PS_SSH_REMOTE_BIND_HOST"), ), - "ssh_pythonanywhere": SSHConfigurableResource( + "ssh_pythonanywhere": SSHResource( remote_host="ssh.pythonanywhere.com", username=EnvVar("PYTHONANYWHERE_SFTP_USERNAME"), password=EnvVar("PYTHONANYWHERE_SFTP_PASSWORD"), ), - "ssh_razkids": SSHConfigurableResource( + "ssh_razkids": SSHResource( remote_host="sftp.learninga-z.com", - remote_port="22224", + remote_port=22224, username=EnvVar("RAZKIDS_SFTP_USERNAME"), password=EnvVar("RAZKIDS_SFTP_PASSWORD"), ), - "ssh_read180": SSHConfigurableResource( + "ssh_read180": SSHResource( remote_host="imports.education.scholastic.com", username=EnvVar("READ180_SFTP_USERNAME"), password=EnvVar("READ180_SFTP_PASSWORD"), diff --git a/src/teamster/staging/definitions.py b/src/teamster/staging/definitions.py index 9b151059fd..caea4d36ab 100644 --- a/src/teamster/staging/definitions.py +++ b/src/teamster/staging/definitions.py @@ -3,7 +3,7 @@ from teamster.core.google.io.resources import gcs_io_manager from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.staging import CODE_LOCATION, GCS_PROJECT_NAME, powerschool resource_config_dir = f"src/teamster/{CODE_LOCATION}/config/resources" @@ -40,9 +40,9 @@ prefetchrows=100000, arraysize=100000, ), - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="psteam.kippnj.org", - remote_port=EnvVar("STAGING_PS_SSH_PORT"), + remote_port=EnvVar("STAGING_PS_SSH_PORT").get_value(), username=EnvVar("STAGING_PS_SSH_USERNAME"), password=EnvVar("STAGING_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("STAGING_PS_SSH_REMOTE_BIND_HOST"), diff --git a/tests/powerschool/test_powerschool_resources.py b/tests/powerschool/test_powerschool_resources.py index 513eba9546..eab5ead320 100644 --- a/tests/powerschool/test_powerschool_resources.py +++ b/tests/powerschool/test_powerschool_resources.py @@ -4,13 +4,13 @@ from sqlalchemy import literal_column, select, table, text from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource with build_resources( resources={ - "ssh_powerschool": SSHConfigurableResource( + "ssh_powerschool": SSHResource( remote_host="teamacademy.clgpstest.com", - remote_port=EnvVar("STAGING_PS_SSH_PORT"), + remote_port=EnvVar("STAGING_PS_SSH_PORT").get_value(), username=EnvVar("STAGING_PS_SSH_USERNAME"), password=EnvVar("STAGING_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("STAGING_PS_SSH_REMOTE_BIND_HOST"), @@ -31,7 +31,7 @@ ), } ) as resources: - SSH_POWERSCHOOL: SSHConfigurableResource = resources.ssh_powerschool + SSH_POWERSCHOOL: SSHResource = resources.ssh_powerschool DB_POWERSCHOOL: OracleResource = resources.db_powerschool diff --git a/tests/powerschool/test_powerschool_schedules.py b/tests/powerschool/test_powerschool_schedules.py index 197ff6870d..31a8b306d7 100644 --- a/tests/powerschool/test_powerschool_schedules.py +++ b/tests/powerschool/test_powerschool_schedules.py @@ -1,7 +1,7 @@ from dagster import EnvVar, build_schedule_context from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.core.utils.functions import get_dagster_cloud_instance from teamster.kippnewark.powerschool.schedules import last_modified_schedule @@ -13,9 +13,9 @@ def test_schedule(): output = last_modified_schedule( context=context, - ssh_powerschool=SSHConfigurableResource( + ssh_powerschool=SSHResource( remote_host="psteam.kippnj.org", - remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT"), + remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT").get_value(), username=EnvVar("KIPPNEWARK_PS_SSH_USERNAME"), password=EnvVar("KIPPNEWARK_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("KIPPNEWARK_PS_SSH_REMOTE_BIND_HOST"), diff --git a/tests/powerschool/test_powerschool_sensor.py b/tests/powerschool/test_powerschool_sensor.py index d02584f400..ca6e02fc7a 100644 --- a/tests/powerschool/test_powerschool_sensor.py +++ b/tests/powerschool/test_powerschool_sensor.py @@ -4,7 +4,7 @@ from teamster.core.powerschool.sensors import build_partition_sensor from teamster.core.sqlalchemy.resources import OracleResource, SqlAlchemyEngineResource -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.kippnewark import LOCAL_TIMEZONE from teamster.kippnewark.powerschool.assets import partition_assets @@ -28,9 +28,9 @@ def test_sensor(): sensor_results = dynamic_partition_sensor( context=context, - ssh_powerschool=SSHConfigurableResource( + ssh_powerschool=SSHResource( remote_host="psteam.kippnj.org", - remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT"), + remote_port=EnvVar("KIPPNEWARK_PS_SSH_PORT").get_value(), username=EnvVar("KIPPNEWARK_PS_SSH_USERNAME"), password=EnvVar("KIPPNEWARK_PS_SSH_PASSWORD"), tunnel_remote_host=EnvVar("KIPPNEWARK_PS_SSH_REMOTE_BIND_HOST"), diff --git a/tests/sftp/test_sftp_assets.py b/tests/sftp/test_sftp_assets.py index 80a5df35b2..586428364e 100644 --- a/tests/sftp/test_sftp_assets.py +++ b/tests/sftp/test_sftp_assets.py @@ -1,11 +1,11 @@ import random import shutil -from dagster import EnvVar, config_from_files, materialize +from dagster import EnvVar, materialize from dagster_gcp import GCSResource from teamster.core.google.io.resources import gcs_io_manager -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource from teamster.staging import GCS_PROJECT_NAME @@ -20,9 +20,10 @@ def _test_assets(asset, ssh_resource): resources={ "gcs": GCSResource(project=GCS_PROJECT_NAME), "io_manager_gcs_avro": gcs_io_manager.configured( - config_from_files( - ["src/teamster/staging/config/resources/io_avro.yaml"] - ) + config_or_config_fn={ + "gcs_bucket": "teamster-staging", + "io_format": "avro", + } ), **ssh_resource, }, @@ -40,7 +41,7 @@ def test_assets_renlearn(): _test_assets( asset=asset, ssh_resource={ - "ssh_renlearn": SSHConfigurableResource( + "ssh_renlearn": SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPMIAMI_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPMIAMI_RENLEARN_SFTP_PASSWORD"), @@ -54,7 +55,7 @@ def test_assets_renlearn(): _test_assets( asset=asset, ssh_resource={ - "ssh_renlearn": SSHConfigurableResource( + "ssh_renlearn": SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPNJ_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPNJ_RENLEARN_SFTP_PASSWORD"), @@ -70,7 +71,7 @@ def test_assets_fldoe(): _test_assets( asset=asset, ssh_resource={ - "ssh_couchdrop": SSHConfigurableResource( + "ssh_couchdrop": SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), @@ -82,7 +83,7 @@ def test_assets_fldoe(): def test_assets_iready(): from teamster.kippmiami.iready import assets - ssh_iready = SSHConfigurableResource( + ssh_iready = SSHResource( remote_host="prod-sftp-1.aws.cainc.com", username=EnvVar("IREADY_SFTP_USERNAME"), password=EnvVar("IREADY_SFTP_PASSWORD"), @@ -104,7 +105,7 @@ def test_assets_edplan(): _test_assets( asset=asset, ssh_resource={ - "ssh_edplan": SSHConfigurableResource( + "ssh_edplan": SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_PASSWORD"), @@ -118,7 +119,7 @@ def test_assets_edplan(): _test_assets( asset=asset, ssh_resource={ - "ssh_edplan": SSHConfigurableResource( + "ssh_edplan": SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPNEWARK_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_EDPLAN_SFTP_PASSWORD"), @@ -134,7 +135,7 @@ def test_assets_titan(): _test_assets( asset=asset, ssh_resource={ - "ssh_titan": SSHConfigurableResource( + "ssh_titan": SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPCAMDEN_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_TITAN_SFTP_PASSWORD"), @@ -148,7 +149,7 @@ def test_assets_titan(): _test_assets( asset=asset, ssh_resource={ - "ssh_titan": SSHConfigurableResource( + "ssh_titan": SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPNEWARK_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_TITAN_SFTP_PASSWORD"), @@ -164,7 +165,7 @@ def test_assets_achieve3k(): _test_assets( asset=assets, ssh_resource={ - "ssh_achieve3k": SSHConfigurableResource( + "ssh_achieve3k": SSHResource( remote_host="xfer.achieve3000.com", username=EnvVar("ACHIEVE3K_SFTP_USERNAME"), password=EnvVar("ACHIEVE3K_SFTP_PASSWORD"), @@ -179,7 +180,7 @@ def test_assets_clever(): _test_assets( asset=assets, ssh_resource={ - "ssh_clever_reports": SSHConfigurableResource( + "ssh_clever_reports": SSHResource( remote_host="reports-sftp.clever.com", username=EnvVar("CLEVER_REPORTS_SFTP_USERNAME"), password=EnvVar("CLEVER_REPORTS_SFTP_PASSWORD"), diff --git a/tests/sftp/test_sftp_resource.py b/tests/sftp/test_sftp_resource.py index e14cfb9da8..392ad3347f 100644 --- a/tests/sftp/test_sftp_resource.py +++ b/tests/sftp/test_sftp_resource.py @@ -1,14 +1,14 @@ from dagster import EnvVar, build_asset_context, build_resources from teamster.core.sftp.assets import match_sftp_files -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def _test(ssh_configurable_resource, remote_file_regex_composed, remote_dir="."): context = build_asset_context() with build_resources(resources={"ssh": ssh_configurable_resource}) as resources: - ssh: SSHConfigurableResource = resources.ssh + ssh: SSHResource = resources.ssh # find matching file for partition file_matches = match_sftp_files( @@ -36,7 +36,7 @@ def _test(ssh_configurable_resource, remote_file_regex_composed, remote_dir=".") def test_iready_nj(): _test( - ssh_configurable_resource=SSHConfigurableResource( + ssh_configurable_resource=SSHResource( remote_host="prod-sftp-1.aws.cainc.com", username=EnvVar("IREADY_SFTP_USERNAME"), password=EnvVar("IREADY_SFTP_PASSWORD"), @@ -48,7 +48,7 @@ def test_iready_nj(): def test_renlearn_miami(): _test( - ssh_configurable_resource=SSHConfigurableResource( + ssh_configurable_resource=SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPMIAMI_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPMIAMI_RENLEARN_SFTP_PASSWORD"), @@ -59,7 +59,7 @@ def test_renlearn_miami(): def test_fldoe(): _test( - ssh_configurable_resource=SSHConfigurableResource( + ssh_configurable_resource=SSHResource( remote_host="kipptaf.couchdrop.io", username=EnvVar("COUCHDROP_SFTP_USERNAME"), password=EnvVar("COUCHDROP_SFTP_PASSWORD"), diff --git a/tests/sftp/test_sftp_sensors.py b/tests/sftp/test_sftp_sensors.py index e2d115155f..085446d16c 100644 --- a/tests/sftp/test_sftp_sensors.py +++ b/tests/sftp/test_sftp_sensors.py @@ -2,7 +2,7 @@ from dagster import EnvVar, SensorResult, build_sensor_context -from teamster.core.ssh.resources import SSHConfigurableResource +from teamster.core.ssh.resources import SSHResource def _test_sensor(sftp_sensor, **kwargs): @@ -29,7 +29,7 @@ def test_sensor_edplan(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_edplan=SSHConfigurableResource( + ssh_edplan=SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_EDPLAN_SFTP_PASSWORD"), @@ -40,7 +40,7 @@ def test_sensor_edplan(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_edplan=SSHConfigurableResource( + ssh_edplan=SSHResource( remote_host="secureftp.easyiep.com", username=EnvVar("KIPPNEWARK_EDPLAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_EDPLAN_SFTP_PASSWORD"), @@ -53,7 +53,7 @@ def test_sensor_titan(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_titan=SSHConfigurableResource( + ssh_titan=SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPCAMDEN_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPCAMDEN_TITAN_SFTP_PASSWORD"), @@ -64,7 +64,7 @@ def test_sensor_titan(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_titan=SSHConfigurableResource( + ssh_titan=SSHResource( remote_host="sftp.titank12.com", username=EnvVar("KIPPNEWARK_TITAN_SFTP_USERNAME"), password=EnvVar("KIPPNEWARK_TITAN_SFTP_PASSWORD"), @@ -73,7 +73,7 @@ def test_sensor_titan(): def test_sensor_iready(): - ssh_iready = SSHConfigurableResource( + ssh_iready = SSHResource( remote_host="prod-sftp-1.aws.cainc.com", username=EnvVar("IREADY_SFTP_USERNAME"), password=EnvVar("IREADY_SFTP_PASSWORD"), @@ -93,7 +93,7 @@ def test_sensor_renlearn(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_renlearn=SSHConfigurableResource( + ssh_renlearn=SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPMIAMI_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPMIAMI_RENLEARN_SFTP_PASSWORD"), @@ -104,7 +104,7 @@ def test_sensor_renlearn(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_renlearn=SSHConfigurableResource( + ssh_renlearn=SSHResource( remote_host="sftp.renaissance.com", username=EnvVar("KIPPNJ_RENLEARN_SFTP_USERNAME"), password=EnvVar("KIPPNJ_RENLEARN_SFTP_PASSWORD"), @@ -117,7 +117,7 @@ def test_sensor_achieve3k(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_achieve3k=SSHConfigurableResource( + ssh_achieve3k=SSHResource( remote_host="xfer.achieve3000.com", username=EnvVar("ACHIEVE3K_SFTP_USERNAME"), password=EnvVar("ACHIEVE3K_SFTP_PASSWORD"), @@ -130,7 +130,7 @@ def test_sensor_clever_reports(): _test_sensor( sftp_sensor=sftp_sensor, - ssh_clever_reports=SSHConfigurableResource( + ssh_clever_reports=SSHResource( remote_host="reports-sftp.clever.com", username=EnvVar("CLEVER_REPORTS_SFTP_USERNAME"), password=EnvVar("CLEVER_REPORTS_SFTP_PASSWORD"), @@ -138,15 +138,17 @@ def test_sensor_clever_reports(): ) +""" # ip restricted def test_sensor_adp(): from teamster.kipptaf.adp.sensors import sftp_sensor _test_sensor( sftp_sensor=sftp_sensor, - ssh_adp_workforce_now=SSHConfigurableResource( + ssh_adp_workforce_now=SSHResource( remote_host="sftp.kippnj.org", username=EnvVar("ADP_SFTP_USERNAME"), password=EnvVar("ADP_SFTP_PASSWORD"), ), ) +"""