diff --git a/README.md b/README.md index e1a43f6..021096b 100644 --- a/README.md +++ b/README.md @@ -14,15 +14,14 @@ This hook handles the authentication and request to Intercom. Based on [python-i This operator composes the logic for this plugin. It fetches the intercom specified object and saves the result in a S3 Bucket, under a specified key, in njson format. The parameters it can accept include the following. -`intercom_conn_id`: The intercom connection id from Airflow -`intercom_obj`: Intercom object to query -`intercom_method`: *optional* Method from python-intercom. -`s3_conn_id`: S3 connection id from Airflow. -`s3_bucket`: The output s3 bucket. -`s3_key`: The input s3 key. -`output`: Name of the temporary file where the results should be saved -`fields`: *optional* list of fields that you want to get from the object. If *None*, then this will get all fields for the object -`replication_key`: *optional* name of the replication key, if needed. -`replication_key_value`: *(optional)* value of the replication key, if needed. The operator will import only results with the property from replication_key grater than the value of this param. -`intercom_method`: *(optional)* method to call from python-intercom. Default to "all". -`**kwargs`: replication key and value, if replication_key parameter is given and extra params for intercom method if needed. \ No newline at end of file +- `intercom_conn_id`: The intercom connection id from Airflow +- `intercom_obj`: Intercom object to query +- `intercom_method`: *optional* Method from python-intercom. +- `s3_conn_id`: S3 connection id from Airflow. +- `s3_bucket`: The output s3 bucket. +- `s3_key`: The input s3 key. +- `fields`: *optional* list of fields that you want to get from the object. If *None*, then this will get all fields for the object +- `replication_key`: *optional* name of the replication key, if needed. +- `replication_key_value`: *(optional)* value of the replication key, if needed. The operator will import only results with the property from replication_key grater than the value of this param. +- `intercom_method`: *(optional)* method to call from python-intercom. Default to "all". +- `**kwargs`: replication key and value, if replication_key parameter is given and extra params for intercom method if needed. \ No newline at end of file diff --git a/operators/intercom_to_s3_operator.py b/operators/intercom_to_s3_operator.py index 9569304..868ce08 100644 --- a/operators/intercom_to_s3_operator.py +++ b/operators/intercom_to_s3_operator.py @@ -13,6 +13,7 @@ class IntercomToS3Operator(BaseOperator): """ Make a query against Intercom and write the resulting data to s3. """ + template_field = ('s3_key', ) @apply_defaults def __init__( @@ -23,7 +24,6 @@ def __init__( s3_conn_id='', s3_bucket='', s3_key='', - output='', fields=None, replication_key_name=None, replication_key_value=0, @@ -39,9 +39,7 @@ def __init__( :param s3_conn_id: name of the Airflow connection that has your Amazon S3 conection params :param s3_bucket: name of the destination S3 bucket - :param s3_key: name of the destination file from bucket - :param output: name of the temporary file where the results - should be saved + :param s3_key: name of the destination file from bucket :param fields: *(optional)* list of fields that you want to get from the object. If *None*, then this will get all fields @@ -68,7 +66,6 @@ def __init__( self.s3_conn_id = s3_conn_id self.s3_bucket = s3_bucket self.s3_key = s3_key - self.output = output self.fields = fields self.replication_key_name = replication_key_name @@ -147,7 +144,7 @@ def execute(self, context): dest_s3 = S3Hook(s3_conn_id=self.s3_conn_id) dest_s3.load_file( filename=tmp.name, - key=self.output, + key=self.s3_key, bucket_name=self.s3_bucket, replace=True