-
Notifications
You must be signed in to change notification settings - Fork 8
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #7 from project-flogo/s3-activity
feat: added s3 activity
- Loading branch information
Showing
7 changed files
with
449 additions
and
0 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,38 @@ | ||
| # Amazon S3 | ||
|
|
||
| Upload or Download files from Amazon Simple Storage Service (S3) | ||
|
|
||
|
|
||
| ## Installation | ||
|
|
||
| ```bash | ||
| flogo install github.com/project-flogo/aws-contrib/activity/s3 | ||
| ``` | ||
|
|
||
| Link for flogo web: | ||
|
|
||
| ```bash | ||
| https://github.comproject-flogo/aws-contrib/activity/s3 | ||
| ``` | ||
|
|
||
| ## Settings | ||
|
|
||
| | Setting | Description | | ||
| |:-------------------|:---------------| | ||
| | awsRegion | The AWS region your S3 bucket is in | | ||
|
|
||
| ## Inputs | ||
|
|
||
| | Input | Description | | ||
| |:-------------------|:---------------| | ||
| | action | The action you want to take, either `download`, `upload`, `delete`, or `copy` | | ||
| | s3BucketName | The name of your S3 bucket | | ||
| | s3Location | The file location on S3, this should be a full path (like `/bla/temp.txt`) | | ||
| | localLocation | The `localLocation` is the full path to a file (like `/bla/temp.txt`) when uploading a file or the full path to a directory (like `./tmp`) when downloading a file | | ||
| | s3NewLocation | The new file location on S3 of you want to copy a file, this should be a full path (like `/bla/temp.txt`) | | ||
|
|
||
| ## Ouputs | ||
|
|
||
| | Output | Description | | ||
| |:----------|:---------------| | ||
| | result | The result will contain OK if the action was carried out successfully or will contain an error message | |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,202 @@ | ||
| package s3 | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "path/filepath" | ||
| "strings" | ||
|
|
||
| "github.com/aws/aws-sdk-go/aws" | ||
| "github.com/aws/aws-sdk-go/aws/session" | ||
| "github.com/aws/aws-sdk-go/service/s3" | ||
| "github.com/aws/aws-sdk-go/service/s3/s3manager" | ||
| "github.com/project-flogo/core/activity" | ||
| "github.com/project-flogo/core/data/metadata" | ||
| ) | ||
|
|
||
| func init() { | ||
| _ = activity.Register(&Activity{}, New) | ||
| } | ||
|
|
||
| const ( | ||
| ovResult = "result" | ||
| ) | ||
|
|
||
| type Activity struct { | ||
| settings *Settings | ||
| awsSession *session.Session | ||
| } | ||
|
|
||
| var activityMd = activity.ToMetadata(&Input{}, &Output{}) | ||
|
|
||
| func New(ctx activity.InitContext) (activity.Activity, error) { | ||
| s := &Settings{} | ||
| err := metadata.MapToStruct(ctx.Settings(), s, true) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
|
|
||
| act := &Activity{settings: s} | ||
|
|
||
| if s.AWSRegion != "" { | ||
| region, err := getRegion(s.AWSRegion) | ||
| if err != nil { | ||
| return nil, err | ||
| } | ||
| act.awsSession = session.Must(session.NewSession(&aws.Config{ | ||
| Region: aws.String(region), | ||
| })) | ||
| } else { | ||
| act.awsSession = session.Must(session.NewSession(&aws.Config{})) | ||
| } | ||
|
|
||
| return act, nil | ||
| } | ||
|
|
||
| func (a *Activity) Metadata() *activity.Metadata { | ||
| return activityMd | ||
| } | ||
|
|
||
| func (a *Activity) Eval(ctx activity.Context) (done bool, err error) { | ||
| in := &Input{} | ||
| err = ctx.GetInputObject(in) | ||
| if err != nil { | ||
| return false, err | ||
| } | ||
|
|
||
| var s3err error | ||
| switch in.Action { | ||
| case "download": | ||
| s3err = downloadFileFromS3(a.awsSession, in.LocalLocation, in.S3Location, in.S3BucketName) | ||
| case "upload": | ||
| s3err = uploadFileToS3(a.awsSession, in.LocalLocation, in.S3Location, in.S3BucketName) | ||
| case "delete": | ||
| s3err = deleteFileFromS3(a.awsSession, in.S3Location, in.S3BucketName) | ||
| case "copy": | ||
| s3err = copyFileOnS3(a.awsSession, in.S3Location, in.S3BucketName, in.S3NewLocation) | ||
| } | ||
| if s3err != nil { | ||
| // Set the output value in the context | ||
| ctx.SetOutput(ovResult, s3err.Error()) | ||
| return true, s3err | ||
| } | ||
|
|
||
| // Set the output value in the context | ||
| ctx.SetOutput(ovResult, "OK") | ||
|
|
||
| return true, nil | ||
| } | ||
|
|
||
| // Function to download a file from an S3 bucket | ||
| func downloadFileFromS3(awsSession *session.Session, directory string, s3Location string, s3BucketName string) error { | ||
| // Create an instance of the S3 Manager | ||
| s3Downloader := s3manager.NewDownloader(awsSession) | ||
|
|
||
| // Create a new temporary file | ||
| f, err := os.Create(filepath.Join(directory, s3Location)) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| // Prepare the download | ||
| objectInput := &s3.GetObjectInput{ | ||
| Bucket: aws.String(s3BucketName), | ||
| Key: aws.String(s3Location), | ||
| } | ||
|
|
||
| // Download the file to disk | ||
| _, err = s3Downloader.Download(f, objectInput) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Function to delete a file from an S3 bucket | ||
| func deleteFileFromS3(awsSession *session.Session, s3Location string, s3BucketName string) error { | ||
| // Create an instance of the S3 Manager | ||
| s3Session := s3.New(awsSession) | ||
|
|
||
| objectDelete := &s3.DeleteObjectInput{ | ||
| Bucket: aws.String(s3BucketName), | ||
| Key: aws.String(s3Location), | ||
| } | ||
|
|
||
| // Delete the file from S3 | ||
| _, err := s3Session.DeleteObject(objectDelete) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Function to upload a file from an S3 bucket | ||
| func uploadFileToS3(awsSession *session.Session, localFile string, s3Location string, s3BucketName string) error { | ||
| // Create an instance of the S3 Manager | ||
| s3Uploader := s3manager.NewUploader(awsSession) | ||
|
|
||
| // Create a file pointer to the source | ||
| reader, err := os.Open(localFile) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| defer reader.Close() | ||
|
|
||
| // Prepare the upload | ||
| uploadInput := &s3manager.UploadInput{ | ||
| Bucket: aws.String(s3BucketName), | ||
| Key: aws.String(s3Location), | ||
| Body: reader, | ||
| } | ||
|
|
||
| // Upload the file | ||
| _, err = s3Uploader.Upload(uploadInput) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // Function to copy a file in an S3 bucket | ||
| func copyFileOnS3(awsSession *session.Session, s3Location string, s3BucketName string, s3NewLocation string) error { | ||
| // Create an instance of the S3 Session | ||
| s3Session := s3.New(awsSession) | ||
|
|
||
| // Prepare the copy object | ||
| objectInput := &s3.CopyObjectInput{ | ||
| Bucket: aws.String(s3BucketName), | ||
| CopySource: aws.String(fmt.Sprintf("/%s/%s", s3BucketName, s3Location)), | ||
| Key: aws.String(s3NewLocation), | ||
| } | ||
|
|
||
| // Copy the object | ||
| _, err := s3Session.CopyObject(objectInput) | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| func getRegion(regionSetting string) (string, error) { | ||
|
|
||
| var awsRegions = []string{"us-east-1", "us-east-2", "us-west-1", "us-west-2", "ap-northeast-1", "ap-northeast-2", "ap-northeast-3", "ap-south-1", "ap-southeast-1", "ap-southeast-2", "ca-central-1", "cn-north-1", "cn-northwest-1", "eu-central-1", "eu-north-1", "eu-west-1", "eu-west-2", "eu-west-2", "sa-east-1"} | ||
|
|
||
| region := strings.ToLower(regionSetting) | ||
| valid := false | ||
| for _, aRegion := range awsRegions { | ||
| if region == aRegion { | ||
| valid = true | ||
| break | ||
| } | ||
| } | ||
|
|
||
| if !valid { | ||
| return "", fmt.Errorf("unsupported region: %s", regionSetting) | ||
| } | ||
|
|
||
| return region, nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,16 @@ | ||
| package s3 | ||
|
|
||
| import ( | ||
| "testing" | ||
|
|
||
| "github.com/project-flogo/core/activity" | ||
| "github.com/stretchr/testify/assert" | ||
| ) | ||
|
|
||
| func TestRegister(t *testing.T) { | ||
|
|
||
| ref := activity.GetRef(&Activity{}) | ||
| act := activity.Get(ref) | ||
|
|
||
| assert.NotNil(t, act) | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,93 @@ | ||
| { | ||
| "name": "flogo-aws-s3", | ||
| "type": "flogo:activity", | ||
| "version": "0.1.0", | ||
| "title": "Amazon S3", | ||
| "description": "Upload or Download files from Amazon Simple Storage Service (S3)", | ||
| "homepage": "https://github.com/project-flogo/aws-contrib/tree/master/activity/s3", | ||
| "settings": [ | ||
| { | ||
| "name": "awsAccessKeyID", | ||
| "type": "string", | ||
| "required": "false", | ||
| "description": "Your AWS Access Key (only needed if you don't give your Lambda function rights to invoke S3 reads and puts)" | ||
| }, | ||
| { | ||
| "name": "awsSecretAccessKey", | ||
| "type": "string", | ||
| "required": "false", | ||
| "description": "Your AWS Secret Key (only needed if you don't give your Lambda function rights to invoke S3 reads and puts)" | ||
| }, | ||
| { | ||
| "name": "awsRegion", | ||
| "type": "string", | ||
| "allowed": [ | ||
| "us-east-1", | ||
| "us-east-2", | ||
| "us-west-1", | ||
| "us-west-2", | ||
| "ap-northeast-1", | ||
| "ap-northeast-2", | ||
| "ap-northeast-3", | ||
| "ap-south-1", | ||
| "ap-southeast-1", | ||
| "ap-southeast-2", | ||
| "ca-central-1", | ||
| "cn-north-1", | ||
| "cn-northwest-1", | ||
| "eu-central-1", | ||
| "eu-north-1", | ||
| "eu-west-1", | ||
| "eu-west-2", | ||
| "eu-west-2", | ||
| "sa-east-1" | ||
| ], | ||
| "description": "The AWS region, uses environment setting by default" | ||
| } | ||
| ] | ||
| "input": [ | ||
| { | ||
| "name": "action", | ||
| "type": "string", | ||
| "allowed": [ | ||
| "download", | ||
| "upload", | ||
| "delete", | ||
| "copy" | ||
| ], | ||
| "required": "true", | ||
| "description": "The action you want to perform" | ||
| }, | ||
| { | ||
| "name": "s3BucketName", | ||
| "type": "string", | ||
| "required": "true", | ||
| "description": "The message subject" | ||
| }, | ||
| { | ||
| "name": "s3Location", | ||
| "type": "string", | ||
| "required": "true", | ||
| "description": "The file location on S3, this should be a full path (like /bla/temp.txt)" | ||
| }, | ||
| { | ||
| "name": "localLocation", | ||
| "type": "string", | ||
| "required": "false", | ||
| "description": "The localLocation is the full path to a file (like /bla/temp.txt) when uploading a file or the full path to a directory (like ./tmp) when downloading a file" | ||
| }, | ||
| { | ||
| "name": "s3NewLocation", | ||
| "type": "string", | ||
| "required": "false", | ||
| "description": "The new file location on S3 of you want to copy a file, this should be a full path (like /bla/temp.txt)" | ||
| } | ||
| ], | ||
| "output": [ | ||
| { | ||
| "name": "result", | ||
| "type": "string", | ||
| "description": "The result will contain OK if the action was carried out successfully or will contain an error message" | ||
| } | ||
| ] | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,10 @@ | ||
| module github.com/project-flogo/aws-contrib/activity/s3 | ||
|
|
||
| go 1.12 | ||
|
|
||
| require ( | ||
| github.com/aws/aws-sdk-go v1.25.34 | ||
| github.com/pkg/errors v0.8.1 // indirect | ||
| github.com/project-flogo/core v0.9.4 | ||
| github.com/stretchr/testify v1.3.0 | ||
| ) |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,22 @@ | ||
| github.com/aws/aws-sdk-go v1.25.34 h1:roL040qe1npx1ToFeXYHOGp/nOpLbcIQHKZ5UeDIyIM= | ||
| github.com/aws/aws-sdk-go v1.25.34/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN924inxo= | ||
| github.com/davecgh/go-spew v1.1.0 h1:ZDRjVQ15GmhC3fiQ8ni8+OwkZQO4DARzQgrnXU1Liz8= | ||
| github.com/davecgh/go-spew v1.1.0/go.mod h1:J7Y8YcW2NihsgmVo/mv3lAwl/skON4iLHjSsI+c5H38= | ||
| github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af h1:pmfjZENx5imkbgOkpRUYLnmbU7UEFbjtDA2hxJ1ichM= | ||
| github.com/jmespath/go-jmespath v0.0.0-20180206201540-c2b33e8439af/go.mod h1:Nht3zPeWKUH0NzdCt2Blrr5ys8VGpn0CEB0cQHVjt7k= | ||
| github.com/pkg/errors v0.8.1 h1:iURUrRGxPUNPdy5/HRSm+Yj6okJ6UtLINN0Q9M4+h3I= | ||
| github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= | ||
| github.com/pmezard/go-difflib v1.0.0 h1:4DBwDE0NGyQoBHbLQYPwSUPoCMWR5BEzIk/f1lZbAQM= | ||
| github.com/pmezard/go-difflib v1.0.0/go.mod h1:iKH77koFhYxTK1pcRnkKkqfTogsbg7gZNVY4sRDYZ/4= | ||
| github.com/project-flogo/core v0.9.4 h1:WGvXbfVi00Weys+jR2pR3M2hEkxjPpO4NW+WiJJ6Lgg= | ||
| github.com/project-flogo/core v0.9.4/go.mod h1:QGWi7TDLlhGUaYH3n/16ImCuulbEHGADYEXyrcHhX7U= | ||
| github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= | ||
| github.com/stretchr/testify v1.3.0 h1:TivCn/peBQ7UY8ooIcPgZFpTNSz0Q2U6UrFlUfqbe0Q= | ||
| github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= | ||
| github.com/xeipuuv/gojsonschema v1.1.0/go.mod h1:5yf86TLmAcydyeJq5YvxkGPE2fm/u4myDekKRoLuqhs= | ||
| go.uber.org/atomic v1.4.0 h1:cxzIVoETapQEqDhQu3QfnvXAV4AlzcvUCxkVUFw3+EU= | ||
| go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE= | ||
| go.uber.org/multierr v1.1.0 h1:HoEmRHQPVSqub6w2z2d2EOVs2fjyFRGyofhKuyDq0QI= | ||
| go.uber.org/multierr v1.1.0/go.mod h1:wR5kodmAFQ0UK8QlbwjlSNy0Z68gJhDJUG5sjR94q/0= | ||
| go.uber.org/zap v1.9.1 h1:XCJQEf3W6eZaVwhRBof6ImoYGJSITeKWsyeh3HFu/5o= | ||
| go.uber.org/zap v1.9.1/go.mod h1:vwi/ZaCAaUcBkycHslxD9B2zi4UTXhF60s6SWpuDF0Q= |
Oops, something went wrong.