From 406842dff1cd082453cb611802e898c515ea9c71 Mon Sep 17 00:00:00 2001 From: Benevolent Benjamin Powell Date: Thu, 2 Feb 2023 20:41:28 -0600 Subject: [PATCH 1/5] feat(artifacts): Add ArtifactStore to Kork This commit adds a new feature called ArtifactStore. This allows for storage of embedded/base64 types to some storage, eg S3. So instead of having duplication of some artifact in a context, it'll now be referenced by a artifact URI. Also, there's a nice README that explains various settings as well as how to setup a local S3 for testing artifact storage [README link](kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md ) Signed-off-by: benjamin-j-powell --- kork-artifacts/kork-artifacts.gradle | 12 ++ .../kork/artifacts/ArtifactTypes.java | 28 +++ .../spinnaker/kork/artifacts/README.md | 130 +++++++++++ .../artifactstore/ArtifactDecorator.java | 23 ++ .../artifactstore/ArtifactDeserializer.java | 71 ++++++ .../artifactstore/ArtifactStore.java | 50 +++++ .../ArtifactStoreConfiguration.java | 113 ++++++++++ .../ArtifactStoreConfigurationProperties.java | 44 ++++ .../ArtifactStoreURIBuilder.java | 35 +++ .../ArtifactStoreURISHA256Builder.java | 34 +++ .../EmbeddedArtifactSerializer.java | 63 ++++++ .../artifactstore/s3/S3ArtifactStore.java | 203 ++++++++++++++++++ .../kork/artifacts/model/Artifact.java | 15 ++ .../artifacts/model/ExpectedArtifact.java | 61 ++++-- .../ArtifactDeserializerTest.java | 74 +++++++ .../EmbeddedArtifactSerializerTest.java | 100 +++++++++ .../artifacts/model/ExpectedArtifactTest.java | 23 ++ kork-expressions/kork-expressions.gradle | 2 + .../ArtifactUriToReferenceConverter.java | 62 ++++++ .../kork/expressions/ExpressionsSupport.java | 3 + .../expressions/ExpressionsSupportTest.java | 58 +++++ .../engines/SecretsManagerSecretEngine.java | 4 +- .../spinnaker-dependencies.gradle | 2 + 23 files changed, 1195 insertions(+), 15 deletions(-) create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java create mode 100644 kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java create mode 100644 kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java create mode 100644 kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java create mode 100644 kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java diff --git a/kork-artifacts/kork-artifacts.gradle b/kork-artifacts/kork-artifacts.gradle index 8be01c05e..c61b3d885 100644 --- a/kork-artifacts/kork-artifacts.gradle +++ b/kork-artifacts/kork-artifacts.gradle @@ -5,11 +5,23 @@ dependencies { api(platform(project(":spinnaker-dependencies"))) implementation project(":kork-annotations") + implementation project(":kork-exceptions") + implementation project(":kork-security") implementation "com.fasterxml.jackson.core:jackson-databind" + implementation platform('software.amazon.awssdk:bom') + api 'software.amazon.awssdk:s3' + api 'software.amazon.awssdk:sts' + implementation "org.apache.httpcomponents:httpclient" + implementation 'software.amazon.awssdk:s3' + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "org.springframework.security:spring-security-core" + implementation 'org.apache.logging.log4j:log4j-api' + implementation 'org.apache.logging.log4j:log4j-core' api "com.hubspot.jinjava:jinjava" testImplementation "org.assertj:assertj-core" testImplementation "org.junit.jupiter:junit-jupiter-api" testImplementation "org.junit.jupiter:junit-jupiter-params" + testImplementation "org.mockito:mockito-core" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java new file mode 100644 index 000000000..9d3d0404c --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ArtifactTypes { + EMBEDDED_BASE64("embedded/base64"), + REMOTE_BASE64("remote/base64"), + ; + + @Getter private final String mimeType; +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md new file mode 100644 index 000000000..a3ed06516 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md @@ -0,0 +1,130 @@ +# Artifact Storage + +Artifact Storage is a feature that allows for embedded aritfacts to be +persisted to some storage, eg the S3ArtifactStore. Spinnaker keeps a history, +which is called the pipeline context, that contains everything that an +execution had done. Pipeline contexts can be very large, especially with +complicated pipeines, and size can be further increased when users have large +artifacts. Spinnaker will duplicate +these artifacts whenever any stage uses any of those artifacts. Using an +artifact store reduces this overhead by providing a reference link of, +`ref://`. This reduces the context size +tremendously, but will vary depending on the size of the pipeline, as well as +how that artifact is used, but we've seen improvements of 80% for some +pipelines. + +## Architecture + + +-----------+ + | | + | Orca | + | | + +-----------+ + | ^ + | | (outgoing artifact compressed) + | +----------------------+ + | (bake request) | + +---------------------+ | + v | + +---------------+ (fetch) +-------------+ + | |<-----------------------------| | + | Clouddriver | | Rosco | + | | | | + | s3 get | (full artifact returned) | s3 stores | + | artifacts |----------------------------->| artifacts | + +---------------+ +-------------+ + + +Artifact storage is divided into two operations of get and store, and there are +primarily two services that utilizes each of these operations. Further the +artifact storage system relies on Spring's (de)serializing to call these +operations to limit the amount of code changes needed within these services. + +When bootstrapping Spring we add in custom bean serializers and deserializers to +handle storage or retrieval of an artifact. + +Rosco is primarily used for baking artifacts which will generate something +deployable. When Spring sends a request back to Orca for a bake request, the +custom serializer injected at Orca startup, will use the artifact storage and +store the artifact, and instead return a `remote/base64` artifact instead of the +usual `embedded/base64`. + +Clouddriver, for this document, handles mostly deployment and has some endpoints +regarding artifacts. It does do a little more than this, but we only care about +these two particular operations. When any request comes in, Spring will use its +custom deserializers to expand any artifact in its payload since any request +with artifacts, probably wants to do some operation on those artifacts. Further +Clouddriver also allows for fetching of artifacts. Orca and Rosco both make +calls to the `/artifact/fetch` endpoint. Where Rosco uses it to fetch an +artifact to be baked, and Orca uses it primarily when dealing with deploy +manifests. When a request is sent to the fetch endpoint, Clouddriver will always +return the full `embedded/base64` artifact back to the service. It is up to the +service receiving the artifact to compress it. Luckily, for Orca, we don't have +to worry about compression, since this no longer becomes an artifact, but a +manifest instead. + +Orca is a special case as it mostly does orchestration, but does cause some +artifacts to be duplicated when handling the expected artifact logic. We inject +some logic to handle the duplication along with ensuring that matching against +expected artifacts still works properly. So if a `embedded/base64` type needs to +match against a `remote/base64` type, Orca will use the artifact store to +retrieve that artifact, and do the comparison. In addition, Orca will store any +expected artifacts, to limit the context size. + +Orca also handles SpEL evaluation which means our new `remote/base64` type +should be backwards compatible with existing pipelines. To ensure this, we +utilized the Spring converters, and injected our own custom converter that will +check if some `String` is a remote base64 URI, and if it is, retrieve it. + +## Configuration + +To enable artifact storage, simple add this to your `spinnaker-local.yml` file + +```yaml +artifactStore: + enabled: true + s3: + enabled: true + bucket: some-artifact-store-bucket +``` + +## Storage Options + +### S3 + +[S3](https://aws.amazon.com/s3/) is an object store provided by AWS. The +currently S3ArtifactStore implementation provides various ways to authenticate +against AWS. + +```yaml +artifactStore: + enabled: true + s3: + enabled: true + profile: dev # if you want to authenticate using a certain profile + region: us-west-2 # allows for specified regions + bucket: some-artifact-store-bucket +``` + +While the implementation is S3 specific, this does not limit usages of other S3 +compatible storage engines. For example, something like +[SeaweedFS](https://github.com/seaweedfs/seaweedfs) can be used to test locally +with + +## Local Testing + +To test the artifact store locally, we will use SeaweedFS. To start the storage simply run +`docker run -p 8333:8333 chrislusf/seaweedfs server -s3` + +Next enable the configuration + +```yaml +artifactStore: + enabled: true + s3: + enabled: true + url: http://localhost:8333 # this URL will be used to make S3 API requests to + bucket: some-artifact-store-bucket +``` + +Start Spinnaker, and you should see reference links in your pipeline contexts. diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java new file mode 100644 index 000000000..1bbb68cb9 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java @@ -0,0 +1,23 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +/** Primarily used to set any custom fields to an artifact */ +public interface ArtifactDecorator { + Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder); +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java new file mode 100644 index 000000000..5ae97d7ce --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java @@ -0,0 +1,71 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import org.springframework.beans.factory.annotation.Qualifier; + +/** + * ArtifactDeserializer is a custom deserializer that will allow for artifacts to be fetched from + * some artifact store as long as the referenceLink field is set and the reference field is null. + */ +public class ArtifactDeserializer extends StdDeserializer { + private final ObjectMapper defaultObjectMapper; + private final ArtifactStore storage; + + public ArtifactDeserializer( + @Qualifier(value = "artifactObjectMapper") ObjectMapper defaultObjectMapper, + ArtifactStore storage) { + super(Artifact.class); + this.defaultObjectMapper = defaultObjectMapper; + this.storage = storage; + } + + @Override + public Artifact deserialize(JsonParser parser, DeserializationContext ctx) throws IOException { + Artifact artifact = defaultObjectMapper.readValue(parser, Artifact.class); + String ref = artifact.getReference(); + if (artifact.getType().equals(ArtifactTypes.REMOTE_BASE64.getMimeType())) { + return storage.get(ref, new ArtifactMergeReferenceDecorator(artifact)); + } + + return artifact; + } + + /** + * ArtifactMergeReferenceDecorator is used to take some artifact and replace its reference with + * the reference from another artifact. + */ + public static class ArtifactMergeReferenceDecorator implements ArtifactDecorator { + private final Artifact artifactToCopy; + + private ArtifactMergeReferenceDecorator(Artifact artifactToCopy) { + this.artifactToCopy = artifactToCopy; + } + + @Override + public Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder) { + Artifact retrieved = builder.build(); + return artifactToCopy.toBuilder().reference(retrieved.getReference()); + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java new file mode 100644 index 000000000..f0dad92b7 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; + +/** + * ArtifactStore is an interface that allows for different types of artifact storage to be used + * during runtime + */ +public abstract class ArtifactStore { + /** ensures the singleton has only been set once */ + private static final AtomicBoolean singletonSet = new AtomicBoolean(false); + + @Getter private static ArtifactStore instance = null; + + public abstract Artifact store(Artifact artifact); + /** + * get is used to return an artifact with some id, while also decorating that artifact with any + * necessary fields needed which should be then be returned by the artifact store. + */ + public abstract Artifact get(String id, ArtifactDecorator... decorators); + + public static void setInstance(ArtifactStore storage) { + if (!singletonSet.compareAndSet(false, true)) { + throw new IllegalStateException("Multiple attempts at setting ArtifactStore's singleton"); + } + + ArtifactStore.instance = storage; + } + + public boolean isArtifactURI(String value) { + return value.startsWith(ArtifactStoreURIBuilder.uriScheme + "://"); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java new file mode 100644 index 000000000..296742ef8 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java @@ -0,0 +1,113 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.artifactstore.s3.S3ArtifactStore; +import java.net.URI; +import java.util.Optional; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.access.PermissionEvaluator; +import software.amazon.awssdk.auth.credentials.*; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +@Configuration +@ConditionalOnExpression("${artifact-store.enabled:false}") +@EnableConfigurationProperties(ArtifactStoreConfigurationProperties.class) +@Log4j2 +public class ArtifactStoreConfiguration { + /** + * this is strictly used due to Spring and Jackson not behaving nicely together. + * Unfortunately, @JsonDeserializer will construct its own deserializer utilizing beans and thus + * not using the object mapper we want to use + */ + @Bean + public ObjectMapper artifactObjectMapper() { + return new ObjectMapper(); + } + + @Bean + public ArtifactStoreURIBuilder artifactStoreURIBuilder() { + return new ArtifactStoreURISHA256Builder(); + } + + @Bean + @ConditionalOnExpression("${artifact-store.s3.enabled:false}") + public ArtifactStore s3ArtifactStore( + Optional permissionEvaluator, + ArtifactStoreConfigurationProperties properties, + @Qualifier("artifactS3Client") S3Client s3Client, + ArtifactStoreURIBuilder artifactStoreURIBuilder) { + + if (permissionEvaluator.isEmpty()) { + log.warn( + "PermissionEvaluator is not present. This means anyone will be able to access any artifact in the store."); + } + + ArtifactStore storage = + new S3ArtifactStore( + s3Client, + permissionEvaluator.orElse(null), + properties.getS3().getBucket(), + artifactStoreURIBuilder, + properties.getApplicationsRegex()); + + ArtifactStore.setInstance(storage); + return storage; + } + + @Bean + @ConditionalOnExpression("${artifact-store.s3.enabled:false}") + public S3Client artifactS3Client(ArtifactStoreConfigurationProperties properties) { + S3ClientBuilder builder = S3Client.builder(); + ArtifactStoreConfigurationProperties.S3ClientConfig config = properties.getS3(); + + // Overwriting the URL is primarily used for S3 compatible object stores + // like seaweedfs + if (config.getUrl() != null) { + builder = + builder + .credentialsProvider(getCredentialsProvider(config)) + .forcePathStyle(config.isForcePathStyle()) + .endpointOverride(URI.create(config.getUrl())); + } else if (config.getProfile() != null) { + builder = builder.credentialsProvider(ProfileCredentialsProvider.create(config.getProfile())); + } + + if (config.getRegion() != null) { + builder = builder.region(Region.of(config.getRegion())); + } + + return builder.build(); + } + + private AwsCredentialsProvider getCredentialsProvider( + ArtifactStoreConfigurationProperties.S3ClientConfig config) { + if (config.getAccessKey() != null) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey())); + } else { + return AnonymousCredentialsProvider.create(); + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java new file mode 100644 index 000000000..e3fbf51be --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties("artifact-store") +public class ArtifactStoreConfigurationProperties { + private String applicationsRegex = null; + /** Configuration for an s3 client which will utilize credentials in the AWS credentials file. */ + @Data + public static class S3ClientConfig { + private boolean enabled = false; + private String profile = null; + private String region = null; + /** + * Url may be used to override the contact URL to an s3 compatible object store. This is useful + * for testing utilizing things like seaweedfs. + */ + private String url = null; + + private String bucket = null; + private String accessKey = null; + private String secretKey = null; + private boolean forcePathStyle = true; + } + + private S3ClientConfig s3 = null; +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java new file mode 100644 index 000000000..17932218e --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java @@ -0,0 +1,35 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +public abstract class ArtifactStoreURIBuilder { + /** + * uriScheme is used as an HTTP scheme to let us further distinguish a String that is a URI to an + * artifact. This is helpful in determining what is an artifact since sometimes we are only given + * a string rather than a full artifact. + */ + public static final String uriScheme = "ref"; + + public abstract String buildArtifactURI(String context, Artifact artifact); + + /** + * buildRawURI is used when you have the raw path and context. This method just simply return the + * properly formatted URI + */ + public abstract String buildRawURI(String context, String raw); +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java new file mode 100644 index 000000000..773fbe873 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java @@ -0,0 +1,34 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.google.common.hash.Hashing; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +/** + * Helper class to abstract away the need for other classes to know the {@link * #uriPrefix} format. + */ +public class ArtifactStoreURISHA256Builder extends ArtifactStoreURIBuilder { + public String buildArtifactURI(String context, Artifact artifact) { + return String.format( + "%s://%s/%s", + uriScheme, context, Hashing.sha256().hashBytes(artifact.getReference().getBytes())); + } + + public String buildRawURI(String context, String raw) { + return String.format("%s://%s/%s", uriScheme, context, raw); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java new file mode 100644 index 000000000..ab70d1b77 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; + +/** + * EmbeddedArtifactSerializer will store any embedded/base64 artifact into the ArtifactStore + * assuming that artifact has a reference link. + */ +public class EmbeddedArtifactSerializer extends StdSerializer { + private final ObjectMapper defaultObjectMapper; + private final ArtifactStore storage; + + public EmbeddedArtifactSerializer(ObjectMapper defaultObjectMapper, ArtifactStore storage) { + super(Artifact.class); + this.defaultObjectMapper = defaultObjectMapper; + this.storage = storage; + } + + @Override + public void serialize(Artifact artifact, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (!shouldStoreArtifact(artifact)) { + defaultObjectMapper.writeValue(gen, artifact); + return; + } + + Artifact stored = storage.store(artifact); + defaultObjectMapper.writeValue(gen, stored); + } + + /** + * shouldStore will return whether we want to store the reference in the ArtifactStore or not. + * This checks to ensure the reference isn't null or an empty string. Further we only care about + * 'embedded/base64' artifact types, since that is directly embedding the artifacts into the + * context + */ + private static boolean shouldStoreArtifact(Artifact artifact) { + String ref = artifact.getReference(); + return artifact.getType().equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + && !(ref == null || ref.isEmpty()); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java new file mode 100644 index 000000000..a9a9315fa --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java @@ -0,0 +1,203 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore.s3; + +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDecorator; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreURIBuilder; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.security.AuthenticatedRequest; +import java.util.Base64; +import java.util.regex.Pattern; +import lombok.extern.log4j.Log4j2; +import org.springframework.security.access.PermissionEvaluator; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.Tag; +import software.amazon.awssdk.services.s3.model.Tagging; + +/** + * S3ArtifactStore will store artifacts in a s3 compatible service + * + *

Note: It is very important that the S3 bucket has object lock on it to prevent multiple writes + * {@see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock-overview.html} + */ +@Log4j2 +public class S3ArtifactStore extends ArtifactStore { + private final S3Client s3Client; + private final PermissionEvaluator permissionEvaluator; + private final String bucket; + private final ArtifactStoreURIBuilder uriBuilder; + private final String applicationsRegex; + private static final String ENFORCE_PERMS_KEY = "application"; + + public S3ArtifactStore( + S3Client s3Client, + PermissionEvaluator permissionEvaluator, + String bucket, + ArtifactStoreURIBuilder uriBuilder, + String applicationsRegex) { + this.s3Client = s3Client; + this.bucket = bucket; + this.permissionEvaluator = permissionEvaluator; + this.uriBuilder = uriBuilder; + this.applicationsRegex = applicationsRegex; + } + + /** + * Will store the artifact using the {@link #s3Client} in some {@link #bucket} + * + *

This method also persists "permissions" by storing the execution id that made the original + * store call. In the event a service wants to retrieve said artifact, they will also need to + * provide the proper execution id + */ + @Override + public Artifact store(Artifact artifact) { + String application = AuthenticatedRequest.getSpinnakerApplication().orElse(null); + if (application == null) { + return artifact; + } + + if (applicationsRegex != null && !Pattern.matches(applicationsRegex, application)) { + return artifact; + } + + String ref = uriBuilder.buildArtifactURI(application, artifact); + if (objectExists(ref)) { + return artifact.toBuilder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(ref) + .build(); + } + + // purpose of tagging is to ensure some sort of identity is persisted to + // enforce permissions when retrieving the artifact + Tag accountTag = Tag.builder().key(ENFORCE_PERMS_KEY).value(application).build(); + + PutObjectRequest request = + PutObjectRequest.builder() + .bucket(bucket) + .key(ref) + .tagging(Tagging.builder().tagSet(accountTag).build()) + .build(); + + s3Client.putObject(request, RequestBody.fromBytes(getReferenceAsBytes(artifact))); + return artifact.toBuilder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(ref) + .build(); + } + + public byte[] getReferenceAsBytes(Artifact artifact) { + String reference = artifact.getReference(); + if (reference == null) { + return null; + } + + String type = artifact.getType(); + if (type.endsWith("/base64")) { + return Base64.getDecoder().decode(reference); + } + + return reference.getBytes(); + } + + /** + * get will return the Artifact with the provided id, and will lastly run the {@link + * ArtifactDecorator} to further populate the artifact for returning + */ + @Override + public Artifact get(String id, ArtifactDecorator... decorators) { + hasAuthorization(id, AuthenticatedRequest.getSpinnakerUser().orElse(null)); + + GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(id).build(); + + ResponseBytes resp = s3Client.getObjectAsBytes(request); + Artifact.ArtifactBuilder builder = + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(Base64.getEncoder().encodeToString(resp.asByteArray())); + + if (decorators == null) { + return builder.build(); + } + + for (ArtifactDecorator decorator : decorators) { + builder = decorator.decorate(builder); + } + + return builder.build(); + } + + /** + * hasAuthorization will ensure that the user has proper permissions for retrieving the stored + * artifact + */ + private void hasAuthorization(String id, String userId) { + GetObjectTaggingRequest request = + GetObjectTaggingRequest.builder().bucket(bucket).key(id).build(); + + GetObjectTaggingResponse resp = s3Client.getObjectTagging(request); + Tag tag = + resp.tagSet().stream() + .filter(t -> t.key().equals(ENFORCE_PERMS_KEY)) + .findFirst() + .orElse(null); + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + if (tag == null + || (permissionEvaluator != null + && !permissionEvaluator.hasPermission(auth, tag.value(), "application", "READ"))) { + log.error( + "Could not authenticate to retrieve artifact user={} applicationOfStoredArtifact={}", + userId, + (tag == null) ? "(none)" : tag.value()); + throw new AuthenticationServiceException( + userId + " does not have permission to access this artifact"); + } + } + + /** + * Helper method to check whether the object exists. This is not thread safe, nor would it help in + * a distributed system due to how S3 works (no conditional statements). If preventing multiple + * writes of the same object is important, another filestore/db needs to be used, possibly + * dynamodb. + */ + private boolean objectExists(String id) { + HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(id).build(); + try { + s3Client.headObject(request); + return true; + } catch (NoSuchKeyException e) { + // pretty gross that we need to use exceptions as control flow, but the + // java SDK doesn't have any other way of check if an object exists in s3 + log.warn("Artifact does not exist reference={}", id, e); + return false; + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java index 2adedc1d1..0ee1492e2 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java @@ -29,9 +29,12 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; +import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; +import lombok.extern.jackson.Jacksonized; @Getter @ToString @@ -101,5 +104,17 @@ public ArtifactBuilder putMetadata(String key, Object value) { metadata.put(key, value); return this; } + + public String getReference() { + return reference; + } + } + + @Data + @Builder + @Jacksonized + @RequiredArgsConstructor + public static class StoredView { + private final String reference; } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java index 0f42393de..2aebcef23 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java @@ -20,6 +20,9 @@ import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.common.base.Strings; import com.netflix.spinnaker.kork.annotations.NonnullByDefault; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import java.io.UncheckedIOException; import java.util.Optional; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -47,7 +50,13 @@ private ExpectedArtifact( boolean useDefaultArtifact, Artifact defaultArtifact, String id, - Artifact boundArtifact) { + Artifact boundArtifact) + throws UncheckedIOException { + + defaultArtifact = store(defaultArtifact); + boundArtifact = store(boundArtifact); + matchArtifact = store(matchArtifact); + this.matchArtifact = Optional.ofNullable(matchArtifact).orElseGet(() -> Artifact.builder().build()); this.usePriorArtifact = usePriorArtifact; @@ -65,9 +74,7 @@ private ExpectedArtifact( * @return true i.f.f. the artifacts match */ public boolean matches(Artifact other) { - String thisType = matchArtifact.getType(); - String otherType = other.getType(); - if (!matches(thisType, otherType)) { + if (!matchTypes(matchArtifact.getType(), other.getType())) { return false; } @@ -89,25 +96,53 @@ public boolean matches(Artifact other) { return false; } - String thisReference = matchArtifact.getReference(); - String otherReference = other.getReference(); - if (!matches(thisReference, otherReference)) { - return false; - } - - // Explicitly avoid matching on UUID, provenance & artifactAccount - - return true; + return matches(matchArtifact.getReference(), other.getReference()); } private boolean matches(@Nullable String us, @Nullable String other) { return StringUtils.isEmpty(us) || (other != null && patternMatches(us, other)); } + /** + * matchTypes will check to see if embedded types are used and if they are, see if they have been + * stored. If the type is embedded and stored, they are a valid match. + */ + private boolean matchTypes(String us, String other) { + if (matches(us, other)) { + return true; + } + + if (us.equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType())) { + return other.equals(ArtifactTypes.REMOTE_BASE64.getMimeType()); + } + + if (other.equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType())) { + return us.equals(ArtifactTypes.REMOTE_BASE64.getMimeType()); + } + + return false; + } + private boolean patternMatches(String us, String other) { return Pattern.compile(us).matcher(other).matches(); } + /** Helper store method to easily store the artifact if needed */ + private static Artifact store(Artifact artifact) { + ArtifactStore storage = ArtifactStore.getInstance(); + if (artifact == null + || storage == null + || !ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(artifact.getType())) { + return artifact; + } + + if (artifact.getReference() != null && !artifact.getReference().isEmpty()) { + return storage.store(artifact); + } + + return artifact; + } + @JsonPOJOBuilder(withPrefix = "") public static final class ExpectedArtifactBuilder {} } diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java new file mode 100644 index 000000000..b4833af3f --- /dev/null +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ArtifactDeserializerTest { + private class InMemoryArtifactStore extends ArtifactStore { + public Map storageMap = new HashMap<>(); + + public InMemoryArtifactStore put(String id, Artifact artifact) { + storageMap.put(id, artifact); + return this; + } + + @Override + public Artifact store(Artifact artifact) { + storageMap.put(artifact.getReference(), artifact); + return artifact; + } + + @Override + public Artifact get(String id, ArtifactDecorator... decorator) { + return storageMap.get(id); + } + } + + @Test + public void simpleDeserialization() throws IOException { + String artifactJSON = + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"stored://link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}"; + String expectedReference = "foobar"; + Artifact expectedArtifact = + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(expectedReference) + .build(); + InMemoryArtifactStore storage = + new InMemoryArtifactStore().put("stored://link", expectedArtifact); + ArtifactDeserializer deserializer = new ArtifactDeserializer(new ObjectMapper(), storage); + + // We avoid using an object mapper here since the Artifact class has a + // deserializer annotation which causes our deserializer to be ignored. So + // rather than using a mixin and setting all that up, this is easier. + JsonParser parser = new JsonFactory().createParser(artifactJSON); + Artifact receivedArtifact = deserializer.deserialize(parser, null); + assertNotNull(receivedArtifact); + assertEquals(expectedArtifact.getReference(), receivedArtifact.getReference()); + assertEquals(ArtifactTypes.REMOTE_BASE64.getMimeType(), receivedArtifact.getType()); + } +} diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java new file mode 100644 index 000000000..57de35abf --- /dev/null +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java @@ -0,0 +1,100 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.s3.S3ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import org.apache.commons.codec.binary.Base64; +import org.junit.jupiter.api.Test; +import org.mockito.Mockito; + +class EmbeddedArtifactSerializerTest { + private static class TestCase { + public String name = null; + public String expectedJSON = null; + public Artifact artifact = null; + public Artifact mockArtifact = null; + + public TestCase(String name, String expectedJSON, Artifact artifact, Artifact mockArtifact) { + this.name = name; + this.expectedJSON = expectedJSON; + this.artifact = artifact; + this.mockArtifact = mockArtifact; + } + } + + @Test + public void serializeEmbeddedBase64Artifact_test() throws IOException { + TestCase[] cases = + new TestCase[] { + new TestCase( + "simple", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + new TestCase( + "stored", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + new TestCase( + "does-not-exist", + "{\"type\":\"nonexistent-type\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"Zm9v\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type("nonexistent-type") + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + }; + + for (TestCase testCase : cases) { + ArtifactStore storage = Mockito.mock(S3ArtifactStore.class); + when(storage.store(Mockito.any())).thenReturn(testCase.mockArtifact); + + EmbeddedArtifactSerializer serializer = + new EmbeddedArtifactSerializer(new ObjectMapper(), storage); + ObjectMapper objectMapper = new ObjectMapper(); + SimpleModule module = new SimpleModule(); + module.addSerializer(Artifact.class, serializer); + objectMapper.registerModule(module); + + String result = objectMapper.writeValueAsString(testCase.artifact); + assertEquals(testCase.expectedJSON, result, testCase.name); + } + } +} diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java index 18e15f62b..596e6b906 100644 --- a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; import java.io.IOException; import java.util.function.Function; import java.util.stream.Stream; @@ -69,6 +70,28 @@ void roundTripSerialization() throws IOException { assertThat(deserializedArtifact).isEqualTo(artifact); } + @Test + void checkEmbeddedStoredTypesMatch() { + Artifact embeddedTypeArtifact = + Artifact.builder().type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()).build(); + + Artifact storedTypeArtifact = + Artifact.builder().type(ArtifactTypes.REMOTE_BASE64.getMimeType()).build(); + + Artifact noMatchTypeArtifact = Artifact.builder().type("does-not-exist").build(); + + ExpectedArtifact expectedArtifact = + ExpectedArtifact.builder().matchArtifact(embeddedTypeArtifact).build(); + assertThat(expectedArtifact.matches(storedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(embeddedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(noMatchTypeArtifact)).isFalse(); + + expectedArtifact = ExpectedArtifact.builder().matchArtifact(storedTypeArtifact).build(); + assertThat(expectedArtifact.matches(embeddedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(storedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(noMatchTypeArtifact)).isFalse(); + } + private String fullExpectedArtifactJson() { return jsonFactory .objectNode() diff --git a/kork-expressions/kork-expressions.gradle b/kork-expressions/kork-expressions.gradle index 5eb64e79f..67903e659 100644 --- a/kork-expressions/kork-expressions.gradle +++ b/kork-expressions/kork-expressions.gradle @@ -6,6 +6,7 @@ dependencies { api(platform(project(":spinnaker-dependencies"))) api project(":kork-api") + api project(":kork-artifacts") api project(":kork-plugins-api") api project(":kork-exceptions") api "com.fasterxml.jackson.core:jackson-databind" @@ -14,6 +15,7 @@ dependencies { implementation "org.springframework.boot:spring-boot" + testImplementation project(":kork-artifacts") testImplementation "org.assertj:assertj-core" testImplementation "org.junit.jupiter:junit-jupiter-api" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" diff --git a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java new file mode 100644 index 000000000..a6f83813d --- /dev/null +++ b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java @@ -0,0 +1,62 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.expressions; + +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import org.jetbrains.annotations.NotNull; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.expression.TypeConverter; +import org.springframework.expression.spel.support.StandardTypeConverter; + +/** + * This converter is used to check if a String is a Artifact reference URI. If it is, this will then + * pull the reference from the artifact store and return the reference back base64 encoded. + */ +public class ArtifactUriToReferenceConverter implements TypeConverter { + + private final ArtifactStore artifactStore; + + public ArtifactUriToReferenceConverter(ArtifactStore artifactStore) { + this.artifactStore = artifactStore; + } + + private final StandardTypeConverter defaultTypeConverter = new StandardTypeConverter(); + + @Override + public boolean canConvert(TypeDescriptor sourceType, @NotNull TypeDescriptor targetType) { + if (sourceType == null) { + return false; + } + + return sourceType.getObjectType() == String.class && targetType.getObjectType() == String.class; + } + + @Override + public Object convertValue( + Object value, TypeDescriptor sourceType, @NotNull TypeDescriptor targetType) { + // For some obscene reason(s), SpEL does not use this in the + // FunctionReference call when calling a method. So we call it internally + if (!canConvert(sourceType, targetType)) { + return defaultTypeConverter.convertValue(value, sourceType, targetType); + } + + if (artifactStore == null || !artifactStore.isArtifactURI((String) value)) { + return defaultTypeConverter.convertValue(value, sourceType, targetType); + } + + return artifactStore.get((String) value).getReference(); + } +} diff --git a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java index 5d76ba82f..ad7e1347d 100644 --- a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java +++ b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.kork.api.expressions.ExpressionFunctionProvider; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; import com.netflix.spinnaker.kork.expressions.allowlist.AllowListTypeLocator; import com.netflix.spinnaker.kork.expressions.allowlist.FilteredMethodResolver; import com.netflix.spinnaker.kork.expressions.allowlist.FilteredPropertyAccessor; @@ -163,6 +164,8 @@ private StandardEvaluationContext createEvaluationContext( StandardEvaluationContext evaluationContext = new StandardEvaluationContext(rootObject); evaluationContext.setTypeLocator(new AllowListTypeLocator()); + evaluationContext.setTypeConverter( + new ArtifactUriToReferenceConverter(ArtifactStore.getInstance())); evaluationContext.setMethodResolvers( Collections.singletonList(new FilteredMethodResolver(returnTypeRestrictor))); evaluationContext.setPropertyAccessors( diff --git a/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java b/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java index 792aa0850..041b7c8dc 100644 --- a/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java +++ b/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java @@ -17,11 +17,18 @@ package com.netflix.spinnaker.kork.expressions; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDecorator; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.expressions.config.ExpressionProperties; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.function.Function; import org.junit.jupiter.api.Test; @@ -29,6 +36,7 @@ import org.springframework.expression.ParserContext; import org.springframework.expression.common.TemplateParserContext; import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; public class ExpressionsSupportTest { private final ExpressionParser parser = new SpelExpressionParser(); @@ -97,4 +105,54 @@ public void testToJsonWhenComposedExpressionAndEvaluationContext() { assertThat(evaluated).isEqualTo("{\"json_file\":\"${#toJson(#doNotEval(file_json))}\"}"); } + + @Test + public void artifactReferenceInSpEL() { + MockArtifactStore artifactStore = new MockArtifactStore(); + ArtifactStore.setInstance(artifactStore); + ExpressionProperties expressionProperties = new ExpressionProperties(); + String expectedValue = "Hello world"; + artifactStore.cache.put("ref://app/sha", expectedValue); + String expr = "${#fromBase64(\"ref://app/sha\")}"; + Map testContext = + Collections.singletonMap( + "artifactReference", Collections.singletonMap("artifactReference", expr)); + + ExpressionsSupport expressionsSupport = new ExpressionsSupport(null, expressionProperties); + + StandardEvaluationContext evaluationContext = + expressionsSupport.buildEvaluationContext( + new ExpressionTransformTest.Pipeline(new ExpressionTransformTest.Trigger(123)), true); + + String evaluated = + new ExpressionTransform(parserContext, parser, Function.identity()) + .transformString(expr, evaluationContext, new ExpressionEvaluationSummary()); + + assertThat(evaluated).isEqualTo(expectedValue); + } + + public class MockArtifactStore extends ArtifactStore { + public Map cache = new HashMap<>(); + + @Override + public Artifact store(Artifact artifact) { + return null; + } + + @Override + public Artifact get(String id, ArtifactDecorator... decorators) { + String reference = cache.get(id); + Artifact.ArtifactBuilder builder = + Artifact.builder() + .reference( + Base64.getEncoder().encodeToString(reference.getBytes(StandardCharsets.UTF_8))); + if (decorators != null) { + for (ArtifactDecorator decorator : decorators) { + builder = decorator.decorate(builder); + } + } + + return builder.build(); + } + } } diff --git a/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java b/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java index 9409ddfc1..0dc6badc2 100644 --- a/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java +++ b/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java @@ -73,10 +73,10 @@ public class SecretsManagerSecretEngine implements SecretEngine { private final SecretsManagerClientProvider clientProvider; public SecretsManagerSecretEngine( - ObjectMapper mapper, + ObjectMapper objectMapper, UserSecretSerdeFactory userSecretSerdeFactory, SecretsManagerClientProvider clientProvider) { - this.mapper = mapper; + this.mapper = objectMapper; this.userSecretSerdeFactory = userSecretSerdeFactory; this.clientProvider = clientProvider; } diff --git a/spinnaker-dependencies/spinnaker-dependencies.gradle b/spinnaker-dependencies/spinnaker-dependencies.gradle index af5e58801..d3dfece34 100644 --- a/spinnaker-dependencies/spinnaker-dependencies.gradle +++ b/spinnaker-dependencies/spinnaker-dependencies.gradle @@ -8,6 +8,7 @@ ext { versions = [ arrow : "0.13.2", aws : "1.12.176", + awsv2 : "2.19.0", bouncycastle : "1.70", brave : "5.12.3", gcp : "25.3.0", @@ -78,6 +79,7 @@ dependencies { api(platform("com.amazonaws:aws-java-sdk-bom:${versions.aws}")) api(platform("com.google.protobuf:protobuf-bom:${versions.protobuf}")) api(platform("com.google.cloud:libraries-bom:${versions.gcp}")) + api(platform("software.amazon.awssdk:bom:${versions.awsv2}")) api(platform("com.google.cloud:google-cloud-secretmanager:2.1.7")) api(platform("org.springframework.cloud:spring-cloud-dependencies:${versions.springCloud}")) api(platform("io.strikt:strikt-bom:0.31.0")) From b67339572e7c310fd1956a3b9d24ab159b97dc02 Mon Sep 17 00:00:00 2001 From: xibz Date: Thu, 29 Jun 2023 22:18:57 -0500 Subject: [PATCH 2/5] Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: Nemesis Osorio Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: Nemesis Osorio Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java Co-authored-by: Nemesis Osorio Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java Co-authored-by: Sergio Quintero Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: Sergio Quintero Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: Sergio Quintero Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: Sergio Quintero --- .../spinnaker/kork/artifacts/README.md | 16 +-- .../artifactstore/ArtifactDeserializer.java | 5 +- .../ArtifactStoreConfiguration.java | 6 +- .../ArtifactStoreURIBuilder.java | 30 ++++- .../ArtifactStoreURISHA256Builder.java | 2 + .../EmbeddedArtifactSerializer.java | 2 +- .../artifactstore/s3/S3ArtifactStore.java | 29 +++-- .../artifacts/model/ExpectedArtifact.java | 14 +- .../EmbeddedArtifactSerializerTest.java | 120 ++++++++---------- 9 files changed, 128 insertions(+), 96 deletions(-) diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md index a3ed06516..5cbd83af1 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md @@ -1,14 +1,14 @@ # Artifact Storage -Artifact Storage is a feature that allows for embedded aritfacts to be +Artifact Storage is a feature that allows for embedded artifacts to be persisted to some storage, eg the S3ArtifactStore. Spinnaker keeps a history, which is called the pipeline context, that contains everything that an execution had done. Pipeline contexts can be very large, especially with -complicated pipeines, and size can be further increased when users have large +complicated pipelines, and size can be further increased when users have large artifacts. Spinnaker will duplicate these artifacts whenever any stage uses any of those artifacts. Using an artifact store reduces this overhead by providing a reference link of, -`ref://`. This reduces the context size +`ref:///`. This reduces the context size tremendously, but will vary depending on the size of the pipeline, as well as how that artifact is used, but we've seen improvements of 80% for some pipelines. @@ -81,7 +81,7 @@ check if some `String` is a remote base64 URI, and if it is, retrieve it. To enable artifact storage, simple add this to your `spinnaker-local.yml` file ```yaml -artifactStore: +artifact-store: enabled: true s3: enabled: true @@ -93,11 +93,11 @@ artifactStore: ### S3 [S3](https://aws.amazon.com/s3/) is an object store provided by AWS. The -currently S3ArtifactStore implementation provides various ways to authenticate +current S3ArtifactStore implementation provides various ways to authenticate against AWS. ```yaml -artifactStore: +artifact-store: enabled: true s3: enabled: true @@ -108,7 +108,7 @@ artifactStore: While the implementation is S3 specific, this does not limit usages of other S3 compatible storage engines. For example, something like -[SeaweedFS](https://github.com/seaweedfs/seaweedfs) can be used to test locally +[SeaweedFS](https://github.com/seaweedfs/seaweedfs) can be used to test locally. with ## Local Testing @@ -119,7 +119,7 @@ To test the artifact store locally, we will use SeaweedFS. To start the storage Next enable the configuration ```yaml -artifactStore: +artifact-store: enabled: true s3: enabled: true diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java index 5ae97d7ce..688a18bda 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java @@ -43,9 +43,8 @@ public ArtifactDeserializer( @Override public Artifact deserialize(JsonParser parser, DeserializationContext ctx) throws IOException { Artifact artifact = defaultObjectMapper.readValue(parser, Artifact.class); - String ref = artifact.getReference(); - if (artifact.getType().equals(ArtifactTypes.REMOTE_BASE64.getMimeType())) { - return storage.get(ref, new ArtifactMergeReferenceDecorator(artifact)); + if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(artifact.getType())) { + return storage.get(artifact.getReference(), new ArtifactMergeReferenceDecorator(artifact)); } return artifact; diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java index 296742ef8..0b122384a 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java @@ -26,7 +26,11 @@ import org.springframework.context.annotation.Bean; import org.springframework.context.annotation.Configuration; import org.springframework.security.access.PermissionEvaluator; -import software.amazon.awssdk.auth.credentials.*; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; import software.amazon.awssdk.regions.Region; import software.amazon.awssdk.services.s3.S3Client; import software.amazon.awssdk.services.s3.S3ClientBuilder; diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java index 17932218e..746944086 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java @@ -25,11 +25,39 @@ public abstract class ArtifactStoreURIBuilder { */ public static final String uriScheme = "ref"; + /** + * Returns the remote artifact URI that will be associated with some artifact. + * + * @param context is the context in which this artifact was run in, e.g. the application. + * @param artifact that will be associated with the generated URI. + * @return the remote URI + */ public abstract String buildArtifactURI(String context, Artifact artifact); /** * buildRawURI is used when you have the raw path and context. This method just simply return the - * properly formatted URI + * properly formatted URI using the URI builder that extends this class. + * + *

This function is primarily used in clouddriver when deck is asking for the raw artifact to + * be displayed. Since we don't have the artifact, but only the context and some raw ID from the + * gate endpoint, + * + * 

/context/hash
+ * + *

we need to reconstruct the full remote URI in clouddriver. + * + * 

{@code
+   * String application = "my-spinnaker-application";
+   * String artifactSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+   *
+   * ArtifactStoreURIBuilder uriBuilder = new ArtifactStoreURISHA256Builder();
+   * // returns ref://my-spinnaker-application/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+   * String uriString = uriBuilder.buildRawURI(application, artifactSHA256);
+   * }
+ * + * @param context is the context in which this artifact was run in, e.g. the application. + * @param raw is the identifier used in the URL, e.g. the hash. + * @return a properly formatted artifact store URI */ public abstract String buildRawURI(String context, String raw); } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java index 773fbe873..60a42dc37 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java @@ -22,12 +22,14 @@ * Helper class to abstract away the need for other classes to know the {@link * #uriPrefix} format. */ public class ArtifactStoreURISHA256Builder extends ArtifactStoreURIBuilder { + @Override public String buildArtifactURI(String context, Artifact artifact) { return String.format( "%s://%s/%s", uriScheme, context, Hashing.sha256().hashBytes(artifact.getReference().getBytes())); } + @Override public String buildRawURI(String context, String raw) { return String.format("%s://%s/%s", uriScheme, context, raw); } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java index ab70d1b77..2c07a2276 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java @@ -57,7 +57,7 @@ public void serialize(Artifact artifact, JsonGenerator gen, SerializerProvider p */ private static boolean shouldStoreArtifact(Artifact artifact) { String ref = artifact.getReference(); - return artifact.getType().equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + return ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(artifact.getType()) && !(ref == null || ref.isEmpty()); } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java index a9a9315fa..88532c0fe 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java @@ -22,6 +22,7 @@ import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.security.AuthenticatedRequest; import java.util.Base64; +import java.util.NoSuchElementException; import java.util.regex.Pattern; import lombok.extern.log4j.Log4j2; import org.springframework.security.access.PermissionEvaluator; @@ -80,6 +81,7 @@ public S3ArtifactStore( public Artifact store(Artifact artifact) { String application = AuthenticatedRequest.getSpinnakerApplication().orElse(null); if (application == null) { + log.warn("failed to retrieve application from request artifact={}", artifact.getName()); return artifact; } @@ -88,11 +90,11 @@ public Artifact store(Artifact artifact) { } String ref = uriBuilder.buildArtifactURI(application, artifact); + Artifact remoteArtifact = + artifact.toBuilder().type(ArtifactTypes.REMOTE_BASE64.getMimeType()).reference(ref).build(); + if (objectExists(ref)) { - return artifact.toBuilder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference(ref) - .build(); + return remoteArtifact; } // purpose of tagging is to ensure some sort of identity is persisted to @@ -107,20 +109,17 @@ public Artifact store(Artifact artifact) { .build(); s3Client.putObject(request, RequestBody.fromBytes(getReferenceAsBytes(artifact))); - return artifact.toBuilder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference(ref) - .build(); + return remoteArtifact; } - public byte[] getReferenceAsBytes(Artifact artifact) { + private byte[] getReferenceAsBytes(Artifact artifact) { String reference = artifact.getReference(); if (reference == null) { - return null; + throw new IllegalArgumentException("reference cannot be null"); } String type = artifact.getType(); - if (type.endsWith("/base64")) { + if (type != null && type.endsWith("/base64")) { return Base64.getDecoder().decode(reference); } @@ -133,7 +132,11 @@ public byte[] getReferenceAsBytes(Artifact artifact) { */ @Override public Artifact get(String id, ArtifactDecorator... decorators) { - hasAuthorization(id, AuthenticatedRequest.getSpinnakerUser().orElse(null)); + hasAuthorization( + id, + AuthenticatedRequest.getSpinnakerUser() + .orElseThrow( + () -> new NoSuchElementException("Could not authenticate due to missing user id"))); GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(id).build(); @@ -157,6 +160,8 @@ public Artifact get(String id, ArtifactDecorator... decorators) { /** * hasAuthorization will ensure that the user has proper permissions for retrieving the stored * artifact + * + * @throws AuthenticationServiceException when user does not have correct permissions */ private void hasAuthorization(String id, String userId) { GetObjectTaggingRequest request = diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java index 2aebcef23..3debe5bfc 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java @@ -22,7 +22,6 @@ import com.netflix.spinnaker.kork.annotations.NonnullByDefault; import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; -import java.io.UncheckedIOException; import java.util.Optional; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -50,8 +49,7 @@ private ExpectedArtifact( boolean useDefaultArtifact, Artifact defaultArtifact, String id, - Artifact boundArtifact) - throws UncheckedIOException { + Artifact boundArtifact) { defaultArtifact = store(defaultArtifact); boundArtifact = store(boundArtifact); @@ -104,8 +102,14 @@ private boolean matches(@Nullable String us, @Nullable String other) { } /** - * matchTypes will check to see if embedded types are used and if they are, see if they have been - * stored. If the type is embedded and stored, they are a valid match. + * Checks to see if artifact types are compatible/matchable. This handles the four known cases of: + * + * 
+   * type_a matches type_b
+   * type_a is embedded/base and type_b is remote/base64
+   * type_b is embedded/base and type_a is remote/base64
+   * and false otherwise
+   *
*/ private boolean matchTypes(String us, String other) { if (matches(us, other)) { diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java index 57de35abf..97ecc2174 100644 --- a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java @@ -24,77 +24,67 @@ import com.netflix.spinnaker.kork.artifacts.artifactstore.s3.S3ArtifactStore; import com.netflix.spinnaker.kork.artifacts.model.Artifact; import java.io.IOException; +import java.util.stream.Stream; import org.apache.commons.codec.binary.Base64; -import org.junit.jupiter.api.Test; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; import org.mockito.Mockito; class EmbeddedArtifactSerializerTest { - private static class TestCase { - public String name = null; - public String expectedJSON = null; - public Artifact artifact = null; - public Artifact mockArtifact = null; + @ParameterizedTest(name = "{index} {0}") + @MethodSource("generateTestCase") + public void serializeEmbeddedBase64Artifact_test( + String name, String expectedJson, Artifact artifact, Artifact mockArtifact) + throws IOException { + ArtifactStore storage = Mockito.mock(S3ArtifactStore.class); + when(storage.store(Mockito.any())).thenReturn(mockArtifact); - public TestCase(String name, String expectedJSON, Artifact artifact, Artifact mockArtifact) { - this.name = name; - this.expectedJSON = expectedJSON; - this.artifact = artifact; - this.mockArtifact = mockArtifact; - } - } - - @Test - public void serializeEmbeddedBase64Artifact_test() throws IOException { - TestCase[] cases = - new TestCase[] { - new TestCase( - "simple", - "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", - Artifact.builder() - .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) - .reference(Base64.encodeBase64String("foo".getBytes())) - .build(), - Artifact.builder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference("link") - .build()), - new TestCase( - "stored", - "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", - Artifact.builder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference("link") - .build(), - Artifact.builder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference("link") - .build()), - new TestCase( - "does-not-exist", - "{\"type\":\"nonexistent-type\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"Zm9v\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", - Artifact.builder() - .type("nonexistent-type") - .reference(Base64.encodeBase64String("foo".getBytes())) - .build(), - Artifact.builder() - .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) - .reference("link") - .build()), - }; + EmbeddedArtifactSerializer serializer = + new EmbeddedArtifactSerializer(new ObjectMapper(), storage); + ObjectMapper objectMapper = new ObjectMapper(); + SimpleModule module = new SimpleModule(); + module.addSerializer(Artifact.class, serializer); + objectMapper.registerModule(module); - for (TestCase testCase : cases) { - ArtifactStore storage = Mockito.mock(S3ArtifactStore.class); - when(storage.store(Mockito.any())).thenReturn(testCase.mockArtifact); - - EmbeddedArtifactSerializer serializer = - new EmbeddedArtifactSerializer(new ObjectMapper(), storage); - ObjectMapper objectMapper = new ObjectMapper(); - SimpleModule module = new SimpleModule(); - module.addSerializer(Artifact.class, serializer); - objectMapper.registerModule(module); + String result = objectMapper.writeValueAsString(artifact); + assertEquals(expectedJson, result); + } - String result = objectMapper.writeValueAsString(testCase.artifact); - assertEquals(testCase.expectedJSON, result, testCase.name); - } + private static Stream generateTestCase() { + return Stream.of( + Arguments.of( + "simple", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + Arguments.of( + "stored", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + Arguments.of( + "does-not-exist", + "{\"type\":\"nonexistent-type\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"Zm9v\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type("nonexistent-type") + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build())); } } From c8d1468b4e3621abddbe262a964aa63a74bc1282 Mon Sep 17 00:00:00 2001 From: xibz Date: Wed, 5 Jul 2023 17:30:17 -0500 Subject: [PATCH 3/5] Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> --- .../kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java index 746944086..4fee70cf0 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java @@ -35,7 +35,7 @@ public abstract class ArtifactStoreURIBuilder { public abstract String buildArtifactURI(String context, Artifact artifact); /** - * buildRawURI is used when you have the raw path and context. This method just simply return the + * buildRawURI is used when you have the raw path and context. This method just simply returns the * properly formatted URI using the URI builder that extends this class. * *

This function is primarily used in clouddriver when deck is asking for the raw artifact to From 22b11cd5297c9ffbd067c3d1acd9ba3a1c836c92 Mon Sep 17 00:00:00 2001 From: xibz Date: Wed, 5 Jul 2023 17:30:37 -0500 Subject: [PATCH 4/5] Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> Update kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md Co-authored-by: David Byron <82477955+dbyron-sf@users.noreply.github.com> --- .../java/com/netflix/spinnaker/kork/artifacts/README.md | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md index 5cbd83af1..39433b123 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md @@ -36,7 +36,7 @@ pipelines. Artifact storage is divided into two operations of get and store, and there are -primarily two services that utilizes each of these operations. Further the +primarily two services that utilize each of these operations. Further the artifact storage system relies on Spring's (de)serializing to call these operations to limit the amount of code changes needed within these services. @@ -44,10 +44,9 @@ When bootstrapping Spring we add in custom bean serializers and deserializers to handle storage or retrieval of an artifact. Rosco is primarily used for baking artifacts which will generate something -deployable. When Spring sends a request back to Orca for a bake request, the -custom serializer injected at Orca startup, will use the artifact storage and -store the artifact, and instead return a `remote/base64` artifact instead of the -usual `embedded/base64`. +deployable. When Rosco responds to a bake request, the custom serializer +injected in Rosco at startup stores the artifact and returns a `remote/base64` +artifact instead of the usual `embedded/base64`. Clouddriver, for this document, handles mostly deployment and has some endpoints regarding artifacts. It does do a little more than this, but we only care about From 61eabc8a9867de2b00d9de33635e3bc13f675b6d Mon Sep 17 00:00:00 2001 From: benjamin-j-powell Date: Wed, 5 Jul 2023 18:19:17 -0500 Subject: [PATCH 5/5] fixup! feat(artifacts): Add ArtifactStore to Kork Signed-off-by: benjamin-j-powell --- kork-artifacts/kork-artifacts.gradle | 2 -- 1 file changed, 2 deletions(-) diff --git a/kork-artifacts/kork-artifacts.gradle b/kork-artifacts/kork-artifacts.gradle index c61b3d885..9a12ae570 100644 --- a/kork-artifacts/kork-artifacts.gradle +++ b/kork-artifacts/kork-artifacts.gradle @@ -8,11 +8,9 @@ dependencies { implementation project(":kork-exceptions") implementation project(":kork-security") implementation "com.fasterxml.jackson.core:jackson-databind" - implementation platform('software.amazon.awssdk:bom') api 'software.amazon.awssdk:s3' api 'software.amazon.awssdk:sts' implementation "org.apache.httpcomponents:httpclient" - implementation 'software.amazon.awssdk:s3' implementation "org.springframework.boot:spring-boot-autoconfigure" implementation "org.springframework.security:spring-security-core" implementation 'org.apache.logging.log4j:log4j-api'