diff --git a/kork-artifacts/kork-artifacts.gradle b/kork-artifacts/kork-artifacts.gradle index 8be01c05e..9a12ae570 100644 --- a/kork-artifacts/kork-artifacts.gradle +++ b/kork-artifacts/kork-artifacts.gradle @@ -5,11 +5,21 @@ dependencies { api(platform(project(":spinnaker-dependencies"))) implementation project(":kork-annotations") + implementation project(":kork-exceptions") + implementation project(":kork-security") implementation "com.fasterxml.jackson.core:jackson-databind" + api 'software.amazon.awssdk:s3' + api 'software.amazon.awssdk:sts' + implementation "org.apache.httpcomponents:httpclient" + implementation "org.springframework.boot:spring-boot-autoconfigure" + implementation "org.springframework.security:spring-security-core" + implementation 'org.apache.logging.log4j:log4j-api' + implementation 'org.apache.logging.log4j:log4j-core' api "com.hubspot.jinjava:jinjava" testImplementation "org.assertj:assertj-core" testImplementation "org.junit.jupiter:junit-jupiter-api" testImplementation "org.junit.jupiter:junit-jupiter-params" + testImplementation "org.mockito:mockito-core" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java new file mode 100644 index 000000000..9d3d0404c --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/ArtifactTypes.java @@ -0,0 +1,28 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts; + +import lombok.Getter; +import lombok.RequiredArgsConstructor; + +@RequiredArgsConstructor +public enum ArtifactTypes { + EMBEDDED_BASE64("embedded/base64"), + REMOTE_BASE64("remote/base64"), + ; + + @Getter private final String mimeType; +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md new file mode 100644 index 000000000..39433b123 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/README.md @@ -0,0 +1,129 @@ +# Artifact Storage + +Artifact Storage is a feature that allows for embedded artifacts to be +persisted to some storage, eg the S3ArtifactStore. Spinnaker keeps a history, +which is called the pipeline context, that contains everything that an +execution had done. Pipeline contexts can be very large, especially with +complicated pipelines, and size can be further increased when users have large +artifacts. Spinnaker will duplicate +these artifacts whenever any stage uses any of those artifacts. Using an +artifact store reduces this overhead by providing a reference link of, +`ref:///`. This reduces the context size +tremendously, but will vary depending on the size of the pipeline, as well as +how that artifact is used, but we've seen improvements of 80% for some +pipelines. + +## Architecture + + +-----------+ + | | + | Orca | + | | + +-----------+ + | ^ + | | (outgoing artifact compressed) + | +----------------------+ + | (bake request) | + +---------------------+ | + v | + +---------------+ (fetch) +-------------+ + | |<-----------------------------| | + | Clouddriver | | Rosco | + | | | | + | s3 get | (full artifact returned) | s3 stores | + | artifacts |----------------------------->| artifacts | + +---------------+ +-------------+ + + +Artifact storage is divided into two operations of get and store, and there are +primarily two services that utilize each of these operations. Further the +artifact storage system relies on Spring's (de)serializing to call these +operations to limit the amount of code changes needed within these services. + +When bootstrapping Spring we add in custom bean serializers and deserializers to +handle storage or retrieval of an artifact. + +Rosco is primarily used for baking artifacts which will generate something +deployable. When Rosco responds to a bake request, the custom serializer +injected in Rosco at startup stores the artifact and returns a `remote/base64` +artifact instead of the usual `embedded/base64`. + +Clouddriver, for this document, handles mostly deployment and has some endpoints +regarding artifacts. It does do a little more than this, but we only care about +these two particular operations. When any request comes in, Spring will use its +custom deserializers to expand any artifact in its payload since any request +with artifacts, probably wants to do some operation on those artifacts. Further +Clouddriver also allows for fetching of artifacts. Orca and Rosco both make +calls to the `/artifact/fetch` endpoint. Where Rosco uses it to fetch an +artifact to be baked, and Orca uses it primarily when dealing with deploy +manifests. When a request is sent to the fetch endpoint, Clouddriver will always +return the full `embedded/base64` artifact back to the service. It is up to the +service receiving the artifact to compress it. Luckily, for Orca, we don't have +to worry about compression, since this no longer becomes an artifact, but a +manifest instead. + +Orca is a special case as it mostly does orchestration, but does cause some +artifacts to be duplicated when handling the expected artifact logic. We inject +some logic to handle the duplication along with ensuring that matching against +expected artifacts still works properly. So if a `embedded/base64` type needs to +match against a `remote/base64` type, Orca will use the artifact store to +retrieve that artifact, and do the comparison. In addition, Orca will store any +expected artifacts, to limit the context size. + +Orca also handles SpEL evaluation which means our new `remote/base64` type +should be backwards compatible with existing pipelines. To ensure this, we +utilized the Spring converters, and injected our own custom converter that will +check if some `String` is a remote base64 URI, and if it is, retrieve it. + +## Configuration + +To enable artifact storage, simple add this to your `spinnaker-local.yml` file + +```yaml +artifact-store: + enabled: true + s3: + enabled: true + bucket: some-artifact-store-bucket +``` + +## Storage Options + +### S3 + +[S3](https://aws.amazon.com/s3/) is an object store provided by AWS. The +current S3ArtifactStore implementation provides various ways to authenticate +against AWS. + +```yaml +artifact-store: + enabled: true + s3: + enabled: true + profile: dev # if you want to authenticate using a certain profile + region: us-west-2 # allows for specified regions + bucket: some-artifact-store-bucket +``` + +While the implementation is S3 specific, this does not limit usages of other S3 +compatible storage engines. For example, something like +[SeaweedFS](https://github.com/seaweedfs/seaweedfs) can be used to test locally. +with + +## Local Testing + +To test the artifact store locally, we will use SeaweedFS. To start the storage simply run +`docker run -p 8333:8333 chrislusf/seaweedfs server -s3` + +Next enable the configuration + +```yaml +artifact-store: + enabled: true + s3: + enabled: true + url: http://localhost:8333 # this URL will be used to make S3 API requests to + bucket: some-artifact-store-bucket +``` + +Start Spinnaker, and you should see reference links in your pipeline contexts. diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java new file mode 100644 index 000000000..1bbb68cb9 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDecorator.java @@ -0,0 +1,23 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +/** Primarily used to set any custom fields to an artifact */ +public interface ArtifactDecorator { + Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder); +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java new file mode 100644 index 000000000..688a18bda --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializer.java @@ -0,0 +1,70 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.DeserializationContext; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.deser.std.StdDeserializer; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import org.springframework.beans.factory.annotation.Qualifier; + +/** + * ArtifactDeserializer is a custom deserializer that will allow for artifacts to be fetched from + * some artifact store as long as the referenceLink field is set and the reference field is null. + */ +public class ArtifactDeserializer extends StdDeserializer { + private final ObjectMapper defaultObjectMapper; + private final ArtifactStore storage; + + public ArtifactDeserializer( + @Qualifier(value = "artifactObjectMapper") ObjectMapper defaultObjectMapper, + ArtifactStore storage) { + super(Artifact.class); + this.defaultObjectMapper = defaultObjectMapper; + this.storage = storage; + } + + @Override + public Artifact deserialize(JsonParser parser, DeserializationContext ctx) throws IOException { + Artifact artifact = defaultObjectMapper.readValue(parser, Artifact.class); + if (ArtifactTypes.REMOTE_BASE64.getMimeType().equals(artifact.getType())) { + return storage.get(artifact.getReference(), new ArtifactMergeReferenceDecorator(artifact)); + } + + return artifact; + } + + /** + * ArtifactMergeReferenceDecorator is used to take some artifact and replace its reference with + * the reference from another artifact. + */ + public static class ArtifactMergeReferenceDecorator implements ArtifactDecorator { + private final Artifact artifactToCopy; + + private ArtifactMergeReferenceDecorator(Artifact artifactToCopy) { + this.artifactToCopy = artifactToCopy; + } + + @Override + public Artifact.ArtifactBuilder decorate(Artifact.ArtifactBuilder builder) { + Artifact retrieved = builder.build(); + return artifactToCopy.toBuilder().reference(retrieved.getReference()); + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java new file mode 100644 index 000000000..f0dad92b7 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStore.java @@ -0,0 +1,50 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.util.concurrent.atomic.AtomicBoolean; +import lombok.Getter; + +/** + * ArtifactStore is an interface that allows for different types of artifact storage to be used + * during runtime + */ +public abstract class ArtifactStore { + /** ensures the singleton has only been set once */ + private static final AtomicBoolean singletonSet = new AtomicBoolean(false); + + @Getter private static ArtifactStore instance = null; + + public abstract Artifact store(Artifact artifact); + /** + * get is used to return an artifact with some id, while also decorating that artifact with any + * necessary fields needed which should be then be returned by the artifact store. + */ + public abstract Artifact get(String id, ArtifactDecorator... decorators); + + public static void setInstance(ArtifactStore storage) { + if (!singletonSet.compareAndSet(false, true)) { + throw new IllegalStateException("Multiple attempts at setting ArtifactStore's singleton"); + } + + ArtifactStore.instance = storage; + } + + public boolean isArtifactURI(String value) { + return value.startsWith(ArtifactStoreURIBuilder.uriScheme + "://"); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java new file mode 100644 index 000000000..0b122384a --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfiguration.java @@ -0,0 +1,117 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.artifactstore.s3.S3ArtifactStore; +import java.net.URI; +import java.util.Optional; +import lombok.extern.log4j.Log4j2; +import org.springframework.beans.factory.annotation.Qualifier; +import org.springframework.boot.autoconfigure.condition.ConditionalOnExpression; +import org.springframework.boot.context.properties.EnableConfigurationProperties; +import org.springframework.context.annotation.Bean; +import org.springframework.context.annotation.Configuration; +import org.springframework.security.access.PermissionEvaluator; +import software.amazon.awssdk.auth.credentials.AnonymousCredentialsProvider; +import software.amazon.awssdk.auth.credentials.AwsBasicCredentials; +import software.amazon.awssdk.auth.credentials.AwsCredentialsProvider; +import software.amazon.awssdk.auth.credentials.ProfileCredentialsProvider; +import software.amazon.awssdk.auth.credentials.StaticCredentialsProvider; +import software.amazon.awssdk.regions.Region; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.S3ClientBuilder; + +@Configuration +@ConditionalOnExpression("${artifact-store.enabled:false}") +@EnableConfigurationProperties(ArtifactStoreConfigurationProperties.class) +@Log4j2 +public class ArtifactStoreConfiguration { + /** + * this is strictly used due to Spring and Jackson not behaving nicely together. + * Unfortunately, @JsonDeserializer will construct its own deserializer utilizing beans and thus + * not using the object mapper we want to use + */ + @Bean + public ObjectMapper artifactObjectMapper() { + return new ObjectMapper(); + } + + @Bean + public ArtifactStoreURIBuilder artifactStoreURIBuilder() { + return new ArtifactStoreURISHA256Builder(); + } + + @Bean + @ConditionalOnExpression("${artifact-store.s3.enabled:false}") + public ArtifactStore s3ArtifactStore( + Optional permissionEvaluator, + ArtifactStoreConfigurationProperties properties, + @Qualifier("artifactS3Client") S3Client s3Client, + ArtifactStoreURIBuilder artifactStoreURIBuilder) { + + if (permissionEvaluator.isEmpty()) { + log.warn( + "PermissionEvaluator is not present. This means anyone will be able to access any artifact in the store."); + } + + ArtifactStore storage = + new S3ArtifactStore( + s3Client, + permissionEvaluator.orElse(null), + properties.getS3().getBucket(), + artifactStoreURIBuilder, + properties.getApplicationsRegex()); + + ArtifactStore.setInstance(storage); + return storage; + } + + @Bean + @ConditionalOnExpression("${artifact-store.s3.enabled:false}") + public S3Client artifactS3Client(ArtifactStoreConfigurationProperties properties) { + S3ClientBuilder builder = S3Client.builder(); + ArtifactStoreConfigurationProperties.S3ClientConfig config = properties.getS3(); + + // Overwriting the URL is primarily used for S3 compatible object stores + // like seaweedfs + if (config.getUrl() != null) { + builder = + builder + .credentialsProvider(getCredentialsProvider(config)) + .forcePathStyle(config.isForcePathStyle()) + .endpointOverride(URI.create(config.getUrl())); + } else if (config.getProfile() != null) { + builder = builder.credentialsProvider(ProfileCredentialsProvider.create(config.getProfile())); + } + + if (config.getRegion() != null) { + builder = builder.region(Region.of(config.getRegion())); + } + + return builder.build(); + } + + private AwsCredentialsProvider getCredentialsProvider( + ArtifactStoreConfigurationProperties.S3ClientConfig config) { + if (config.getAccessKey() != null) { + return StaticCredentialsProvider.create( + AwsBasicCredentials.create(config.getAccessKey(), config.getSecretKey())); + } else { + return AnonymousCredentialsProvider.create(); + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java new file mode 100644 index 000000000..e3fbf51be --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreConfigurationProperties.java @@ -0,0 +1,44 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import lombok.Data; +import org.springframework.boot.context.properties.ConfigurationProperties; + +@Data +@ConfigurationProperties("artifact-store") +public class ArtifactStoreConfigurationProperties { + private String applicationsRegex = null; + /** Configuration for an s3 client which will utilize credentials in the AWS credentials file. */ + @Data + public static class S3ClientConfig { + private boolean enabled = false; + private String profile = null; + private String region = null; + /** + * Url may be used to override the contact URL to an s3 compatible object store. This is useful + * for testing utilizing things like seaweedfs. + */ + private String url = null; + + private String bucket = null; + private String accessKey = null; + private String secretKey = null; + private boolean forcePathStyle = true; + } + + private S3ClientConfig s3 = null; +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java new file mode 100644 index 000000000..4fee70cf0 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURIBuilder.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +public abstract class ArtifactStoreURIBuilder { + /** + * uriScheme is used as an HTTP scheme to let us further distinguish a String that is a URI to an + * artifact. This is helpful in determining what is an artifact since sometimes we are only given + * a string rather than a full artifact. + */ + public static final String uriScheme = "ref"; + + /** + * Returns the remote artifact URI that will be associated with some artifact. + * + * @param context is the context in which this artifact was run in, e.g. the application. + * @param artifact that will be associated with the generated URI. + * @return the remote URI + */ + public abstract String buildArtifactURI(String context, Artifact artifact); + + /** + * buildRawURI is used when you have the raw path and context. This method just simply returns the + * properly formatted URI using the URI builder that extends this class. + * + *

This function is primarily used in clouddriver when deck is asking for the raw artifact to + * be displayed. Since we don't have the artifact, but only the context and some raw ID from the + * gate endpoint, + * + * 

/context/hash
+ * + *

we need to reconstruct the full remote URI in clouddriver. + * + * 

{@code
+   * String application = "my-spinnaker-application";
+   * String artifactSHA256 = "e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855";
+   *
+   * ArtifactStoreURIBuilder uriBuilder = new ArtifactStoreURISHA256Builder();
+   * // returns ref://my-spinnaker-application/e3b0c44298fc1c149afbf4c8996fb92427ae41e4649b934ca495991b7852b855
+   * String uriString = uriBuilder.buildRawURI(application, artifactSHA256);
+   * }
+ * + * @param context is the context in which this artifact was run in, e.g. the application. + * @param raw is the identifier used in the URL, e.g. the hash. + * @return a properly formatted artifact store URI + */ + public abstract String buildRawURI(String context, String raw); +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java new file mode 100644 index 000000000..60a42dc37 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactStoreURISHA256Builder.java @@ -0,0 +1,36 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.google.common.hash.Hashing; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; + +/** + * Helper class to abstract away the need for other classes to know the {@link * #uriPrefix} format. + */ +public class ArtifactStoreURISHA256Builder extends ArtifactStoreURIBuilder { + @Override + public String buildArtifactURI(String context, Artifact artifact) { + return String.format( + "%s://%s/%s", + uriScheme, context, Hashing.sha256().hashBytes(artifact.getReference().getBytes())); + } + + @Override + public String buildRawURI(String context, String raw) { + return String.format("%s://%s/%s", uriScheme, context, raw); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java new file mode 100644 index 000000000..2c07a2276 --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializer.java @@ -0,0 +1,63 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import com.fasterxml.jackson.core.JsonGenerator; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.SerializerProvider; +import com.fasterxml.jackson.databind.ser.std.StdSerializer; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; + +/** + * EmbeddedArtifactSerializer will store any embedded/base64 artifact into the ArtifactStore + * assuming that artifact has a reference link. + */ +public class EmbeddedArtifactSerializer extends StdSerializer { + private final ObjectMapper defaultObjectMapper; + private final ArtifactStore storage; + + public EmbeddedArtifactSerializer(ObjectMapper defaultObjectMapper, ArtifactStore storage) { + super(Artifact.class); + this.defaultObjectMapper = defaultObjectMapper; + this.storage = storage; + } + + @Override + public void serialize(Artifact artifact, JsonGenerator gen, SerializerProvider provider) + throws IOException { + if (!shouldStoreArtifact(artifact)) { + defaultObjectMapper.writeValue(gen, artifact); + return; + } + + Artifact stored = storage.store(artifact); + defaultObjectMapper.writeValue(gen, stored); + } + + /** + * shouldStore will return whether we want to store the reference in the ArtifactStore or not. + * This checks to ensure the reference isn't null or an empty string. Further we only care about + * 'embedded/base64' artifact types, since that is directly embedding the artifacts into the + * context + */ + private static boolean shouldStoreArtifact(Artifact artifact) { + String ref = artifact.getReference(); + return ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(artifact.getType()) + && !(ref == null || ref.isEmpty()); + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java new file mode 100644 index 000000000..88532c0fe --- /dev/null +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/artifactstore/s3/S3ArtifactStore.java @@ -0,0 +1,208 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore.s3; + +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDecorator; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStoreURIBuilder; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import com.netflix.spinnaker.security.AuthenticatedRequest; +import java.util.Base64; +import java.util.NoSuchElementException; +import java.util.regex.Pattern; +import lombok.extern.log4j.Log4j2; +import org.springframework.security.access.PermissionEvaluator; +import org.springframework.security.authentication.AuthenticationServiceException; +import org.springframework.security.core.Authentication; +import org.springframework.security.core.context.SecurityContextHolder; +import software.amazon.awssdk.core.ResponseBytes; +import software.amazon.awssdk.core.sync.RequestBody; +import software.amazon.awssdk.services.s3.S3Client; +import software.amazon.awssdk.services.s3.model.GetObjectRequest; +import software.amazon.awssdk.services.s3.model.GetObjectResponse; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingRequest; +import software.amazon.awssdk.services.s3.model.GetObjectTaggingResponse; +import software.amazon.awssdk.services.s3.model.HeadObjectRequest; +import software.amazon.awssdk.services.s3.model.NoSuchKeyException; +import software.amazon.awssdk.services.s3.model.PutObjectRequest; +import software.amazon.awssdk.services.s3.model.Tag; +import software.amazon.awssdk.services.s3.model.Tagging; + +/** + * S3ArtifactStore will store artifacts in a s3 compatible service + * + *

Note: It is very important that the S3 bucket has object lock on it to prevent multiple writes + * {@see https://docs.aws.amazon.com/AmazonS3/latest/userguide/object-lock-overview.html} + */ +@Log4j2 +public class S3ArtifactStore extends ArtifactStore { + private final S3Client s3Client; + private final PermissionEvaluator permissionEvaluator; + private final String bucket; + private final ArtifactStoreURIBuilder uriBuilder; + private final String applicationsRegex; + private static final String ENFORCE_PERMS_KEY = "application"; + + public S3ArtifactStore( + S3Client s3Client, + PermissionEvaluator permissionEvaluator, + String bucket, + ArtifactStoreURIBuilder uriBuilder, + String applicationsRegex) { + this.s3Client = s3Client; + this.bucket = bucket; + this.permissionEvaluator = permissionEvaluator; + this.uriBuilder = uriBuilder; + this.applicationsRegex = applicationsRegex; + } + + /** + * Will store the artifact using the {@link #s3Client} in some {@link #bucket} + * + *

This method also persists "permissions" by storing the execution id that made the original + * store call. In the event a service wants to retrieve said artifact, they will also need to + * provide the proper execution id + */ + @Override + public Artifact store(Artifact artifact) { + String application = AuthenticatedRequest.getSpinnakerApplication().orElse(null); + if (application == null) { + log.warn("failed to retrieve application from request artifact={}", artifact.getName()); + return artifact; + } + + if (applicationsRegex != null && !Pattern.matches(applicationsRegex, application)) { + return artifact; + } + + String ref = uriBuilder.buildArtifactURI(application, artifact); + Artifact remoteArtifact = + artifact.toBuilder().type(ArtifactTypes.REMOTE_BASE64.getMimeType()).reference(ref).build(); + + if (objectExists(ref)) { + return remoteArtifact; + } + + // purpose of tagging is to ensure some sort of identity is persisted to + // enforce permissions when retrieving the artifact + Tag accountTag = Tag.builder().key(ENFORCE_PERMS_KEY).value(application).build(); + + PutObjectRequest request = + PutObjectRequest.builder() + .bucket(bucket) + .key(ref) + .tagging(Tagging.builder().tagSet(accountTag).build()) + .build(); + + s3Client.putObject(request, RequestBody.fromBytes(getReferenceAsBytes(artifact))); + return remoteArtifact; + } + + private byte[] getReferenceAsBytes(Artifact artifact) { + String reference = artifact.getReference(); + if (reference == null) { + throw new IllegalArgumentException("reference cannot be null"); + } + + String type = artifact.getType(); + if (type != null && type.endsWith("/base64")) { + return Base64.getDecoder().decode(reference); + } + + return reference.getBytes(); + } + + /** + * get will return the Artifact with the provided id, and will lastly run the {@link + * ArtifactDecorator} to further populate the artifact for returning + */ + @Override + public Artifact get(String id, ArtifactDecorator... decorators) { + hasAuthorization( + id, + AuthenticatedRequest.getSpinnakerUser() + .orElseThrow( + () -> new NoSuchElementException("Could not authenticate due to missing user id"))); + + GetObjectRequest request = GetObjectRequest.builder().bucket(bucket).key(id).build(); + + ResponseBytes resp = s3Client.getObjectAsBytes(request); + Artifact.ArtifactBuilder builder = + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(Base64.getEncoder().encodeToString(resp.asByteArray())); + + if (decorators == null) { + return builder.build(); + } + + for (ArtifactDecorator decorator : decorators) { + builder = decorator.decorate(builder); + } + + return builder.build(); + } + + /** + * hasAuthorization will ensure that the user has proper permissions for retrieving the stored + * artifact + * + * @throws AuthenticationServiceException when user does not have correct permissions + */ + private void hasAuthorization(String id, String userId) { + GetObjectTaggingRequest request = + GetObjectTaggingRequest.builder().bucket(bucket).key(id).build(); + + GetObjectTaggingResponse resp = s3Client.getObjectTagging(request); + Tag tag = + resp.tagSet().stream() + .filter(t -> t.key().equals(ENFORCE_PERMS_KEY)) + .findFirst() + .orElse(null); + Authentication auth = SecurityContextHolder.getContext().getAuthentication(); + + if (tag == null + || (permissionEvaluator != null + && !permissionEvaluator.hasPermission(auth, tag.value(), "application", "READ"))) { + log.error( + "Could not authenticate to retrieve artifact user={} applicationOfStoredArtifact={}", + userId, + (tag == null) ? "(none)" : tag.value()); + throw new AuthenticationServiceException( + userId + " does not have permission to access this artifact"); + } + } + + /** + * Helper method to check whether the object exists. This is not thread safe, nor would it help in + * a distributed system due to how S3 works (no conditional statements). If preventing multiple + * writes of the same object is important, another filestore/db needs to be used, possibly + * dynamodb. + */ + private boolean objectExists(String id) { + HeadObjectRequest request = HeadObjectRequest.builder().bucket(bucket).key(id).build(); + try { + s3Client.headObject(request); + return true; + } catch (NoSuchKeyException e) { + // pretty gross that we need to use exceptions as control flow, but the + // java SDK doesn't have any other way of check if an object exists in s3 + log.warn("Artifact does not exist reference={}", id, e); + return false; + } + } +} diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java index 2adedc1d1..0ee1492e2 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/Artifact.java @@ -29,9 +29,12 @@ import javax.annotation.Nonnull; import javax.annotation.Nullable; import lombok.Builder; +import lombok.Data; import lombok.EqualsAndHashCode; import lombok.Getter; +import lombok.RequiredArgsConstructor; import lombok.ToString; +import lombok.extern.jackson.Jacksonized; @Getter @ToString @@ -101,5 +104,17 @@ public ArtifactBuilder putMetadata(String key, Object value) { metadata.put(key, value); return this; } + + public String getReference() { + return reference; + } + } + + @Data + @Builder + @Jacksonized + @RequiredArgsConstructor + public static class StoredView { + private final String reference; } } diff --git a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java index 0f42393de..3debe5bfc 100644 --- a/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java +++ b/kork-artifacts/src/main/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifact.java @@ -20,6 +20,8 @@ import com.fasterxml.jackson.databind.annotation.JsonPOJOBuilder; import com.google.common.base.Strings; import com.netflix.spinnaker.kork.annotations.NonnullByDefault; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; import java.util.Optional; import java.util.regex.Pattern; import javax.annotation.Nullable; @@ -48,6 +50,11 @@ private ExpectedArtifact( Artifact defaultArtifact, String id, Artifact boundArtifact) { + + defaultArtifact = store(defaultArtifact); + boundArtifact = store(boundArtifact); + matchArtifact = store(matchArtifact); + this.matchArtifact = Optional.ofNullable(matchArtifact).orElseGet(() -> Artifact.builder().build()); this.usePriorArtifact = usePriorArtifact; @@ -65,9 +72,7 @@ private ExpectedArtifact( * @return true i.f.f. the artifacts match */ public boolean matches(Artifact other) { - String thisType = matchArtifact.getType(); - String otherType = other.getType(); - if (!matches(thisType, otherType)) { + if (!matchTypes(matchArtifact.getType(), other.getType())) { return false; } @@ -89,25 +94,59 @@ public boolean matches(Artifact other) { return false; } - String thisReference = matchArtifact.getReference(); - String otherReference = other.getReference(); - if (!matches(thisReference, otherReference)) { - return false; - } - - // Explicitly avoid matching on UUID, provenance & artifactAccount - - return true; + return matches(matchArtifact.getReference(), other.getReference()); } private boolean matches(@Nullable String us, @Nullable String other) { return StringUtils.isEmpty(us) || (other != null && patternMatches(us, other)); } + /** + * Checks to see if artifact types are compatible/matchable. This handles the four known cases of: + * + * 

+   * type_a matches type_b
+   * type_a is embedded/base and type_b is remote/base64
+   * type_b is embedded/base and type_a is remote/base64
+   * and false otherwise
+   *
+ */ + private boolean matchTypes(String us, String other) { + if (matches(us, other)) { + return true; + } + + if (us.equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType())) { + return other.equals(ArtifactTypes.REMOTE_BASE64.getMimeType()); + } + + if (other.equals(ArtifactTypes.EMBEDDED_BASE64.getMimeType())) { + return us.equals(ArtifactTypes.REMOTE_BASE64.getMimeType()); + } + + return false; + } + private boolean patternMatches(String us, String other) { return Pattern.compile(us).matcher(other).matches(); } + /** Helper store method to easily store the artifact if needed */ + private static Artifact store(Artifact artifact) { + ArtifactStore storage = ArtifactStore.getInstance(); + if (artifact == null + || storage == null + || !ArtifactTypes.EMBEDDED_BASE64.getMimeType().equals(artifact.getType())) { + return artifact; + } + + if (artifact.getReference() != null && !artifact.getReference().isEmpty()) { + return storage.store(artifact); + } + + return artifact; + } + @JsonPOJOBuilder(withPrefix = "") public static final class ExpectedArtifactBuilder {} } diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java new file mode 100644 index 000000000..b4833af3f --- /dev/null +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/ArtifactDeserializerTest.java @@ -0,0 +1,74 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import static org.junit.jupiter.api.Assertions.*; + +import com.fasterxml.jackson.core.JsonFactory; +import com.fasterxml.jackson.core.JsonParser; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import java.util.HashMap; +import java.util.Map; +import org.junit.jupiter.api.Test; + +class ArtifactDeserializerTest { + private class InMemoryArtifactStore extends ArtifactStore { + public Map storageMap = new HashMap<>(); + + public InMemoryArtifactStore put(String id, Artifact artifact) { + storageMap.put(id, artifact); + return this; + } + + @Override + public Artifact store(Artifact artifact) { + storageMap.put(artifact.getReference(), artifact); + return artifact; + } + + @Override + public Artifact get(String id, ArtifactDecorator... decorator) { + return storageMap.get(id); + } + } + + @Test + public void simpleDeserialization() throws IOException { + String artifactJSON = + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"stored://link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}"; + String expectedReference = "foobar"; + Artifact expectedArtifact = + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference(expectedReference) + .build(); + InMemoryArtifactStore storage = + new InMemoryArtifactStore().put("stored://link", expectedArtifact); + ArtifactDeserializer deserializer = new ArtifactDeserializer(new ObjectMapper(), storage); + + // We avoid using an object mapper here since the Artifact class has a + // deserializer annotation which causes our deserializer to be ignored. So + // rather than using a mixin and setting all that up, this is easier. + JsonParser parser = new JsonFactory().createParser(artifactJSON); + Artifact receivedArtifact = deserializer.deserialize(parser, null); + assertNotNull(receivedArtifact); + assertEquals(expectedArtifact.getReference(), receivedArtifact.getReference()); + assertEquals(ArtifactTypes.REMOTE_BASE64.getMimeType(), receivedArtifact.getType()); + } +} diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java new file mode 100644 index 000000000..97ecc2174 --- /dev/null +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/artifactstore/EmbeddedArtifactSerializerTest.java @@ -0,0 +1,90 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.artifacts.artifactstore; + +import static org.junit.jupiter.api.Assertions.*; +import static org.mockito.Mockito.when; + +import com.fasterxml.jackson.databind.ObjectMapper; +import com.fasterxml.jackson.databind.module.SimpleModule; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; +import com.netflix.spinnaker.kork.artifacts.artifactstore.s3.S3ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; +import java.io.IOException; +import java.util.stream.Stream; +import org.apache.commons.codec.binary.Base64; +import org.junit.jupiter.params.ParameterizedTest; +import org.junit.jupiter.params.provider.Arguments; +import org.junit.jupiter.params.provider.MethodSource; +import org.mockito.Mockito; + +class EmbeddedArtifactSerializerTest { + @ParameterizedTest(name = "{index} {0}") + @MethodSource("generateTestCase") + public void serializeEmbeddedBase64Artifact_test( + String name, String expectedJson, Artifact artifact, Artifact mockArtifact) + throws IOException { + ArtifactStore storage = Mockito.mock(S3ArtifactStore.class); + when(storage.store(Mockito.any())).thenReturn(mockArtifact); + + EmbeddedArtifactSerializer serializer = + new EmbeddedArtifactSerializer(new ObjectMapper(), storage); + ObjectMapper objectMapper = new ObjectMapper(); + SimpleModule module = new SimpleModule(); + module.addSerializer(Artifact.class, serializer); + objectMapper.registerModule(module); + + String result = objectMapper.writeValueAsString(artifact); + assertEquals(expectedJson, result); + } + + private static Stream generateTestCase() { + return Stream.of( + Arguments.of( + "simple", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + Arguments.of( + "stored", + "{\"type\":\"remote/base64\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"link\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()) + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build()), + Arguments.of( + "does-not-exist", + "{\"type\":\"nonexistent-type\",\"customKind\":false,\"name\":null,\"version\":null,\"location\":null,\"reference\":\"Zm9v\",\"metadata\":{},\"artifactAccount\":null,\"provenance\":null,\"uuid\":null}", + Artifact.builder() + .type("nonexistent-type") + .reference(Base64.encodeBase64String("foo".getBytes())) + .build(), + Artifact.builder() + .type(ArtifactTypes.REMOTE_BASE64.getMimeType()) + .reference("link") + .build())); + } +} diff --git a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java index 18e15f62b..596e6b906 100644 --- a/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java +++ b/kork-artifacts/src/test/java/com/netflix/spinnaker/kork/artifacts/model/ExpectedArtifactTest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.node.JsonNodeFactory; import com.fasterxml.jackson.databind.node.ObjectNode; +import com.netflix.spinnaker.kork.artifacts.ArtifactTypes; import java.io.IOException; import java.util.function.Function; import java.util.stream.Stream; @@ -69,6 +70,28 @@ void roundTripSerialization() throws IOException { assertThat(deserializedArtifact).isEqualTo(artifact); } + @Test + void checkEmbeddedStoredTypesMatch() { + Artifact embeddedTypeArtifact = + Artifact.builder().type(ArtifactTypes.EMBEDDED_BASE64.getMimeType()).build(); + + Artifact storedTypeArtifact = + Artifact.builder().type(ArtifactTypes.REMOTE_BASE64.getMimeType()).build(); + + Artifact noMatchTypeArtifact = Artifact.builder().type("does-not-exist").build(); + + ExpectedArtifact expectedArtifact = + ExpectedArtifact.builder().matchArtifact(embeddedTypeArtifact).build(); + assertThat(expectedArtifact.matches(storedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(embeddedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(noMatchTypeArtifact)).isFalse(); + + expectedArtifact = ExpectedArtifact.builder().matchArtifact(storedTypeArtifact).build(); + assertThat(expectedArtifact.matches(embeddedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(storedTypeArtifact)).isTrue(); + assertThat(expectedArtifact.matches(noMatchTypeArtifact)).isFalse(); + } + private String fullExpectedArtifactJson() { return jsonFactory .objectNode() diff --git a/kork-expressions/kork-expressions.gradle b/kork-expressions/kork-expressions.gradle index 5eb64e79f..67903e659 100644 --- a/kork-expressions/kork-expressions.gradle +++ b/kork-expressions/kork-expressions.gradle @@ -6,6 +6,7 @@ dependencies { api(platform(project(":spinnaker-dependencies"))) api project(":kork-api") + api project(":kork-artifacts") api project(":kork-plugins-api") api project(":kork-exceptions") api "com.fasterxml.jackson.core:jackson-databind" @@ -14,6 +15,7 @@ dependencies { implementation "org.springframework.boot:spring-boot" + testImplementation project(":kork-artifacts") testImplementation "org.assertj:assertj-core" testImplementation "org.junit.jupiter:junit-jupiter-api" testRuntimeOnly "org.junit.jupiter:junit-jupiter-engine" diff --git a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java new file mode 100644 index 000000000..a6f83813d --- /dev/null +++ b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ArtifactUriToReferenceConverter.java @@ -0,0 +1,62 @@ +/* + * Copyright 2023 Apple Inc. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package com.netflix.spinnaker.kork.expressions; + +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import org.jetbrains.annotations.NotNull; +import org.springframework.core.convert.TypeDescriptor; +import org.springframework.expression.TypeConverter; +import org.springframework.expression.spel.support.StandardTypeConverter; + +/** + * This converter is used to check if a String is a Artifact reference URI. If it is, this will then + * pull the reference from the artifact store and return the reference back base64 encoded. + */ +public class ArtifactUriToReferenceConverter implements TypeConverter { + + private final ArtifactStore artifactStore; + + public ArtifactUriToReferenceConverter(ArtifactStore artifactStore) { + this.artifactStore = artifactStore; + } + + private final StandardTypeConverter defaultTypeConverter = new StandardTypeConverter(); + + @Override + public boolean canConvert(TypeDescriptor sourceType, @NotNull TypeDescriptor targetType) { + if (sourceType == null) { + return false; + } + + return sourceType.getObjectType() == String.class && targetType.getObjectType() == String.class; + } + + @Override + public Object convertValue( + Object value, TypeDescriptor sourceType, @NotNull TypeDescriptor targetType) { + // For some obscene reason(s), SpEL does not use this in the + // FunctionReference call when calling a method. So we call it internally + if (!canConvert(sourceType, targetType)) { + return defaultTypeConverter.convertValue(value, sourceType, targetType); + } + + if (artifactStore == null || !artifactStore.isArtifactURI((String) value)) { + return defaultTypeConverter.convertValue(value, sourceType, targetType); + } + + return artifactStore.get((String) value).getReference(); + } +} diff --git a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java index 5d76ba82f..ad7e1347d 100644 --- a/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java +++ b/kork-expressions/src/main/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupport.java @@ -20,6 +20,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.netflix.spinnaker.kork.api.expressions.ExpressionFunctionProvider; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; import com.netflix.spinnaker.kork.expressions.allowlist.AllowListTypeLocator; import com.netflix.spinnaker.kork.expressions.allowlist.FilteredMethodResolver; import com.netflix.spinnaker.kork.expressions.allowlist.FilteredPropertyAccessor; @@ -163,6 +164,8 @@ private StandardEvaluationContext createEvaluationContext( StandardEvaluationContext evaluationContext = new StandardEvaluationContext(rootObject); evaluationContext.setTypeLocator(new AllowListTypeLocator()); + evaluationContext.setTypeConverter( + new ArtifactUriToReferenceConverter(ArtifactStore.getInstance())); evaluationContext.setMethodResolvers( Collections.singletonList(new FilteredMethodResolver(returnTypeRestrictor))); evaluationContext.setPropertyAccessors( diff --git a/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java b/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java index 792aa0850..041b7c8dc 100644 --- a/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java +++ b/kork-expressions/src/test/java/com/netflix/spinnaker/kork/expressions/ExpressionsSupportTest.java @@ -17,11 +17,18 @@ package com.netflix.spinnaker.kork.expressions; import static org.assertj.core.api.Assertions.assertThat; +import static org.junit.Assert.assertThat; import static org.junit.jupiter.api.Assertions.assertEquals; import static org.junit.jupiter.api.Assertions.assertThrows; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactDecorator; +import com.netflix.spinnaker.kork.artifacts.artifactstore.ArtifactStore; +import com.netflix.spinnaker.kork.artifacts.model.Artifact; import com.netflix.spinnaker.kork.expressions.config.ExpressionProperties; +import java.nio.charset.StandardCharsets; +import java.util.Base64; import java.util.Collections; +import java.util.HashMap; import java.util.Map; import java.util.function.Function; import org.junit.jupiter.api.Test; @@ -29,6 +36,7 @@ import org.springframework.expression.ParserContext; import org.springframework.expression.common.TemplateParserContext; import org.springframework.expression.spel.standard.SpelExpressionParser; +import org.springframework.expression.spel.support.StandardEvaluationContext; public class ExpressionsSupportTest { private final ExpressionParser parser = new SpelExpressionParser(); @@ -97,4 +105,54 @@ public void testToJsonWhenComposedExpressionAndEvaluationContext() { assertThat(evaluated).isEqualTo("{\"json_file\":\"${#toJson(#doNotEval(file_json))}\"}"); } + + @Test + public void artifactReferenceInSpEL() { + MockArtifactStore artifactStore = new MockArtifactStore(); + ArtifactStore.setInstance(artifactStore); + ExpressionProperties expressionProperties = new ExpressionProperties(); + String expectedValue = "Hello world"; + artifactStore.cache.put("ref://app/sha", expectedValue); + String expr = "${#fromBase64(\"ref://app/sha\")}"; + Map testContext = + Collections.singletonMap( + "artifactReference", Collections.singletonMap("artifactReference", expr)); + + ExpressionsSupport expressionsSupport = new ExpressionsSupport(null, expressionProperties); + + StandardEvaluationContext evaluationContext = + expressionsSupport.buildEvaluationContext( + new ExpressionTransformTest.Pipeline(new ExpressionTransformTest.Trigger(123)), true); + + String evaluated = + new ExpressionTransform(parserContext, parser, Function.identity()) + .transformString(expr, evaluationContext, new ExpressionEvaluationSummary()); + + assertThat(evaluated).isEqualTo(expectedValue); + } + + public class MockArtifactStore extends ArtifactStore { + public Map cache = new HashMap<>(); + + @Override + public Artifact store(Artifact artifact) { + return null; + } + + @Override + public Artifact get(String id, ArtifactDecorator... decorators) { + String reference = cache.get(id); + Artifact.ArtifactBuilder builder = + Artifact.builder() + .reference( + Base64.getEncoder().encodeToString(reference.getBytes(StandardCharsets.UTF_8))); + if (decorators != null) { + for (ArtifactDecorator decorator : decorators) { + builder = decorator.decorate(builder); + } + } + + return builder.build(); + } + } } diff --git a/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java b/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java index 9409ddfc1..0dc6badc2 100644 --- a/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java +++ b/kork-secrets-aws/src/main/java/com/netflix/spinnaker/kork/secrets/engines/SecretsManagerSecretEngine.java @@ -73,10 +73,10 @@ public class SecretsManagerSecretEngine implements SecretEngine { private final SecretsManagerClientProvider clientProvider; public SecretsManagerSecretEngine( - ObjectMapper mapper, + ObjectMapper objectMapper, UserSecretSerdeFactory userSecretSerdeFactory, SecretsManagerClientProvider clientProvider) { - this.mapper = mapper; + this.mapper = objectMapper; this.userSecretSerdeFactory = userSecretSerdeFactory; this.clientProvider = clientProvider; } diff --git a/spinnaker-dependencies/spinnaker-dependencies.gradle b/spinnaker-dependencies/spinnaker-dependencies.gradle index af5e58801..d3dfece34 100644 --- a/spinnaker-dependencies/spinnaker-dependencies.gradle +++ b/spinnaker-dependencies/spinnaker-dependencies.gradle @@ -8,6 +8,7 @@ ext { versions = [ arrow : "0.13.2", aws : "1.12.176", + awsv2 : "2.19.0", bouncycastle : "1.70", brave : "5.12.3", gcp : "25.3.0", @@ -78,6 +79,7 @@ dependencies { api(platform("com.amazonaws:aws-java-sdk-bom:${versions.aws}")) api(platform("com.google.protobuf:protobuf-bom:${versions.protobuf}")) api(platform("com.google.cloud:libraries-bom:${versions.gcp}")) + api(platform("software.amazon.awssdk:bom:${versions.awsv2}")) api(platform("com.google.cloud:google-cloud-secretmanager:2.1.7")) api(platform("org.springframework.cloud:spring-cloud-dependencies:${versions.springCloud}")) api(platform("io.strikt:strikt-bom:0.31.0"))