Merge branch 'main' into features/superbeeny/addCredentialGeneration

.github/workflows/devops-boards.yaml

@@ -35,7 +35,7 @@ jobs:
           # https://learn.microsoft.com/azure/devops/integrate/get-started/authentication/service-principal-managed-identity
           echo "ADO_TOKEN=$(az account get-access-token --resource 499b84ac-1321-427f-aa17-267ca6975798 --query "accessToken" --output tsv)" >> $GITHUB_ENV
       - name: Sync issue to Azure DevOps
-        uses: danhellem/github-actions-issue-to-work-item@v2.3
+        uses: danhellem/github-actions-issue-to-work-item@v2.4
         env:
           ado_token: ${{ env.ADO_TOKEN }}
           github_token: '${{ secrets.GH_RAD_CI_BOT_PAT }}'

.github/workflows/functional-test-cloud.yaml

@@ -657,6 +657,50 @@ jobs:
             --set global.azureWorkloadIdentity.enabled=true \
             --set global.aws.irsa.enabled=true 
           
+          echo "*** Verify manifests are registered ***"
+          rm -f registermanifest_logs.txt
+          # Find the pod with container "ucp"
+          POD_NAME=$(
+            kubectl get pods -n radius-system \
+              -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
+            | grep "ucp" \
+            | head -n1 \
+            | cut -d" " -f1
+          )
+          echo "Found ucp pod: $POD_NAME"
+
+          if [ -z "$POD_NAME" ]; then
+            echo "No pod with container 'ucp' found in namespace radius-system."
+            exit 1
+          fi
+
+          # Poll logs for up to  iterations, 30 seconds each (upto 3 minutes total)
+          for i in {1..6}; do
+            kubectl logs "$POD_NAME" -n radius-system | tee registermanifest_logs.txt > /dev/null
+
+            # Exit on error
+            if grep -qi "Service initializer terminated with error" registermanifest_logs.txt; then
+              echo "Error found in ucp logs:"
+              grep -i "Service initializer terminated with error" registermanifest_logs.txt
+              exit 1
+            fi
+
+            # Check for success
+            if grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+              echo "Successfully registered manifests - message found."
+              break
+            fi
+
+            echo "Logs not ready, waiting 30 seconds..."
+            sleep 30
+          done
+
+          # Final check to ensure success message was found
+          if ! grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+            echo "Manifests not registered after 3 minutes."
+            exit 1
+          fi
+
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
           rad group create kind-radius
@@ -678,7 +722,7 @@ jobs:
           
           rad credential register aws irsa \
             --iam-role ${{ secrets.FUNC_TEST_RAD_IRSA_ROLE }}
-          
+
       - uses: marocchino/sticky-pull-request-comment@v2
         if: failure() && env.PR_NUMBER != ''
         continue-on-error: true
@@ -903,4 +947,4 @@ jobs:
               title: `Scheduled functional test failed - Run ID: ${context.runId}`,
               labels: ['bug', 'test-failure'],
               body: `## Bug information \n\nThis issue is automatically generated if the scheduled functional test fails. The Radius functional test operates on a schedule of every 4 hours during weekdays and every 12 hours over the weekend. It's important to understand that the test may fail due to workflow infrastructure issues, like network problems, rather than the flakiness of the test itself. For the further investigation, please visit [here](${process.env.ACTION_LINK}).`
-            })
+            })

.github/workflows/functional-test-noncloud.yaml

@@ -126,7 +126,7 @@ jobs:
     strategy:
       fail-fast: true
       matrix:
-        os: [ubuntu-latest-m]
+        os: [ubuntu-latest]
         name:
           [
             cli-noncloud,
@@ -179,25 +179,7 @@ jobs:
         uses: actions/setup-go@v5
         with:
           go-version: ${{ env.GOVER }}
-          cache: false
-
-      - name: Get Go Cache path
-        id: go-cache-paths
-        run: |
-          echo "go-build=$(go env GOCACHE)" >> $GITHUB_OUTPUT
-          sudo rm -rf $(go env GOCACHE)
-
-          echo "go-mod=$(go env GOMODCACHE)" >> $GITHUB_OUTPUT
-          sudo rm -rf $(go env GOMODCACHE)
-
-      - uses: actions/cache@v4
-        with:
-          path: |
-            ${{ steps.go-cache-paths.outputs.go-build }}
-            ${{ steps.go-cache-paths.outputs.go-mod }}
-          key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }}
-          restore-keys: |
-            ${{ runner.os }}-go-
+          cache: true
 
       - name: Setup Node.js
         uses: actions/setup-node@v4
@@ -290,6 +272,50 @@ jobs:
           echo "*** Installing Radius to Kubernetes ***"
           eval $RAD_COMMAND
 
+          echo "*** Verify manifests are registered ***"
+          rm -f registermanifest_logs.txt
+          # Find the pod with container "ucp"
+          POD_NAME=$(
+            kubectl get pods -n radius-system \
+              -o jsonpath='{range .items[*]}{.metadata.name}{" "}{.spec.containers[*].name}{"\n"}{end}' \
+            | grep "ucp" \
+            | head -n1 \
+            | cut -d" " -f1
+          )
+          echo "Found ucp pod: $POD_NAME"
+
+          if [ -z "$POD_NAME" ]; then
+            echo "No pod with container 'ucp' found in namespace radius-system."
+            exit 1
+          fi
+
+          # Poll logs for up to  iterations, 30 seconds each (upto 3 minutes total)
+          for i in {1..6}; do
+            kubectl logs "$POD_NAME" -n radius-system | tee registermanifest_logs.txt > /dev/null
+            
+            # Exit on error
+            if grep -qi "Service initializer terminated with error" registermanifest_logs.txt; then
+              echo "Error found in ucp logs."
+              grep -i "Service initializer terminated with error" registermanifest_logs.txt
+              exit 1
+            fi
+
+            # Check for success
+            if grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+              echo "Successfully registered manifests - message found."
+              break
+            fi
+
+            echo "Logs not ready, waiting 30 seconds..."
+            sleep 30
+          done
+
+          # Final check to ensure success message was found
+          if ! grep -q "Successfully registered manifests" registermanifest_logs.txt; then
+            echo "Manifests not registered after 3 minutes."
+            exit 1
+          fi
+
           echo "*** Create workspace, group and environment for test ***"
           rad workspace create kubernetes
           rad group create kind-radius
@@ -298,6 +324,8 @@ jobs:
           # The functional test is designed to use default namespace. So you must create the environment for default namespace.
           rad env create kind-radius --namespace default
           rad env switch kind-radius
+        continue-on-error: true  
+
         env:
           USE_CERT_FILE: "true"
           TEMP_CERT_DIR: ${{ steps.create-local-registry.outputs.temp-cert-dir }}

.github/workflows/release.yaml

@@ -46,7 +46,7 @@ jobs:
         uses: actions/checkout@v4
       - name: Get supported versions from versions.yaml
         id: get-supported-versions
-        uses: mikefarah/yq@v4.44.6
+        uses: mikefarah/yq@v4.45.1
         with:
           # Get a comma-separated list of supported versions
           cmd: yq '.supported[].version' versions.yaml | tr '\n' ',' | sed 's/,$//'
@@ -165,7 +165,7 @@ jobs:
           git config --global user.email "[email protected]"
       - name: Get supported versions from versions.yaml
         id: get-supported-versions
-        uses: mikefarah/yq@v4.44.6
+        uses: mikefarah/yq@v4.45.1
         with:
           # Get a comma-separated list of supported versions
           cmd: yq '.supported[].version' ./radius/versions.yaml | tr '\n' ',' | sed 's/,$//'

build/docker.mk

@@ -17,6 +17,7 @@
 DOCKER_REGISTRY?=$(shell whoami)
 DOCKER_TAG_VERSION?=latest
 IMAGE_SRC?=https://github.com/radius-project/radius
+MANIFEST_DIR?=deploy/manifest/built-in-providers/self-hosted
 
 ##@ Docker Images
 
@@ -106,6 +107,17 @@ APPS_MAP := ucpd:./deploy/images/ucpd \
 	testrp:./test/testrp \
 	magpiego:./test/magpiego
 
+# copy_manifests copies the manifests to the output directory
+.PHONY: copy-manifests
+copy-manifests:
+	@if [ ! -d "$(MANIFEST_DIR)" ] || [ -z "$$(ls -A $(MANIFEST_DIR))" ]; then \
+		echo "MANIFEST_DIR '$(MANIFEST_DIR)' does not exist or is empty"; \
+		exit 1; \
+	fi
+	@mkdir -p $(OUT_DIR)/manifest/built-in-providers/
+	@echo "Copying manifests from $(MANIFEST_DIR) to $(OUT_DIR)/manifest/built-in-providers/"
+	@cp -v $(MANIFEST_DIR)/* $(OUT_DIR)/manifest/built-in-providers/
+
 # Function to extract the name and the directory of the Dockerfile from the app string
 define parseApp
 $(eval NAME := $(shell echo $(1) | cut -d: -f1))
@@ -132,15 +144,15 @@ DOCKER_PUSH_MULTI_TARGETS := $(foreach APP,$(APPS_MAP),$(eval $(call parseApp,$(
 
 # targets to build development images
 .PHONY: docker-build
-docker-build: $(DOCKER_BUILD_TARGETS) ## Builds all Docker images.
+docker-build: copy-manifests $(DOCKER_BUILD_TARGETS) ## Builds all Docker images.
 
 .PHONY: docker-push
 docker-push: $(DOCKER_PUSH_TARGETS) ## Pushes all Docker images (without building).
 
 # targets to build and push multi arch images. If you run this target in your machine,
 # ensure you have qemu and buildx installed by running make configure-buildx.
 .PHONY: docker-multi-arch-build
-docker-multi-arch-build: $(DOCKER_BUILD_MULTI_TARGETS) ## Builds all docker images for multiple architectures.
+docker-multi-arch-build: copy-manifests $(DOCKER_BUILD_MULTI_TARGETS) ## Builds all docker images for multiple architectures.
 
 .PHONY: docker-multi-arch-push
-docker-multi-arch-push: $(DOCKER_PUSH_MULTI_TARGETS) ## Pushes all docker images for multiple architectures after building.
+docker-multi-arch-push: copy-manifests $(DOCKER_PUSH_MULTI_TARGETS) ## Pushes all docker images for multiple architectures after building.

cmd/ucpd/ucp-dev.yaml

@@ -53,7 +53,7 @@ initialization:
           Microsoft.Resources: "http://localhost:5017"
         kind: "UCPNative"
   # This is the directory location which contains manifests to be registered.
-  manifestDirectory: ""      
+  manifestDirectory: "manifest/built-in-providers/"      
 
 identity:
   authMethod: default

deploy/Chart/templates/ucp/configmaps.yaml

@@ -50,7 +50,7 @@ data:
         - id: "/planes/aws/aws"
           properties:
             kind: "AWS"
-      manifestDirectory: ""      
+      manifestDirectory: "/manifest/built-in-providers"
 
     identity:
       authMethod: UCPCredential

deploy/images/ucpd/Dockerfile

@@ -10,6 +10,9 @@ WORKDIR /
 # Copy the application binary for the specified architecture
 COPY ./linux_${TARGETARCH:-amd64}/release/ucpd /
 
+# Copy the manifest files for the built-in providers
+COPY ./manifest/built-in-providers/ /manifest/built-in-providers/
+
 # Set the user to non-root (65532:65532 is the default non-root user in distroless)
 USER 65532:65532
 

deploy/manifest/built-in-providers/applications_core.yaml → deploy/manifest/built-in-providers/dev/applications_core.yaml

@@ -1,37 +1,40 @@
 name: Applications.Core
+location:
+  global:
+    "http://localhost:8080"
 types:
   containers:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   applications:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   environments:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   gateways:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   secretStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []
   extenders:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   volumes:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: []

deploy/manifest/built-in-providers/applications_dapr.yaml → deploy/manifest/built-in-providers/dev/applications_dapr.yaml

@@ -1,22 +1,25 @@
 name: Applications.Dapr
+location:
+  global:
+    "http://localhost:8080"
 types:
   configurationStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   pubSubBrokers:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   secretStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   stateStores:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/applications_datastores.yaml → deploy/manifest/built-in-providers/dev/applications_datastores.yaml

@@ -1,17 +1,20 @@
 name: Applications.Datastores
+location:
+  global:
+    "http://localhost:8080"
 types:
   mongoDatabases:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   sqlDatabases:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]
   redisCaches:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/applications_messaging.yaml → deploy/manifest/built-in-providers/dev/applications_messaging.yaml

@@ -1,7 +1,10 @@
 name: Applications.Messaging
+location:
+  global:
+    "http://localhost:8080"
 types:
   rabbitMQQueues:
     apiVersions:
-      "2025-01-01-preview":
+      "2023-10-01-preview":
         schema: {}
     capabilities: ["SupportsRecipes"]

deploy/manifest/built-in-providers/dev/microsoft_resources.yaml

@@ -0,0 +1,12 @@
+name: Microsoft.Resources
+location:
+  global:
+    "http://localhost:5017"
+types:
+  deployments:
+    apiVersions:
+      "2020-10-01":
+        schema: {}
+      "2022-09-01":
+        schema: {}  
+    capabilities: []