Global floatingUserId to support OpenShift (#9835)

* Add helm values * Add new defaults to "1-gloo-deployment.yaml" * test working, no KubeGateway * Pod level test * GWParams * updates * cleanup template * Update _helpers.tpl * Update utils.go * Update utils.go * Create consistent-floating-user-id.yaml * Pass globals.securitySettings to helper * update casing * Update 5-resource-configmap.yaml * Adding changelog file to new location * Deleting changelog file from old location * Update Dockerfile * skip test * Update _helpers.tpl * test fix * updated helper and tests * move gg2 helper * add floatingUserId to GatewayParameters * Adding changelog file to new location * Deleting changelog file from old location * gg floatingUserId * GG2 floatingUserId helm test * Deployer Tests * Remove global fsGroup * Refactor template param names * Update merge.go * update documentation * Update to make `floatingUserId` also disable selected pod security contexts * whitespace fix * Update docs * Update cluster_setup.md * PR feedback #1 * Add ConvertUnstructured first pass * Pass matchers to ValidateSecurityContexts * Update unstructured_test.go * Update unstructured_test.go * Update install/test/helm_test.go Co-authored-by: Sam Heilbron <[email protected]> * Update helm_test.go * Update k8sgateway_test.go * PR Feedback * Update unstructured_test.go * Update unstructured_test.go --------- Co-authored-by: soloio-bulldozer[bot] <48420018+soloio-bulldozer[bot]@users.noreply.github.com> Co-authored-by: changelog-bot <changelog-bot> Co-authored-by: Sam Heilbron <[email protected]>
solo-io · Aug 9, 2024 · df93f5f · df93f5f
1 parent 4ae2093
commit df93f5f
Show file tree

Hide file tree

Showing 40 changed files with 976 additions and 350 deletions.
diff --git a/changelog/v1.18.0-beta14/consistent-floating-user-id.yaml b/changelog/v1.18.0-beta14/consistent-floating-user-id.yaml
@@ -0,0 +1,21 @@
+changelog:
+  - type: NEW_FEATURE
+    issueLink: https://github.com/solo-io/gloo/issues/5034
+    resolvesIssue: false
+    description: >-
+      Provide a Helm field `global.securitySettings.floatingUserId` to apply floatingUserId logic, which unsets `runAsUser`
+      for security contexts, for all deployments in the Gloo Helm chart. The global field will also cause templates to be rendered
+      as if deployments with "enablePodSecurityContext` fields have their value set to `false` to allow for easy OpenShift deployment.
+      This functionality has also been added to Gloo Gateway via the GatewayParameters resource. If `floatingUserId` is set in 
+      GatewayParameters, it will be applied to all deployments in the Gloo Gateway Helm chart, unless a deployment-specific value is set.
+  - type: HELM
+    issueLink: https://github.com/solo-io/gloo/issues/5034
+    resolvesIssue: false
+    description: >-
+      Add a new field `global.securitySettings.floatingUserId` to the Gloo Helm chart that when set to `true` has the same effect as
+      setting `floatingUserId=true` for all deployment-specific `floatingUserId`s, as well as setting `discovery.deployment.enablePodSecurityContext=false` 
+      and gatewayProxies.gatewayProxy.podTemplate.enablePodSecurityContext=false`to allow for easy OpenShift deployment.
+      The global value will override any local settings.
+  - type: NON_USER_FACING
+    description:
+      Update `kubectl` image to use USER_ID 10101 to allow resource rollout jobs to run when user id is not set
diff --git a/docs/content/installation/platform_configuration/cluster_setup.md b/docs/content/installation/platform_configuration/cluster_setup.md
@@ -209,73 +209,21 @@ You can use this Helm chart `values.yaml` file while following the [Gloo Gateway
 {{< tabs >}}
 {{< tab name="Enterprise" codelang="yaml" >}}
 global:
-  extensions:
-    extAuth:
-      deployment:
-        floatingUserId: true
-        fsGroup: ""
-    rateLimit:
-      deployment:
-        floatingUserId: true
-oberservability:
-  deployment:
+  securitySettings:
     floatingUserId: true
-redis:
-  deployment:
-    floatingUserId: true
-    enablePodSecurityContext: false
-gloo:
-  gloo:
-    deployment:
-      floatingUserId: true
-  discovery:
-    deployment:
-      floatingUserId: true
-      enablePodSecurityContext: false
-  gateway:
-    deployment:
-      floatingUserId: true
-    certGenJob:
-      floatingUserId: true
-  observability:
-    deployment:
-      floatingUserId: true
-  gatewayProxies:
-    gatewayProxy:
-      podTemplate:
-        floatingUserId: true
-        enablePodSecurityContext: false
 gloo-fed:
   enabled: false
   glooFedApiserver:
     enable: false #optionally disable the admin console, too
-    floatingUserId: true
 prometheus:
   enabled: false
 grafana:
   defaultInstallationEnabled: false
 {{< /tab >}}
 {{< tab name="Open Source" codelang="yaml">}}
-gloo:
-  deployment:
-    floatingUserId: true
-discovery:
-  deployment:
-    floatingUserId: true
-    enablePodSecurityContext: false
-gateway:
-  deployment:
-    floatingUserId: true
-  certGenJob:
-    floatingUserId: true
-observability:
-  deployment:
+global:
+  securitySettings:
     floatingUserId: true
-gatewayProxies:
-  gatewayProxy:
-    podTemplate:
-      floatingUserId: true
-      enablePodSecurityContext: false
 {{< /tab >}}
 {{< /tabs >}}
 

diff --git a/docs/content/reference/values.txt b/docs/content/reference/values.txt
@@ -171,6 +171,7 @@
 |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].containerPort|int32|||
 |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].protocol|string|||
 |kubeGateway.gatewayParameters.glooGateway.aiExtension.ports[].hostIP|string|||
+|kubeGateway.gatewayParameters.glooGateway.floatingUserId|bool||If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false.|
 |settings.watchNamespaces[]|string||whitelist of namespaces for Gloo Edge to watch for services and CRDs. Empty list means all namespaces|
 |settings.writeNamespace|string||namespace where intermediary CRDs will be written to, e.g. Upstreams written by Gloo Edge Discovery.|
 |settings.integrations.knative.enabled|bool|false|enabled knative components|
@@ -1650,3 +1651,4 @@
 |global.additionalLabels.NAME|string||Additional labels to add to all gloo resources.|
 |global.podSecurityStandards.container.enableRestrictedContainerDefaults|bool||Set to true to default all containers to a security policy that minimally conforms to a [restricted container security policy](https://kubernetes.io/docs/concepts/security/pod-security-standards/#restricted). |
 |global.podSecurityStandards.container.defaultSeccompProfileType|string||The seccomp profile type to use for default restricted container securityContexts. Valid values are 'RuntimeDefault' and 'Localhost'. Default is 'RuntimeDefault'. Has no effect if enableRestrictedContainerDefaults is false.|
+|global.securitySettings.floatingUserId|bool||If true, use 'true' as default value for all instances of floatingUserId. In OSS, has the additional effects of rendering charts as if 'discovery.deployment.enablePodSecurityContext=false' and 'gatewayProxies.gatewayProxy.podTemplate.enablePodSecurityContext=false'. In EE templates  has the additional effects of rendering charts as if 'redis.deployment.enablePodSecurityContext=false', and in the ExtAuth deployment's podSecurityContext, behavior will match the local 'floatingUserId' and fsGroup will not be rendered.|
diff --git a/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml b/install/helm/gloo/crds/gateway.gloo.solo.io_gatewayparameters.yaml
@@ -352,6 +352,8 @@ spec:
                             type: object
                         type: object
                     type: object
+                  floatingUserId:
+                    type: boolean
                   istio:
                     properties:
                       customSidecars:

diff --git a/install/helm/gloo/generate/values.go b/install/helm/gloo/generate/values.go
@@ -40,6 +40,11 @@ type Global struct {
 	ExtraCustomResources *bool                 `json:"extraCustomResources,omitempty" desc:"Add additional custom resources to create, as defined by a helm partial. Defaults to false in open source, and true in enterprise."`
 	AdditionalLabels     map[string]string     `json:"additionalLabels,omitempty" desc:"Additional labels to add to all gloo resources."`
 	PodSecurityStandards *PodSecurityStandards `json:"podSecurityStandards,omitempty" desc:"Configuration related to [Pod Security Standards](https://kubernetes.io/docs/concepts/security/pod-security-standards/)."`
+	SecuritySettings     *SecuritySettings     `json:"securitySettings,omitempty" desc:"Global settings for pod and container security contexts"`
+}
+
+type SecuritySettings struct {
+	FloatingUserId *bool `json:"floatingUserId,omitempty" desc:"If true, use 'true' as default value for all instances of floatingUserId. In OSS, has the additional effects of rendering charts as if 'discovery.deployment.enablePodSecurityContext=false' and 'gatewayProxies.gatewayProxy.podTemplate.enablePodSecurityContext=false'. In EE templates  has the additional effects of rendering charts as if 'redis.deployment.enablePodSecurityContext=false', and in the ExtAuth deployment's podSecurityContext, behavior will match the local 'floatingUserId' and fsGroup will not be rendered."`
 }
 
 type PodSecurityStandards struct {
@@ -326,6 +331,7 @@ type GatewayParameters struct {
 	Istio           *Istio                     `json:"istio,omitempty" desc:"Configs used to manage Istio integration."`
 	Stats           *GatewayParamsStatsConfig  `json:"stats,omitempty" desc:"Config used to manage the stats endpoints exposed on the deployed proxies"`
 	AIExtension     *GatewayParamsAIExtension  `json:"aiExtension,omitempty" desc:"Config used to manage the Gloo Gateway AI extension."`
+	FloatingUserId  *bool                      `json:"floatingUserId,omitempty" desc:"If true, allows the cluster to dynamically assign a user ID for the processes running in the container. Default is false."`
 	// TODO(npolshak): Add support for GlooMtls
 }
 

diff --git a/install/helm/gloo/templates/1-gloo-deployment.yaml b/install/helm/gloo/templates/1-gloo-deployment.yaml
@@ -42,8 +42,12 @@ spec:
         {{- end }}
         {{- end }}
     spec:
-{{ with .Values.gloo.deployment.podSecurityContext }}
-{{- include "gloo.securityContext" (dict "values" . "defaults" (dict)) | indent 6 -}}
+{{- $podSecCtx := dict -}}
+{{- with .Values.gloo.deployment.podSecurityContext -}}
+  {{- $podSecCtx = . -}}
+{{- end -}}
+{{- if $podSecCtx  -}}
+{{- include "gloo.securityContext" (dict "values" $podSecCtx "globalSec" .Values.global.securitySettings) | indent 6  -}}
 {{- end }}
 {{- include "gloo.pullSecret" $image | nindent 6 -}}
       serviceAccountName: gloo
@@ -84,7 +88,7 @@ spec:
         {{- if not .Values.gloo.deployment.floatingUserId }}
           {{- $_ := set $securityDefaults "runAsUser" .Values.gloo.deployment.runAsUser }}
         {{- end }}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.global.glooMtls.envoy.securityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.global.glooMtls.envoy.securityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.gloo.deployment.xdsPort }}
           name: grpc-xds
@@ -127,7 +131,7 @@ spec:
         {{- if not .Values.gloo.deployment.floatingUserId -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.gloo.deployment.runAsUser}}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.global.glooMtls.sds.securityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.global.glooMtls.sds.securityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.gloo.deployment.validationPort }}
           name: validation
@@ -168,7 +172,7 @@ spec:
         {{- if not .Values.gloo.deployment.floatingUserId -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.gloo.deployment.runAsUser }}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.gloo.deployment.glooContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.gloo.deployment.glooContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
 {{- if not .Values.global.glooMtls.enabled }}
         - containerPort: {{ .Values.gloo.deployment.xdsPort }}
@@ -271,4 +275,4 @@ spec:
 {{- end }} {{/* define "gloo.deploymentSpec "*/}}
 
 {{/* Render template with yaml overrides */}}
-{{- include "gloo.util.merge" (list . .Values.gloo.deployment.kubeResourceOverride "gloo.deploymentSpec") -}}
+{{- include "gloo.util.merge" (list . .Values.gloo.deployment.kubeResourceOverride "gloo.deploymentSpec") -}}
diff --git a/install/helm/gloo/templates/10-ingress-deployment.yaml b/install/helm/gloo/templates/10-ingress-deployment.yaml
@@ -47,7 +47,7 @@ spec:
       - image: {{template "gloo.image" $image}}
         imagePullPolicy: {{ $image.pullPolicy }}
         name: ingress
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.ingress.deployment.ingressContainerSecurityContext "defaults" (dict) "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.ingress.deployment.ingressContainerSecurityContext "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
 {{- if .Values.ingress.deployment.resources }}
         resources:
 {{ toYaml .Values.ingress.deployment.resources | indent 10}}

diff --git a/install/helm/gloo/templates/11-ingress-proxy-deployment.yaml b/install/helm/gloo/templates/11-ingress-proxy-deployment.yaml
@@ -70,7 +70,7 @@ spec:
         {{- if .Values.ingressProxy.deployment.runAsUser  -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.ingressProxy.deployment.runAsUser }}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.ingressProxy.deployment.ingressProxyContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.ingressProxy.deployment.ingressProxyContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.ingressProxy.deployment.httpPort }}
           name: http

diff --git a/install/helm/gloo/templates/14-clusteringress-proxy-deployment.yaml b/install/helm/gloo/templates/14-clusteringress-proxy-deployment.yaml
@@ -64,7 +64,7 @@ spec:
             "allowPrivilegeEscalation" false
             "capabilities" $capabilities
         }}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.settings.integrations.knative.proxy.httpPort }}
           name: http
@@ -80,7 +80,7 @@ spec:
           name: clusteringress-envoy-config
         name: envoy-config
 
-{{- end }}
+{{- end }} {{/*  if (semverCompare "< 0.8.0" .Values.settings.integrations.knative.version ) */}}
 {{- end }} {{/* if .Values.settings.integrations.knative.enabled */}}
 {{- end }} {{/* define clusterIngressProxy.deploymentSpec */}}
 

diff --git a/install/helm/gloo/templates/19-gloo-mtls-certgen-cronjob.yaml b/install/helm/gloo/templates/19-gloo-mtls-certgen-cronjob.yaml
@@ -48,7 +48,7 @@ spec:
               {{- if not .Values.gateway.certGenJob.floatingUserId }}
                 {{- $_ := set $securityDefaults "runAsUser" .Values.gateway.certGenJob.runAsUser }}
               {{- end }}
-              {{- include "gloo.containerSecurityContext" (dict "values" .Values.gateway.certGenJob.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 14) }}
+              {{- include "gloo.containerSecurityContext" (dict "values" .Values.gateway.certGenJob.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 14 "globalSec" .Values.global.securitySettings) }}
               {{- with .Values.gateway.certGenJob.resources }}
               resources: {{ toYaml . | nindent 16 }}
               {{- end }}

diff --git a/install/helm/gloo/templates/19-gloo-mtls-certgen-job.yaml b/install/helm/gloo/templates/19-gloo-mtls-certgen-job.yaml
@@ -56,7 +56,7 @@ spec:
           {{- if not .Values.gateway.certGenJob.floatingUserId }}
             {{- $_ := set $securityDefaults "runAsUser" .Values.gateway.certGenJob.runAsUser }}
           {{- end }}
-          {{- include "gloo.containerSecurityContext" (dict "values" .Values.gateway.certGenJob.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 10) }}
+          {{- include "gloo.containerSecurityContext" (dict "values" .Values.gateway.certGenJob.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 10 "globalSec" .Values.global.securitySettings) }}
           {{- with .Values.gateway.certGenJob.resources }}
           resources: {{ toYaml . | nindent 12}}
           {{- end }}

diff --git a/install/helm/gloo/templates/26-knative-external-proxy-deployment.yaml b/install/helm/gloo/templates/26-knative-external-proxy-deployment.yaml
@@ -67,7 +67,7 @@ spec:
        {{- if .Values.settings.integrations.knative.proxy.runAsUser -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.settings.integrations.knative.proxy.runAsUser }}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.settings.integrations.knative.proxy.httpPort }}
           name: http

diff --git a/install/helm/gloo/templates/29-knative-internal-proxy-deployment.yaml b/install/helm/gloo/templates/29-knative-internal-proxy-deployment.yaml
@@ -67,7 +67,7 @@ spec:
        {{- if .Values.settings.integrations.knative.proxy.runAsUser -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.settings.integrations.knative.proxy.runAsUser }}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.settings.integrations.knative.proxy.containerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         ports:
         - containerPort: {{ .Values.settings.integrations.knative.proxy.httpPort }}
           name: http

diff --git a/install/helm/gloo/templates/3-discovery-deployment.yaml b/install/helm/gloo/templates/3-discovery-deployment.yaml
@@ -67,7 +67,7 @@ spec:
         {{- if not .Values.discovery.deployment.floatingUserId -}}
           {{- $_ := set $securityDefaults "runAsUser" .Values.discovery.deployment.runAsUser }}
         {{- end -}}
-        {{- include "gloo.containerSecurityContext" (dict "values" .Values.discovery.deployment.discoveryContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8) }}
+        {{- include "gloo.containerSecurityContext" (dict "values" .Values.discovery.deployment.discoveryContainerSecurityContext "defaults" $securityDefaults "podSecurityStandards" .Values.global.podSecurityStandards "indent" 8 "globalSec" .Values.global.securitySettings) }}
         env:
 {{- if .Values.license_secret_name }}
           - name: GLOO_LICENSE_KEY
@@ -96,15 +96,16 @@ spec:
           - name: http-monitoring
             containerPort: 9091
         {{- end }}
-      # Pod security context
-      {{- if .Values.discovery.deployment.enablePodSecurityContext }}
-      securityContext:
+      {{- /* Pod Security Context */ -}}
+      {{- if and .Values.discovery.deployment.enablePodSecurityContext (not .Values.global.securitySettings.floatingUserId) }}
+        {{- $securityDefaults := dict -}}
         {{- if .Values.discovery.deployment.fsGroup }}
-        fsGroup: {{ printf "%.0f" (float64 .Values.discovery.deployment.fsGroup) }}
-        {{- end}}
+          {{- $_ := set $securityDefaults "fsGroup" (int64 (printf "%.0f" (float64 .Values.discovery.deployment.fsGroup))) -}}
+        {{- end -}}
         {{- if .Values.discovery.deployment.runAsUser }}
-        runAsUser: {{ printf "%.0f" (float64 .Values.discovery.deployment.runAsUser) }}
+          {{- $_ := set $securityDefaults "runAsUser" (int64 (printf "%.0f" (float64 .Values.discovery.deployment.runAsUser))) -}}
         {{- end }}
+        {{- include "gloo.securityContext" (dict "defaults" $securityDefaults "indent" 6 "globalSec" .Values.global.securitySettings) -}}
       {{- end}}
 {{- end }}
 {{- end }} {{/* define "discovery.deploymentSpec"*/}}

diff --git a/install/helm/gloo/templates/43-gatewayparameters.yaml b/install/helm/gloo/templates/43-gatewayparameters.yaml
@@ -31,6 +31,7 @@ spec:
 {{- end }}{{/* if $gg.service */}}
     service:
       type: {{ $serviceType }}
+    floatingUserId: {{ hasKey $global.securitySettings "floatingUserId" | ternary $global.securitySettings.floatingUserId (or $gg.floatingUserId false) }}
     envoyContainer:
       image:
         {{- $envoyImage := dict -}}
@@ -116,4 +117,4 @@ spec:
 {{ toYaml $gg.aiExtension.ports | indent 6 }}
 {{- end }} {{/* if $gg.aiExtension.ports */}}
 {{- end }}{{/* if $gg.aiExtension */}}
-{{- end }}{{/* if .Values.kubeGateway.enabled */}}
+{{- end }}{{/* if .Values.kubeGateway.enabled */}}