Merge branch 'apache:master' into YUNIKORN-1901

.github/workflows/pre-commit.yml

@@ -4,6 +4,7 @@ on:
   pull_request:
     branches:
       - master
+  workflow_dispatch: {}
 
 jobs:
   build:

.golangci.yml

@@ -20,13 +20,7 @@ run:
   issues-exit-code: 1
   skip-dirs-use-default: true
   modules-download-mode: readonly
-  deadline: 10m
-  skip-dirs:
-    - pkg/client/clientset
-    - pkg/client/informers
-    - pkg/client/listers
-  skip-files:
-    - pkg/apis/yunikorn.apache.org/v1alpha1/zz_generated.deepcopy.go
+  timeout: 10m
 
 # settings of specific linters
 linters-settings:

Makefile

@@ -159,7 +159,6 @@ REGISTRY := apache
 endif
 
 # Force Go modules even when checked out inside GOPATH
-export ACK_GINKGO_DEPRECATIONS=2.9.0
 GO111MODULE := on
 export GO111MODULE
 
@@ -511,7 +510,7 @@ bench:
 fsm_graph:
 	@echo "generating FSM graphs"
 	"$(GO)" clean -testcache
-	"$(GO)" test -tags graphviz -run 'Test.*FsmGraph' ./pkg/shim ./pkg/cache
+	"$(GO)" test -tags graphviz -run 'Test.*FsmGraph' ./pkg/cache
 	scripts/generate-fsm-graph-images.sh
 
 # Remove generated build artifacts

deployments/examples/placements/README.md

@@ -28,12 +28,12 @@ The sleep pod is described in the example file and the partition is provided in
 
 ## How to adopt the queues in `config.yaml`
 Before deploying the pods, make sure that the data in the `yunikorn-configs` configmap is correct.
-For example, `queue.yaml` in the `yunikorn-configs` configmap should be updated before starting fixed example.
+For example, `queues.yaml` in the `yunikorn-configs` configmap should be updated before starting fixed example.
 `yunikorn-configs` configmap should contains following information.
----
-**_NOTE:_** 
-`queue.yaml` should be __full__ queue config and then legal configuration would be updated to Yunikorn.
----
+
+> **_NOTE:_**
+> `queues.yaml` should be __full__ queue config and then legal configuration would be updated to Yunikorn.
+
 ```
 yunikornDefaults:
   queues.yaml: |

deployments/examples/placements/fixed/config.yaml

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update this queue yaml to the yunikorn-configs configmap before trying this example.
-queue.yaml: |
+# Update the queues.yaml to the yunikorn-configs configmap before trying this example.
+queues.yaml: |
   partitions:
     - name: default
       placementrules:
@@ -27,4 +27,3 @@ queue.yaml: |
         submitacl: '*'
         queues:
         - name: last_resort
-

deployments/examples/placements/fixed/fixed_example.yaml

@@ -20,9 +20,10 @@ kind: Pod
 metadata:
   labels:
     app: sleep
-    applicationId: "prvoided-rule-example01"
+    applicationId: "fixed-rule-example01"
     queue: "my_special_queue"
-    yunikorn.apache.org/username: developer
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"developer\"}"
   name: task0
 spec:
   schedulerName: yunikorn
@@ -34,4 +35,3 @@ spec:
         requests:
           cpu: "100m"
           memory: "500M"
-

deployments/examples/placements/provided/config.yaml

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update this queue yaml to the yunikorn-configs configmap before trying this example.
-queue.yaml: |
+# Update the queues.yaml to the yunikorn-configs configmap before trying this example.
+queues.yaml: |
   partitions:
     - name: default
       placementrules:
@@ -28,4 +28,3 @@ queue.yaml: |
       queues:
       - name: root
         submitacl: '*'
-

deployments/examples/placements/provided/provided_example.yaml

@@ -22,7 +22,8 @@ metadata:
     app: sleep
     applicationId: "prvoided-rule-example01"
     queue: "my_special_queue"
-    yunikorn.apache.org/username: "developer"
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"developer\"}"
   name: task0
 spec:
   schedulerName: yunikorn
@@ -34,4 +35,3 @@ spec:
         requests:
           cpu: "100m"
           memory: "500M"
-

deployments/examples/placements/tag/config.yaml

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update this queue yaml to the yunikorn-configs configmap before trying this example.
-queue.yaml: |
+# Update the queues.yaml to the yunikorn-configs configmap before trying this example.
+queues.yaml: |
   partitions:
     - name: default
       placementrules:
@@ -26,4 +26,3 @@ queue.yaml: |
       queues:
       - name: root
         submitacl: '*'
-

deployments/examples/placements/tag/tag_example.yaml

@@ -20,8 +20,9 @@ kind: Pod
 metadata:
   labels:
     app: sleep
-    applicationId: "prvoided-rule-example01"
-    yunikorn.apache.org/username: developer
+    applicationId: "tag-rule-example01"
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"developer\"}"
   name: task0
 spec:
   schedulerName: yunikorn
@@ -40,8 +41,9 @@ metadata:
   namespace: testing
   labels:
     app: sleep
-    applicationId: "prvoided-rule-example02"
-    yunikorn.apache.org/username: developer
+    applicationId: "tag-rule-example02"
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"developer\"}"
   name: task1
 spec:
   schedulerName: yunikorn
@@ -53,4 +55,3 @@ spec:
         requests:
           cpu: "100m"
           memory: "500M"
-

deployments/examples/placements/username/config.yaml

@@ -15,8 +15,8 @@
 # See the License for the specific language governing permissions and
 # limitations under the License.
 
-# Update this queue yaml to the yunikorn-configs configmap before trying this example.
-queue.yaml: |
+# Update the queues.yaml to the yunikorn-configs configmap before trying this example.
+queues.yaml: |
   partitions:
     - name: default
       placementrules:
@@ -27,4 +27,3 @@ queue.yaml: |
         submitacl: '*'
         queues:
         - name: finance_dot_test
-

deployments/examples/placements/username/username_example.yaml

@@ -21,7 +21,8 @@ metadata:
   labels:
     app: sleep
     applicationId: "username-rule-example01"
-    yunikorn.apache.org/username: finance.test
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"finance.test\"}"
   name: task0
 spec:
   schedulerName: yunikorn
@@ -40,7 +41,8 @@ metadata:
   labels:
     app: sleep
     applicationId: "username-rule-example02"
-    yunikorn.apache.org/username: developer
+  annotations:
+    yunikorn.apache.org/user.info: "{\"user\": \"developer\"}"
   name: task1
 spec:
   schedulerName: yunikorn
@@ -52,4 +54,3 @@ spec:
         requests:
           cpu: "100m"
           memory: "500M"
-

go.mod

@@ -22,8 +22,8 @@ go 1.20
 
 require (
 	github.com/GoogleCloudPlatform/spark-on-k8s-operator v0.0.0-20201215015655-2e8b733f5ad0
-	github.com/apache/yunikorn-core v0.0.0-20230831174655-2feb6ecc1384
-	github.com/apache/yunikorn-scheduler-interface v0.0.0-20230831133811-e7622cf54e95
+	github.com/apache/yunikorn-core v0.0.0-20231025072825-bc7c00124c4c
+	github.com/apache/yunikorn-scheduler-interface v0.0.0-20231020041412-6f80d179257c
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.3.0
 	github.com/looplab/fsm v1.0.1
@@ -52,7 +52,7 @@ require (
 	github.com/asaskevich/govalidator v0.0.0-20190424111038-f61b66f89f4a // indirect
 	github.com/beorn7/perks v1.0.1 // indirect
 	github.com/blang/semver/v4 v4.0.0 // indirect
-	github.com/cenkalti/backoff/v4 v4.1.3 // indirect
+	github.com/cenkalti/backoff/v4 v4.2.1 // indirect
 	github.com/cespare/xxhash/v2 v2.2.0 // indirect
 	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/coreos/go-systemd/v22 v22.4.0 // indirect
@@ -80,7 +80,7 @@ require (
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
 	github.com/gregjones/httpcache v0.0.0-20190212212710-3befbb6ad0cc // indirect
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 // indirect
-	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.16.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
 	github.com/inconshreveable/mousetrap v1.0.1 // indirect
 	github.com/josharian/intern v1.0.0 // indirect
@@ -115,31 +115,32 @@ require (
 	go.etcd.io/etcd/client/pkg/v3 v3.5.7 // indirect
 	go.etcd.io/etcd/client/v3 v3.5.7 // indirect
 	go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.35.0 // indirect
-	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.35.1 // indirect
-	go.opentelemetry.io/otel v1.10.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.10.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.10.0 // indirect
-	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.10.0 // indirect
-	go.opentelemetry.io/otel/metric v0.31.0 // indirect
-	go.opentelemetry.io/otel/sdk v1.10.0 // indirect
-	go.opentelemetry.io/otel/trace v1.10.0 // indirect
-	go.opentelemetry.io/proto/otlp v0.19.0 // indirect
+	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.44.0 // indirect
+	go.opentelemetry.io/otel v1.19.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 // indirect
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 // indirect
+	go.opentelemetry.io/otel/metric v1.19.0 // indirect
+	go.opentelemetry.io/otel/sdk v1.19.0 // indirect
+	go.opentelemetry.io/otel/trace v1.19.0 // indirect
+	go.opentelemetry.io/proto/otlp v1.0.0 // indirect
 	go.starlark.net v0.0.0-20200306205701-8dd3e2ee1dd5 // indirect
 	go.uber.org/atomic v1.7.0 // indirect
 	go.uber.org/multierr v1.6.0 // indirect
-	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/net v0.14.0 // indirect
-	golang.org/x/oauth2 v0.7.0 // indirect
+	golang.org/x/crypto v0.14.0 // indirect
+	golang.org/x/net v0.17.0 // indirect
+	golang.org/x/oauth2 v0.10.0 // indirect
 	golang.org/x/sync v0.3.0 // indirect
-	golang.org/x/sys v0.11.0 // indirect
-	golang.org/x/term v0.11.0 // indirect
-	golang.org/x/text v0.12.0 // indirect
+	golang.org/x/sys v0.13.0 // indirect
+	golang.org/x/term v0.13.0 // indirect
+	golang.org/x/text v0.13.0 // indirect
 	golang.org/x/time v0.3.0 // indirect
-	golang.org/x/tools v0.9.3 // indirect
+	golang.org/x/tools v0.13.0 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
-	google.golang.org/genproto v0.0.0-20230410155749-daa745c078e1 // indirect
-	google.golang.org/grpc v1.56.0 // indirect
-	google.golang.org/protobuf v1.30.0 // indirect
+	google.golang.org/genproto v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/api v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/genproto/googleapis/rpc v0.0.0-20230711160842-782d3b101e98 // indirect
+	google.golang.org/grpc v1.58.3 // indirect
+	google.golang.org/protobuf v1.31.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/natefinch/lumberjack.v2 v2.0.0 // indirect
 	gopkg.in/yaml.v2 v2.4.0 // indirect
@@ -163,12 +164,12 @@ require (
 )
 
 replace (
-	golang.org/x/crypto => golang.org/x/crypto v0.12.0
+	golang.org/x/crypto => golang.org/x/crypto v0.14.0
 	golang.org/x/lint => golang.org/x/lint v0.0.0-20210508222113-6edffad5e616
-	golang.org/x/net => golang.org/x/net v0.14.0
-	golang.org/x/sys => golang.org/x/sys v0.12.0
+	golang.org/x/net => golang.org/x/net v0.17.0
+	golang.org/x/sys => golang.org/x/sys v0.13.0
 	golang.org/x/text => golang.org/x/text v0.13.0
-	golang.org/x/tools => golang.org/x/tools v0.12.0
+	golang.org/x/tools => golang.org/x/tools v0.13.0
 	k8s.io/api => k8s.io/api v0.27.3
 	k8s.io/apiextensions-apiserver => k8s.io/apiextensions-apiserver v0.27.3
 	k8s.io/apimachinery => k8s.io/apimachinery v0.27.3