Merge branch 'master' into test_coverage_submit.go

.github/workflows/build.yml

@@ -30,8 +30,6 @@ jobs:
           DOCKER_REPO: "gresearch"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
-          DOCKER_BUILDX_CACHE_FROM: "type=gha"
-          DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max"
 
       - name: Output full commit sha
         if: github.event_name == 'push' && github.ref == 'refs/heads/master'

.github/workflows/release.yml

@@ -90,8 +90,6 @@ jobs:
           DOCKER_REPO: "gresearch"
           GITHUB_TOKEN: "${{ secrets.GITHUB_TOKEN }}"
           DOCKER_BUILDX_BUILDER: "${{ steps.buildx.outputs.name }}"
-          DOCKER_BUILDX_CACHE_FROM: "type=gha"
-          DOCKER_BUILDX_CACHE_TO: "type=gha,mode=max"
   invoke-chart-push:
     name: Invoke Chart push
     needs: release

.goreleaser.yml

@@ -19,11 +19,6 @@ env:
   # To use a builder other than "default", set this variable.
   # Necessary for, e.g., GitHub actions cache integration.
   - DOCKER_BUILDX_BUILDER={{ if index .Env "DOCKER_BUILDX_BUILDER"  }}{{ .Env.DOCKER_BUILDX_BUILDER }}{{ else }}default{{ end }}
-  # Setup to enable Docker to use, e.g., the GitHub actions cache; see
-  # https://docs.docker.com/build/building/cache/backends/
-  # https://github.com/moby/buildkit#export-cache
-  - DOCKER_BUILDX_CACHE_FROM={{ if index .Env "DOCKER_BUILDX_CACHE_FROM"  }}{{ .Env.DOCKER_BUILDX_CACHE_FROM }}{{ else }}type=inline{{ end }}
-  - DOCKER_BUILDX_CACHE_TO={{ if index .Env "DOCKER_BUILDX_CACHE_TO"  }}{{ .Env.DOCKER_BUILDX_CACHE_TO }}{{ else }}type=inline{{ end }}
   - GOVERSION={{ if index .Env "GOVERSION"  }}{{ .Env.GOVERSION }}{{ else }}go1.20{{ end }}
 
 builds:
@@ -205,8 +200,6 @@ dockers:
       - "{{ .Env.DOCKER_REPO }}armada-bundle:{{ .Version }}"
     build_flag_templates: &BUILD_FLAG_TEMPLATES
       - --builder={{ .Env.DOCKER_BUILDX_BUILDER }}
-      - --cache-to={{ .Env.DOCKER_BUILDX_CACHE_TO }}
-      - --cache-from={{ .Env.DOCKER_BUILDX_CACHE_FROM }}
       - --label=org.opencontainers.image.source=https://github.com/armadaproject/armada
       - --label=org.opencontainers.image.version={{ .Version }}
       - --label=org.opencontainers.image.created={{ time "2006-01-02T15:04:05Z07:00" }}

client/DotNet/Armada.Client/ClientGenerated.cs

@@ -2077,6 +2077,31 @@ public partial class ApiJobUtilisationEvent
         public System.Collections.Generic.IDictionary<string, string> TotalCumulativeUsage { get; set; }
 
 
+    }
+
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
+    public partial class ApiPriorityClassPoolResourceLimits 
+    {
+        [Newtonsoft.Json.JsonProperty("maximumResourceFraction", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, double> MaximumResourceFraction { get; set; }
+
+
+    }
+
+    [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
+    public partial class ApiPriorityClassResourceLimits 
+    {
+        /// <summary>Limits resources assigned to jobs of this priority class.
+        /// Specifically, jobs of this priority class are only scheduled if doing so does not exceed this limit.</summary>
+        [Newtonsoft.Json.JsonProperty("maximumResourceFraction", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, double> MaximumResourceFraction { get; set; }
+
+        /// <summary>Per-pool override of maximum_resource_fraction.
+        /// If missing for a particular pool, maximum_resource_fraction is used instead for that pool.</summary>
+        [Newtonsoft.Json.JsonProperty("maximumResourceFractionByPool", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, ApiPriorityClassPoolResourceLimits> MaximumResourceFractionByPool { get; set; }
+
+
     }
 
     [System.CodeDom.Compiler.GeneratedCode("NJsonSchema", "10.0.27.0 (Newtonsoft.Json v12.0.0.0)")]
@@ -2097,6 +2122,11 @@ public partial class ApiQueue
         [Newtonsoft.Json.JsonProperty("resourceLimits", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public System.Collections.Generic.IDictionary<string, double> ResourceLimits { get; set; }
 
+        /// <summary>Map from priority class name to resource limit overrides for this queue and priority class.
+        /// If provided for a priority class, global limits for that priority class do not apply to this queue.</summary>
+        [Newtonsoft.Json.JsonProperty("resourceLimitsByPriorityClassName", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
+        public System.Collections.Generic.IDictionary<string, ApiPriorityClassResourceLimits> ResourceLimitsByPriorityClassName { get; set; }
+
         [Newtonsoft.Json.JsonProperty("userOwners", Required = Newtonsoft.Json.Required.Default, NullValueHandling = Newtonsoft.Json.NullValueHandling.Ignore)]
         public System.Collections.Generic.ICollection<string> UserOwners { get; set; }
 

cmd/armadactl/cmd/queue.go

@@ -2,7 +2,6 @@ package cmd
 
 import (
 	"fmt"
-	"strconv"
 
 	"github.com/spf13/cobra"
 
@@ -47,17 +46,11 @@ Job priority is evaluated inside queue, queue has its own priority.`,
 				return fmt.Errorf("error reading groupOwners: %s", err)
 			}
 
-			resourceLimits, err := flagGetStringToString(cmd.Flags().GetStringToString).toFloat64("resourceLimits")
-			if err != nil {
-				return fmt.Errorf("error reading resourceLimits: %s", err)
-			}
-
 			queue, err := queue.NewQueue(&api.Queue{
 				Name:           name,
 				PriorityFactor: priorityFactor,
 				UserOwners:     owners,
 				GroupOwners:    groups,
-				ResourceLimits: resourceLimits,
 			})
 			if err != nil {
 				return fmt.Errorf("invalid queue data: %s", err)
@@ -69,9 +62,6 @@ Job priority is evaluated inside queue, queue has its own priority.`,
 	cmd.Flags().Float64("priorityFactor", 1, "Set queue priority factor - lower number makes queue more important, must be > 0.")
 	cmd.Flags().StringSlice("owners", []string{}, "Comma separated list of queue owners, defaults to current user.")
 	cmd.Flags().StringSlice("groupOwners", []string{}, "Comma separated list of queue group owners, defaults to empty list.")
-	cmd.Flags().StringToString("resourceLimits", map[string]string{},
-		"Command separated list of resource limits pairs, defaults to empty list.\nExample: --resourceLimits cpu=0.3,memory=0.2",
-	)
 	return cmd
 }
 
@@ -151,17 +141,11 @@ func queueUpdateCmdWithApp(a *armadactl.App) *cobra.Command {
 				return fmt.Errorf("error reading groupOwners: %s", err)
 			}
 
-			resourceLimits, err := flagGetStringToString(cmd.Flags().GetStringToString).toFloat64("resourceLimits")
-			if err != nil {
-				return fmt.Errorf("error reading resourceLimits: %s", err)
-			}
-
 			queue, err := queue.NewQueue(&api.Queue{
 				Name:           name,
 				PriorityFactor: priorityFactor,
 				UserOwners:     owners,
 				GroupOwners:    groups,
-				ResourceLimits: resourceLimits,
 			})
 			if err != nil {
 				return fmt.Errorf("invalid queue data: %s", err)
@@ -174,28 +158,5 @@ func queueUpdateCmdWithApp(a *armadactl.App) *cobra.Command {
 	cmd.Flags().Float64("priorityFactor", 1, "Set queue priority factor - lower number makes queue more important, must be > 0.")
 	cmd.Flags().StringSlice("owners", []string{}, "Comma separated list of queue owners, defaults to current user.")
 	cmd.Flags().StringSlice("groupOwners", []string{}, "Comma separated list of queue group owners, defaults to empty list.")
-	cmd.Flags().StringToString("resourceLimits", map[string]string{},
-		"Command separated list of resource limits pairs, defaults to empty list. Example: --resourceLimits cpu=0.3,memory=0.2",
-	)
 	return cmd
 }
-
-type flagGetStringToString func(string) (map[string]string, error)
-
-func (f flagGetStringToString) toFloat64(flagName string) (map[string]float64, error) {
-	limits, err := f(flagName)
-	if err != nil {
-		return nil, err
-	}
-
-	result := make(map[string]float64, len(limits))
-	for resourceName, limit := range limits {
-		limitFloat, err := strconv.ParseFloat(limit, 64)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse %s as float64. %s", resourceName, err)
-		}
-		result[resourceName] = limitFloat
-	}
-
-	return result, nil
-}

cmd/armadactl/cmd/queue_test.go

@@ -40,13 +40,11 @@ func TestCreate(t *testing.T) {
 		PriorityFactor *float64
 		Owners         []string
 		GroupOwners    []string
-		ResourceLimits map[string]float64
 	}{
-		"default flags":         {nil, nil, nil, nil, nil},
-		"valid priority":        {[]flag{{"priorityFactor", "1.0"}}, makeFloat64Pointer(1.0), nil, nil, nil},
-		"valid owners":          {[]flag{{"owners", "user1,user2"}}, nil, []string{"user1", "user2"}, nil, nil},
-		"valid group owners":    {[]flag{{"groupOwners", "group1,group2"}}, nil, nil, []string{"group1", "group2"}, nil},
-		"valid resource limits": {[]flag{{"resourceLimits", "cpu=0.3,memory=0.2"}}, nil, nil, nil, map[string]float64{"cpu": 0.3, "memory": 0.2}},
+		"default flags":      {nil, nil, nil, nil},
+		"valid priority":     {[]flag{{"priorityFactor", "1.0"}}, makeFloat64Pointer(1.0), nil, nil},
+		"valid owners":       {[]flag{{"owners", "user1,user2"}}, nil, []string{"user1", "user2"}, nil},
+		"valid group owners": {[]flag{{"groupOwners", "group1,group2"}}, nil, nil, []string{"group1", "group2"}},
 	}
 
 	for name, test := range tests {
@@ -75,11 +73,6 @@ func TestCreate(t *testing.T) {
 						require.True(t, reflect.DeepEqual(q.Permissions, permissions))
 					}
 
-					if test.ResourceLimits != nil {
-						for resourceName, resourceLimit := range q.ResourceLimits {
-							require.Equal(t, test.ResourceLimits[string(resourceName)], float64(resourceLimit), "resource limit mismatch")
-						}
-					}
 					return nil
 				}
 				return nil
@@ -127,13 +120,11 @@ func TestUpdate(t *testing.T) {
 		PriorityFactor *float64
 		Owners         []string
 		GroupOwners    []string
-		ResourceLimits map[string]float64
 	}{
-		"default flags":         {nil, nil, nil, nil, nil},
-		"valid priority":        {[]flag{{"priorityFactor", "1.0"}}, makeFloat64Pointer(1.0), nil, nil, nil},
-		"valid owners":          {[]flag{{"owners", "user1,user2"}}, nil, []string{"user1", "user2"}, nil, nil},
-		"valid group owners":    {[]flag{{"groupOwners", "group1,group2"}}, nil, nil, []string{"group1", "group2"}, nil},
-		"valid resource limits": {[]flag{{"resourceLimits", "cpu=0.3,memory=0.2"}}, nil, nil, nil, map[string]float64{"cpu": 0.3, "memory": 0.2}},
+		"default flags":      {nil, nil, nil, nil},
+		"valid priority":     {[]flag{{"priorityFactor", "1.0"}}, makeFloat64Pointer(1.0), nil, nil},
+		"valid owners":       {[]flag{{"owners", "user1,user2"}}, nil, []string{"user1", "user2"}, nil},
+		"valid group owners": {[]flag{{"groupOwners", "group1,group2"}}, nil, nil, []string{"group1", "group2"}},
 	}
 
 	for name, test := range tests {
@@ -159,12 +150,6 @@ func TestUpdate(t *testing.T) {
 					if test.Owners != nil {
 						require.True(t, reflect.DeepEqual(q.Permissions, permissions))
 					}
-
-					if test.ResourceLimits != nil {
-						for resourceName, resourceLimit := range q.ResourceLimits {
-							require.Equal(t, test.ResourceLimits[string(resourceName)], float64(resourceLimit), "resource limit mismatch")
-						}
-					}
 					return nil
 				}
 				return nil

config/scheduler/config.yaml

@@ -24,6 +24,10 @@ schedulerMetrics:
     - "memory"
     - "ephemeral-storage"
     - "nvidia.com/gpu"
+  resourceRenaming:
+    nvidia.com/gpu: "gpu"
+    amd.com/gpu: "gpu"
+    ephemeral-storage: "ephemeralStorage"
   matchedRegexIndexByErrorMessageCacheSize: 100
   resetInterval: "1h"
 pulsar:

go.mod

@@ -22,14 +22,14 @@ require (
 	github.com/golang/protobuf v1.5.3
 	github.com/google/go-cmp v0.5.9
 	github.com/google/uuid v1.4.0
-	github.com/grpc-ecosystem/go-grpc-middleware v1.3.0
+	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/grpc-ecosystem/grpc-gateway v1.16.0
 	github.com/hashicorp/go-memdb v1.3.4
 	github.com/hashicorp/go-multierror v1.1.1
 	github.com/hashicorp/golang-lru v1.0.2
 	github.com/instrumenta/kubeval v0.0.0-20190918223246-8d013ec9fc56
-	github.com/jackc/pgtype v1.14.1
+	github.com/jackc/pgtype v1.14.2
 	github.com/jackc/pgx/v4 v4.17.2 // indirect
 	github.com/jolestar/go-commons-pool v2.0.0+incompatible
 	github.com/jstemmer/go-junit-report/v2 v2.0.0
@@ -50,8 +50,8 @@ require (
 	github.com/stretchr/testify v1.8.4
 	github.com/weaveworks/promrus v1.2.0
 	golang.org/x/exp v0.0.0-20240222234643-814bf88cf225
-	golang.org/x/net v0.21.0
-	golang.org/x/oauth2 v0.16.0
+	golang.org/x/net v0.22.0
+	golang.org/x/oauth2 v0.18.0
 	golang.org/x/sync v0.6.0
 	golang.org/x/tools v0.18.0 // indirect
 	google.golang.org/genproto v0.0.0-20230526161137-0005af68ea54 // indirect
@@ -91,7 +91,6 @@ require (
 	golang.org/x/time v0.3.0
 	gonum.org/v1/gonum v0.14.0
 	google.golang.org/genproto/googleapis/api v0.0.0-20230525234035-dd9d682886f9
-	gopkg.in/yaml.v3 v3.0.1
 )
 
 require (
@@ -185,10 +184,10 @@ require (
 	github.com/xitongsys/parquet-go-source v0.0.0-20200817004010-026bad9b25d0 // indirect
 	go.mongodb.org/mongo-driver v1.13.1 // indirect
 	go.uber.org/atomic v1.9.0 // indirect
-	golang.org/x/crypto v0.19.0 // indirect
+	golang.org/x/crypto v0.21.0 // indirect
 	golang.org/x/mod v0.15.0 // indirect
-	golang.org/x/sys v0.17.0 // indirect
-	golang.org/x/term v0.17.0 // indirect
+	golang.org/x/sys v0.18.0 // indirect
+	golang.org/x/term v0.18.0 // indirect
 	golang.org/x/text v0.14.0 // indirect
 	golang.org/x/xerrors v0.0.0-20220907171357-04be3eba64a2 // indirect
 	google.golang.org/appengine v1.6.7 // indirect
@@ -197,6 +196,7 @@ require (
 	gopkg.in/inf.v0 v0.9.1 // indirect
 	gopkg.in/ini.v1 v1.67.0 // indirect
 	gopkg.in/square/go-jose.v2 v2.6.0 // indirect
+	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/klog/v2 v2.100.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20211109043538-20434351676c // indirect
 	lukechampine.com/uint128 v1.2.0 // indirect