From 3ec16cccd31cf80ee2448d2a78070ce9cf461923 Mon Sep 17 00:00:00 2001 From: peng9808 Date: Fri, 14 Jun 2024 13:47:37 +0800 Subject: [PATCH] Added dataset-manager installation module Signed-off-by: peng9808 --- .gitignore | 7 +- Dockerfile | 1 + Makefile | 8 +- api/v1alpha1/cluster_types.go | 29 + api/v1alpha1/zz_generated.deepcopy.go | 115 + config/crd/bases/hwameistor.io_clusters.yaml | 1346 ++- controllers/cluster_controller.go | 20 + go.mod | 19 +- go.sum | 39 +- helm/operator/templates/_helpers.tpl | 10 + .../operator/templates/hwameistorcluster.yaml | 19 +- helm/operator/values.extra.prod.yaml | 19 + helm/operator/values.yaml | 15 + .../dataloadmanager/dataloadmanager.go | 283 + pkg/install/datasetmanager/datasetmanager.go | 188 + tool.go | 4 + vendor/github.com/google/uuid/.travis.yml | 9 - vendor/github.com/google/uuid/CHANGELOG.md | 28 + vendor/github.com/google/uuid/CONTRIBUTING.md | 16 + vendor/github.com/google/uuid/README.md | 10 +- vendor/github.com/google/uuid/hash.go | 4 +- vendor/github.com/google/uuid/node_js.go | 2 +- vendor/github.com/google/uuid/null.go | 118 + vendor/github.com/google/uuid/sql.go | 2 +- vendor/github.com/google/uuid/time.go | 21 +- vendor/github.com/google/uuid/uuid.go | 144 +- vendor/github.com/google/uuid/version4.go | 35 +- vendor/github.com/google/uuid/version6.go | 56 + vendor/github.com/google/uuid/version7.go | 75 + .../github.com/hwameistor/datastore/LICENSE | 202 + .../crds/datastore.io_basemodels_crd.yaml | 107 + .../crds/datastore.io_checkpoints_crd.yaml | 112 + .../datastore.io_dataloadrequests_crd.yaml | 84 + .../crds/datastore.io_datasets_crd.yaml | 152 + .../datastore/deploy/crds/module.go | 4 + vendor/github.com/sirupsen/logrus/README.md | 8 +- vendor/github.com/sirupsen/logrus/writer.go | 34 +- .../x/crypto/pkcs12/internal/rc2/rc2.go | 53 +- vendor/golang.org/x/net/context/go17.go | 1 - vendor/golang.org/x/net/context/go19.go | 1 - vendor/golang.org/x/net/context/pre_go17.go | 1 - vendor/golang.org/x/net/context/pre_go19.go | 1 - vendor/golang.org/x/net/html/doc.go | 21 + vendor/golang.org/x/net/html/escape.go | 81 + vendor/golang.org/x/net/html/render.go | 30 +- vendor/golang.org/x/net/html/token.go | 19 +- vendor/golang.org/x/net/http2/Dockerfile | 51 - vendor/golang.org/x/net/http2/Makefile | 3 - vendor/golang.org/x/net/http2/databuffer.go | 59 +- vendor/golang.org/x/net/http2/go111.go | 30 - vendor/golang.org/x/net/http2/go115.go | 27 - vendor/golang.org/x/net/http2/go118.go | 17 - vendor/golang.org/x/net/http2/not_go111.go | 21 - vendor/golang.org/x/net/http2/not_go115.go | 31 - vendor/golang.org/x/net/http2/not_go118.go | 17 - vendor/golang.org/x/net/http2/pipe.go | 6 +- vendor/golang.org/x/net/http2/server.go | 126 +- vendor/golang.org/x/net/http2/transport.go | 114 +- vendor/golang.org/x/net/http2/writesched.go | 3 +- .../x/net/http2/writesched_roundrobin.go | 119 + vendor/golang.org/x/net/idna/go118.go | 1 - vendor/golang.org/x/net/idna/idna10.0.0.go | 1 - vendor/golang.org/x/net/idna/idna9.0.0.go | 3 +- vendor/golang.org/x/net/idna/pre_go118.go | 1 - vendor/golang.org/x/net/idna/tables10.0.0.go | 1 - vendor/golang.org/x/net/idna/tables11.0.0.go | 1 - vendor/golang.org/x/net/idna/tables12.0.0.go | 1 - vendor/golang.org/x/net/idna/tables13.0.0.go | 2987 ++++--- vendor/golang.org/x/net/idna/tables15.0.0.go | 5144 +++++++++++ vendor/golang.org/x/net/idna/tables9.0.0.go | 1 - vendor/golang.org/x/net/idna/trie.go | 21 - vendor/golang.org/x/net/idna/trie12.0.0.go | 30 + vendor/golang.org/x/net/idna/trie13.0.0.go | 30 + .../golang.org/x/net/internal/socks/socks.go | 2 +- .../sys/internal/unsafeheader/unsafeheader.go | 30 - .../golang.org/x/sys/plan9/pwd_go15_plan9.go | 1 - vendor/golang.org/x/sys/plan9/pwd_plan9.go | 1 - vendor/golang.org/x/sys/plan9/race.go | 1 - vendor/golang.org/x/sys/plan9/race0.go | 1 - vendor/golang.org/x/sys/plan9/str.go | 1 - vendor/golang.org/x/sys/plan9/syscall.go | 1 - .../x/sys/plan9/zsyscall_plan9_386.go | 1 - .../x/sys/plan9/zsyscall_plan9_amd64.go | 1 - .../x/sys/plan9/zsyscall_plan9_arm.go | 1 - vendor/golang.org/x/sys/unix/aliases.go | 2 - vendor/golang.org/x/sys/unix/asm_aix_ppc64.s | 1 - vendor/golang.org/x/sys/unix/asm_bsd_386.s | 2 - vendor/golang.org/x/sys/unix/asm_bsd_amd64.s | 2 - vendor/golang.org/x/sys/unix/asm_bsd_arm.s | 2 - vendor/golang.org/x/sys/unix/asm_bsd_arm64.s | 2 - vendor/golang.org/x/sys/unix/asm_bsd_ppc64.s | 2 - .../golang.org/x/sys/unix/asm_bsd_riscv64.s | 2 - vendor/golang.org/x/sys/unix/asm_linux_386.s | 1 - .../golang.org/x/sys/unix/asm_linux_amd64.s | 1 - vendor/golang.org/x/sys/unix/asm_linux_arm.s | 1 - .../golang.org/x/sys/unix/asm_linux_arm64.s | 3 - .../golang.org/x/sys/unix/asm_linux_loong64.s | 3 - .../golang.org/x/sys/unix/asm_linux_mips64x.s | 3 - .../golang.org/x/sys/unix/asm_linux_mipsx.s | 3 - .../golang.org/x/sys/unix/asm_linux_ppc64x.s | 3 - .../golang.org/x/sys/unix/asm_linux_riscv64.s | 2 - .../golang.org/x/sys/unix/asm_linux_s390x.s | 3 - .../x/sys/unix/asm_openbsd_mips64.s | 1 - .../golang.org/x/sys/unix/asm_solaris_amd64.s | 1 - vendor/golang.org/x/sys/unix/asm_zos_s390x.s | 3 - vendor/golang.org/x/sys/unix/cap_freebsd.go | 1 - vendor/golang.org/x/sys/unix/constants.go | 1 - vendor/golang.org/x/sys/unix/dev_aix_ppc.go | 1 - vendor/golang.org/x/sys/unix/dev_aix_ppc64.go | 1 - vendor/golang.org/x/sys/unix/dev_zos.go | 1 - vendor/golang.org/x/sys/unix/dirent.go | 1 - vendor/golang.org/x/sys/unix/endian_big.go | 1 - vendor/golang.org/x/sys/unix/endian_little.go | 1 - vendor/golang.org/x/sys/unix/env_unix.go | 1 - vendor/golang.org/x/sys/unix/epoll_zos.go | 1 - vendor/golang.org/x/sys/unix/fcntl.go | 3 +- .../x/sys/unix/fcntl_linux_32bit.go | 1 - vendor/golang.org/x/sys/unix/fdset.go | 1 - vendor/golang.org/x/sys/unix/fstatfs_zos.go | 1 - vendor/golang.org/x/sys/unix/gccgo.go | 1 - vendor/golang.org/x/sys/unix/gccgo_c.c | 1 - .../x/sys/unix/gccgo_linux_amd64.go | 1 - vendor/golang.org/x/sys/unix/ifreq_linux.go | 1 - vendor/golang.org/x/sys/unix/ioctl_linux.go | 5 + vendor/golang.org/x/sys/unix/ioctl_signed.go | 69 + .../sys/unix/{ioctl.go => ioctl_unsigned.go} | 20 +- vendor/golang.org/x/sys/unix/ioctl_zos.go | 21 +- vendor/golang.org/x/sys/unix/mkall.sh | 2 +- vendor/golang.org/x/sys/unix/mkerrors.sh | 18 +- vendor/golang.org/x/sys/unix/mmap_nomremap.go | 13 + vendor/golang.org/x/sys/unix/mremap.go | 52 + vendor/golang.org/x/sys/unix/pagesize_unix.go | 1 - .../golang.org/x/sys/unix/pledge_openbsd.go | 92 +- vendor/golang.org/x/sys/unix/ptrace_darwin.go | 1 - vendor/golang.org/x/sys/unix/ptrace_ios.go | 1 - vendor/golang.org/x/sys/unix/race.go | 1 - vendor/golang.org/x/sys/unix/race0.go | 1 - .../x/sys/unix/readdirent_getdents.go | 1 - .../x/sys/unix/readdirent_getdirentries.go | 1 - vendor/golang.org/x/sys/unix/sockcmsg_unix.go | 1 - .../x/sys/unix/sockcmsg_unix_other.go | 1 - vendor/golang.org/x/sys/unix/syscall.go | 1 - vendor/golang.org/x/sys/unix/syscall_aix.go | 28 +- .../golang.org/x/sys/unix/syscall_aix_ppc.go | 2 - .../x/sys/unix/syscall_aix_ppc64.go | 2 - vendor/golang.org/x/sys/unix/syscall_bsd.go | 20 +- .../golang.org/x/sys/unix/syscall_darwin.go | 251 +- .../x/sys/unix/syscall_darwin_amd64.go | 1 - .../x/sys/unix/syscall_darwin_arm64.go | 1 - .../x/sys/unix/syscall_darwin_libSystem.go | 1 - .../x/sys/unix/syscall_dragonfly.go | 200 +- .../x/sys/unix/syscall_dragonfly_amd64.go | 1 - .../golang.org/x/sys/unix/syscall_freebsd.go | 236 +- .../x/sys/unix/syscall_freebsd_386.go | 18 +- .../x/sys/unix/syscall_freebsd_amd64.go | 18 +- .../x/sys/unix/syscall_freebsd_arm.go | 16 +- .../x/sys/unix/syscall_freebsd_arm64.go | 16 +- .../x/sys/unix/syscall_freebsd_riscv64.go | 16 +- vendor/golang.org/x/sys/unix/syscall_hurd.go | 9 +- .../golang.org/x/sys/unix/syscall_hurd_386.go | 1 - .../golang.org/x/sys/unix/syscall_illumos.go | 1 - vendor/golang.org/x/sys/unix/syscall_linux.go | 299 +- .../x/sys/unix/syscall_linux_386.go | 28 - .../x/sys/unix/syscall_linux_alarm.go | 2 - .../x/sys/unix/syscall_linux_amd64.go | 4 +- .../x/sys/unix/syscall_linux_amd64_gc.go | 1 - .../x/sys/unix/syscall_linux_arm.go | 28 - .../x/sys/unix/syscall_linux_arm64.go | 13 +- .../golang.org/x/sys/unix/syscall_linux_gc.go | 1 - .../x/sys/unix/syscall_linux_gc_386.go | 1 - .../x/sys/unix/syscall_linux_gc_arm.go | 1 - .../x/sys/unix/syscall_linux_gccgo_386.go | 1 - .../x/sys/unix/syscall_linux_gccgo_arm.go | 1 - .../x/sys/unix/syscall_linux_loong64.go | 8 +- .../x/sys/unix/syscall_linux_mips64x.go | 5 +- .../x/sys/unix/syscall_linux_mipsx.go | 29 - .../x/sys/unix/syscall_linux_ppc.go | 28 - .../x/sys/unix/syscall_linux_ppc64x.go | 3 - .../x/sys/unix/syscall_linux_riscv64.go | 15 +- .../x/sys/unix/syscall_linux_s390x.go | 2 - .../x/sys/unix/syscall_linux_sparc64.go | 2 - .../golang.org/x/sys/unix/syscall_netbsd.go | 279 +- .../x/sys/unix/syscall_netbsd_386.go | 1 - .../x/sys/unix/syscall_netbsd_amd64.go | 1 - .../x/sys/unix/syscall_netbsd_arm.go | 1 - .../x/sys/unix/syscall_netbsd_arm64.go | 1 - .../golang.org/x/sys/unix/syscall_openbsd.go | 121 +- .../x/sys/unix/syscall_openbsd_386.go | 1 - .../x/sys/unix/syscall_openbsd_amd64.go | 1 - .../x/sys/unix/syscall_openbsd_arm.go | 1 - .../x/sys/unix/syscall_openbsd_arm64.go | 1 - .../x/sys/unix/syscall_openbsd_libc.go | 1 - .../x/sys/unix/syscall_openbsd_ppc64.go | 1 - .../x/sys/unix/syscall_openbsd_riscv64.go | 1 - .../golang.org/x/sys/unix/syscall_solaris.go | 73 +- .../x/sys/unix/syscall_solaris_amd64.go | 1 - vendor/golang.org/x/sys/unix/syscall_unix.go | 19 +- .../golang.org/x/sys/unix/syscall_unix_gc.go | 2 - .../x/sys/unix/syscall_unix_gc_ppc64x.go | 3 - .../x/sys/unix/syscall_zos_s390x.go | 24 +- vendor/golang.org/x/sys/unix/sysvshm_linux.go | 1 - vendor/golang.org/x/sys/unix/sysvshm_unix.go | 1 - .../x/sys/unix/sysvshm_unix_other.go | 1 - vendor/golang.org/x/sys/unix/timestruct.go | 1 - .../golang.org/x/sys/unix/unveil_openbsd.go | 41 +- vendor/golang.org/x/sys/unix/xattr_bsd.go | 1 - .../golang.org/x/sys/unix/zerrors_aix_ppc.go | 1 - .../x/sys/unix/zerrors_aix_ppc64.go | 1 - .../x/sys/unix/zerrors_darwin_amd64.go | 20 +- .../x/sys/unix/zerrors_darwin_arm64.go | 20 +- .../x/sys/unix/zerrors_dragonfly_amd64.go | 1 - .../x/sys/unix/zerrors_freebsd_386.go | 1 - .../x/sys/unix/zerrors_freebsd_amd64.go | 1 - .../x/sys/unix/zerrors_freebsd_arm.go | 1 - .../x/sys/unix/zerrors_freebsd_arm64.go | 1 - .../x/sys/unix/zerrors_freebsd_riscv64.go | 1 - vendor/golang.org/x/sys/unix/zerrors_linux.go | 90 +- .../x/sys/unix/zerrors_linux_386.go | 12 +- .../x/sys/unix/zerrors_linux_amd64.go | 12 +- .../x/sys/unix/zerrors_linux_arm.go | 12 +- .../x/sys/unix/zerrors_linux_arm64.go | 14 +- .../x/sys/unix/zerrors_linux_loong64.go | 15 +- .../x/sys/unix/zerrors_linux_mips.go | 12 +- .../x/sys/unix/zerrors_linux_mips64.go | 12 +- .../x/sys/unix/zerrors_linux_mips64le.go | 12 +- .../x/sys/unix/zerrors_linux_mipsle.go | 12 +- .../x/sys/unix/zerrors_linux_ppc.go | 12 +- .../x/sys/unix/zerrors_linux_ppc64.go | 12 +- .../x/sys/unix/zerrors_linux_ppc64le.go | 12 +- .../x/sys/unix/zerrors_linux_riscv64.go | 15 +- .../x/sys/unix/zerrors_linux_s390x.go | 12 +- .../x/sys/unix/zerrors_linux_sparc64.go | 60 +- .../x/sys/unix/zerrors_netbsd_386.go | 1 - .../x/sys/unix/zerrors_netbsd_amd64.go | 1 - .../x/sys/unix/zerrors_netbsd_arm.go | 1 - .../x/sys/unix/zerrors_netbsd_arm64.go | 1 - .../x/sys/unix/zerrors_openbsd_386.go | 1 - .../x/sys/unix/zerrors_openbsd_amd64.go | 1 - .../x/sys/unix/zerrors_openbsd_arm.go | 1 - .../x/sys/unix/zerrors_openbsd_arm64.go | 1 - .../x/sys/unix/zerrors_openbsd_mips64.go | 1 - .../x/sys/unix/zerrors_openbsd_ppc64.go | 1 - .../x/sys/unix/zerrors_openbsd_riscv64.go | 1 - .../x/sys/unix/zerrors_solaris_amd64.go | 1 - .../x/sys/unix/zerrors_zos_s390x.go | 1 - .../x/sys/unix/zptrace_armnn_linux.go | 10 +- .../x/sys/unix/zptrace_linux_arm64.go | 4 +- .../x/sys/unix/zptrace_mipsnn_linux.go | 10 +- .../x/sys/unix/zptrace_mipsnnle_linux.go | 10 +- .../x/sys/unix/zptrace_x86_linux.go | 10 +- .../golang.org/x/sys/unix/zsyscall_aix_ppc.go | 46 +- .../x/sys/unix/zsyscall_aix_ppc64.go | 47 +- .../x/sys/unix/zsyscall_aix_ppc64_gc.go | 18 +- .../x/sys/unix/zsyscall_aix_ppc64_gccgo.go | 19 +- .../x/sys/unix/zsyscall_darwin_amd64.go | 72 +- .../x/sys/unix/zsyscall_darwin_amd64.s | 160 +- .../x/sys/unix/zsyscall_darwin_arm64.go | 72 +- .../x/sys/unix/zsyscall_darwin_arm64.s | 160 +- .../x/sys/unix/zsyscall_dragonfly_amd64.go | 43 +- .../x/sys/unix/zsyscall_freebsd_386.go | 53 +- .../x/sys/unix/zsyscall_freebsd_amd64.go | 53 +- .../x/sys/unix/zsyscall_freebsd_arm.go | 53 +- .../x/sys/unix/zsyscall_freebsd_arm64.go | 53 +- .../x/sys/unix/zsyscall_freebsd_riscv64.go | 53 +- .../x/sys/unix/zsyscall_illumos_amd64.go | 11 +- .../golang.org/x/sys/unix/zsyscall_linux.go | 115 +- .../x/sys/unix/zsyscall_linux_386.go | 11 - .../x/sys/unix/zsyscall_linux_amd64.go | 11 - .../x/sys/unix/zsyscall_linux_arm.go | 11 - .../x/sys/unix/zsyscall_linux_arm64.go | 11 - .../x/sys/unix/zsyscall_linux_loong64.go | 1 - .../x/sys/unix/zsyscall_linux_mips.go | 11 - .../x/sys/unix/zsyscall_linux_mips64.go | 11 - .../x/sys/unix/zsyscall_linux_mips64le.go | 11 - .../x/sys/unix/zsyscall_linux_mipsle.go | 11 - .../x/sys/unix/zsyscall_linux_ppc.go | 11 - .../x/sys/unix/zsyscall_linux_ppc64.go | 11 - .../x/sys/unix/zsyscall_linux_ppc64le.go | 11 - .../x/sys/unix/zsyscall_linux_riscv64.go | 27 +- .../x/sys/unix/zsyscall_linux_s390x.go | 11 - .../x/sys/unix/zsyscall_linux_sparc64.go | 11 - .../x/sys/unix/zsyscall_netbsd_386.go | 50 +- .../x/sys/unix/zsyscall_netbsd_amd64.go | 50 +- .../x/sys/unix/zsyscall_netbsd_arm.go | 50 +- .../x/sys/unix/zsyscall_netbsd_arm64.go | 50 +- .../x/sys/unix/zsyscall_openbsd_386.go | 118 +- .../x/sys/unix/zsyscall_openbsd_386.s | 35 +- .../x/sys/unix/zsyscall_openbsd_amd64.go | 118 +- .../x/sys/unix/zsyscall_openbsd_amd64.s | 35 +- .../x/sys/unix/zsyscall_openbsd_arm.go | 118 +- .../x/sys/unix/zsyscall_openbsd_arm.s | 35 +- .../x/sys/unix/zsyscall_openbsd_arm64.go | 118 +- .../x/sys/unix/zsyscall_openbsd_arm64.s | 35 +- .../x/sys/unix/zsyscall_openbsd_mips64.go | 118 +- .../x/sys/unix/zsyscall_openbsd_mips64.s | 35 +- .../x/sys/unix/zsyscall_openbsd_ppc64.go | 118 +- .../x/sys/unix/zsyscall_openbsd_ppc64.s | 42 +- .../x/sys/unix/zsyscall_openbsd_riscv64.go | 118 +- .../x/sys/unix/zsyscall_openbsd_riscv64.s | 35 +- .../x/sys/unix/zsyscall_solaris_amd64.go | 281 +- .../x/sys/unix/zsyscall_zos_s390x.go | 24 +- .../x/sys/unix/zsysctl_openbsd_386.go | 1 - .../x/sys/unix/zsysctl_openbsd_amd64.go | 1 - .../x/sys/unix/zsysctl_openbsd_arm.go | 1 - .../x/sys/unix/zsysctl_openbsd_arm64.go | 1 - .../x/sys/unix/zsysctl_openbsd_mips64.go | 1 - .../x/sys/unix/zsysctl_openbsd_ppc64.go | 1 - .../x/sys/unix/zsysctl_openbsd_riscv64.go | 1 - .../x/sys/unix/zsysnum_darwin_amd64.go | 1 - .../x/sys/unix/zsysnum_darwin_arm64.go | 1 - .../x/sys/unix/zsysnum_dragonfly_amd64.go | 1 - .../x/sys/unix/zsysnum_freebsd_386.go | 1 - .../x/sys/unix/zsysnum_freebsd_amd64.go | 1 - .../x/sys/unix/zsysnum_freebsd_arm.go | 1 - .../x/sys/unix/zsysnum_freebsd_arm64.go | 1 - .../x/sys/unix/zsysnum_freebsd_riscv64.go | 1 - .../x/sys/unix/zsysnum_linux_386.go | 3 +- .../x/sys/unix/zsysnum_linux_amd64.go | 4 +- .../x/sys/unix/zsysnum_linux_arm.go | 3 +- .../x/sys/unix/zsysnum_linux_arm64.go | 3 +- .../x/sys/unix/zsysnum_linux_loong64.go | 3 +- .../x/sys/unix/zsysnum_linux_mips.go | 3 +- .../x/sys/unix/zsysnum_linux_mips64.go | 3 +- .../x/sys/unix/zsysnum_linux_mips64le.go | 3 +- .../x/sys/unix/zsysnum_linux_mipsle.go | 3 +- .../x/sys/unix/zsysnum_linux_ppc.go | 3 +- .../x/sys/unix/zsysnum_linux_ppc64.go | 3 +- .../x/sys/unix/zsysnum_linux_ppc64le.go | 3 +- .../x/sys/unix/zsysnum_linux_riscv64.go | 5 +- .../x/sys/unix/zsysnum_linux_s390x.go | 4 +- .../x/sys/unix/zsysnum_linux_sparc64.go | 3 +- .../x/sys/unix/zsysnum_netbsd_386.go | 1 - .../x/sys/unix/zsysnum_netbsd_amd64.go | 1 - .../x/sys/unix/zsysnum_netbsd_arm.go | 1 - .../x/sys/unix/zsysnum_netbsd_arm64.go | 1 - .../x/sys/unix/zsysnum_openbsd_386.go | 1 - .../x/sys/unix/zsysnum_openbsd_amd64.go | 1 - .../x/sys/unix/zsysnum_openbsd_arm.go | 1 - .../x/sys/unix/zsysnum_openbsd_arm64.go | 1 - .../x/sys/unix/zsysnum_openbsd_mips64.go | 1 - .../x/sys/unix/zsysnum_openbsd_ppc64.go | 1 - .../x/sys/unix/zsysnum_openbsd_riscv64.go | 1 - .../x/sys/unix/zsysnum_zos_s390x.go | 1 - .../golang.org/x/sys/unix/ztypes_aix_ppc.go | 1 - .../golang.org/x/sys/unix/ztypes_aix_ppc64.go | 1 - .../x/sys/unix/ztypes_darwin_amd64.go | 12 +- .../x/sys/unix/ztypes_darwin_arm64.go | 12 +- .../x/sys/unix/ztypes_dragonfly_amd64.go | 1 - .../x/sys/unix/ztypes_freebsd_386.go | 3 +- .../x/sys/unix/ztypes_freebsd_amd64.go | 3 +- .../x/sys/unix/ztypes_freebsd_arm.go | 3 +- .../x/sys/unix/ztypes_freebsd_arm64.go | 3 +- .../x/sys/unix/ztypes_freebsd_riscv64.go | 3 +- vendor/golang.org/x/sys/unix/ztypes_linux.go | 270 +- .../golang.org/x/sys/unix/ztypes_linux_386.go | 5 +- .../x/sys/unix/ztypes_linux_amd64.go | 5 +- .../golang.org/x/sys/unix/ztypes_linux_arm.go | 5 +- .../x/sys/unix/ztypes_linux_arm64.go | 5 +- .../x/sys/unix/ztypes_linux_loong64.go | 5 +- .../x/sys/unix/ztypes_linux_mips.go | 5 +- .../x/sys/unix/ztypes_linux_mips64.go | 5 +- .../x/sys/unix/ztypes_linux_mips64le.go | 5 +- .../x/sys/unix/ztypes_linux_mipsle.go | 5 +- .../golang.org/x/sys/unix/ztypes_linux_ppc.go | 5 +- .../x/sys/unix/ztypes_linux_ppc64.go | 5 +- .../x/sys/unix/ztypes_linux_ppc64le.go | 5 +- .../x/sys/unix/ztypes_linux_riscv64.go | 32 +- .../x/sys/unix/ztypes_linux_s390x.go | 5 +- .../x/sys/unix/ztypes_linux_sparc64.go | 5 +- .../x/sys/unix/ztypes_netbsd_386.go | 1 - .../x/sys/unix/ztypes_netbsd_amd64.go | 1 - .../x/sys/unix/ztypes_netbsd_arm.go | 1 - .../x/sys/unix/ztypes_netbsd_arm64.go | 1 - .../x/sys/unix/ztypes_openbsd_386.go | 1 - .../x/sys/unix/ztypes_openbsd_amd64.go | 1 - .../x/sys/unix/ztypes_openbsd_arm.go | 1 - .../x/sys/unix/ztypes_openbsd_arm64.go | 1 - .../x/sys/unix/ztypes_openbsd_mips64.go | 1 - .../x/sys/unix/ztypes_openbsd_ppc64.go | 1 - .../x/sys/unix/ztypes_openbsd_riscv64.go | 1 - .../x/sys/unix/ztypes_solaris_amd64.go | 1 - .../golang.org/x/sys/unix/ztypes_zos_s390x.go | 1 - vendor/golang.org/x/sys/windows/aliases.go | 1 - vendor/golang.org/x/sys/windows/empty.s | 1 - .../golang.org/x/sys/windows/env_windows.go | 6 +- vendor/golang.org/x/sys/windows/eventlog.go | 1 - .../golang.org/x/sys/windows/exec_windows.go | 92 +- vendor/golang.org/x/sys/windows/mksyscall.go | 1 - vendor/golang.org/x/sys/windows/race.go | 1 - vendor/golang.org/x/sys/windows/race0.go | 1 - .../x/sys/windows/security_windows.go | 21 +- vendor/golang.org/x/sys/windows/service.go | 12 +- vendor/golang.org/x/sys/windows/str.go | 1 - vendor/golang.org/x/sys/windows/syscall.go | 1 - .../x/sys/windows/syscall_windows.go | 82 +- .../golang.org/x/sys/windows/types_windows.go | 130 +- .../x/sys/windows/zsyscall_windows.go | 126 +- vendor/golang.org/x/term/term_unix.go | 3 +- vendor/golang.org/x/term/term_unix_bsd.go | 1 - vendor/golang.org/x/term/term_unix_other.go | 1 - vendor/golang.org/x/term/term_unsupported.go | 1 - .../x/text/encoding/internal/internal.go | 2 +- .../text/internal/language/compact/tables.go | 356 +- .../x/text/internal/language/tables.go | 4686 +++++----- vendor/golang.org/x/text/language/match.go | 2 +- vendor/golang.org/x/text/language/tables.go | 138 +- .../x/text/secure/bidirule/bidirule10.0.0.go | 1 - .../x/text/secure/bidirule/bidirule9.0.0.go | 1 - .../x/text/unicode/bidi/tables10.0.0.go | 1 - .../x/text/unicode/bidi/tables11.0.0.go | 1 - .../x/text/unicode/bidi/tables12.0.0.go | 1 - .../x/text/unicode/bidi/tables13.0.0.go | 3 +- .../x/text/unicode/bidi/tables15.0.0.go | 2042 +++++ .../x/text/unicode/bidi/tables9.0.0.go | 1 - .../x/text/unicode/norm/forminfo.go | 2 +- .../x/text/unicode/norm/tables10.0.0.go | 1 - .../x/text/unicode/norm/tables11.0.0.go | 1 - .../x/text/unicode/norm/tables12.0.0.go | 1 - .../x/text/unicode/norm/tables13.0.0.go | 3 +- .../x/text/unicode/norm/tables15.0.0.go | 7907 +++++++++++++++++ .../x/text/unicode/norm/tables9.0.0.go | 1 - vendor/golang.org/x/text/unicode/norm/trie.go | 2 +- vendor/modules.txt | 34 +- 423 files changed, 26198 insertions(+), 7882 deletions(-) create mode 100644 pkg/install/dataloadmanager/dataloadmanager.go create mode 100644 pkg/install/datasetmanager/datasetmanager.go create mode 100644 tool.go delete mode 100644 vendor/github.com/google/uuid/.travis.yml create mode 100644 vendor/github.com/google/uuid/CHANGELOG.md create mode 100644 vendor/github.com/google/uuid/null.go create mode 100644 vendor/github.com/google/uuid/version6.go create mode 100644 vendor/github.com/google/uuid/version7.go create mode 100644 vendor/github.com/hwameistor/datastore/LICENSE create mode 100644 vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_basemodels_crd.yaml create mode 100644 vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_checkpoints_crd.yaml create mode 100644 vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_dataloadrequests_crd.yaml create mode 100644 vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_datasets_crd.yaml create mode 100644 vendor/github.com/hwameistor/datastore/deploy/crds/module.go delete mode 100644 vendor/golang.org/x/net/http2/Dockerfile delete mode 100644 vendor/golang.org/x/net/http2/Makefile delete mode 100644 vendor/golang.org/x/net/http2/go111.go delete mode 100644 vendor/golang.org/x/net/http2/go115.go delete mode 100644 vendor/golang.org/x/net/http2/go118.go delete mode 100644 vendor/golang.org/x/net/http2/not_go111.go delete mode 100644 vendor/golang.org/x/net/http2/not_go115.go delete mode 100644 vendor/golang.org/x/net/http2/not_go118.go create mode 100644 vendor/golang.org/x/net/http2/writesched_roundrobin.go create mode 100644 vendor/golang.org/x/net/idna/tables15.0.0.go create mode 100644 vendor/golang.org/x/net/idna/trie12.0.0.go create mode 100644 vendor/golang.org/x/net/idna/trie13.0.0.go delete mode 100644 vendor/golang.org/x/sys/internal/unsafeheader/unsafeheader.go create mode 100644 vendor/golang.org/x/sys/unix/ioctl_signed.go rename vendor/golang.org/x/sys/unix/{ioctl.go => ioctl_unsigned.go} (76%) create mode 100644 vendor/golang.org/x/sys/unix/mmap_nomremap.go create mode 100644 vendor/golang.org/x/sys/unix/mremap.go create mode 100644 vendor/golang.org/x/text/unicode/bidi/tables15.0.0.go create mode 100644 vendor/golang.org/x/text/unicode/norm/tables15.0.0.go diff --git a/.gitignore b/.gitignore index 9445a02b..eea58a31 100644 --- a/.gitignore +++ b/.gitignore @@ -1,5 +1,6 @@ - +#_build +_build/ # Binaries for programs and plugins *.exe @@ -28,3 +29,7 @@ testbin/* # Dependency directories (remove the comment below to include it) # vendor/ + +# macOS system files +.DS_Store +helm/.DS_Store \ No newline at end of file diff --git a/Dockerfile b/Dockerfile index 02e78a63..4f63d59d 100644 --- a/Dockerfile +++ b/Dockerfile @@ -3,6 +3,7 @@ FROM centos:7 RUN yum upgrade nss -y COPY ./vendor/github.com/hwameistor/hwameistor/deploy/crds /hwameistorcrds +COPY ./vendor/github.com/hwameistor/datastore/deploy/crds /hwameistorcrds COPY ./scheduler-config.yaml /scheduler-config.yaml COPY ./_build/operator /operator diff --git a/Makefile b/Makefile index 4624553e..b726c303 100644 --- a/Makefile +++ b/Makefile @@ -277,7 +277,7 @@ e2e-test: pr-test: bash test/pr-test.sh - - - - +.PHONY: apis +apis: + make generate + make manifests \ No newline at end of file diff --git a/api/v1alpha1/cluster_types.go b/api/v1alpha1/cluster_types.go index b3d7bcc8..ccef7973 100644 --- a/api/v1alpha1/cluster_types.go +++ b/api/v1alpha1/cluster_types.go @@ -69,6 +69,10 @@ type ClusterSpec struct { RBAC *RBACSpec `json:"rbac,omitempty"` StorageClass *StorageClassSpec `json:"storageClass,omitempty"` + + DataLoadManager *DataLoadManagerSpec `json:"dataLoadManager,omitempty"` + + DataSetManager *DataSetManagerSpec `json:"dataSetManager,omitempty"` } type DiskReserveConfiguration struct { @@ -245,6 +249,18 @@ type StorageClassSpec struct { FSType string `json:"fsType,omitempty"` } +type DataLoadManagerSpec struct { + Disable bool `json:"disable,omitempty"` + Common *PodCommonSpec `json:"common,omitempty"` + DataLoadManagerContainer *ContainerCommonSpec `json:"dataLoadManagerContainer,omitempty"` +} + +type DataSetManagerSpec struct { + Disable bool `json:"disable,omitempty"` + Common *PodCommonSpec `json:"common,omitempty"` + DataSetManagerContainer *ContainerCommonSpec `json:"dataSetManagerContainer,omitempty"` +} + // ClusterStatus defines the observed state of Cluster type ClusterStatus struct { // INSERT ADDITIONAL STATUS FIELD - define observed state of cluster @@ -254,6 +270,7 @@ type ClusterStatus struct { DRBDAdapterCreated bool `json:"drbdAdapterCreated"` DRBDAdapterCreatedJobNum int `json:"drbdAdapterCreatedJobNum"` DiskReserveState string `json:"diskReserveState,omitempty"` + DatasetDefaultPoolClass string `json:"datasetDefaultPoolClass,omitempty"` Phase string `json:"phase,omitempty"` ComponentStatus ComponentStatus `json:"componentStatus"` } @@ -270,6 +287,8 @@ type ComponentStatus struct { PVCAutoResizer *PVCAutoResizerStatus `json:"pvcAutoResizer,omitempty"` LocalDiskActionController *LocalDiskActionControllerStatus `json:"localDiskActionController,omitempty"` Exporter *ExporterStatus `json:"exporter,omitempty"` + DataLoadManager *DataLoadManagerStatus `json:"dataLoadManager,omitempty"` + DataSetManager *DataSetManagerStatus `json:"dataSetManager,omitempty"` } type DeployStatus struct { @@ -343,6 +362,16 @@ type ExporterStatus struct { Health string `json:"health,omitempty"` } +type DataLoadManagerStatus struct { + Instances *DeployStatus `json:"instances,omitempty"` + Health string `json:"health,omitempty"` +} + +type DataSetManagerStatus struct { + Instances *DeployStatus `json:"instances,omitempty"` + Health string `json:"health,omitempty"` +} + //+kubebuilder:object:root=true //+kubebuilder:subresource:status diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 3c491e43..dff35949 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -400,6 +400,16 @@ func (in *ClusterSpec) DeepCopyInto(out *ClusterSpec) { *out = new(StorageClassSpec) **out = **in } + if in.DataLoadManager != nil { + in, out := &in.DataLoadManager, &out.DataLoadManager + *out = new(DataLoadManagerSpec) + (*in).DeepCopyInto(*out) + } + if in.DataSetManager != nil { + in, out := &in.DataSetManager, &out.DataSetManager + *out = new(DataSetManagerSpec) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ClusterSpec. @@ -486,6 +496,16 @@ func (in *ComponentStatus) DeepCopyInto(out *ComponentStatus) { *out = new(ExporterStatus) (*in).DeepCopyInto(*out) } + if in.DataLoadManager != nil { + in, out := &in.DataLoadManager, &out.DataLoadManager + *out = new(DataLoadManagerStatus) + (*in).DeepCopyInto(*out) + } + if in.DataSetManager != nil { + in, out := &in.DataSetManager, &out.DataSetManager + *out = new(DataSetManagerStatus) + (*in).DeepCopyInto(*out) + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ComponentStatus. @@ -526,6 +546,11 @@ func (in *ContainerCommonSpec) DeepCopy() *ContainerCommonSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DRBDSpec) DeepCopyInto(out *DRBDSpec) { *out = *in + if in.Shipper != nil { + in, out := &in.Shipper, &out.Shipper + *out = new(ImageSpec) + **out = **in + } if in.NodeAffinity != nil { in, out := &in.NodeAffinity, &out.NodeAffinity *out = new(v1.NodeAffinity) @@ -543,6 +568,96 @@ func (in *DRBDSpec) DeepCopy() *DRBDSpec { return out } +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataLoadManagerSpec) DeepCopyInto(out *DataLoadManagerSpec) { + *out = *in + if in.Common != nil { + in, out := &in.Common, &out.Common + *out = new(PodCommonSpec) + (*in).DeepCopyInto(*out) + } + if in.DataLoadManagerContainer != nil { + in, out := &in.DataLoadManagerContainer, &out.DataLoadManagerContainer + *out = new(ContainerCommonSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataLoadManagerSpec. +func (in *DataLoadManagerSpec) DeepCopy() *DataLoadManagerSpec { + if in == nil { + return nil + } + out := new(DataLoadManagerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataLoadManagerStatus) DeepCopyInto(out *DataLoadManagerStatus) { + *out = *in + if in.Instances != nil { + in, out := &in.Instances, &out.Instances + *out = new(DeployStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataLoadManagerStatus. +func (in *DataLoadManagerStatus) DeepCopy() *DataLoadManagerStatus { + if in == nil { + return nil + } + out := new(DataLoadManagerStatus) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSetManagerSpec) DeepCopyInto(out *DataSetManagerSpec) { + *out = *in + if in.Common != nil { + in, out := &in.Common, &out.Common + *out = new(PodCommonSpec) + (*in).DeepCopyInto(*out) + } + if in.DataSetManagerContainer != nil { + in, out := &in.DataSetManagerContainer, &out.DataSetManagerContainer + *out = new(ContainerCommonSpec) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSetManagerSpec. +func (in *DataSetManagerSpec) DeepCopy() *DataSetManagerSpec { + if in == nil { + return nil + } + out := new(DataSetManagerSpec) + in.DeepCopyInto(out) + return out +} + +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *DataSetManagerStatus) DeepCopyInto(out *DataSetManagerStatus) { + *out = *in + if in.Instances != nil { + in, out := &in.Instances, &out.Instances + *out = new(DeployStatus) + (*in).DeepCopyInto(*out) + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new DataSetManagerStatus. +func (in *DataSetManagerStatus) DeepCopy() *DataSetManagerStatus { + if in == nil { + return nil + } + out := new(DataSetManagerStatus) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *DeployStatus) DeepCopyInto(out *DeployStatus) { *out = *in diff --git a/config/crd/bases/hwameistor.io_clusters.yaml b/config/crd/bases/hwameistor.io_clusters.yaml index 5b75a19b..c606a539 100644 --- a/config/crd/bases/hwameistor.io_clusters.yaml +++ b/config/crd/bases/hwameistor.io_clusters.yaml @@ -1930,6 +1930,1252 @@ spec: required: - replicas type: object + dataLoadManager: + properties: + common: + properties: + nodeAffinity: + description: Node affinity is a group of node affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity + scheduling rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + priorityClassName: + type: string + tolerations: + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value should + be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + dataLoadManagerContainer: + properties: + image: + properties: + registry: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + disable: + type: boolean + type: object + dataSetManager: + properties: + common: + properties: + nodeAffinity: + description: Node affinity is a group of node affinity scheduling + rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node matches the corresponding matchExpressions; + the node(s) with the highest sum are the most preferred. + items: + description: An empty preferred scheduling term matches + all objects with implicit weight 0 (i.e. it's a no-op). + A null preferred scheduling term matches no objects + (i.e. is also a no-op). + properties: + preference: + description: A node selector term, associated with + the corresponding weight. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + weight: + description: Weight associated with matching the + corresponding nodeSelectorTerm, in the range 1-100. + format: int32 + type: integer + required: + - preference + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to an update), the system + may or may not try to eventually evict the pod from + its node. + properties: + nodeSelectorTerms: + description: Required. A list of node selector terms. + The terms are ORed. + items: + description: A null or empty node selector term + matches no objects. The requirements of them are + ANDed. The TopologySelectorTerm type implements + a subset of the NodeSelectorTerm. + properties: + matchExpressions: + description: A list of node selector requirements + by node's labels. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchFields: + description: A list of node selector requirements + by node's fields. + items: + description: A node selector requirement is + a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: The label key that the selector + applies to. + type: string + operator: + description: Represents a key's relationship + to a set of values. Valid operators + are In, NotIn, Exists, DoesNotExist. + Gt, and Lt. + type: string + values: + description: An array of string values. + If the operator is In or NotIn, the + values array must be non-empty. If the + operator is Exists or DoesNotExist, + the values array must be empty. If the + operator is Gt or Lt, the values array + must have a single element, which will + be interpreted as an integer. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + type: object + type: array + required: + - nodeSelectorTerms + type: object + type: object + podAffinity: + description: Pod affinity is a group of inter pod affinity + scheduling rules. + properties: + preferredDuringSchedulingIgnoredDuringExecution: + description: The scheduler will prefer to schedule pods + to nodes that satisfy the affinity expressions specified + by this field, but it may choose a node that violates + one or more of the expressions. The node that is most + preferred is the one with the greatest sum of weights, + i.e. for each node that meets all of the scheduling + requirements (resource request, requiredDuringScheduling + affinity expressions, etc.), compute a sum by iterating + through the elements of this field and adding "weight" + to the sum if the node has pods which matches the corresponding + podAffinityTerm; the node(s) with the highest sum are + the most preferred. + items: + description: The weights of all of the matched WeightedPodAffinityTerm + fields are added per-node to find the most preferred + node(s) + properties: + podAffinityTerm: + description: Required. A pod affinity term, associated + with the corresponding weight. + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by + this field and the ones listed in the namespaces + field. null selector and null or empty namespaces + list means "this pod's namespace". An empty + selector ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list + of label selector requirements. The requirements + are ANDed. + items: + description: A label selector requirement + is a selector that contains values, + a key, and an operator that relates + the key and values. + properties: + key: + description: key is the label key + that the selector applies to. + type: string + operator: + description: operator represents a + key's relationship to a set of values. + Valid operators are In, NotIn, Exists + and DoesNotExist. + type: string + values: + description: values is an array of + string values. If the operator is + In or NotIn, the values array must + be non-empty. If the operator is + Exists or DoesNotExist, the values + array must be empty. This array + is replaced during a strategic merge + patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator + is "In", and the values array contains + only "value". The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. + The term is applied to the union of the namespaces + listed in this field and the ones selected + by namespaceSelector. null or empty namespaces + list and null namespaceSelector means "this + pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the + pods matching the labelSelector in the specified + namespaces, where co-located is defined as + running on a node whose value of the label + with key topologyKey matches that of any node + on which any of the selected pods is running. + Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + weight: + description: weight associated with matching the + corresponding podAffinityTerm, in the range 1-100. + format: int32 + type: integer + required: + - podAffinityTerm + - weight + type: object + type: array + requiredDuringSchedulingIgnoredDuringExecution: + description: If the affinity requirements specified by + this field are not met at scheduling time, the pod will + not be scheduled onto the node. If the affinity requirements + specified by this field cease to be met at some point + during pod execution (e.g. due to a pod label update), + the system may or may not try to eventually evict the + pod from its node. When there are multiple elements, + the lists of nodes corresponding to each podAffinityTerm + are intersected, i.e. all terms must be satisfied. + items: + description: Defines a set of pods (namely those matching + the labelSelector relative to the given namespace(s)) + that this pod should be co-located (affinity) or not + co-located (anti-affinity) with, where co-located + is defined as running on a node whose value of the + label with key matches that of any node + on which a pod of the set of pods is running + properties: + labelSelector: + description: A label query over a set of resources, + in this case pods. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaceSelector: + description: A label query over the set of namespaces + that the term applies to. The term is applied + to the union of the namespaces selected by this + field and the ones listed in the namespaces field. + null selector and null or empty namespaces list + means "this pod's namespace". An empty selector + ({}) matches all namespaces. + properties: + matchExpressions: + description: matchExpressions is a list of label + selector requirements. The requirements are + ANDed. + items: + description: A label selector requirement + is a selector that contains values, a key, + and an operator that relates the key and + values. + properties: + key: + description: key is the label key that + the selector applies to. + type: string + operator: + description: operator represents a key's + relationship to a set of values. Valid + operators are In, NotIn, Exists and + DoesNotExist. + type: string + values: + description: values is an array of string + values. If the operator is In or NotIn, + the values array must be non-empty. + If the operator is Exists or DoesNotExist, + the values array must be empty. This + array is replaced during a strategic + merge patch. + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + description: matchLabels is a map of {key,value} + pairs. A single {key,value} in the matchLabels + map is equivalent to an element of matchExpressions, + whose key field is "key", the operator is + "In", and the values array contains only "value". + The requirements are ANDed. + type: object + type: object + namespaces: + description: namespaces specifies a static list + of namespace names that the term applies to. The + term is applied to the union of the namespaces + listed in this field and the ones selected by + namespaceSelector. null or empty namespaces list + and null namespaceSelector means "this pod's namespace". + items: + type: string + type: array + topologyKey: + description: This pod should be co-located (affinity) + or not co-located (anti-affinity) with the pods + matching the labelSelector in the specified namespaces, + where co-located is defined as running on a node + whose value of the label with key topologyKey + matches that of any node on which any of the selected + pods is running. Empty topologyKey is not allowed. + type: string + required: + - topologyKey + type: object + type: array + type: object + priorityClassName: + type: string + tolerations: + items: + description: The pod this Toleration is attached to tolerates + any taint that matches the triple using + the matching operator . + properties: + effect: + description: Effect indicates the taint effect to match. + Empty means match all taint effects. When specified, + allowed values are NoSchedule, PreferNoSchedule and + NoExecute. + type: string + key: + description: Key is the taint key that the toleration + applies to. Empty means match all taint keys. If the + key is empty, operator must be Exists; this combination + means to match all values and all keys. + type: string + operator: + description: Operator represents a key's relationship + to the value. Valid operators are Exists and Equal. + Defaults to Equal. Exists is equivalent to wildcard + for value, so that a pod can tolerate all taints of + a particular category. + type: string + tolerationSeconds: + description: TolerationSeconds represents the period + of time the toleration (which must be of effect NoExecute, + otherwise this field is ignored) tolerates the taint. + By default, it is not set, which means tolerate the + taint forever (do not evict). Zero and negative values + will be treated as 0 (evict immediately) by the system. + format: int64 + type: integer + value: + description: Value is the taint value the toleration + matches to. If the operator is Exists, the value should + be empty, otherwise just a regular string. + type: string + type: object + type: array + type: object + dataSetManagerContainer: + properties: + image: + properties: + registry: + type: string + repository: + type: string + tag: + type: string + type: object + resources: + description: ResourceRequirements describes the compute resource + requirements. + properties: + limits: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Limits describes the maximum amount of compute + resources allowed. More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + requests: + additionalProperties: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + description: 'Requests describes the minimum amount of + compute resources required. If Requests is omitted for + a container, it defaults to Limits if that is explicitly + specified, otherwise to an implementation-defined value. + More info: https://kubernetes.io/docs/concepts/configuration/manage-resources-containers/' + type: object + type: object + type: object + disable: + type: boolean + type: object diskReserveConfigurations: items: properties: @@ -1946,15 +3192,6 @@ spec: type: array drbd: properties: - shipper: - properties: - registry: - type: string - repository: - type: string - tag: - type: string - type: object chartVersion: type: string checkHostName: @@ -1967,10 +3204,6 @@ spec: type: string imagePullPolicy: type: string - imageRegistry: - type: string - imageRepoOwner: - type: string nodeAffinity: description: Node affinity is a group of node affinity scheduling rules. @@ -2175,6 +3408,15 @@ spec: - nodeSelectorTerms type: object type: object + shipper: + properties: + registry: + type: string + repository: + type: string + tag: + type: string + type: object upgrade: type: string useAffinity: @@ -9871,6 +11113,82 @@ spec: - workloadType type: object type: object + dataLoadManager: + properties: + health: + type: string + instances: + properties: + availablePodCount: + format: int32 + type: integer + desiredPodCount: + format: int32 + type: integer + pods: + items: + properties: + name: + type: string + node: + type: string + status: + type: string + required: + - name + - node + - status + type: object + type: array + workloadName: + type: string + workloadType: + type: string + required: + - availablePodCount + - desiredPodCount + - workloadName + - workloadType + type: object + type: object + dataSetManager: + properties: + health: + type: string + instances: + properties: + availablePodCount: + format: int32 + type: integer + desiredPodCount: + format: int32 + type: integer + pods: + items: + properties: + name: + type: string + node: + type: string + status: + type: string + required: + - name + - node + - status + type: object + type: array + workloadName: + type: string + workloadType: + type: string + required: + - availablePodCount + - desiredPodCount + - workloadName + - workloadType + type: object + type: object evictor: properties: health: @@ -10242,6 +11560,8 @@ spec: type: object type: object type: object + datasetDefaultPoolClass: + type: string diskReserveState: type: string drbdAdapterCreated: diff --git a/controllers/cluster_controller.go b/controllers/cluster_controller.go index b5759b8f..a6d082b5 100644 --- a/controllers/cluster_controller.go +++ b/controllers/cluster_controller.go @@ -18,6 +18,8 @@ package controllers import ( "context" + "github.com/hwameistor/hwameistor-operator/pkg/install/dataloadmanager" + "github.com/hwameistor/hwameistor-operator/pkg/install/datasetmanager" "reflect" "time" @@ -411,6 +413,22 @@ func (r *ClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ct } } + if !newInstance.Spec.DataLoadManager.Disable { + newInstance, err = dataloadmanager.NewMaintainer(r.Client, newInstance).Ensure() + if err != nil { + log.Errorf("Ensure DataLoadManager err: %v", err) + return ctrl.Result{}, err + } + } + + if !newInstance.Spec.DataSetManager.Disable { + newInstance, err = datasetmanager.NewMaintainer(r.Client, newInstance).Ensure() + if err != nil { + log.Errorf("Ensure DataSetManager err: %v", err) + return ctrl.Result{}, err + } + } + // Use Phase to adopt DiskReserveState, notice that don't lose the value of DiskReserveState if newInstance.Status.DiskReserveState != "" && newInstance.Status.Phase == "" { newInstance.Status.Phase = newInstance.Status.DiskReserveState @@ -503,6 +521,8 @@ func FulfillClusterInstance(clusterInstance *hwameistoroperatorv1alpha1.Cluster) newClusterInstance = ui.FulfillUISpec(newClusterInstance) newClusterInstance = storageclass.FulfillStorageClassSpec(newClusterInstance) newClusterInstance = drbd.FulfillDRBDSpec(newClusterInstance) + newClusterInstance = dataloadmanager.FulfillDataLoadManagerSpec(newClusterInstance) + newClusterInstance = datasetmanager.FulfillDataSetManagerSpec(newClusterInstance) return newClusterInstance } diff --git a/go.mod b/go.mod index a879b69d..edae90a8 100644 --- a/go.mod +++ b/go.mod @@ -12,8 +12,9 @@ require ( ) require ( - github.com/hwameistor/hwameistor v0.14.3 - github.com/sirupsen/logrus v1.9.0 + github.com/hwameistor/datastore v0.0.0-20240613033429-35e712326d4d + github.com/hwameistor/hwameistor v0.14.4 + github.com/sirupsen/logrus v1.9.3 k8s.io/api v0.26.0 k8s.io/apiextensions-apiserver v0.23.0 ) @@ -45,11 +46,10 @@ require ( github.com/google/gnostic v0.6.9 // indirect github.com/google/go-cmp v0.5.9 // indirect github.com/google/gofuzz v1.1.0 // indirect - github.com/google/uuid v1.1.2 // indirect + github.com/google/uuid v1.5.0 // indirect github.com/imdario/mergo v0.3.12 // indirect github.com/josharian/intern v1.0.0 // indirect github.com/json-iterator/go v1.1.12 // indirect - github.com/kr/pretty v0.3.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.2-0.20181231171920-c182affec369 // indirect github.com/moby/spdystream v0.2.0 // indirect @@ -62,17 +62,16 @@ require ( github.com/prometheus/client_model v0.2.0 // indirect github.com/prometheus/common v0.37.0 // indirect github.com/prometheus/procfs v0.8.0 // indirect - github.com/rogpeppe/go-internal v1.8.0 // indirect github.com/spf13/pflag v1.0.5 // indirect go.uber.org/atomic v1.7.0 // indirect go.uber.org/multierr v1.6.0 // indirect go.uber.org/zap v1.19.1 // indirect - golang.org/x/crypto v0.1.0 // indirect - golang.org/x/net v0.7.0 // indirect + golang.org/x/crypto v0.16.0 // indirect + golang.org/x/net v0.19.0 // indirect golang.org/x/oauth2 v0.1.0 // indirect - golang.org/x/sys v0.5.0 // indirect - golang.org/x/term v0.5.0 // indirect - golang.org/x/text v0.7.0 // indirect + golang.org/x/sys v0.15.0 // indirect + golang.org/x/term v0.15.0 // indirect + golang.org/x/text v0.14.0 // indirect golang.org/x/time v0.0.0-20220210224613-90d013bbcef8 // indirect gomodules.xyz/jsonpatch/v2 v2.2.0 // indirect google.golang.org/appengine v1.6.7 // indirect diff --git a/go.sum b/go.sum index b74e4d75..6f06a2f1 100644 --- a/go.sum +++ b/go.sum @@ -275,8 +275,9 @@ github.com/google/pprof v0.0.0-20210601050228-01bbb1931b22/go.mod h1:kpwsk12EmLe github.com/google/pprof v0.0.0-20210609004039-a478d1d731e9/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/pprof v0.0.0-20210720184732-4bb14d4b1be1/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= -github.com/google/uuid v1.1.2 h1:EVhdT+1Kseyi1/pUmXKaFxYsDNy9RQYkMWRH68J/W7Y= github.com/google/uuid v1.1.2/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= +github.com/google/uuid v1.5.0 h1:1p67kYwdtXjb0gL0BPiP1Av9wiZPo5A8z2cWkTZ+eyU= +github.com/google/uuid v1.5.0/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo= github.com/googleapis/enterprise-certificate-proxy v0.0.0-20220520183353-fd19c99a87aa/go.mod h1:17drOmN3MwGY7t0e+Ei9b45FFGA3fBs3x36SsCg1hq8= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= github.com/googleapis/gax-go/v2 v2.0.5/go.mod h1:DWXyrwAJ9X0FpwwEdw+IPEYBICEFu5mhpdKc/us6bOk= @@ -316,10 +317,10 @@ github.com/hashicorp/mdns v1.0.0/go.mod h1:tL+uN++7HEJ6SQLQ2/p+z2pH24WQKWjBPkE0m github.com/hashicorp/memberlist v0.1.3/go.mod h1:ajVTdAv/9Im8oMAAj5G31PhhMCZJV2pPBoIllUwCN7I= github.com/hashicorp/serf v0.8.2/go.mod h1:6hOLApaqBFA1NXqRQAsxw9QxuDEvNxSQRwA/JwenrHc= github.com/hpcloud/tail v1.0.0/go.mod h1:ab1qPbhIpdTxEkNHXyeSf5vhxWSCs/tWer42PpOxQnU= -github.com/hwameistor/hwameistor v0.14.2 h1:8PYNeAgNJZBL4VImpTAZ0ankgAdAYp+uDjI8uLGeChs= -github.com/hwameistor/hwameistor v0.14.2/go.mod h1:uuYPiN78aba8NgqQ2Wv35phP33VKIJ1YCOS8K5koJ/k= -github.com/hwameistor/hwameistor v0.14.3 h1:PkcKppIwjtVgy6A9sIuBsI6/tnV8pidLDJq6/SyefVY= -github.com/hwameistor/hwameistor v0.14.3/go.mod h1:d+J2bzNIU2UFIKaihRK3rh2gN45NJcOA6D4Qv2fiEi4= +github.com/hwameistor/datastore v0.0.0-20240613033429-35e712326d4d h1:9sO715yVL5S6gAvaCZPQCEEF/oIRjrjAUjhPqNF9vZ4= +github.com/hwameistor/datastore v0.0.0-20240613033429-35e712326d4d/go.mod h1:7cc6H7Tvnp9afDstUBNVJtNEM+1LSfM+GN/utl4xf48= +github.com/hwameistor/hwameistor v0.14.4 h1:MKblPQraGLp0NUWuS7+jTv0yIGhR411sagYppbjeziY= +github.com/hwameistor/hwameistor v0.14.4/go.mod h1:d+J2bzNIU2UFIKaihRK3rh2gN45NJcOA6D4Qv2fiEi4= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= @@ -354,7 +355,6 @@ github.com/kr/pretty v0.1.0/go.mod h1:dAy3ld7l9f0ibDNOQOHHMYYIIbhfbHSm3C4ZsoJORN github.com/kr/pretty v0.2.0/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.2.1/go.mod h1:ipq/a2n7PKx3OHsz4KJII5eveXtPO4qwEXGdVfWzfnI= github.com/kr/pretty v0.3.0 h1:WgNl7dwNpEZ6jJ9k1snq4pZsg7DOEN8hP9Xw0Tsjwk0= -github.com/kr/pretty v0.3.0/go.mod h1:640gp4NfQd8pI5XOwp5fnNeVWj67G7CFk/SaSQn7NBk= github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= @@ -418,7 +418,6 @@ github.com/opentracing/opentracing-go v1.1.0/go.mod h1:UkNAQd3GIcIGf0SeVgPpRdFSt github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= github.com/peterbourgon/diskv v2.0.1+incompatible/go.mod h1:uqqh8zWWbv1HBMNONnaR/tNboyR3/BZd58JJSHlUSCU= -github.com/pkg/diff v0.0.0-20210226163009-20ebb0f2a09e/go.mod h1:pJLUxLENpZxwdsKMEsNbx1VGcRFpLqf3715MtcvvzbA= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4= @@ -461,9 +460,7 @@ github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40T github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= -github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= github.com/rogpeppe/go-internal v1.8.0 h1:FCbCCtXNOY3UtUuHUYaghJg4y7Fd14rXifAYUAtL9R8= -github.com/rogpeppe/go-internal v1.8.0/go.mod h1:WmiCO8CzOY8rg0OYDC4/i/2WRWAB6poM+XZ2dLUbcbE= github.com/russross/blackfriday/v2 v2.0.1/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -474,8 +471,8 @@ github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6Mwd github.com/sirupsen/logrus v1.6.0/go.mod h1:7uNnSEd1DgxDLC74fIahvMZmmYsHGZGEOFrfsX/uA88= github.com/sirupsen/logrus v1.7.0/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= github.com/sirupsen/logrus v1.8.1/go.mod h1:yWOB1SBYBC5VeMP7gHvWumXLIWorT60ONWic61uBYv0= -github.com/sirupsen/logrus v1.9.0 h1:trlNQbNUG3OdDrDil03MCb1H2o9nJ1x4/5LYw7byDE0= -github.com/sirupsen/logrus v1.9.0/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= +github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= +github.com/sirupsen/logrus v1.9.3/go.mod h1:naHLuLoDiP4jHNo9R0sCBMtWGeIprob74mVsIT4qYEQ= github.com/smartystreets/assertions v0.0.0-20180927180507-b2de0cb4f26d/go.mod h1:OnSkiWE9lh6wB0YB77sQom3nweQdgAjqCqsofrRNTgc= github.com/smartystreets/goconvey v1.6.4/go.mod h1:syvi0/a8iFYH4r/RixwvyeAJjdLS9QV7WQ/tjFTllLA= github.com/soheilhy/cmux v0.1.4/go.mod h1:IM3LyeVVIOuxMH7sFAkER9+bJ4dT7Ms6E4xg4kGIyLM= @@ -577,8 +574,8 @@ golang.org/x/crypto v0.0.0-20201002170205-7f63de1d35b0/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.0.0-20220214200702-86341886e292/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= golang.org/x/crypto v0.0.0-20220722155217-630584e8d5aa/go.mod h1:IxCIyHEi3zRg3s0A5j5BB6A9Jmi73HwBIUl50j+osU4= -golang.org/x/crypto v0.1.0 h1:MDRAIl0xIo9Io2xV565hzXHw3zVseKrJKodhohM5CjU= -golang.org/x/crypto v0.1.0/go.mod h1:RecgLatLF4+eUMCP1PoPZQb+cVrJcOPbHkTkbkB9sbw= +golang.org/x/crypto v0.16.0 h1:mMMrFzRSCF0GvB7Ne27XVtVAaXLrPmgPC7/v0tkwHaY= +golang.org/x/crypto v0.16.0/go.mod h1:gCAAfMLgwOJRpTjQ2zCCt2OcSfYMTeZVSRtQlPC7Nq4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8= @@ -668,8 +665,8 @@ golang.org/x/net v0.0.0-20220325170049-de3da57026de/go.mod h1:CfG3xpIq0wQ8r1q4Su golang.org/x/net v0.0.0-20220412020605-290c469a71a5/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220425223048-2871e0cb64e4/go.mod h1:CfG3xpIq0wQ8r1q4Su4UZFWDARRcnwPjda9FqA0JpMk= golang.org/x/net v0.0.0-20220607020251-c690dde0001d/go.mod h1:XRhObCWvk6IyKnWLug+ECip1KBveYUHfp+8e9klMJ9c= -golang.org/x/net v0.7.0 h1:rJrUqqhjsgNp7KqAIc25s9pZnjU7TUcSY7HcVZjdn1g= -golang.org/x/net v0.7.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= +golang.org/x/net v0.19.0 h1:zTwKpTd2XuCqf8huc7Fo2iSy+4RHPd10s4KzeTnVr1c= +golang.org/x/net v0.19.0/go.mod h1:CfAk/cbD4CthTvqiEl8NpboMuiuOYsAr/7NOjZJtv1U= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -785,12 +782,12 @@ golang.org/x/sys v0.0.0-20220503163025-988cb79eb6c6/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20220520151302-bc2c85ada10a/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220610221304-9f5ed59c137d/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220715151400-c0bba94af5f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.5.0 h1:MUK/U/4lj1t1oPg0HfuXDN/Z1wv31ZJ/YcPiGccS4DU= -golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.15.0 h1:h48lPFYpsTvQJZF4EKyI4aLHaev3CxivZmv7yZig9pc= +golang.org/x/sys v0.15.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= -golang.org/x/term v0.5.0 h1:n2a8QNdAb0sZNpU9R1ALUXBbY+w51fCQDN+7EdxNBsY= -golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k= +golang.org/x/term v0.15.0 h1:y/Oo/a/q3IXu26lQgl04j/gjuBDOBlx7X6Om1j2CPW4= +golang.org/x/term v0.15.0/go.mod h1:BDl952bC7+uMoWR75FIrCDx79TPU9oHkTZ9yRbYOrX0= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -800,8 +797,8 @@ golang.org/x/text v0.3.4/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.5/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.6/go.mod h1:5Zoc/QRtKVWzQhOtBMvqHzDpF6irO9z98xDceosuGiQ= golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ= -golang.org/x/text v0.7.0 h1:4BRB4x83lYWy72KwLD/qYDuTu7q9PjSagHvijDw7cLo= -golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= +golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= +golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/helm/operator/templates/_helpers.tpl b/helm/operator/templates/_helpers.tpl index 1b06be54..74f60c2f 100644 --- a/helm/operator/templates/_helpers.tpl +++ b/helm/operator/templates/_helpers.tpl @@ -125,3 +125,13 @@ Create the name of the service account to use {{- define "hwameistor.uiImageTag" -}} {{- default .Values.global.hwameistorVersion .Values.ui.tag -}} {{- end -}} + +{{/* Allow dataLoadManager image tag to be overridden. */}} +{{- define "hwameistor.dataLoalManagerImageTag" -}} + {{- default .Values.global.hwameistorVersion .Values.dataLoadManager.tag -}} +{{- end -}} + +{{/* Allow dataSetManager image tag to be overridden. */}} +{{- define "hwameistor.dataSetManagerImageTag" -}} + {{- default .Values.global.hwameistorVersion .Values.dataSetManager.tag -}} +{{- end -}} \ No newline at end of file diff --git a/helm/operator/templates/hwameistorcluster.yaml b/helm/operator/templates/hwameistorcluster.yaml index 082dd552..57f05860 100644 --- a/helm/operator/templates/hwameistorcluster.yaml +++ b/helm/operator/templates/hwameistorcluster.yaml @@ -11,6 +11,7 @@ spec: - {{ . }} {{- end }} {{- end }} + datasetDefaultPoolClass: {{ $.Values.datasetDefaultPoolClass }} notClaimDisk: {{ $.Values.global.notClaimDisk }} targetNamespace: {{ $.Values.global.targetNamespace }} rbac: @@ -255,4 +256,20 @@ spec: storageClass: allowVolumeExpansion: true reclaimPolicy: "Delete" - fsType: "xfs" \ No newline at end of file + fsType: "xfs" + + dataLoadManager: + disable: {{ $.Values.dataLoadManager.disable }} + dataLoadManagerContainer: + image: + registry: {{ $.Values.global.hwameistorImageRegistry }} + repository: {{ $.Values.dataLoadManager.imageRepository }} + tag: {{ template "hwameistor.dataLoalManagerImageTag" . }} + + dataSetManager: + disable: {{ $.Values.dataSetManager.disable }} + dataSetManagerContainer: + image: + registry: {{ $.Values.global.hwameistorImageRegistry }} + repository: {{ $.Values.dataSetManager.imageRepository }} + tag: {{ template "hwameistor.dataSetManagerImageTag" . }} \ No newline at end of file diff --git a/helm/operator/values.extra.prod.yaml b/helm/operator/values.extra.prod.yaml index b443ecf6..963ce311 100644 --- a/helm/operator/values.extra.prod.yaml +++ b/helm/operator/values.extra.prod.yaml @@ -186,6 +186,25 @@ pvcAutoResizer: memory: 20Mi localDiskActionController: + resources: + limits: + cpu: 300m + memory: 300Mi + requests: + cpu: 1m + memory: 20Mi + + +dataLoadManager: + resources: + limits: + cpu: 300m + memory: 300Mi + requests: + cpu: 1m + memory: 20Mi + +dataSetManager: resources: limits: cpu: 300m diff --git a/helm/operator/values.yaml b/helm/operator/values.yaml index e5deac00..6d62d781 100644 --- a/helm/operator/values.yaml +++ b/helm/operator/values.yaml @@ -167,6 +167,21 @@ drbdBionic: drbdFocal: imageRepository: hwameistor/drbd9-focal +dataLoadManager: + disable: true + tolerationOnMaster: true + imageRepository: hwameistor/dataload-manager + tag: v1.0.0 + +dataSetManager: + disable: true + tolerationOnMaster: true + imageRepository: hwameistor/dataset-manager + tag: v1.0.0 + + +#datasetDefaultPoolClass: NVMe > SSD > HDD + # diskReserve example # diskReserve: # - nodeName: node1 diff --git a/pkg/install/dataloadmanager/dataloadmanager.go b/pkg/install/dataloadmanager/dataloadmanager.go new file mode 100644 index 00000000..97aa8471 --- /dev/null +++ b/pkg/install/dataloadmanager/dataloadmanager.go @@ -0,0 +1,283 @@ +package dataloadmanager + +import ( + "context" + hwameistoriov1alpha1 "github.com/hwameistor/hwameistor-operator/api/v1alpha1" + "github.com/hwameistor/hwameistor-operator/pkg/install" + log "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "k8s.io/apimachinery/pkg/util/intstr" + "reflect" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var dlmDaemonSetLabelSelectorKey = "app" +var dlmDaemonSetLabelSelectorValue = "hwameistor-dataload-manager" +var dlmContainerName = "dataload-manager" +var defaultDLMDaemonsetImageRegistry = "ghcr.m.daocloud.io" +var defaultDLMDaemonsetImageRepository = "hwameistor/dataload-manager" +var defaultDLMDaemonsetImageTag = "v1.0.0" + +type DataLoadManagerMaintainer struct { + Client client.Client + ClusterInstance *hwameistoriov1alpha1.Cluster +} + +func NewMaintainer(cli client.Client, clusterInstance *hwameistoriov1alpha1.Cluster) *DataLoadManagerMaintainer { + return &DataLoadManagerMaintainer{ + Client: cli, + ClusterInstance: clusterInstance, + } +} + +var dlmDaemonSetTemplate = appsv1.DaemonSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "hwameistor-dataload-manager", + }, + Spec: appsv1.DaemonSetSpec{ + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + dlmDaemonSetLabelSelectorKey: dlmDaemonSetLabelSelectorValue, + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + dlmDaemonSetLabelSelectorKey: dlmDaemonSetLabelSelectorValue, + }, + }, + Spec: corev1.PodSpec{ + Affinity: &corev1.Affinity{ + NodeAffinity: &corev1.NodeAffinity{ + RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{ + NodeSelectorTerms: []corev1.NodeSelectorTerm{ + { + MatchExpressions: []corev1.NodeSelectorRequirement{ + { + Key: "lvm.hwameistor.io/enable", + Operator: corev1.NodeSelectorOpNotIn, + Values: []string{"false"}, + }, + }, + }, + }, + }, + }, + }, + Containers: []corev1.Container{ + { + Name: dlmContainerName, + //Image: "10.6.118.138:5000/hwameistor:dataload_manager_99.9-dev", + ImagePullPolicy: corev1.PullIfNotPresent, + SecurityContext: &corev1.SecurityContext{ + Privileged: &install.SecurityContextPrivilegedTrue, + }, + Env: []corev1.EnvVar{ + { + Name: "MY_NODENAME", + ValueFrom: &corev1.EnvVarSource{ + FieldRef: &corev1.ObjectFieldSelector{ + APIVersion: "v1", + FieldPath: "spec.nodeName", + }, + }, + }, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "host-dev", + MountPath: "/dev", + MountPropagation: nil, + }, + { + Name: "host-mnt", + MountPath: "/mnt", + MountPropagation: &[]corev1.MountPropagationMode{corev1.MountPropagationBidirectional}[0], + }, + }, + Args: []string{"--nodename=$(MY_NODENAME)"}, + }, + }, + DNSPolicy: corev1.DNSClusterFirstWithHostNet, + HostPID: true, + Volumes: []corev1.Volume{ + { + Name: "host-dev", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/dev", + }, + }, + }, + { + Name: "host-mnt", + VolumeSource: corev1.VolumeSource{ + HostPath: &corev1.HostPathVolumeSource{ + Path: "/mnt", + Type: &[]corev1.HostPathType{corev1.HostPathDirectoryOrCreate}[0], + }, + }, + }, + }, + }, + }, + UpdateStrategy: appsv1.DaemonSetUpdateStrategy{ + RollingUpdate: &appsv1.RollingUpdateDaemonSet{ + MaxUnavailable: &[]intstr.IntOrString{intstr.FromInt(1)}[0], + }, + Type: appsv1.RollingUpdateDaemonSetStrategyType, + }, + }, +} + +func SetDLMDaemonSet(clusterInstance *hwameistoriov1alpha1.Cluster) *appsv1.DaemonSet { + dlmDaemonSetToCreate := dlmDaemonSetTemplate.DeepCopy() + dlmDaemonSetToCreate.OwnerReferences = append(dlmDaemonSetToCreate.OwnerReferences, *metav1.NewControllerRef(clusterInstance, clusterInstance.GroupVersionKind())) + dlmDaemonSetToCreate.Namespace = clusterInstance.Spec.TargetNamespace + dlmDaemonSetToCreate.Spec.Template.Spec.ServiceAccountName = clusterInstance.Spec.RBAC.ServiceAccountName + dlmDaemonSetToCreate = setDLMDaemonSetContainers(clusterInstance, dlmDaemonSetToCreate) + + return dlmDaemonSetToCreate + +} + +func setDLMDaemonSetContainers(clusterInstance *hwameistoriov1alpha1.Cluster, dlmDaemonSetToCreate *appsv1.DaemonSet) *appsv1.DaemonSet { + for i, container := range dlmDaemonSetToCreate.Spec.Template.Spec.Containers { + if container.Name == dlmContainerName { + container.Image = getDLMContainerRegistrarImageStringFromClusterInstance(clusterInstance) + } + dlmDaemonSetToCreate.Spec.Template.Spec.Containers[i] = container + } + return dlmDaemonSetToCreate +} + +func getDLMContainerRegistrarImageStringFromClusterInstance(clusterInstance *hwameistoriov1alpha1.Cluster) string { + imageSpec := clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image + return imageSpec.Registry + "/" + imageSpec.Repository + ":" + imageSpec.Tag +} + +func needOrNotToUpdateDLMDaemonset(cluster *hwameistoriov1alpha1.Cluster, gotten appsv1.DaemonSet) (bool, *appsv1.DaemonSet) { + ds := gotten.DeepCopy() + var needToUpdate bool + + for i, container := range ds.Spec.Template.Spec.Containers { + if container.Name == dlmContainerName { + wantedImage := getDLMContainerRegistrarImageStringFromClusterInstance(cluster) + if container.Image != wantedImage { + container.Image = wantedImage + ds.Spec.Template.Spec.Containers[i] = container + needToUpdate = true + } + } + } + + return needToUpdate, ds +} + +func (m *DataLoadManagerMaintainer) Ensure() (*hwameistoriov1alpha1.Cluster, error) { + newClusterInstance := m.ClusterInstance.DeepCopy() + dlmDaemonSetToCreate := SetDLMDaemonSet(newClusterInstance) + key := types.NamespacedName{ + Namespace: dlmDaemonSetToCreate.Namespace, + Name: dlmDaemonSetToCreate.Name, + } + var gottenDS appsv1.DaemonSet + if err := m.Client.Get(context.TODO(), key, &gottenDS); err != nil { + if errors.IsNotFound(err) { + if errCreate := m.Client.Create(context.TODO(), dlmDaemonSetToCreate); errCreate != nil { + log.Errorf("Create DataLoadManager DaemonSet err: %v", errCreate) + return newClusterInstance, errCreate + } + return newClusterInstance, nil + } else { + log.Errorf("Get DataLoadManager DaemonSet err: %v", err) + return newClusterInstance, err + } + } + needToUpdate, dsToUpdate := needOrNotToUpdateDLMDaemonset(newClusterInstance, gottenDS) + if needToUpdate { + log.Infof("need to update DataLoadManager daemonset") + if err := m.Client.Update(context.TODO(), dsToUpdate); err != nil { + log.Errorf("Update DataLoadManager daemonset err: %v", err) + return newClusterInstance, err + } + } + + var podList corev1.PodList + if err := m.Client.List(context.TODO(), &podList, &client.ListOptions{Namespace: dlmDaemonSetToCreate.Namespace}); err != nil { + log.Errorf("List pods err: %v", err) + return newClusterInstance, err + } + + var podsManaged []corev1.Pod + for _, pod := range podList.Items { + if pod.Labels[dlmDaemonSetLabelSelectorKey] == dlmDaemonSetLabelSelectorValue { + podsManaged = append(podsManaged, pod) + } + } + podsStatus := make([]hwameistoriov1alpha1.PodStatus, 0) + for _, pod := range podsManaged { + podStatus := hwameistoriov1alpha1.PodStatus{ + Name: pod.Name, + Node: pod.Spec.NodeName, + Status: string(pod.Status.Phase), + } + podsStatus = append(podsStatus, podStatus) + } + + instancesDeployStatus := hwameistoriov1alpha1.DeployStatus{ + Pods: podsStatus, + DesiredPodCount: gottenDS.Status.DesiredNumberScheduled, + AvailablePodCount: gottenDS.Status.NumberAvailable, + WorkloadType: "DaemonSet", + WorkloadName: gottenDS.Name, + } + + if newClusterInstance.Status.ComponentStatus.DataLoadManager == nil { + newClusterInstance.Status.ComponentStatus.DataLoadManager = &hwameistoriov1alpha1.DataLoadManagerStatus{ + Instances: &instancesDeployStatus, + } + return newClusterInstance, nil + } else { + if newClusterInstance.Status.ComponentStatus.DataLoadManager.Instances == nil { + newClusterInstance.Status.ComponentStatus.DataLoadManager.Instances = &instancesDeployStatus + return newClusterInstance, nil + } else { + if !reflect.DeepEqual(newClusterInstance.Status.ComponentStatus.DataLoadManager.Instances, instancesDeployStatus) { + newClusterInstance.Status.ComponentStatus.DataLoadManager.Instances = &instancesDeployStatus + return newClusterInstance, nil + } + } + } + return newClusterInstance, nil +} + +func FulfillDataLoadManagerSpec(clusterInstance *hwameistoriov1alpha1.Cluster) *hwameistoriov1alpha1.Cluster { + if clusterInstance.Spec.DataLoadManager == nil { + clusterInstance.Spec.DataLoadManager = &hwameistoriov1alpha1.DataLoadManagerSpec{} + } + + if clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer == nil { + clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer = &hwameistoriov1alpha1.ContainerCommonSpec{} + } + + if clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image == nil { + clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image = &hwameistoriov1alpha1.ImageSpec{} + } + + if clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Registry == "" { + clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Registry = defaultDLMDaemonsetImageRegistry + } + + if clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Repository == "" { + clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Repository = defaultDLMDaemonsetImageRepository + } + if clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Tag == "" { + clusterInstance.Spec.DataLoadManager.DataLoadManagerContainer.Image.Tag = defaultDLMDaemonsetImageTag + } + return clusterInstance +} diff --git a/pkg/install/datasetmanager/datasetmanager.go b/pkg/install/datasetmanager/datasetmanager.go new file mode 100644 index 00000000..9f3b11d9 --- /dev/null +++ b/pkg/install/datasetmanager/datasetmanager.go @@ -0,0 +1,188 @@ +package datasetmanager + +import ( + "context" + hwameistoriov1alpha1 "github.com/hwameistor/hwameistor-operator/api/v1alpha1" + log "github.com/sirupsen/logrus" + appsv1 "k8s.io/api/apps/v1" + corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/errors" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/types" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +var dsmDeploymentLabelSelectorKey = "app" +var dsmDeploymentLabelSelectorValue = "hwameistor-dataset-manager" +var dsmContainerName = "dataset-manager" +var defaultDSMDeploymentImageRegistry = "ghcr.m.daocloud.io" +var defaultDSMDeploymentImageRepository = "hwameistor/dataset-manager" +var defaultDSMDeploymentImageTag = "v1.0.0" + +type DataSetManagerMaintainer struct { + Client client.Client + ClusterInstance *hwameistoriov1alpha1.Cluster +} + +func NewMaintainer(cli client.Client, clusterInstance *hwameistoriov1alpha1.Cluster) *DataSetManagerMaintainer { + return &DataSetManagerMaintainer{ + Client: cli, + ClusterInstance: clusterInstance, + } +} + +var dsmDeploymentTemplate = appsv1.Deployment{ + + ObjectMeta: metav1.ObjectMeta{ + Name: "hwameistor-dataset-manager", + Labels: map[string]string{ + "app": "hwameistor-dataset-manager", + }, + }, + Spec: appsv1.DeploymentSpec{ + Replicas: int32Ptr(1), + Selector: &metav1.LabelSelector{ + MatchLabels: map[string]string{ + "app": "hwameistor-dataset-manager", + }, + }, + Template: corev1.PodTemplateSpec{ + ObjectMeta: metav1.ObjectMeta{ + Labels: map[string]string{ + "app": "hwameistor-dataset-manager", + }, + }, + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: "dataset-manager", + Args: []string{"--v=4", "--leader-election=true"}, + ImagePullPolicy: corev1.PullIfNotPresent, + }, + }, + RestartPolicy: corev1.RestartPolicyAlways, + DNSPolicy: corev1.DNSClusterFirst, + TerminationGracePeriodSeconds: int64Ptr(30), + }, + }, + ProgressDeadlineSeconds: int32Ptr(600), + RevisionHistoryLimit: int32Ptr(10), + Strategy: appsv1.DeploymentStrategy{ + Type: appsv1.RecreateDeploymentStrategyType, + }, + }, +} + +func int32Ptr(i int32) *int32 { return &i } + +func int64Ptr(i int64) *int64 { return &i } + +func SetDSMDeployment(clusterInstance *hwameistoriov1alpha1.Cluster) *appsv1.Deployment { + dsmDeploymentToCreate := dsmDeploymentTemplate.DeepCopy() + dsmDeploymentToCreate.OwnerReferences = append(dsmDeploymentToCreate.OwnerReferences, *metav1.NewControllerRef(clusterInstance, clusterInstance.GroupVersionKind())) + dsmDeploymentToCreate.Namespace = clusterInstance.Spec.TargetNamespace + dsmDeploymentToCreate.Spec.Template.Spec.ServiceAccountName = clusterInstance.Spec.RBAC.ServiceAccountName + dsmDeploymentToCreate = setdsmDeploymentContainers(clusterInstance, dsmDeploymentToCreate) + + if clusterInstance.Status.DatasetDefaultPoolClass != "" { + pool_env := corev1.EnvVar{ + Name: "DEFAULT_POOL_CLASS", + Value: clusterInstance.Status.DatasetDefaultPoolClass, + } + dsmDeploymentToCreate.Spec.Template.Spec.Containers[0].Env = append(dsmDeploymentToCreate.Spec.Template.Spec.Containers[0].Env, pool_env) + } + + return dsmDeploymentToCreate + +} + +func setdsmDeploymentContainers(clusterInstance *hwameistoriov1alpha1.Cluster, dsmDeploymentToCreate *appsv1.Deployment) *appsv1.Deployment { + for i, container := range dsmDeploymentToCreate.Spec.Template.Spec.Containers { + if container.Name == dsmContainerName { + container.Image = getDSMContainerRegistrarImageStringFromClusterInstance(clusterInstance) + } + dsmDeploymentToCreate.Spec.Template.Spec.Containers[i] = container + } + return dsmDeploymentToCreate +} + +func getDSMContainerRegistrarImageStringFromClusterInstance(clusterInstance *hwameistoriov1alpha1.Cluster) string { + imageSpec := clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image + return imageSpec.Registry + "/" + imageSpec.Repository + ":" + imageSpec.Tag +} + +func needOrNotToUpdateDSMDeployment(cluster *hwameistoriov1alpha1.Cluster, gotten appsv1.Deployment) (bool, *appsv1.Deployment) { + ds := gotten.DeepCopy() + var needToUpdate bool + + for i, container := range ds.Spec.Template.Spec.Containers { + if container.Name == dsmContainerName { + wantedImage := getDSMContainerRegistrarImageStringFromClusterInstance(cluster) + if container.Image != wantedImage { + container.Image = wantedImage + ds.Spec.Template.Spec.Containers[i] = container + needToUpdate = true + } + } + } + + return needToUpdate, ds +} + +func (m *DataSetManagerMaintainer) Ensure() (*hwameistoriov1alpha1.Cluster, error) { + newClusterInstance := m.ClusterInstance.DeepCopy() + dsmDeploymentToCreate := SetDSMDeployment(newClusterInstance) + key := types.NamespacedName{ + Namespace: dsmDeploymentToCreate.Namespace, + Name: dsmDeploymentToCreate.Name, + } + var gottenDS appsv1.Deployment + if err := m.Client.Get(context.TODO(), key, &gottenDS); err != nil { + if errors.IsNotFound(err) { + if errCreate := m.Client.Create(context.TODO(), dsmDeploymentToCreate); errCreate != nil { + log.Errorf("Create DataSetManager Deployment err: %v", errCreate) + return newClusterInstance, errCreate + } + return newClusterInstance, nil + } else { + log.Errorf("Get DataSetManager Deployment err: %v", err) + return newClusterInstance, err + } + } + needToUpdate, dsToUpdate := needOrNotToUpdateDSMDeployment(newClusterInstance, gottenDS) + if needToUpdate { + log.Infof("need to update DataSetManager Deployment") + if err := m.Client.Update(context.TODO(), dsToUpdate); err != nil { + log.Errorf("Update DataSetManager Deployment err: %v", err) + return newClusterInstance, err + } + } + + return newClusterInstance, nil +} + +func FulfillDataSetManagerSpec(clusterInstance *hwameistoriov1alpha1.Cluster) *hwameistoriov1alpha1.Cluster { + if clusterInstance.Spec.DataSetManager == nil { + clusterInstance.Spec.DataSetManager = &hwameistoriov1alpha1.DataSetManagerSpec{} + } + + if clusterInstance.Spec.DataSetManager.DataSetManagerContainer == nil { + clusterInstance.Spec.DataSetManager.DataSetManagerContainer = &hwameistoriov1alpha1.ContainerCommonSpec{} + } + + if clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image == nil { + clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image = &hwameistoriov1alpha1.ImageSpec{} + } + + if clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Registry == "" { + clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Registry = defaultDSMDeploymentImageRegistry + } + + if clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Repository == "" { + clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Repository = defaultDSMDeploymentImageRepository + } + if clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Tag == "" { + clusterInstance.Spec.DataSetManager.DataSetManagerContainer.Image.Tag = defaultDSMDeploymentImageTag + } + return clusterInstance +} diff --git a/tool.go b/tool.go new file mode 100644 index 00000000..d248c03f --- /dev/null +++ b/tool.go @@ -0,0 +1,4 @@ +package main + +//import _ "github.com/hwameistor/datastore/deploy/crds" +//import _ "github.com/hwameistor/hwameistor/deploy/crds" diff --git a/vendor/github.com/google/uuid/.travis.yml b/vendor/github.com/google/uuid/.travis.yml deleted file mode 100644 index d8156a60..00000000 --- a/vendor/github.com/google/uuid/.travis.yml +++ /dev/null @@ -1,9 +0,0 @@ -language: go - -go: - - 1.4.3 - - 1.5.3 - - tip - -script: - - go test -v ./... diff --git a/vendor/github.com/google/uuid/CHANGELOG.md b/vendor/github.com/google/uuid/CHANGELOG.md new file mode 100644 index 00000000..c9fb829d --- /dev/null +++ b/vendor/github.com/google/uuid/CHANGELOG.md @@ -0,0 +1,28 @@ +# Changelog + +## [1.5.0](https://github.com/google/uuid/compare/v1.4.0...v1.5.0) (2023-12-12) + + +### Features + +* Validate UUID without creating new UUID ([#141](https://github.com/google/uuid/issues/141)) ([9ee7366](https://github.com/google/uuid/commit/9ee7366e66c9ad96bab89139418a713dc584ae29)) + +## [1.4.0](https://github.com/google/uuid/compare/v1.3.1...v1.4.0) (2023-10-26) + + +### Features + +* UUIDs slice type with Strings() convenience method ([#133](https://github.com/google/uuid/issues/133)) ([cd5fbbd](https://github.com/google/uuid/commit/cd5fbbdd02f3e3467ac18940e07e062be1f864b4)) + +### Fixes + +* Clarify that Parse's job is to parse but not necessarily validate strings. (Documents current behavior) + +## [1.3.1](https://github.com/google/uuid/compare/v1.3.0...v1.3.1) (2023-08-18) + + +### Bug Fixes + +* Use .EqualFold() to parse urn prefixed UUIDs ([#118](https://github.com/google/uuid/issues/118)) ([574e687](https://github.com/google/uuid/commit/574e6874943741fb99d41764c705173ada5293f0)) + +## Changelog diff --git a/vendor/github.com/google/uuid/CONTRIBUTING.md b/vendor/github.com/google/uuid/CONTRIBUTING.md index 04fdf09f..a502fdc5 100644 --- a/vendor/github.com/google/uuid/CONTRIBUTING.md +++ b/vendor/github.com/google/uuid/CONTRIBUTING.md @@ -2,6 +2,22 @@ We definitely welcome patches and contribution to this project! +### Tips + +Commits must be formatted according to the [Conventional Commits Specification](https://www.conventionalcommits.org). + +Always try to include a test case! If it is not possible or not necessary, +please explain why in the pull request description. + +### Releasing + +Commits that would precipitate a SemVer change, as described in the Conventional +Commits Specification, will trigger [`release-please`](https://github.com/google-github-actions/release-please-action) +to create a release candidate pull request. Once submitted, `release-please` +will create a release. + +For tips on how to work with `release-please`, see its documentation. + ### Legal requirements In order to protect both you and ourselves, you will need to sign the diff --git a/vendor/github.com/google/uuid/README.md b/vendor/github.com/google/uuid/README.md index f765a46f..3e9a6188 100644 --- a/vendor/github.com/google/uuid/README.md +++ b/vendor/github.com/google/uuid/README.md @@ -1,6 +1,6 @@ -# uuid ![build status](https://travis-ci.org/google/uuid.svg?branch=master) +# uuid The uuid package generates and inspects UUIDs based on -[RFC 4122](http://tools.ietf.org/html/rfc4122) +[RFC 4122](https://datatracker.ietf.org/doc/html/rfc4122) and DCE 1.1: Authentication and Security Services. This package is based on the github.com/pborman/uuid package (previously named @@ -9,10 +9,12 @@ a UUID is a 16 byte array rather than a byte slice. One loss due to this change is the ability to represent an invalid UUID (vs a NIL UUID). ###### Install -`go get github.com/google/uuid` +```sh +go get github.com/google/uuid +``` ###### Documentation -[![GoDoc](https://godoc.org/github.com/google/uuid?status.svg)](http://godoc.org/github.com/google/uuid) +[![Go Reference](https://pkg.go.dev/badge/github.com/google/uuid.svg)](https://pkg.go.dev/github.com/google/uuid) Full `go doc` style documentation for the package can be viewed online without installing this package by using the GoDoc site here: diff --git a/vendor/github.com/google/uuid/hash.go b/vendor/github.com/google/uuid/hash.go index b1746163..b404f4be 100644 --- a/vendor/github.com/google/uuid/hash.go +++ b/vendor/github.com/google/uuid/hash.go @@ -26,8 +26,8 @@ var ( // NewMD5 and NewSHA1. func NewHash(h hash.Hash, space UUID, data []byte, version int) UUID { h.Reset() - h.Write(space[:]) - h.Write(data) + h.Write(space[:]) //nolint:errcheck + h.Write(data) //nolint:errcheck s := h.Sum(nil) var uuid UUID copy(uuid[:], s) diff --git a/vendor/github.com/google/uuid/node_js.go b/vendor/github.com/google/uuid/node_js.go index 24b78edc..b2a0bc87 100644 --- a/vendor/github.com/google/uuid/node_js.go +++ b/vendor/github.com/google/uuid/node_js.go @@ -7,6 +7,6 @@ package uuid // getHardwareInterface returns nil values for the JS version of the code. -// This remvoves the "net" dependency, because it is not used in the browser. +// This removes the "net" dependency, because it is not used in the browser. // Using the "net" library inflates the size of the transpiled JS code by 673k bytes. func getHardwareInterface(name string) (string, []byte) { return "", nil } diff --git a/vendor/github.com/google/uuid/null.go b/vendor/github.com/google/uuid/null.go new file mode 100644 index 00000000..d7fcbf28 --- /dev/null +++ b/vendor/github.com/google/uuid/null.go @@ -0,0 +1,118 @@ +// Copyright 2021 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "bytes" + "database/sql/driver" + "encoding/json" + "fmt" +) + +var jsonNull = []byte("null") + +// NullUUID represents a UUID that may be null. +// NullUUID implements the SQL driver.Scanner interface so +// it can be used as a scan destination: +// +// var u uuid.NullUUID +// err := db.QueryRow("SELECT name FROM foo WHERE id=?", id).Scan(&u) +// ... +// if u.Valid { +// // use u.UUID +// } else { +// // NULL value +// } +// +type NullUUID struct { + UUID UUID + Valid bool // Valid is true if UUID is not NULL +} + +// Scan implements the SQL driver.Scanner interface. +func (nu *NullUUID) Scan(value interface{}) error { + if value == nil { + nu.UUID, nu.Valid = Nil, false + return nil + } + + err := nu.UUID.Scan(value) + if err != nil { + nu.Valid = false + return err + } + + nu.Valid = true + return nil +} + +// Value implements the driver Valuer interface. +func (nu NullUUID) Value() (driver.Value, error) { + if !nu.Valid { + return nil, nil + } + // Delegate to UUID Value function + return nu.UUID.Value() +} + +// MarshalBinary implements encoding.BinaryMarshaler. +func (nu NullUUID) MarshalBinary() ([]byte, error) { + if nu.Valid { + return nu.UUID[:], nil + } + + return []byte(nil), nil +} + +// UnmarshalBinary implements encoding.BinaryUnmarshaler. +func (nu *NullUUID) UnmarshalBinary(data []byte) error { + if len(data) != 16 { + return fmt.Errorf("invalid UUID (got %d bytes)", len(data)) + } + copy(nu.UUID[:], data) + nu.Valid = true + return nil +} + +// MarshalText implements encoding.TextMarshaler. +func (nu NullUUID) MarshalText() ([]byte, error) { + if nu.Valid { + return nu.UUID.MarshalText() + } + + return jsonNull, nil +} + +// UnmarshalText implements encoding.TextUnmarshaler. +func (nu *NullUUID) UnmarshalText(data []byte) error { + id, err := ParseBytes(data) + if err != nil { + nu.Valid = false + return err + } + nu.UUID = id + nu.Valid = true + return nil +} + +// MarshalJSON implements json.Marshaler. +func (nu NullUUID) MarshalJSON() ([]byte, error) { + if nu.Valid { + return json.Marshal(nu.UUID) + } + + return jsonNull, nil +} + +// UnmarshalJSON implements json.Unmarshaler. +func (nu *NullUUID) UnmarshalJSON(data []byte) error { + if bytes.Equal(data, jsonNull) { + *nu = NullUUID{} + return nil // valid null UUID + } + err := json.Unmarshal(data, &nu.UUID) + nu.Valid = err == nil + return err +} diff --git a/vendor/github.com/google/uuid/sql.go b/vendor/github.com/google/uuid/sql.go index f326b54d..2e02ec06 100644 --- a/vendor/github.com/google/uuid/sql.go +++ b/vendor/github.com/google/uuid/sql.go @@ -9,7 +9,7 @@ import ( "fmt" ) -// Scan implements sql.Scanner so UUIDs can be read from databases transparently +// Scan implements sql.Scanner so UUIDs can be read from databases transparently. // Currently, database types that map to string and []byte are supported. Please // consult database-specific driver documentation for matching types. func (uuid *UUID) Scan(src interface{}) error { diff --git a/vendor/github.com/google/uuid/time.go b/vendor/github.com/google/uuid/time.go index e6ef06cd..c3511292 100644 --- a/vendor/github.com/google/uuid/time.go +++ b/vendor/github.com/google/uuid/time.go @@ -108,12 +108,23 @@ func setClockSequence(seq int) { } // Time returns the time in 100s of nanoseconds since 15 Oct 1582 encoded in -// uuid. The time is only defined for version 1 and 2 UUIDs. +// uuid. The time is only defined for version 1, 2, 6 and 7 UUIDs. func (uuid UUID) Time() Time { - time := int64(binary.BigEndian.Uint32(uuid[0:4])) - time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 - time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 - return Time(time) + var t Time + switch uuid.Version() { + case 6: + time := binary.BigEndian.Uint64(uuid[:8]) // Ignore uuid[6] version b0110 + t = Time(time) + case 7: + time := binary.BigEndian.Uint64(uuid[:8]) + t = Time((time>>16)*10000 + g1582ns100) + default: // forward compatible + time := int64(binary.BigEndian.Uint32(uuid[0:4])) + time |= int64(binary.BigEndian.Uint16(uuid[4:6])) << 32 + time |= int64(binary.BigEndian.Uint16(uuid[6:8])&0xfff) << 48 + t = Time(time) + } + return t } // ClockSequence returns the clock sequence encoded in uuid. diff --git a/vendor/github.com/google/uuid/uuid.go b/vendor/github.com/google/uuid/uuid.go index 524404cc..5232b486 100644 --- a/vendor/github.com/google/uuid/uuid.go +++ b/vendor/github.com/google/uuid/uuid.go @@ -12,6 +12,7 @@ import ( "fmt" "io" "strings" + "sync" ) // A UUID is a 128 bit (16 byte) Universal Unique IDentifier as defined in RFC @@ -33,13 +34,37 @@ const ( Future // Reserved for future definition. ) -var rander = rand.Reader // random function +const randPoolSize = 16 * 16 -// Parse decodes s into a UUID or returns an error. Both the standard UUID -// forms of xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and -// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx are decoded as well as the -// Microsoft encoding {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} and the raw hex -// encoding: xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx. +var ( + rander = rand.Reader // random function + poolEnabled = false + poolMu sync.Mutex + poolPos = randPoolSize // protected with poolMu + pool [randPoolSize]byte // protected with poolMu +) + +type invalidLengthError struct{ len int } + +func (err invalidLengthError) Error() string { + return fmt.Sprintf("invalid UUID length: %d", err.len) +} + +// IsInvalidLengthError is matcher function for custom error invalidLengthError +func IsInvalidLengthError(err error) bool { + _, ok := err.(invalidLengthError) + return ok +} + +// Parse decodes s into a UUID or returns an error if it cannot be parsed. Both +// the standard UUID forms defined in RFC 4122 +// (xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx and +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx) are decoded. In addition, +// Parse accepts non-standard strings such as the raw hex encoding +// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx and 38 byte "Microsoft style" encodings, +// e.g. {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx}. Only the middle 36 bytes are +// examined in the latter case. Parse should not be used to validate strings as +// it parses non-standard encodings as indicated above. func Parse(s string) (UUID, error) { var uuid UUID switch len(s) { @@ -48,7 +73,7 @@ func Parse(s string) (UUID, error) { // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx case 36 + 9: - if strings.ToLower(s[:9]) != "urn:uuid:" { + if !strings.EqualFold(s[:9], "urn:uuid:") { return uuid, fmt.Errorf("invalid urn prefix: %q", s[:9]) } s = s[9:] @@ -68,7 +93,7 @@ func Parse(s string) (UUID, error) { } return uuid, nil default: - return uuid, fmt.Errorf("invalid UUID length: %d", len(s)) + return uuid, invalidLengthError{len(s)} } // s is now at least 36 bytes long // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -80,7 +105,8 @@ func Parse(s string) (UUID, error) { 9, 11, 14, 16, 19, 21, - 24, 26, 28, 30, 32, 34} { + 24, 26, 28, 30, 32, 34, + } { v, ok := xtob(s[x], s[x+1]) if !ok { return uuid, errors.New("invalid UUID format") @@ -96,7 +122,7 @@ func ParseBytes(b []byte) (UUID, error) { switch len(b) { case 36: // xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx case 36 + 9: // urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx - if !bytes.Equal(bytes.ToLower(b[:9]), []byte("urn:uuid:")) { + if !bytes.EqualFold(b[:9], []byte("urn:uuid:")) { return uuid, fmt.Errorf("invalid urn prefix: %q", b[:9]) } b = b[9:] @@ -112,7 +138,7 @@ func ParseBytes(b []byte) (UUID, error) { } return uuid, nil default: - return uuid, fmt.Errorf("invalid UUID length: %d", len(b)) + return uuid, invalidLengthError{len(b)} } // s is now at least 36 bytes long // it must be of the form xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx @@ -124,7 +150,8 @@ func ParseBytes(b []byte) (UUID, error) { 9, 11, 14, 16, 19, 21, - 24, 26, 28, 30, 32, 34} { + 24, 26, 28, 30, 32, 34, + } { v, ok := xtob(b[x], b[x+1]) if !ok { return uuid, errors.New("invalid UUID format") @@ -159,6 +186,59 @@ func Must(uuid UUID, err error) UUID { return uuid } +// Validate returns an error if s is not a properly formatted UUID in one of the following formats: +// xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// urn:uuid:xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx +// xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx +// {xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx} +// It returns an error if the format is invalid, otherwise nil. +func Validate(s string) error { + switch len(s) { + // Standard UUID format + case 36: + + // UUID with "urn:uuid:" prefix + case 36 + 9: + if !strings.EqualFold(s[:9], "urn:uuid:") { + return fmt.Errorf("invalid urn prefix: %q", s[:9]) + } + s = s[9:] + + // UUID enclosed in braces + case 36 + 2: + if s[0] != '{' || s[len(s)-1] != '}' { + return fmt.Errorf("invalid bracketed UUID format") + } + s = s[1 : len(s)-1] + + // UUID without hyphens + case 32: + for i := 0; i < len(s); i += 2 { + _, ok := xtob(s[i], s[i+1]) + if !ok { + return errors.New("invalid UUID format") + } + } + + default: + return invalidLengthError{len(s)} + } + + // Check for standard UUID format + if len(s) == 36 { + if s[8] != '-' || s[13] != '-' || s[18] != '-' || s[23] != '-' { + return errors.New("invalid UUID format") + } + for _, x := range []int{0, 2, 4, 6, 9, 11, 14, 16, 19, 21, 24, 26, 28, 30, 32, 34} { + if _, ok := xtob(s[x], s[x+1]); !ok { + return errors.New("invalid UUID format") + } + } + } + + return nil +} + // String returns the string form of uuid, xxxxxxxx-xxxx-xxxx-xxxx-xxxxxxxxxxxx // , or "" if uuid is invalid. func (uuid UUID) String() string { @@ -243,3 +323,43 @@ func SetRand(r io.Reader) { } rander = r } + +// EnableRandPool enables internal randomness pool used for Random +// (Version 4) UUID generation. The pool contains random bytes read from +// the random number generator on demand in batches. Enabling the pool +// may improve the UUID generation throughput significantly. +// +// Since the pool is stored on the Go heap, this feature may be a bad fit +// for security sensitive applications. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func EnableRandPool() { + poolEnabled = true +} + +// DisableRandPool disables the randomness pool if it was previously +// enabled with EnableRandPool. +// +// Both EnableRandPool and DisableRandPool are not thread-safe and should +// only be called when there is no possibility that New or any other +// UUID Version 4 generation function will be called concurrently. +func DisableRandPool() { + poolEnabled = false + defer poolMu.Unlock() + poolMu.Lock() + poolPos = randPoolSize +} + +// UUIDs is a slice of UUID types. +type UUIDs []UUID + +// Strings returns a string slice containing the string form of each UUID in uuids. +func (uuids UUIDs) Strings() []string { + var uuidStrs = make([]string, len(uuids)) + for i, uuid := range uuids { + uuidStrs[i] = uuid.String() + } + return uuidStrs +} diff --git a/vendor/github.com/google/uuid/version4.go b/vendor/github.com/google/uuid/version4.go index c110465d..7697802e 100644 --- a/vendor/github.com/google/uuid/version4.go +++ b/vendor/github.com/google/uuid/version4.go @@ -14,11 +14,21 @@ func New() UUID { return Must(NewRandom()) } +// NewString creates a new random UUID and returns it as a string or panics. +// NewString is equivalent to the expression +// +// uuid.New().String() +func NewString() string { + return Must(NewRandom()).String() +} + // NewRandom returns a Random (Version 4) UUID. // // The strength of the UUIDs is based on the strength of the crypto/rand // package. // +// Uses the randomness pool if it was enabled with EnableRandPool. +// // A note about uniqueness derived from the UUID Wikipedia entry: // // Randomly generated UUIDs have 122 random bits. One's annual risk of being @@ -27,7 +37,10 @@ func New() UUID { // equivalent to the odds of creating a few tens of trillions of UUIDs in a // year and having one duplicate. func NewRandom() (UUID, error) { - return NewRandomFromReader(rander) + if !poolEnabled { + return NewRandomFromReader(rander) + } + return newRandomFromPool() } // NewRandomFromReader returns a UUID based on bytes read from a given io.Reader. @@ -41,3 +54,23 @@ func NewRandomFromReader(r io.Reader) (UUID, error) { uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 return uuid, nil } + +func newRandomFromPool() (UUID, error) { + var uuid UUID + poolMu.Lock() + if poolPos == randPoolSize { + _, err := io.ReadFull(rander, pool[:]) + if err != nil { + poolMu.Unlock() + return Nil, err + } + poolPos = 0 + } + copy(uuid[:], pool[poolPos:(poolPos+16)]) + poolPos += 16 + poolMu.Unlock() + + uuid[6] = (uuid[6] & 0x0f) | 0x40 // Version 4 + uuid[8] = (uuid[8] & 0x3f) | 0x80 // Variant is 10 + return uuid, nil +} diff --git a/vendor/github.com/google/uuid/version6.go b/vendor/github.com/google/uuid/version6.go new file mode 100644 index 00000000..339a959a --- /dev/null +++ b/vendor/github.com/google/uuid/version6.go @@ -0,0 +1,56 @@ +// Copyright 2023 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import "encoding/binary" + +// UUID version 6 is a field-compatible version of UUIDv1, reordered for improved DB locality. +// It is expected that UUIDv6 will primarily be used in contexts where there are existing v1 UUIDs. +// Systems that do not involve legacy UUIDv1 SHOULD consider using UUIDv7 instead. +// +// see https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-03#uuidv6 +// +// NewV6 returns a Version 6 UUID based on the current NodeID and clock +// sequence, and the current time. If the NodeID has not been set by SetNodeID +// or SetNodeInterface then it will be set automatically. If the NodeID cannot +// be set NewV6 set NodeID is random bits automatically . If clock sequence has not been set by +// SetClockSequence then it will be set automatically. If GetTime fails to +// return the current NewV6 returns Nil and an error. +func NewV6() (UUID, error) { + var uuid UUID + now, seq, err := GetTime() + if err != nil { + return uuid, err + } + + /* + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | time_high | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | time_mid | time_low_and_version | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |clk_seq_hi_res | clk_seq_low | node (0-1) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | node (2-5) | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + + binary.BigEndian.PutUint64(uuid[0:], uint64(now)) + binary.BigEndian.PutUint16(uuid[8:], seq) + + uuid[6] = 0x60 | (uuid[6] & 0x0F) + uuid[8] = 0x80 | (uuid[8] & 0x3F) + + nodeMu.Lock() + if nodeID == zeroID { + setNodeInterface("") + } + copy(uuid[10:], nodeID[:]) + nodeMu.Unlock() + + return uuid, nil +} diff --git a/vendor/github.com/google/uuid/version7.go b/vendor/github.com/google/uuid/version7.go new file mode 100644 index 00000000..ba9dd5eb --- /dev/null +++ b/vendor/github.com/google/uuid/version7.go @@ -0,0 +1,75 @@ +// Copyright 2023 Google Inc. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package uuid + +import ( + "io" +) + +// UUID version 7 features a time-ordered value field derived from the widely +// implemented and well known Unix Epoch timestamp source, +// the number of milliseconds seconds since midnight 1 Jan 1970 UTC, leap seconds excluded. +// As well as improved entropy characteristics over versions 1 or 6. +// +// see https://datatracker.ietf.org/doc/html/draft-peabody-dispatch-new-uuid-format-03#name-uuid-version-7 +// +// Implementations SHOULD utilize UUID version 7 over UUID version 1 and 6 if possible. +// +// NewV7 returns a Version 7 UUID based on the current time(Unix Epoch). +// Uses the randomness pool if it was enabled with EnableRandPool. +// On error, NewV7 returns Nil and an error +func NewV7() (UUID, error) { + uuid, err := NewRandom() + if err != nil { + return uuid, err + } + makeV7(uuid[:]) + return uuid, nil +} + +// NewV7FromReader returns a Version 7 UUID based on the current time(Unix Epoch). +// it use NewRandomFromReader fill random bits. +// On error, NewV7FromReader returns Nil and an error. +func NewV7FromReader(r io.Reader) (UUID, error) { + uuid, err := NewRandomFromReader(r) + if err != nil { + return uuid, err + } + + makeV7(uuid[:]) + return uuid, nil +} + +// makeV7 fill 48 bits time (uuid[0] - uuid[5]), set version b0111 (uuid[6]) +// uuid[8] already has the right version number (Variant is 10) +// see function NewV7 and NewV7FromReader +func makeV7(uuid []byte) { + /* + 0 1 2 3 + 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 2 3 4 5 6 7 8 9 0 1 + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | unix_ts_ms | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | unix_ts_ms | ver | rand_a | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + |var| rand_b | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + | rand_b | + +-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+-+ + */ + _ = uuid[15] // bounds check + + t := timeNow().UnixMilli() + + uuid[0] = byte(t >> 40) + uuid[1] = byte(t >> 32) + uuid[2] = byte(t >> 24) + uuid[3] = byte(t >> 16) + uuid[4] = byte(t >> 8) + uuid[5] = byte(t) + + uuid[6] = 0x70 | (uuid[6] & 0x0F) + // uuid[8] has already has right version +} diff --git a/vendor/github.com/hwameistor/datastore/LICENSE b/vendor/github.com/hwameistor/datastore/LICENSE new file mode 100644 index 00000000..21c57fae --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright 2016 The Kubernetes Authors All Rights Reserved + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_basemodels_crd.yaml b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_basemodels_crd.yaml new file mode 100644 index 00000000..5fe3f5aa --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_basemodels_crd.yaml @@ -0,0 +1,107 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: basemodels.datastore.io +spec: + group: datastore.io + names: + kind: BaseModel + listKind: BaseModelList + plural: basemodels + shortNames: + - bm + singular: basemodel + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: BaseModel is the Schema for the basemodels API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: BaseModelSpec defines the desired state of BaseModel + properties: + ftp: + properties: + dir: + default: / + type: string + endpoint: + type: string + password: + type: string + user: + type: string + required: + - dir + - password + - user + type: object + http: + properties: + url: + type: string + required: + - url + type: object + minio: + properties: + accessKey: + type: string + bucket: + type: string + endpoint: + type: string + prefix: + type: string + region: + type: string + secretKey: + type: string + required: + - accessKey + - bucket + - secretKey + type: object + modelName: + type: string + nfs: + properties: + endpoint: + type: string + export: + type: string + rootdir: + default: . + type: string + required: + - export + - rootdir + type: object + proto: + default: minio + type: string + required: + - modelName + - proto + type: object + status: + description: BaseModelStatus defines the observed state of BaseModel + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_checkpoints_crd.yaml b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_checkpoints_crd.yaml new file mode 100644 index 00000000..cfea71c9 --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_checkpoints_crd.yaml @@ -0,0 +1,112 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: checkpoints.datastore.io +spec: + group: datastore.io + names: + kind: Checkpoint + listKind: CheckpointList + plural: checkpoints + shortNames: + - ckpt + singular: checkpoint + scope: Namespaced + versions: + - name: v1alpha1 + schema: + openAPIV3Schema: + description: Checkpoint is the Schema for the checkpoints API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: CheckpointSpec defines the desired state of Checkpoint + properties: + backup: + properties: + minio: + properties: + accessKey: + type: string + bucket: + type: string + endpoint: + type: string + prefix: + type: string + region: + type: string + secretKey: + type: string + required: + - accessKey + - bucket + - secretKey + type: object + proto: + default: minio + type: string + required: + - proto + type: object + retain: + default: 2 + format: int64 + type: integer + required: + - retain + type: object + status: + description: CheckpointStatus defines the observed state of Checkpoint + properties: + records: + items: + properties: + backedup: + default: false + type: boolean + checksum: + type: string + created: + format: date-time + type: string + expired: + format: date-time + type: string + hostdir: + type: string + name: + type: string + node: + type: string + size: + type: string + required: + - backedup + - checksum + - created + - hostdir + - name + - node + - size + type: object + type: array + required: + - records + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_dataloadrequests_crd.yaml b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_dataloadrequests_crd.yaml new file mode 100644 index 00000000..6a4dc4f4 --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_dataloadrequests_crd.yaml @@ -0,0 +1,84 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: dataloadrequests.datastore.io +spec: + group: datastore.io + names: + kind: DataLoadRequest + listKind: DataLoadRequestList + plural: dataloadrequests + shortNames: + - dlr + singular: dataloadrequest + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.dataSet + name: DataSet + type: string + - jsonPath: .spec.subDir + name: SubDir + type: string + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + name: v1alpha1 + schema: + openAPIV3Schema: + description: DataLoadRequest is the Schema for the dataloadrequests API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DataLoadRequestSpec defines the desired state of DataLoadRequest + properties: + dataSet: + description: name of the dataSet source + type: string + dstDir: + type: string + isGlobal: + default: true + description: indicate if the request is for all or not + type: boolean + node: + description: name of the node who will loads the data, and it works + only when isglobal is false + type: string + subDir: + type: string + required: + - dataSet + - isGlobal + type: object + status: + description: DataLoadRequestStatus defines the observed state of DataLoadRequest + properties: + readyNodes: + description: when a node finishes the data loading, record it here + items: + type: string + type: array + state: + description: State of the operation, e.g. submitted, started, completed, + abort, ... + type: string + required: + - readyNodes + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_datasets_crd.yaml b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_datasets_crd.yaml new file mode 100644 index 00000000..f4911e31 --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/deploy/crds/datastore.io_datasets_crd.yaml @@ -0,0 +1,152 @@ +apiVersion: apiextensions.k8s.io/v1 +kind: CustomResourceDefinition +metadata: + name: datasets.datastore.io +spec: + group: datastore.io + names: + kind: DataSet + listKind: DataSetList + plural: datasets + shortNames: + - dsrc + singular: dataset + scope: Namespaced + versions: + - additionalPrinterColumns: + - jsonPath: .spec.type + name: Type + type: string + - jsonPath: .status.lastRefreshTime + name: LastRefreshTime + type: date + - jsonPath: .spec.refresh + name: Refresh + priority: 1 + type: boolean + - jsonPath: .status.refreshCount + name: RefreshCount + priority: 1 + type: integer + - jsonPath: .status.connected + name: Connected + type: boolean + - jsonPath: .metadata.creationTimestamp + name: Age + type: date + - jsonPath: .status.error + name: Error + type: string + name: v1alpha1 + schema: + openAPIV3Schema: + description: DataSet is the Schema for the DataSets API + properties: + apiVersion: + description: 'APIVersion defines the versioned schema of this representation + of an object. Servers should convert recognized schemas to the latest + internal value, and may reject unrecognized values. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#resources' + type: string + kind: + description: 'Kind is a string value representing the REST resource this + object represents. Servers may infer this from the endpoint the client + submits requests to. Cannot be updated. In CamelCase. More info: https://git.k8s.io/community/contributors/devel/sig-architecture/api-conventions.md#types-kinds' + type: string + metadata: + type: object + spec: + description: DataSetSpec defines the desired state of DataSet + properties: + ftp: + properties: + dir: + default: / + type: string + endpoint: + type: string + password: + type: string + user: + type: string + required: + - dir + - password + - user + type: object + minio: + properties: + accessKey: + type: string + bucket: + type: string + endpoint: + type: string + prefix: + type: string + region: + type: string + secretKey: + type: string + required: + - accessKey + - bucket + - secretKey + type: object + nfs: + properties: + endpoint: + type: string + export: + type: string + rootdir: + default: . + type: string + required: + - export + - rootdir + type: object + refresh: + default: true + description: indicate if refresh data or not + type: boolean + type: + default: unknown + description: type of data source + enum: + - minio + - aws-s3 + - nfs + - ftp + - unknown + type: string + required: + - refresh + - type + type: object + status: + description: DataSetStatus defines the observed state of DataSet + properties: + connected: + default: false + description: is the data source connected or accessible? + type: boolean + error: + description: any error message + type: string + lastRefreshTime: + description: last time to refresh the data from the backend + format: date-time + type: string + refreshCount: + default: 0 + description: how many time the data of backend have been refreshed + type: integer + required: + - connected + - refreshCount + type: object + type: object + served: true + storage: true + subresources: + status: {} diff --git a/vendor/github.com/hwameistor/datastore/deploy/crds/module.go b/vendor/github.com/hwameistor/datastore/deploy/crds/module.go new file mode 100644 index 00000000..051026af --- /dev/null +++ b/vendor/github.com/hwameistor/datastore/deploy/crds/module.go @@ -0,0 +1,4 @@ +//go:build module +// +build module + +package crds diff --git a/vendor/github.com/sirupsen/logrus/README.md b/vendor/github.com/sirupsen/logrus/README.md index b042c896..d1d4a85f 100644 --- a/vendor/github.com/sirupsen/logrus/README.md +++ b/vendor/github.com/sirupsen/logrus/README.md @@ -9,7 +9,7 @@ the last thing you want from your Logging library (again...). This does not mean Logrus is dead. Logrus will continue to be maintained for security, (backwards compatible) bug fixes, and performance (where we are -limited by the interface). +limited by the interface). I believe Logrus' biggest contribution is to have played a part in today's widespread use of structured logging in Golang. There doesn't seem to be a @@ -43,7 +43,7 @@ plain text): With `log.SetFormatter(&log.JSONFormatter{})`, for easy parsing by logstash or Splunk: -```json +```text {"animal":"walrus","level":"info","msg":"A group of walrus emerges from the ocean","size":10,"time":"2014-03-10 19:57:38.562264131 -0400 EDT"} @@ -99,7 +99,7 @@ time="2015-03-26T01:27:38-04:00" level=fatal method=github.com/sirupsen/arcticcr ``` Note that this does add measurable overhead - the cost will depend on the version of Go, but is between 20 and 40% in recent tests with 1.6 and 1.7. You can validate this in your -environment via benchmarks: +environment via benchmarks: ``` go test -bench=.*CallerTracing ``` @@ -317,6 +317,8 @@ log.SetLevel(log.InfoLevel) It may be useful to set `log.Level = logrus.DebugLevel` in a debug or verbose environment if your application has that. +Note: If you want different log levels for global (`log.SetLevel(...)`) and syslog logging, please check the [syslog hook README](hooks/syslog/README.md#different-log-levels-for-local-and-remote-logging). + #### Entries Besides the fields added with `WithField` or `WithFields` some fields are diff --git a/vendor/github.com/sirupsen/logrus/writer.go b/vendor/github.com/sirupsen/logrus/writer.go index 72e8e3a1..074fd4b8 100644 --- a/vendor/github.com/sirupsen/logrus/writer.go +++ b/vendor/github.com/sirupsen/logrus/writer.go @@ -4,6 +4,7 @@ import ( "bufio" "io" "runtime" + "strings" ) // Writer at INFO level. See WriterLevel for details. @@ -20,15 +21,18 @@ func (logger *Logger) WriterLevel(level Level) *io.PipeWriter { return NewEntry(logger).WriterLevel(level) } +// Writer returns an io.Writer that writes to the logger at the info log level func (entry *Entry) Writer() *io.PipeWriter { return entry.WriterLevel(InfoLevel) } +// WriterLevel returns an io.Writer that writes to the logger at the given log level func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { reader, writer := io.Pipe() var printFunc func(args ...interface{}) + // Determine which log function to use based on the specified log level switch level { case TraceLevel: printFunc = entry.Trace @@ -48,23 +52,51 @@ func (entry *Entry) WriterLevel(level Level) *io.PipeWriter { printFunc = entry.Print } + // Start a new goroutine to scan the input and write it to the logger using the specified print function. + // It splits the input into chunks of up to 64KB to avoid buffer overflows. go entry.writerScanner(reader, printFunc) + + // Set a finalizer function to close the writer when it is garbage collected runtime.SetFinalizer(writer, writerFinalizer) return writer } +// writerScanner scans the input from the reader and writes it to the logger func (entry *Entry) writerScanner(reader *io.PipeReader, printFunc func(args ...interface{})) { scanner := bufio.NewScanner(reader) + + // Set the buffer size to the maximum token size to avoid buffer overflows + scanner.Buffer(make([]byte, bufio.MaxScanTokenSize), bufio.MaxScanTokenSize) + + // Define a split function to split the input into chunks of up to 64KB + chunkSize := bufio.MaxScanTokenSize // 64KB + splitFunc := func(data []byte, atEOF bool) (int, []byte, error) { + if len(data) >= chunkSize { + return chunkSize, data[:chunkSize], nil + } + + return bufio.ScanLines(data, atEOF) + } + + // Use the custom split function to split the input + scanner.Split(splitFunc) + + // Scan the input and write it to the logger using the specified print function for scanner.Scan() { - printFunc(scanner.Text()) + printFunc(strings.TrimRight(scanner.Text(), "\r\n")) } + + // If there was an error while scanning the input, log an error if err := scanner.Err(); err != nil { entry.Errorf("Error while reading from Writer: %s", err) } + + // Close the reader when we are done reader.Close() } +// WriterFinalizer is a finalizer function that closes then given writer when it is garbage collected func writerFinalizer(writer *io.PipeWriter) { writer.Close() } diff --git a/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go b/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go index 7499e3fb..05de9cc2 100644 --- a/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go +++ b/vendor/golang.org/x/crypto/pkcs12/internal/rc2/rc2.go @@ -14,6 +14,7 @@ package rc2 import ( "crypto/cipher" "encoding/binary" + "math/bits" ) // The rc2 block size in bytes @@ -80,10 +81,6 @@ func expandKey(key []byte, t1 int) [64]uint16 { return k } -func rotl16(x uint16, b uint) uint16 { - return (x >> (16 - b)) | (x << b) -} - func (c *rc2Cipher) Encrypt(dst, src []byte) { r0 := binary.LittleEndian.Uint16(src[0:]) @@ -96,22 +93,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 16 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -124,22 +121,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 40 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -152,22 +149,22 @@ func (c *rc2Cipher) Encrypt(dst, src []byte) { for j <= 60 { // mix r0 r0 = r0 + c.k[j] + (r3 & r2) + ((^r3) & r1) - r0 = rotl16(r0, 1) + r0 = bits.RotateLeft16(r0, 1) j++ // mix r1 r1 = r1 + c.k[j] + (r0 & r3) + ((^r0) & r2) - r1 = rotl16(r1, 2) + r1 = bits.RotateLeft16(r1, 2) j++ // mix r2 r2 = r2 + c.k[j] + (r1 & r0) + ((^r1) & r3) - r2 = rotl16(r2, 3) + r2 = bits.RotateLeft16(r2, 3) j++ // mix r3 r3 = r3 + c.k[j] + (r2 & r1) + ((^r2) & r0) - r3 = rotl16(r3, 5) + r3 = bits.RotateLeft16(r3, 5) j++ } @@ -188,22 +185,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 44 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- } @@ -215,22 +212,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 20 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- @@ -243,22 +240,22 @@ func (c *rc2Cipher) Decrypt(dst, src []byte) { for j >= 0 { // unmix r3 - r3 = rotl16(r3, 16-5) + r3 = bits.RotateLeft16(r3, 16-5) r3 = r3 - c.k[j] - (r2 & r1) - ((^r2) & r0) j-- // unmix r2 - r2 = rotl16(r2, 16-3) + r2 = bits.RotateLeft16(r2, 16-3) r2 = r2 - c.k[j] - (r1 & r0) - ((^r1) & r3) j-- // unmix r1 - r1 = rotl16(r1, 16-2) + r1 = bits.RotateLeft16(r1, 16-2) r1 = r1 - c.k[j] - (r0 & r3) - ((^r0) & r2) j-- // unmix r0 - r0 = rotl16(r0, 16-1) + r0 = bits.RotateLeft16(r0, 16-1) r0 = r0 - c.k[j] - (r3 & r2) - ((^r3) & r1) j-- diff --git a/vendor/golang.org/x/net/context/go17.go b/vendor/golang.org/x/net/context/go17.go index 2cb9c408..0c1b8679 100644 --- a/vendor/golang.org/x/net/context/go17.go +++ b/vendor/golang.org/x/net/context/go17.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build go1.7 -// +build go1.7 package context diff --git a/vendor/golang.org/x/net/context/go19.go b/vendor/golang.org/x/net/context/go19.go index 64d31ecc..e31e35a9 100644 --- a/vendor/golang.org/x/net/context/go19.go +++ b/vendor/golang.org/x/net/context/go19.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build go1.9 -// +build go1.9 package context diff --git a/vendor/golang.org/x/net/context/pre_go17.go b/vendor/golang.org/x/net/context/pre_go17.go index 7b6b6851..065ff3df 100644 --- a/vendor/golang.org/x/net/context/pre_go17.go +++ b/vendor/golang.org/x/net/context/pre_go17.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !go1.7 -// +build !go1.7 package context diff --git a/vendor/golang.org/x/net/context/pre_go19.go b/vendor/golang.org/x/net/context/pre_go19.go index 1f971534..ec5a6380 100644 --- a/vendor/golang.org/x/net/context/pre_go19.go +++ b/vendor/golang.org/x/net/context/pre_go19.go @@ -3,7 +3,6 @@ // license that can be found in the LICENSE file. //go:build !go1.9 -// +build !go1.9 package context diff --git a/vendor/golang.org/x/net/html/doc.go b/vendor/golang.org/x/net/html/doc.go index 822ed42a..2466ae3d 100644 --- a/vendor/golang.org/x/net/html/doc.go +++ b/vendor/golang.org/x/net/html/doc.go @@ -92,6 +92,27 @@ example, to process each anchor node in depth-first order: The relevant specifications include: https://html.spec.whatwg.org/multipage/syntax.html and https://html.spec.whatwg.org/multipage/syntax.html#tokenization + +# Security Considerations + +Care should be taken when parsing and interpreting HTML, whether full documents +or fragments, within the framework of the HTML specification, especially with +regard to untrusted inputs. + +This package provides both a tokenizer and a parser, which implement the +tokenization, and tokenization and tree construction stages of the WHATWG HTML +parsing specification respectively. While the tokenizer parses and normalizes +individual HTML tokens, only the parser constructs the DOM tree from the +tokenized HTML, as described in the tree construction stage of the +specification, dynamically modifying or extending the docuemnt's DOM tree. + +If your use case requires semantically well-formed HTML documents, as defined by +the WHATWG specification, the parser should be used rather than the tokenizer. + +In security contexts, if trust decisions are being made using the tokenized or +parsed content, the input must be re-serialized (for instance by using Render or +Token.String) in order for those trust decisions to hold, as the process of +tokenization or parsing may alter the content. */ package html // import "golang.org/x/net/html" diff --git a/vendor/golang.org/x/net/html/escape.go b/vendor/golang.org/x/net/html/escape.go index d8561396..04c6bec2 100644 --- a/vendor/golang.org/x/net/html/escape.go +++ b/vendor/golang.org/x/net/html/escape.go @@ -193,6 +193,87 @@ func lower(b []byte) []byte { return b } +// escapeComment is like func escape but escapes its input bytes less often. +// Per https://github.com/golang/go/issues/58246 some HTML comments are (1) +// meaningful and (2) contain angle brackets that we'd like to avoid escaping +// unless we have to. +// +// "We have to" includes the '&' byte, since that introduces other escapes. +// +// It also includes those bytes (not including EOF) that would otherwise end +// the comment. Per the summary table at the bottom of comment_test.go, this is +// the '>' byte that, per above, we'd like to avoid escaping unless we have to. +// +// Studying the summary table (and T actions in its '>' column) closely, we +// only need to escape in states 43, 44, 49, 51 and 52. State 43 is at the +// start of the comment data. State 52 is after a '!'. The other three states +// are after a '-'. +// +// Our algorithm is thus to escape every '&' and to escape '>' if and only if: +// - The '>' is after a '!' or '-' (in the unescaped data) or +// - The '>' is at the start of the comment data (after the opening ""); err != nil { @@ -194,9 +194,8 @@ func render1(w writer, n *Node) error { } } - // Render any child nodes. - switch n.Data { - case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "xmp": + // Render any child nodes + if childTextNodesAreLiteral(n) { for c := n.FirstChild; c != nil; c = c.NextSibling { if c.Type == TextNode { if _, err := w.WriteString(c.Data); err != nil { @@ -213,7 +212,7 @@ func render1(w writer, n *Node) error { // last element in the file, with no closing tag. return plaintextAbort } - default: + } else { for c := n.FirstChild; c != nil; c = c.NextSibling { if err := render1(w, c); err != nil { return err @@ -231,6 +230,27 @@ func render1(w writer, n *Node) error { return w.WriteByte('>') } +func childTextNodesAreLiteral(n *Node) bool { + // Per WHATWG HTML 13.3, if the parent of the current node is a style, + // script, xmp, iframe, noembed, noframes, or plaintext element, and the + // current node is a text node, append the value of the node's data + // literally. The specification is not explicit about it, but we only + // enforce this if we are in the HTML namespace (i.e. when the namespace is + // ""). + // NOTE: we also always include noscript elements, although the + // specification states that they should only be rendered as such if + // scripting is enabled for the node (which is not something we track). + if n.Namespace != "" { + return false + } + switch n.Data { + case "iframe", "noembed", "noframes", "noscript", "plaintext", "script", "style", "xmp": + return true + default: + return false + } +} + // writeQuoted writes s to w surrounded by quotes. Normally it will use double // quotes, but if s contains a double quote, it will use single quotes. // It is used for writing the identifiers in a doctype declaration. diff --git a/vendor/golang.org/x/net/html/token.go b/vendor/golang.org/x/net/html/token.go index 50f7c6aa..de67f938 100644 --- a/vendor/golang.org/x/net/html/token.go +++ b/vendor/golang.org/x/net/html/token.go @@ -110,7 +110,7 @@ func (t Token) String() string { case SelfClosingTagToken: return "<" + t.tagString() + "/>" case CommentToken: - return "" + return "" case DoctypeToken: return "" } @@ -598,10 +598,10 @@ scriptDataDoubleEscapeEnd: // readComment reads the next comment token starting with "