Skip to content

Commit

Permalink
repo-sync-2024-03-01T16:44:35+0800 (#239)
Browse files Browse the repository at this point in the history
* repo-sync-2024-03-01T16:44:35+0800

* Update CHANGELOG.md
  • Loading branch information
CodePorterL authored Mar 4, 2024
1 parent 0b6a347 commit 7becc12
Show file tree
Hide file tree
Showing 119 changed files with 3,420 additions and 1,413 deletions.
25 changes: 24 additions & 1 deletion CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -11,8 +11,31 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
`Removed` for now removed features.
`Fixed` for any bug fixes.
`Security` in case of vulnerabilities.
`Breaking Changed` Breaking for backward-incompatible changes that require user intervention.

## [0.6.0.dev240131] - 2023-01-31
## [0.7.0.dev240229] - 2024-02-29
### Added
- add the documents of datasource api
- add kusciaapi errorcode doc
- add kuscia init command generating kuscia config
### Changed
- update domain register and handshake error
- report events when Pod failed to start in RunK mode

### Breaking Changed
- Change the mounting directory for logs and data of Kuscia deploying with Docker
- {{ROOT}}/kuscia-{{DEPLOY_MODE}}-{{DOMAIN_ID}}-data -> {{ROOT}}/{{DOMAIN_CONTAINER_NAME}}/data.
- {{ROOT}}/kuscia-{{DEPLOY_MODE}}-{{DOMAIN_ID}}-logs -> {{ROOT}}/{{DOMAIN_CONTAINER_NAME}}/logs.
- {{ROOT}}/kuscia-{{DEPLOY_MODE}}-{{DOMAIN_ID}}-certs was deleted.
### Fixed
- fix some unit test case
- fix the issue of inconsistent states among multiple parties in KusciaDeployment
- fix the issue of ClusterDefine only having unilateral information under KusciaDeployment in P2P scenarios
- fix kusciaapi grpc role check
- Upgrade certain dependency packages to fix security vulnerabilities in pkg.


## [0.6.0.dev240131] - 2024-01-31
### Added
- Upgrade interconnection protocol from kuscia-alpha to kuscia-beta to support interconnection between Kuscia-Master and Kuscia-Autonomy.
- Kuscia monitor, Kuscia exposes a set of metric data, which can be used as data sources for collection by external monitoring tools (such as Prometheus).
Expand Down
16 changes: 12 additions & 4 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,7 @@ all: build

.PHONY: help
help: ## Display this help.
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-16s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_0-9-]+:.*?##/ { printf " \033[36m%-18s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

##@ Development

Expand All @@ -68,8 +68,16 @@ fmt: ## Run go fmt against code.
vet: ## Run go vet against code.
go vet ./...

.PHONY: verify_error_code
verify_error_code: ## Verify integrity of error code i18n configuration.
bash hack/errorcode/gen_error_code_doc.sh verify pkg/kusciaapi/errorcode/error_code.go hack/errorcode/i18n/errorcode.zh-CN.toml

.PHONY: gen_error_code_doc
gen_error_code_doc: verify_error_code ## Generate error code markdown doc.
bash hack/errorcode/gen_error_code_doc.sh doc pkg/kusciaapi/errorcode/error_code.go hack/errorcode/i18n/errorcode.zh-CN.toml docs/reference/apis/error_code_cn.md

.PHONY: test
test: fmt vet ## Run tests.
test: verify_error_code fmt vet ## Run tests.
rm -rf ./test-results
mkdir -p test-results
go test ./cmd/... -gcflags="all=-N -l" -coverprofile=test-results/cmd.covprofile.out | tee test-results/cmd.output.txt
Expand All @@ -91,11 +99,11 @@ clean: # clean build and test product.
##@ Build

.PHONY: build
build: fmt vet ## Build kuscia binary.
build: verify_error_code fmt vet ## Build kuscia binary.
bash hack/build.sh -t kuscia

.PHONY: docs
docs: ## Build docs.
docs: gen_error_code_doc ## Build docs.
cd docs && pip install -r requirements.txt && make html

.PHONY: deps-image
Expand Down
6 changes: 4 additions & 2 deletions cmd/kuscia/confloader/config.go
Original file line number Diff line number Diff line change
Expand Up @@ -46,8 +46,9 @@ type KusciaConfig struct {
CACertFile string `yaml:"caFile,omitempty"` // Note: for ca cert will be mounted to agent pod
CACertData string `yaml:"caCertData,omitempty"`

LogLevel string `yaml:"logLevel"`
MetricUpdatePeriod uint `yaml:"metricUpdatePeriod,omitempty"` // Unit: second
LogLevel string `yaml:"logLevel"`
Logrotate LogrotateConfig `yaml:"logrotate,omitempty"`
MetricUpdatePeriod uint `yaml:"metricUpdatePeriod,omitempty"` // Unit: second

Debug bool `yaml:"debug"`
DebugPort int `yaml:"debugPort"`
Expand Down Expand Up @@ -133,6 +134,7 @@ func defaultKusciaConfig(rootDir string) KusciaConfig {
EnvoyIP: hostIP,
KusciaAPI: kaconfig.NewDefaultKusciaAPIConfig(rootDir),
MetricUpdatePeriod: defaultMetricUpdatePeriod,
Logrotate: LogrotateConfig{3, 128},
}
}

Expand Down
14 changes: 14 additions & 0 deletions cmd/kuscia/confloader/config_test.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2023 Ant Group Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package confloader

import (
Expand Down
47 changes: 41 additions & 6 deletions cmd/kuscia/confloader/kuscia_config.go
Original file line number Diff line number Diff line change
@@ -1,3 +1,17 @@
// Copyright 2023 Ant Group Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package confloader

import (
Expand Down Expand Up @@ -36,7 +50,7 @@ type LiteKusciaConfig struct {
type MasterKusciaConfig struct {
CommonConfig `yaml:",inline"`
DatastoreEndpoint string `yaml:"datastoreEndpoint"`
ClusterToken string `yaml:"clusterToken"`
ClusterToken string `yaml:"clusterToken,omitempty"`
AdvancedConfig `yaml:",inline"`
}

Expand Down Expand Up @@ -81,7 +95,12 @@ type CommonConfig struct {
DomainID string `yaml:"domainID"`
DomainKeyData string `yaml:"domainKeyData"`
LogLevel string `yaml:"logLevel"`
Protocol common.Protocol `yaml:"protocol"`
Protocol common.Protocol `yaml:"protocol,omitempty"`
}

type LogrotateConfig struct {
MaxFiles int `yaml:"maxFiles"`
MaxFileSizeMB int `yaml:"maxFileSizeMB"`
}

type AdvancedConfig struct {
Expand All @@ -91,9 +110,10 @@ type AdvancedConfig struct {
DataMesh *dmconfig.DataMeshConfig `yaml:"dataMesh,omitempty"`
DomainRoute DomainRouteConfig `yaml:"domainRoute,omitempty"`
Agent config.AgentConfig `yaml:"agent,omitempty"`
Debug bool `yaml:"debug"`
DebugPort int `yaml:"debugPort"`
Debug bool `yaml:"debug,omitempty"`
DebugPort int `yaml:"debugPort,omitempty"`
EnableWorkloadApprove bool `yaml:"enableWorkloadApprove,omitempty"`
Logrotate LogrotateConfig `yaml:"logrotate,omitempty"`
}

func LoadCommonConfig(configFile string) *CommonConfig {
Expand Down Expand Up @@ -146,6 +166,8 @@ func (lite *LiteKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConfig)
kusciaConfig.DomainRoute.DomainCsrData = generateCsrData(lite.DomainID, lite.DomainKeyData, lite.LiteDeployToken)
kusciaConfig.Debug = lite.Debug
kusciaConfig.DebugPort = lite.DebugPort

overwriteKusciaConfigLogrotate(&kusciaConfig.Logrotate, &lite.AdvancedConfig.Logrotate)
}

func (master *MasterKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConfig) {
Expand All @@ -167,6 +189,8 @@ func (master *MasterKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConf
kusciaConfig.Debug = master.Debug
kusciaConfig.DebugPort = master.DebugPort
kusciaConfig.EnableWorkloadApprove = master.AdvancedConfig.EnableWorkloadApprove

overwriteKusciaConfigLogrotate(&kusciaConfig.Logrotate, &master.AdvancedConfig.Logrotate)
}

func (autonomy *AutomonyKusciaConfig) OverwriteKusciaConfig(kusciaConfig *KusciaConfig) {
Expand Down Expand Up @@ -197,9 +221,20 @@ func (autonomy *AutomonyKusciaConfig) OverwriteKusciaConfig(kusciaConfig *Kuscia
kusciaConfig.Master.DatastoreEndpoint = autonomy.DatastoreEndpoint
kusciaConfig.Debug = autonomy.Debug
kusciaConfig.DebugPort = autonomy.DebugPort

kusciaConfig.Agent.DomainKeyData = autonomy.DomainKeyData
kusciaConfig.EnableWorkloadApprove = autonomy.AdvancedConfig.EnableWorkloadApprove

overwriteKusciaConfigLogrotate(&kusciaConfig.Logrotate, &autonomy.AdvancedConfig.Logrotate)
}

func overwriteKusciaConfigLogrotate(kusciaConfig, overwriteLogrotate *LogrotateConfig) {
if overwriteLogrotate != nil {
if overwriteLogrotate.MaxFileSizeMB > 0 {
kusciaConfig.MaxFileSizeMB = overwriteLogrotate.MaxFileSizeMB
}
if overwriteLogrotate.MaxFiles > 0 {
kusciaConfig.MaxFiles = overwriteLogrotate.MaxFiles
}
}
}

func loadConfig(configFile string, conf interface{}) {
Expand Down
1 change: 1 addition & 0 deletions cmd/kuscia/image/cmd/cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,4 +28,5 @@ type Context struct {
func InstallCommands(rootCmd *cobra.Command, cmdCtx *Context) {
rootCmd.AddCommand(loadCommand(cmdCtx))
rootCmd.AddCommand(builtinCommand(cmdCtx))
rootCmd.AddCommand(mountCommand(cmdCtx))
}
59 changes: 59 additions & 0 deletions cmd/kuscia/image/cmd/mount.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,59 @@
// Copyright 2024 Ant Group Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"github.com/spf13/cobra"

"github.com/secretflow/kuscia/pkg/agent/local/store/kii"
"github.com/secretflow/kuscia/pkg/agent/local/store/layout"
"github.com/secretflow/kuscia/pkg/utils/nlog"
)

func mountCommand(cmdCtx *Context) *cobra.Command {
mountCmd := &cobra.Command{
Use: "mount MOUNT_ID IMAGE TARGET_PATH",
Short: "mount a image to path",
DisableFlagsInUseLine: true,
Example: `
# Mount image to dir
kuscia image mount unique_id app:0.1 target_dir/
`,
Args: cobra.ExactArgs(3),
Run: func(cmd *cobra.Command, args []string) {
if err := mountImage(cmdCtx, args[0], args[1], args[2]); err != nil {
nlog.Fatal(err)
}
},
}

return mountCmd
}

func mountImage(cmdCtx *Context, mountID, image, targetPath string) error {
imageName, err := kii.NewImageName(image)
if err != nil {
return err
}

bundle, err := layout.NewBundle(cmdCtx.StorageDir)
if err != nil {
return err
}

rBundle := bundle.GetContainerBundle(mountID)

return cmdCtx.Store.MountImage(imageName, kii.Plain, rBundle.GetFsWorkingDirPath(), targetPath)
}
38 changes: 38 additions & 0 deletions cmd/kuscia/image/cmd/mount_test.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,38 @@
// Copyright 2024 Ant Group Co., Ltd.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package cmd

import (
"path/filepath"
"testing"

"github.com/stretchr/testify/assert"

storetesting "github.com/secretflow/kuscia/pkg/agent/local/store/testing"
)

func TestMountCommand(t *testing.T) {
rootDir := t.TempDir()

context := &Context{
StorageDir: filepath.Join(rootDir, "storage"),
Store: storetesting.NewFakeStore(),
}

cmd := mountCommand(context)
args := []string{"123", "app:0.1", filepath.Join(rootDir, "rootfs")}
cmd.SetArgs(args)
assert.NoError(t, cmd.Execute())
}
Loading

0 comments on commit 7becc12

Please sign in to comment.