Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Sync-up changes - Syslog support, '-plugins' json/yaml, python library to invoke PM #26

Open
wants to merge 3 commits into
base: v2
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
8 changes: 7 additions & 1 deletion .github/workflows/go.yml
Original file line number Diff line number Diff line change
Expand Up @@ -19,7 +19,11 @@ jobs:
go-version: 1.19

- name: Static analysis of code for errors
run: make analyze
env:
PM_LOG_DIR: ${{ github.workspace }}/test-logs
run: |
mkdir -p ${{ github.workspace }}/test-logs
make analyze

# - name: Build
# run: go build -v ./...
Expand All @@ -30,10 +34,12 @@ jobs:
- name: Test - go test
env:
PM_CONF_FILE: ${{ github.workspace }}/sample/pm.config.yaml
PM_LOG_DIR: ${{ github.workspace }}/test-logs
INTEGRATION_TEST: START
INTEG_TEST_BIN: ${{ github.workspace }}
run: |
mkdir -p ${{ github.workspace }}/cover
mkdir -p ${{ github.workspace }}/test-logs
go test -mod=vendor -v ./...

# - name: Test - make test
Expand Down
36 changes: 23 additions & 13 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
# Copyright (c) 2023 Veritas Technologies LLC. All rights reserved. IP63-2828-7171-04-15-9
# Copyright (c) 2024 Veritas Technologies LLC. All rights reserved. IP63-2828-7171-04-15-9

# A Self-Documenting Makefile: http://marmelab.com/blog/2016/02/29/auto-documented-makefile.html
.DEFAULT_GOAL := help
.PHONY: help
help: ## Display this help message.
@grep -E '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'
@grep -Eh '^[a-zA-Z0-9_-]+:.*?## .*$$' $(MAKEFILE_LIST) | sort | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-20s\033[0m %s\n", $$1, $$2}'

TOP=$(CURDIR)
include $(TOP)/Makefile.conf
Expand All @@ -14,7 +14,7 @@ GOSRC=$(TOP)

# Set GOBIN to where binaries get picked up while creating RPM/ISO.
GOBIN?=$(TOP)/bin
GOCOVER=$(GOSRC)/cover
GOCOVERDIR=$(GOSRC)/cover
GOTOOLSBIN=$(TOP)/tools/go/

.SILENT:
Expand All @@ -29,10 +29,9 @@ clean: ## Clean Plugin Manager go build & test artifacts
cd $(GOSRC); \
go clean -i -mod=vendor ./...;
@echo "Cleaning Go test artifacts... ";
-@rm $(GOSRC)/{,.}*{dot,html,log,svg,xml};
-@rm $(GOSRC)/cmd/pm/{,.}*{dot,log,svg};
-@find $(GOSRC) -name "*.dot" -o -name "*.html" -o -name "*.log" -o -name "*.svg" -o -name "*.xml" | xargs -i rm -f {}
-@rm -rf $(GOSRC)/plugins/
-@rm -rf $(GOCOVER);
-@rm -rf $(GOCOVERDIR);

.PHONY: build
build: ## Build source code
Expand Down Expand Up @@ -75,24 +74,25 @@ govet: ## Run go vet
.PHONY: test
test: ## Run all tests
echo "Running Plugin Manager Go Unit Tests...";
mkdir -p $(GOCOVER);
mkdir -p $(GOCOVERDIR);
export GOCOVERDIR=$(GOCOVERDIR); \
export INTEG_TEST_BIN=$(GOSRC); \
export PM_CONF_FILE=$(GOSRC)/sample/pm.config.yaml; \
export INTEGRATION_TEST=START; \
cd $(GOSRC); \
test_failed=0; \
d=pm; \
go test -mod=vendor -v --cover -covermode=count -coverprofile=$(GOCOVER)/$${d}.out ./... | \
go test -mod=vendor -v --cover -covermode=count -coverprofile=$(GOCOVERDIR)/$${d}.out ./... | \
$(GOTOOLSBIN)/go-junit-report > TEST-$${d}.xml; \
ret=$${PIPESTATUS[0]}; \
if [ $${ret} -ne 0 ]; then \
echo "Go unit test failed for $${d}."; \
test_failed=1; \
fi ; \
awk -f $(TOP)/tools/gocoverage-collate.awk $(GOCOVER)/* > $(GOCOVER)/cover.out; \
go tool cover -html=$(GOCOVER)/cover.out -o go-coverage-$${d}.html; \
$(GOTOOLSBIN)/gocov convert $(GOCOVER)/cover.out | $(GOTOOLSBIN)/gocov-xml > go-coverage-$${d}.xml; \
rm -rf $(GOCOVER)/*; \
awk -f $(TOP)/tools/gocoverage-collate.awk $(GOCOVERDIR)/* > $(GOCOVERDIR)/cover.out; \
go tool cover -html=$(GOCOVERDIR)/cover.out -o go-coverage-$${d}.html; \
$(GOTOOLSBIN)/gocov convert $(GOCOVERDIR)/cover.out | $(GOTOOLSBIN)/gocov-xml > go-coverage-$${d}.xml; \
rm -rf $(GOCOVERDIR)/*; \
export INTEGRATION_TEST=DONE; \
if [ $${test_failed} -ne 0 ]; then \
echo "Go unit tests failed."; \
Expand All @@ -104,11 +104,21 @@ go-race: ## Run Go tests with race detector enabled
echo "Checking Go code for race conditions...";
# NOTE: COVER directory should be present, along with INTEGRATION_TEST
# value being set to "START" for integ_test.go to succeed.
mkdir -p $(GOCOVER);
mkdir -p $(GOCOVERDIR);
export GOCOVERDIR=$(GOCOVERDIR); \
export INTEGRATION_TEST=START; \
export INTEG_TEST_BIN=$(GOSRC); \
cd $(GOSRC); \
export PM_CONF_FILE=$(GOSRC)/sample/pm.config.yaml; \
go test -mod=vendor -v -race ./...;

.PHONY: update-go-tools
update-go-tools: ## Update Go thirdparty tools to current go version
export GOBIN=$(GOTOOLSBIN); \
go install github.com/axw/gocov/gocov@latest; \
go install github.com/AlekSi/gocov-xml@latest; \
go install github.com/matm/gocov-html/cmd/gocov-html@latest; \
go install github.com/jstemmer/go-junit-report/v2@latest; \
go install golang.org/x/lint/golint@latest

.NOTPARALLEL:
165 changes: 143 additions & 22 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,10 @@ systemd.
- [Example: Plugin Manager (PM) `list`](#example-plugin-manager-pm-list)
- [Configuring Plugin Manager](#configuring-plugin-manager)
- [Running Plugins](#running-plugins)
- [Example: Plugin Manager (PM) `run`](#example-plugin-manager-pm-run)
- [Example: Plugin Manager (PM) `run -plugins`](#example-plugin-manager-pm-run--plugins)
- [Specify `-plugins` details as a json string](#specify--plugins-details-as-a-json-string)
- [Specify `-plugins` details via json file](#specify--plugins-details-via-json-file)
- [Example: Plugin Manager (PM) `run -type`](#example-plugin-manager-pm-run--type)
- [Example: Plugin Manager (PM) with `sequential` flag](#example-plugin-manager-pm-with-sequential-flag)
- [Example: Overriding Plugin Manager (PM) configuration - `library`, `log-dir` and `log-file`](#example-overriding-plugin-manager-pm-configuration---library-log-dir-and-log-file)
- [Example: Writing plugins result to a `output-file` in `output-format` {json, yaml} format](#example-writing-plugins-result-to-a-output-file-in-output-format-json-yaml-format)
Expand Down Expand Up @@ -87,6 +90,7 @@ The PM list command syntax / usage is as shown below:
```bash
pm list -type <PluginType>
[-library=<PluginsLibraryPath>]
[-log-tag=<TagOfSysLog>]
[-log-dir=<LogDirectory>]
[-log-file=<NameOfLogFile>]
```
Expand All @@ -96,6 +100,9 @@ where
- **`type`**: Indicates the plugin type.
- **`library`**: Indicates the location of plugins library.
**Overrides** value present in PM configuration.
- **`log-tag`**: Indicates the log tag written by rsyslog.
Note: rsyslog is used as default logger for both main and plugin logs.
It will be overwritten if `log-file` option set.
- **`log-dir`**: Indicates the log directory path.
**Overrides** value present in PM configuration.
- **`log-file`**: Indicates the name of the log file.
Expand All @@ -113,7 +120,7 @@ The list of plugins are mapped in .//preupgrade.2020-01-13T15:56:46.725348-08:00

Plugin Manager can be configured to look for plugins at a specific location,
and to write logs to a specific file by specifying those details in the
Plugin Manager configuration file `/etc/asum/pm.config.yml`.
Plugin Manager configuration file `/opt/veritas/appliance/asum/pm.config.yml`.

Instead of updating the default config file, one can choose to provide his/her
own custom config file.
Expand All @@ -138,7 +145,7 @@ PluginManager:
# The timestamp and '.log' extension would be appended to this name.
# I.e., The format of the log file generated would be: "<log file>.<timestamp>.log"
# Example: The below value results in following log file: pm.2020-01-13T16:11:58.6006565-08:00.log
log file: "pm"
log file: "pm.log"
...
```

Expand All @@ -156,9 +163,11 @@ exit value of plugins, the PM exits with 1.
The PM run command syntax / usage is as shown below:

```bash
pm run -type <PluginType>
pm run [-plugins <PluginInformation>]
[-type <PluginType>]
[-library=<PluginsLibraryPath>]
[-sequential[={true|1|false|0}]]
[-log-tag=<TagOfSysLog>]
[-log-dir=<LogDirectory>]
[-log-file=<NameOfLogFile>]
[-output={json|yaml}]
Expand All @@ -167,13 +176,16 @@ pm run -type <PluginType>

where

- **`plugins`**: A json string or a json file containing plugins and its dependencies.
- **`type`**: Indicates the plugin type.
- **`library`**: Indicates the location of plugins library.
**Overrides** value present in PM configuration.
**NOTE** The specified value gets set as an environment variable `PM_LIBRARY` for the plugins being run. The plugin file can access any scripts in the same folder via `PM_LIBRARY` variable.
- **`-sequential`**: Indicates PM to execute only one plugin at a time
regardless of how many plugins' dependencies are met.
**Default: Disabled**. To enable, specify `-sequential=true` or just
`-sequential` while running PM.
- **`log-tag`**: Indicates the log tag written by rsyslog. The `log-tag` option will supercede `log-dir` and `log-file` options.
- **`log-dir`**: Indicates the log directory path.
**Overrides** value present in PM configuration.
- **`log-file`**: Indicates the name of the log file.
Expand All @@ -185,7 +197,86 @@ where
If `output` format is specified, and `output-file` is not specified,
then result will be displayed on console.

### Example: Plugin Manager (PM) `run`
### Example: Plugin Manager (PM) `run -plugins`

```json
$ jq -n "$plugins" | tee sample/plugins-prereboot.json
{
"Plugins": [
{
"Name": "A/a.prereboot",
"Description": "Applying \"A\" settings",
"ExecStart": "/usr/bin/ls -l -t",
"Requires": [
"C/c.prereboot",
"D/d.prereboot"
]
},
{
"Name": "B/b.prereboot",
"Description": "Applying \"B\" settings...",
"ExecStart": "/bin/echo \"Running B...\"",
"RequiredBy": [
"D/d.prereboot"
]
},
{
"Name": "C/c.prereboot",
"Description": "Applying \"C\" settings...",
"ExecStart": "/bin/echo \"Running C...\"",
"RequiredBy": [
"A/a.prereboot"
]
},
{
"Name": "D/d.prereboot",
"Description": "Applying \"D\" settings...",
"ExecStart": "/bin/echo 'Running D...!'",
"RequiredBy": [
"A/a.prereboot"
],
"Requires": [
"B/b.prereboot"
]
}
]
}
$
```

#### Specify `-plugins` details as a json string

```bash
$ $GOBIN/pm run -plugins "$plugins"
Applying "B" settings...: Starting
Applying "C" settings...: Starting
Applying "B" settings...: Succeeded
Applying "D" settings...: Starting
Applying "C" settings...: Succeeded
Applying "D" settings...: Succeeded
Applying "A" settings: Starting
Applying "A" settings: Succeeded
Running plugins: Succeeded
bash-5.1$
```

#### Specify `-plugins` details via json file

```bash
$ $GOBIN/pm run -plugins "./sample/plugins-prereboot.json" -library sample/library/
Applying "C" settings...: Starting
Applying "B" settings...: Starting
Applying "C" settings...: Succeeded
Applying "B" settings...: Succeeded
Applying "D" settings...: Starting
Applying "D" settings...: Succeeded
Applying "A" settings: Starting
Applying "A" settings: Succeeded
Running plugins: Succeeded
$
```

### Example: Plugin Manager (PM) `run -type`

```bash
$ $GOBIN/pm run -type=prereboot
Expand Down Expand Up @@ -253,7 +344,7 @@ $

```bash
$ $GOBIN/pm run -type preupgrade -output-format=json -output-file=a.json -library ./sample/library/
Log: /log/asum/pm.2021-01-29T17:46:57.6904918-08:00.log
Log: /var/log/asum/pm.2021-01-29T17:46:57.6904918-08:00.log

Checking for "D" settings...: Starting
Checking for "D" settings...: Succeeded
Expand All @@ -271,7 +362,7 @@ $ cat a.json
"Plugins": [
{
"Description": "Checking for \"D\" settings...",
"FileName": "D/d.preupgrade",
"Name": "D/d.preupgrade",
"ExecStart": "$PM_LIBRARY/D/preupgrade.sh",
"RequiredBy": [
"A/a.preupgrade"
Expand All @@ -282,7 +373,7 @@ $ cat a.json
},
{
"Description": "Checking for \"A\" settings",
"FileName": "A/a.preupgrade",
"Name": "A/a.preupgrade",
"ExecStart": "/bin/echo \"Checking A...\"",
"RequiredBy": null,
"Requires": [
Expand All @@ -299,7 +390,7 @@ $ cat a.json

```bash
$ $GOBIN/pm run -type preupgrade -output-format=yaml -output-file=a.yaml -library ./sample/library/
Log: /log/asum/pm.2021-01-29T17:53:15.8128937-08:00.log
Log: /var/log/asum/pm.2021-01-29T17:53:15.8128937-08:00.log

Checking for "D" settings...: Starting
Checking for "D" settings...: Failed
Expand All @@ -308,19 +399,49 @@ $
```

```yaml
$ cat a.yaml
type: preupgrade
# cat a.yaml
name: preupgrade
description: ""
requiredby: []
requires: []
execstart: ""
plugins:
- description: Checking for "D" settings...
filename: D/d.preupgrade
execstart: $PM_LIBRARY/D/preupgrade.sh
requiredby:
- A/a.preupgrade
requires: []
status: Failed
stdouterr: "Running preupgrade.sh (path: sample/library//D/preupgrade.sh) with
status(1)...\nDisplaying Plugin Manager (PM) Config file path: \nFail(1)\n"
- name: A/a.preupgrade
description: Checking for "A" settings
requiredby: []
requires:
- D/d.preupgrade
execstart: /bin/echo "Checking A..."
plugins: []
library: ""
runtime:
starttime: 2024-10-28T18:21:17.289968946-05:00
endtime: 2024-10-28T18:21:17.337773824-05:00
duration: 47.804888ms
status: Skipped
stdouterr: []
- name: D/d.preupgrade
description: Checking for "D" settings...
requiredby: []
requires: []
execstart: $PM_LIBRARY/D/preupgrade.sh
plugins: []
library: ""
runtime:
starttime: 2024-10-28T18:21:17.220368224-05:00
endtime: 2024-10-28T18:21:17.289945583-05:00
duration: 69.577293ms
status: Failed
stdouterr:
- 'Running preupgrade.sh (path: sample/library//D/preupgrade.sh) with status(1)...'
- 'Displaying Plugin Manager (PM) Config file path: '
- Fail(1)
library: ""
runtime:
starttime: 2024-10-28T18:21:17.185365352-05:00
endtime: 2024-10-28T18:21:17.337805574-05:00
duration: 152.440222ms
status: Failed
stdouterr: 'Running preupgrade plugins: Failed'
$
stdouterr:
- 'Running preupgrade plugins: Failed'
```
Loading
Loading