From d06ff213fadc099a1d814910aceffbaa922d6aad Mon Sep 17 00:00:00 2001 From: Yota Hamada Date: Fri, 17 Jan 2025 03:48:54 -0800 Subject: [PATCH] doc: Update contributing documentation and README (#793) --- Makefile | 29 ++++- README.md | 272 +++++++++------------------------------- docs/source/contrib.rst | 29 +---- 3 files changed, 88 insertions(+), 242 deletions(-) diff --git a/Makefile b/Makefile index 320266d80..cb53156ea 100644 --- a/Makefile +++ b/Makefile @@ -1,5 +1,3 @@ -.PHONY: run run-server run-server-https run-scheduler test test-coverage lint build certs swagger - ############################################################################## # Arguments ############################################################################## @@ -81,16 +79,19 @@ OPENSSL_CONF=${CONFIG_DIR}/openssl.local.conf ############################################################################## # run starts the frontend server and the scheduler. +.PHONY: run run: ${FE_BUNDLE_JS} @echo "${COLOR_GREEN}Starting the frontend server and the scheduler...${COLOR_RESET}" @go run ./cmd start-all # server build the binary and start the server. +.PHONY: run-server run-server: golangci-lint build-bin @echo "${COLOR_GREEN}Starting the server...${COLOR_RESET}" ${LOCAL_BIN_DIR}/${APP_NAME} server # scheduler build the binary and start the scheduler. +.PHONY: run-scheduler run-scheduler: golangci-lint build-bin @echo "${COLOR_GREEN}Starting the scheduler...${COLOR_RESET}" ${LOCAL_BIN_DIR}/${APP_NAME} scheduler @@ -101,6 +102,7 @@ ${FE_BUNDLE_JS}: @echo "${COLOR_RED}Please run 'make build-ui' before starting the server.${COLOR_RESET}" # https starts the server with the HTTPS protocol. +.PHONY: run-server-https run-server-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE} @echo "${COLOR_GREEN}Starting the server with HTTPS...${COLOR_RESET}" @DAGU_CERT_FILE=${SERVER_CERT_FILE} \ @@ -108,6 +110,7 @@ run-server-https: ${SERVER_CERT_FILE} ${SERVER_KEY_FILE} go run ./cmd start-all # test runs all tests. +.PHONY: test test: build-bin @echo "${COLOR_GREEN}Running tests...${COLOR_RESET}" @GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum} @@ -115,25 +118,31 @@ test: build-bin @${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} ./... # test-coverage runs all tests with coverage. +.PHONY: test-coverage test-coverage: @echo "${COLOR_GREEN}Running tests with coverage...${COLOR_RESET}" @GOBIN=${LOCAL_BIN_DIR} go install ${PKG_gotestsum} @${LOCAL_BIN_DIR}/gotestsum ${GOTESTSUM_ARGS} -- ${GO_TEST_FLAGS} -coverprofile="coverage.txt" -covermode=atomic ./... # open-coverage opens the coverage file +.PHONY: open-coverage open-coverage: @go tool cover -html=coverage.txt # lint runs the linter. +.PHONY: lint lint: golangci-lint # api generates the swagger server code. +.PHONY: swagger api: clean-swagger gen-swagger # certs generates the certificates to use in the development environment. +.PHONY: certs certs: ${CERTS_DIR} ${SERVER_CERT_FILE} ${CLIENT_CERT_FILE} certs-check # build build the binary. +.PHONY: build build: build-ui build-bin # build-image build the docker image and push to the registry. @@ -143,21 +152,26 @@ build: build-ui build-bin # ``` # {version} should be the version number such as "1.13.0". -build-image: build-image-version build-image-latest +.PHONY: build-image +build-image: build-image-version + +.PHONY: build-image-latest build-image-version: ifeq ($(VERSION),) $(error "VERSION is not set") endif echo "${COLOR_GREEN}Building the docker image with the version $(VERSION)...${COLOR_RESET}" - $(DOCKER_CMD) -t ghcr.io/dagu-org/${APP_NAME}:$(VERSION) . + $(DOCKER_CMD) -t ghcr.io/dagu-org/${APP_NAME}:$(VERSION) -t ghcr.io/dagu-org/${APP_NAME}:latest . # build-image-latest build the docker image with the latest tag and push to # the registry. +.PHONY: build-image-latest build-image-latest: @echo "${COLOR_GREEN}Building the docker image...${COLOR_RESET}" $(DOCKER_CMD) -t ghcr.io/dagu-org/${APP_NAME}:latest . # gomerger merges all go files into a single file. +.PHONY: gomerger gomerger: ${LOCAL_DIR}/merged @echo "${COLOR_GREEN}Merging Go files...${COLOR_RESET}" @rm -f ${LOCAL_DIR}/merged/merged_project.go @@ -169,6 +183,7 @@ ${LOCAL_DIR}/merged: @mkdir -p ${LOCAL_DIR}/merged # addlicnese adds license header to all files. +.PHONY: addlicense addlicense: @echo "${COLOR_GREEN}Adding license headers...${COLOR_RESET}" @GOBIN=${LOCAL_BIN_DIR} go install ${PKG_addlicense} @@ -198,12 +213,14 @@ addlicense: ############################################################################## # build-bin builds the go application. +.PHONY: build-bin build-bin: @echo "${COLOR_GREEN}Building the binary...${COLOR_RESET}" @mkdir -p ${BIN_DIR} @go build -ldflags="$(LDFLAGS)" -o ${BIN_DIR}/${APP_NAME} ./cmd # build-ui builds the frontend codes. +.PHONY: build-ui build-ui: @echo "${COLOR_GREEN}Building UI...${COLOR_RESET}" @cd ui; \ @@ -213,18 +230,21 @@ build-ui: @cp ${FE_BUILD_DIR}/* ${FE_ASSETS_DIR} # golangci-lint run linting tool. +.PHONY: golangci-lint golangci-lint: @echo "${COLOR_GREEN}Running linter...${COLOR_RESET}" @GOBIN=${LOCAL_BIN_DIR} go install $(PKG_golangci_lint) @${LOCAL_BIN_DIR}/golangci-lint run ./... # clean-swagger removes generated go files for swagger. +.PHONY: clean-swagger clean-swagger: @echo "${COLOR_GREEN}Cleaning the swagger files...${COLOR_RESET}" @rm -rf ${FE_GEN_DIR}/restapi/models @rm -rf ${FE_GEN_DIR}/restapi/operations # gen-swagger generates go files for the API schema. +.PHONY: gen-swagger gen-swagger: @echo "${COLOR_GREEN}Generating the swagger server code...${COLOR_RESET}" @GOBIN=${LOCAL_BIN_DIR} go install $(PKG_swagger) @@ -271,6 +291,7 @@ ${CERTS_DIR}: @echo "${COLOR_GREEN}Creating the certificates directory...${COLOR_RESET}" @mkdir -p ${CERTS_DIR} +.PHONY: certs-check certs-check: @echo "${COLOR_GREEN}Checking CA certificate...${COLOR_RESET}" @openssl x509 -in ${SERVER_CERT_FILE} -noout -text diff --git a/README.md b/README.md index 3fb2ba0b6..ea796bc99 100644 --- a/README.md +++ b/README.md @@ -26,51 +26,40 @@

Dagu

-A powerful, self-contained Cron alternative with a clean Web UI and a [declarative YAML-based workflow definition](https://dagu.readthedocs.io/en/latest/yaml_format.html). Dagu simplifies complex job dependencies and scheduling with minimal overhead. +Dagu is a powerful Cron alternative that comes with a Web UI. It allows you to define dependencies between commands in a declarative YAML Format. Additionally, Dagu natively supports running Docker containers, making HTTP requests, and executing commands over SSH. Dagu was designed to be easy to use, self-contained, and require no coding, making it ideal for small projects. ---- +## Why Dagu? -## **Highlights** +Dagu is a modern workflow engine that combines simplicity with power, designed for developers who need reliable automation without the overhead. Here's what makes Dagu stand out: -- **Simple Installation**: Single binary, no dependencies -- **Intuitive Web UI**: Visualize, monitor, and control workflows -- **YAML-based**: Define workflows in simple YAML files -- **Built-in Executors**: Support for Docker, HTTP, SSH, and more -- **Zero Config**: No database required, works out of the box - -## **Features** - -- Web UI & CLI -- Web API Interface -- Powerful DAG definition in YAML format: - - Code snippets, parameters, environment variables - - Command substitution, piping, conditional logic - - Redirection of stdout and stderr - - Lifecycle hooks (on failure, on exit, etc.) - - Repeating tasks, automatic/manual retry - - Run sub workflows -- Handy built-in executors: - - Docker containers - - HTTP requests - - Email sending - - JSON query with jq - - SSH remote commands -- Remote Dagu node management -- Email notification -- Scheduling with Cron expressions +- **Language Agnostic**: Run any command or script regardless of programming language. Whether you're working with Python, Node.js, Bash, or any other language, Dagu seamlessly integrates with your existing tools and scripts. +- **Local-First Architecture**: Deploy and run workflows directly on your machine without external dependencies. This local-first approach ensures complete control over your automation while maintaining the flexibility to scale to distributed environments when needed. -## **Why we built Dagu** +- **Zero Configuration**: Get started in minutes with minimal setup. Dagu uses simple YAML files to define workflows, eliminating the need for complex configurations or infrastructure setup. -In many organizations, legacy systems still rely on hundreds of cron jobs running across multiple servers. These jobs are often written in various languages like Perl or Shell scripts, with implicit interdependencies. When one job fails, troubleshooting requires manually logging into servers via SSH and checking individual logs. To perform recovery, one must understand these implicit dependencies, which often rely on tribal knowledge. Dagu was developed to eliminate this complexity by providing a clear and understandable tool for workflow definition and dependency management. +- **Built for Developers**: Designed with software engineers in mind, Dagu provides powerful features like dependency management, retry logic, and parallel execution while maintaining a clean, intuitive interface. -## **A Lightweight and Self-Contained Solution** +- **Cloud Native Ready**: While running perfectly on local environments, Dagu is built to seamlessly integrate with modern cloud infrastructure when you need to scale. -While Cron is lightweight and suitable for simple scheduling, it doesn't scale well for complex workflows or provide features like retries, dependencies, or observability out of the box. On the other hand, tools like Airflow or other workflow engines can be overly complex for smaller projects or legacy environments, with steep learning curves and burdensome to maintain. Dagu strikes a balance: it's easy to use, self-contained, and require no coding, making it ideal for smaller projects. - -## **Built By and For In-House Developers** +## **Features** -Dagu's design philosophy stems from the real-world experience in managing complex jobs across diverse environments, from small startups to enterprise companies. By focusing on simplicity, transparency, and minimal setup overhead, Dagu aims to make life easier for in-house developers who need a robust workflow engine without the heavy lift of a more complex tool. +- Environment variables +- Flexible parameter passing +- Conditional logic with regex and shell commands +- Automatic retries +- Parallel steps +- Repeat steps at regular intervals +- Running sub workflows +- Execution timeouts +- Automatic logging +- Lifecycle hooks (on failure, on exit, etc.) +- Email notifications +- Running Docker containers in steps +- JSON handling support +- Controlling remote Dagu nodes from a single UI +- SSH remote commands in steps +- Flexible scheduling with cron expressions ## **Community** @@ -78,40 +67,6 @@ Dagu's design philosophy stems from the real-world experience in managing comple - Discussion: [GitHub Discussions](https://github.com/dagu-org/dagu/discussions) - Chat: [Discord](https://discord.gg/gpahPUjGRk) -## **Web UI** - -### DAG Details - -Real-time status, logs, and configuration for each DAG. Toggle graph orientation from the top-right corner. - -![example](assets/images/demo.gif?raw=true) - -![Details-TD](assets/images/ui-details2.webp?raw=true) - -### DAGs - -View all DAGs in one place with live status updates. - -![DAGs](assets/images/ui-dags.webp?raw=true) - -### Search - -Search across all DAG definitions. - -![History](assets/images/ui-search.webp?raw=true) - -### Execution History - -Review past DAG executions and logs at a glance. - -![History](assets/images/ui-history.webp?raw=true) - -### Log Viewer - -Examine detailed step-level logs and outputs. - -![DAG Log](assets/images/ui-logoutput.webp?raw=true) - ## **Installation** Dagu can be installed in multiple ways, such as using Homebrew or downloading a single binary from GitHub releases. @@ -187,32 +142,32 @@ steps: You can execute the example by pressing the `Start` button. You can see "Hello Dagu" in the log page in the Web UI. -## **CLI** +## **Usage / Command Line Interface** ```sh # Runs the DAG -dagu start +dagu start # Runs the DAG with named parameters -dagu start [-- = ...] +dagu start [-- = ...] # Runs the DAG with positional parameters -dagu start [-- value1 value2 ...] +dagu start [-- value1 value2 ...] # Displays the current status of the DAG -dagu status +dagu status # Re-runs the specified DAG run -dagu retry --req= +dagu retry --req= # Stops the DAG execution -dagu stop +dagu stop # Restarts the current running DAG -dagu restart +dagu restart # Dry-runs the DAG -dagu dry [-- = ...] +dagu dry [-- = ...] # Launches both the web UI server and scheduler process dagu start-all [--host=] [--port=] [--dags=] @@ -227,63 +182,6 @@ dagu scheduler [--dags=] dagu version ``` -## **Remote Node Management** - -Dagu supports managing multiple Dagu servers from a single UI through its remote node feature. This allows you to: - -- Monitor and manage DAGs across different environments (dev, staging, prod) -- Access multiple Dagu instances from a centralized UI -- Switch between nodes easily through the UI dropdown - -See [Remote Node Configuration](https://dagu.readthedocs.io/en/latest/config_remote.html) for more details. - -### Configuration - -Create `config.yaml` in `$HOME/.config/dagu/`: - -```yaml -remoteNodes: - - name: "prod" - apiBaseUrl: "https://prod.example.com/api/v1" - - name: "staging" - apiBaseUrl: "https://staging.example.com/api/v1" -``` - -## **Documentation** - -- [Installation Instructions](https://dagu.readthedocs.io/en/latest/installation.html) -- ️[Quick Start Guide](https://dagu.readthedocs.io/en/latest/quickstart.html) -- [Command Line Interface](https://dagu.readthedocs.io/en/latest/cli.html) -- [Web User Interface](https://dagu.readthedocs.io/en/latest/web_interface.html) -- Writing DAG - - [Minimal DAG Definition](https://dagu.readthedocs.io/en/latest/yaml_format.html#minimal-dag-definition) - - [Running Arbitrary Code Snippets](https://dagu.readthedocs.io/en/latest/yaml_format.html#running-arbitrary-code-snippets) - - [Environment Variables](https://dagu.readthedocs.io/en/latest/yaml_format.html#defining-environment-variables) - - [Parameters](https://dagu.readthedocs.io/en/latest/yaml_format.html#defining-and-using-parameters) - - [Command Substitution](https://dagu.readthedocs.io/en/latest/yaml_format.html#using-command-substitution) - - [Conditional Logic](https://dagu.readthedocs.io/en/latest/yaml_format.html#adding-conditional-logic) - - [Environment Variables with Standard Output](https://dagu.readthedocs.io/en/latest/yaml_format.html#setting-environment-variables-with-standard-output) - - [Redirecting Stdout and Stderr](https://dagu.readthedocs.io/en/latest/yaml_format.html#redirecting-stdout-and-stderr) - - [Lifecycle Hooks](https://dagu.readthedocs.io/en/latest/yaml_format.html#adding-lifecycle-hooks) - - [Repeating Task](https://dagu.readthedocs.io/en/latest/yaml_format.html#repeating-a-task-at-regular-intervals) - - [Running Sub-workflow](https://dagu.readthedocs.io/en/latest/yaml_format.html#running-sub-dag) - - [All Available Fields for a DAG](https://dagu.readthedocs.io/en/latest/yaml_format.html#all-available-fields-for-dags) - - [All Available Fields for a Step](https://dagu.readthedocs.io/en/latest/yaml_format.html#all-available-fields-for-steps) -- Example DAGs - - [Hello World](https://dagu.readthedocs.io/en/latest/examples.html#hello-world) - - [Conditional Steps](https://dagu.readthedocs.io/en/latest/examples.html#conditional-steps) - - [File Output](https://dagu.readthedocs.io/en/latest/examples.html#file-output) - - [Passing Output to Next Step](https://dagu.readthedocs.io/en/latest/examples.html#passing-output-to-next-step) - - [Running a Container Image](https://dagu.readthedocs.io/en/latest/examples.html#running-a-docker-container) - - [Making HTTP Requests](https://dagu.readthedocs.io/en/latest/examples.html#sending-http-requests) - - [JSON Processing](https://dagu.readthedocs.io/en/latest/examples.html#querying-json-data-with-jq) - - [Email](https://dagu.readthedocs.io/en/latest/examples.html#sending-email) -- [Configurations](https://dagu.readthedocs.io/en/latest/config.html) -- [Remote Node](https://dagu.readthedocs.io/en/latest/config_remote.html) -- [Scheduler](https://dagu.readthedocs.io/en/latest/scheduler.html) -- [Docker Compose](https://dagu.readthedocs.io/en/latest/docker-compose.html) -- [REST API Documentation](https://app.swaggerhub.com/apis/YOHAMTA_1/dagu) - ## **Example DAG** ### Minimal examples @@ -417,85 +315,41 @@ steps: command: echo "hello" ``` -### A bigger example +More examples can be found in the [documentation](https://dagu.readthedocs.io/en/latest/yaml_format.html). -A typical data pipeline for DevOps/Data Engineering scenarios: +## **Web UI** -![Details-TD](assets/images/example.webp?raw=true) +### DAG Details -The YAML code below represents this DAG: +Real-time status, logs, and configuration for each DAG. Toggle graph orientation from the top-right corner. -```yaml -# Environment variables used throughout the pipeline -env: - - DATA_DIR: /data - - SCRIPT_DIR: /scripts - - LOG_DIR: /log - # ... other variables can be added here - -# Handlers to manage errors and cleanup after execution -handlerOn: - failure: - command: "echo error" - exit: - command: "echo clean up" - -# The schedule for the DAG execution in cron format -# This schedule runs the DAG daily at 12:00 AM -schedule: "0 0 * * *" +![example](assets/images/demo.gif?raw=true) -steps: - # Step 1: Pull the latest data from a data source - - name: pull_data - command: "sh" - script: echo `date '+%Y-%m-%d'` - output: DATE - - # Step 2: Cleanse and prepare the data - - name: cleanse_data - command: echo cleansing ${DATA_DIR}/${DATE}.csv - depends: - - pull_data +![Details-TD](assets/images/ui-details2.webp?raw=true) - # Step 3: Transform the data - - name: transform_data - command: echo transforming ${DATA_DIR}/${DATE}_clean.csv - depends: - - cleanse_data +### DAGs - # Parallel Step 1: Load the data into a database - - name: load_data - command: echo loading ${DATA_DIR}/${DATE}_transformed.csv - depends: - - transform_data +View all DAGs in one place with live status updates. - # Parallel Step 2: Generate a statistical report - - name: generate_report - command: echo generating report ${DATA_DIR}/${DATE}_transformed.csv - depends: - - transform_data +![DAGs](assets/images/ui-dags.webp?raw=true) - # Step 4: Run some analytics - - name: run_analytics - command: echo running analytics ${DATA_DIR}/${DATE}_transformed.csv - depends: - - load_data +### Search - # Step 5: Send an email report - - name: send_report - command: echo sending email ${DATA_DIR}/${DATE}_analytics.csv - depends: - - run_analytics - - generate_report +Search across all DAG definitions. - # Step 6: Cleanup temporary files - - name: cleanup - command: echo removing ${DATE}*.csv - depends: - - send_report -``` +![History](assets/images/ui-search.webp?raw=true) -More examples can be found in the [documentation](https://dagu.readthedocs.io/en/latest/yaml_format.html). +### Execution History + +Review past DAG executions and logs at a glance. + +![History](assets/images/ui-history.webp?raw=true) + +### Log Viewer + +Examine detailed step-level logs and outputs. + +![DAG Log](assets/images/ui-logoutput.webp?raw=true) ## **Running as a daemon** @@ -516,15 +370,9 @@ fi exit ``` -## **How It Works** - -Dagu is a single command line tool that uses the local file system to store data, so no database management system or cloud service is required. DAGs are defined in a declarative YAML format, and existing programs can be used without modification. - ---- - -Feel free to contribute in any way you want! Share ideas, questions, submit issues, and create pull requests. Check out our [Contribution Guide](https://dagu.readthedocs.io/en/latest/contrib.html) for help getting started. +## Contributing -We welcome any and all contributions! +We welcome new contributors! Check out our [Contribution Guide](https://dagu.readthedocs.io/en/latest/contrib.html) for guidelines on how to get started. ## **Contributors** @@ -534,4 +382,4 @@ We welcome any and all contributions! ## **License** -This project is licensed under the GNU GPLv3. +Dagu is released under the [GNU GPLv3](./LICENSE.md). diff --git a/docs/source/contrib.rst b/docs/source/contrib.rst index 3b3d82303..217e68b57 100644 --- a/docs/source/contrib.rst +++ b/docs/source/contrib.rst @@ -1,12 +1,12 @@ Contribution Guide =================== -We welcome contributions of any size and skill level. If you have an idea for a new feature or have found a bug, please open an issue on the GitHub repository. +We welcome any contributions to the `Dagu` project. If you have an idea for a new feature or have found a bug, please open an issue on the GitHub repository. Prerequisite ------------- -* `Go version 1.19 or later. `_ +* `Go version 1.23 or later. `_ * Latest version of `Node.js `_. * `yarn `_ package manager. @@ -41,24 +41,6 @@ Running Tests make test -Code Structure ---------------- - -- ``ui``: Frontend code for the Web UI. -- ``cmd``: Contains the main application entry point. -- ``docs``: Contains the documentation for the project. -- ``examples``: Contains the example DAGs. -- ``internal``: Contains the internal code for the project. - - - ``agent``: Contains the code for running the DAGs. - - ``config``: Contains the code for loading the configuration. - - ``dag``: Contains the code for parsing the DAG definition. - - ``client``: Contains the code for managing the DAGs. - - ``persistence``: Contains the code for interacting with the database. - - ``sock``: Contains the code for interacting with the socket. - - ``frontend``: Contains the code for the frontend Web UI. - - ``scheduler``: Contains the code for scheduler. - Setting up your local environment for front end development ------------------------------------------------------------- @@ -73,7 +55,7 @@ Setting up your local environment for front end development .. code-block:: sh - go run main.go server + go run ./cmd/ server #. Navigate to ``ui`` directory and run the following command to install the dependencies: @@ -85,8 +67,3 @@ Setting up your local environment for front end development #. Open the browser and navigate to http://localhost:8081. #. Make changes to the source code and refresh the browser to see the changes. - -Branches ---------- - -* ``main``: The main branch where the source code always reflects a production-ready state.