diff --git a/.gitignore b/.gitignore index 1bda0842..5bc605b9 100644 --- a/.gitignore +++ b/.gitignore @@ -2,6 +2,8 @@ /node_modules # Build +/.task +/build /dist # For bundle size analysis diff --git a/.gitmodules b/.gitmodules new file mode 100644 index 00000000..2d96d0d8 --- /dev/null +++ b/.gitmodules @@ -0,0 +1,3 @@ +[submodule "tools/yscope-dev-utils"] + path = tools/yscope-dev-utils + url = https://github.com/y-scope/yscope-dev-utils.git diff --git a/README.md b/README.md index d98baf65..3d92eeda 100644 --- a/README.md +++ b/README.md @@ -4,83 +4,52 @@ ![Open feature requests][feature-requests-shield] ![CLP on Zulip][zulip-shield] -`yscope-log-viewer` is a tool that can be used to view log files. It currently -supports [CLP][clp-repo]'s compressed log files (IR streams) and JSON log files. -The viewer can be used to navigate the log file, filter by log level, view the -logs with syntax highlighting, and generate direct links to specific log events. +The YScope Log Viewer is a tool that can be used to view log files. It currently supports +[CLP][clp-repo]'s compressed log files (IR streams) and JSON log files. The viewer can be used to +navigate the log file, filter by log level, view the logs with syntax highlighting, and generate +direct links to specific log events. -See the [online demo](#online-demo) section to try out the log viewer on a -sample log file. To set up a local server, follow the -[build guide](docs/dev-guide/building.md). +See the [demo](#demo) section to try the log viewer on a sample log file. Or you can generate your +own compressed log files using one of our [libraries][docs-site/generating-ir-stream-logs]. -See the [features in development](#features-in-development) section for upcoming -features. +Want to report a bug or request a feature? Check out the [feedback](#providing-feedback) section. +A list of [features in development][docs-site/features-in-development] is available on our +[docs site][docs-site]. -# Online Demo +# Demo + +A demo of the viewer available [here][online-demo]. -* A demo of the log viewer can be found [here][online-demo]. * The demo loads a Hadoop YARN log file from the [hive-24hrs] log dataset. * More info on the dataset and other datasets can be found [here][datasets]. -* To open an IR stream, drag and drop it onto the log viewer or use the open - file dialog. - -# Generating IR Stream Logs - -IR stream log files can currently be generated using these libraries: - -* [Log4j Logging Library][log4j1-appenders] -* [Logback Logging Library][logback-appenders] -* [Python Logging Library][clp-loglib-py] -* Golang Logging Library (in development) - -# How does it work? +* To open an IR stream, drag and drop it onto the log viewer or use the open file dialog. -The log viewer is written using the ReactJS framework and uses the open source -[monaco-editor] and [clp-ffi-js]. +# Docs -The viewer spawns a worker to assist with computationally intensive tasks such -as: +The log viewer's docs are available [here][docs-site]. -* Deserializing the file and creating an index of logs events. -* Paginating the indexed logs. -* Decoding the deserialized log events into plain text. +# Providing feedback -Tasks are passed to the worker as needed and changes are rendered in the UI. - -# Providing Feedback - -You can use GitHub issues to [report a bug][report-bug] or -[request a feature][request-feature]. +You can use GitHub issues to [report a bug][bug-report] or [request a feature][feature-req]. Join us on [Zulip][zulip] to chat with developers and other community members. # Contributing -See the docs in our [developer guide](docs/dev-guide). - -# Features in Development - -* Pretty printing to enhance the readability of structured data in the logs. -* A dashboard to visualize the distribution of log types and log levels. -* Support for plain text, archived log files, and other requested formats. -* Searching within a file, multiple files, or within provided time ranges. -* Infinite scrolling instead of pagination. -* Log correlation with sync by timestamp across multiple editors. -* Automatic conversion of text log files to IR stream format in the browser. -* Deployment of components via NPM. +See the docs in our [developer guide][docs-site/dev-guide]. +[bug-report]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=bug&template=bug-report.yml [bugs-shield]: https://img.shields.io/github/issues/y-scope/yscope-log-viewer/bug?label=bugs -[clp-ffi-js]: https://github.com/y-scope/clp-ffi-js -[clp-loglib-py]: https://github.com/y-scope/clp-loglib-py [clp-repo]: https://github.com/y-scope/clp [datasets]: https://docs.yscope.com/clp/main/user-guide/resources-datasets +[docs-site]: https://docs.yscope.com/yscope-log-viewer/main/ +[docs-site/building]: https://docs.yscope.com/yscope-log-viewer/main/dev-guide/building-getting-started +[docs-site/dev-guide]: https://docs.yscope.com/yscope-log-viewer/main/dev-guide/index +[docs-site/features-in-development]: https://docs.yscope.com/yscope-log-viewer/main/index#features-in-development +[docs-site/generating-ir-stream-logs]: https://docs.yscope.com/yscope-log-viewer/main/index#generating-ir-stream-logs +[feature-req]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=enhancement&template=feature-request.yml [feature-requests-shield]: https://img.shields.io/github/issues/y-scope/yscope-log-viewer/enhancement?label=feature-requests -[hive-24hrs]: https://zenodo.org/record/7094921#.Y5JbH33MKHs -[log4j1-appenders]: https://github.com/y-scope/log4j1-appenders -[logback-appenders]: https://github.com/y-scope/logback-appenders -[monaco-editor]: https://microsoft.github.io/monaco-editor/ +[hive-24hrs]: https://zenodo.org/records/7094921#.Y5JbH33MKHs [online-demo]: https://y-scope.github.io/yscope-log-viewer/?filePath=https://yscope.s3.us-east-2.amazonaws.com/sample-logs/yarn-ubuntu-resourcemanager-ip-172-31-17-135.log.1.clp.zst -[report-bug]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=bug&template=bug-report.yml -[request-feature]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=enhancement&template=feature-request.yml [zulip]: https://yscope-clp.zulipchat.com/ [zulip-shield]: https://img.shields.io/badge/zulip-yscope--clp%20chat-1888FA?logo=zulip diff --git a/docs/README.md b/docs/README.md new file mode 100644 index 00000000..1638e3d4 --- /dev/null +++ b/docs/README.md @@ -0,0 +1,46 @@ +# Docs + +This directory contains the files necessary to generate a Sphinx-based documentation website for +this project: + +* `conf` - Configuration files +* `src` - The actual docs + +## Requirements + +* [Git LFS][git-lfs] + * We use Git LFS to store the images in the docs; this is to avoid significant increases in + the size of repo as we add and update images. +* [Node.js] >= 16 to be able to [view the output](#viewing-the-output) +* Python 3.10 or later +* [Task] >= 3.38.0 + +## Build commands + +* Build the site incrementally: + + ```shell + task docs:site + ``` + + * The output of the build will be in `../build/docs/html`. + +* Clean up the build: + + ```shell + task docs:clean + ``` + +## Viewing the output + +```shell +task docs:serve +``` + +The command above will install [http-server] and serve the built docs site; `http-server` will print +the address it binds to (usually http://localhost:8080). + +[git-lfs]: https://git-lfs.com +[http-server]: https://www.npmjs.com/package/http-server +[Node.js]: https://nodejs.org/en/download/current +[Task]: https://taskfile.dev/ diff --git a/docs/conf/conf.py b/docs/conf/conf.py new file mode 100644 index 00000000..6d177cd4 --- /dev/null +++ b/docs/conf/conf.py @@ -0,0 +1,78 @@ +# -- Project information ------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#project-information + +project = "YScope Log Viewer" + +# NOTE: We don't include a period after "Inc" since the theme adds one already. +copyright = "2022-2024 YScope Inc" + +# -- General configuration ----------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#general-configuration + +extensions = [ + "myst_parser", + "sphinx_copybutton", + "sphinx_design", + "sphinx.ext.autodoc", + "sphinx.ext.viewcode", + "sphinxcontrib.mermaid", +] + +# -- MyST extensions ----------------------------------------------------------- +# https://myst-parser.readthedocs.io/en/stable/syntax/optional.html +myst_enable_extensions = [ + "attrs_block", + "colon_fence", +] + +myst_heading_anchors = 4 + +# -- Sphinx autodoc options ---------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/extensions/autodoc.html#configuration + +autoclass_content = "class" +autodoc_class_signature = "separated" +autodoc_typehints = "description" + +# -- HTML output options ------------------------------------------------------- +# https://www.sphinx-doc.org/en/master/usage/configuration.html#options-for-html-output + +html_favicon = "https://docs.yscope.com/_static/favicon.ico" +html_title = project +html_show_copyright = True + +html_static_path = ["../src/_static"] + +html_theme = "pydata_sphinx_theme" + +# -- Theme options ------------------------------------------------------------- +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/layout.html + +html_theme_options = { + "footer_start": ["copyright"], + "footer_center": [], + "footer_end": ["theme-version"], + "navbar_start": ["navbar-logo"], + "navbar_end": ["navbar-icon-links", "theme-switcher"], + "primary_sidebar_end": [], + "secondary_sidebar_items": ["page-toc", "edit-this-page"], + "show_prev_next": False, + "use_edit_page_button": True, +} + +# -- Theme source buttons ------------------------------------------------------ +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/source-buttons.html + +html_context = { + "github_user": "y-scope", + "github_repo": "yscope-log-viewer", + "github_version": "main", + "doc_path": "docs/src", +} + +# -- Theme custom CSS and JS --------------------------------------------------- +# https://pydata-sphinx-theme.readthedocs.io/en/stable/user_guide/static_assets.html + + +def setup(app): + app.add_css_file("custom.css") diff --git a/docs/dev-guide/building.md b/docs/dev-guide/building.md deleted file mode 100644 index 0758d967..00000000 --- a/docs/dev-guide/building.md +++ /dev/null @@ -1,30 +0,0 @@ -# Quick Start - -## Requirements -* Node.js via [prebuilt installers][nodejs-prebuilt-installer] / [nvm][nvm] / -[nvm-windows][nvm-windows] - -## Install the dependencies - -```shell -$ npm install -``` - -## Running in development - -```shell -$ npm start -``` -The application should now be served in debug mode at http://localhost:3010. - -# Distribute - -To create a build, run the following command and the build will be placed in the `dist` folder: - -```shell -$ npm run build -``` - -[nodejs-prebuilt-installer]: https://nodejs.org/en/download/prebuilt-installer -[nvm]: https://github.com/nvm-sh/nvm -[nvm-windows]: https://github.com/coreybutler/nvm-windows \ No newline at end of file diff --git a/docs/dev-guide/state-and-stateRef.md b/docs/dev-guide/state-and-stateRef.md deleted file mode 100644 index 6072ba23..00000000 --- a/docs/dev-guide/state-and-stateRef.md +++ /dev/null @@ -1,6 +0,0 @@ -# Using state references to maintain current values in React Hooks - -Whenever we encounter a `stateRef` reference alongside a `state` variable (e.g., `logEventNumRef` -and `logEventNum` in `StateContextProvider`), the reference is meant to hold the current value of -the state variable. We create these references (mirrors) to use the current state values in React -hooks without including the state variables in the dependency arrays. diff --git a/docs/dev-guide/validation.md b/docs/dev-guide/validation.md deleted file mode 100644 index f64381e9..00000000 --- a/docs/dev-guide/validation.md +++ /dev/null @@ -1,16 +0,0 @@ -# Validation - -Currently, there is no automated testing in place. While automation is being considered for future -development, the following tests should be performed manually: - -* Verify that the following features work: - * Changing display themes - * Changing the number of events per page - * Navigating to the first/last/next/previous page - * Loading a log file using the open file dialog and dragging & dropping - * Copying a link to a log event - * Changing the log level filter - * Exporting all logs to a file - * Toggling tabbed panels in the sidebar - * Using keyboard shortcuts -* Perform a build and verify that all features are functional diff --git a/docs/dev-guide/var-naming.md b/docs/dev-guide/var-naming.md deleted file mode 100644 index 7f3ab922..00000000 --- a/docs/dev-guide/var-naming.md +++ /dev/null @@ -1,16 +0,0 @@ -# Variable naming - -To differentiate variables that use different starting indexes (0 vs. 1), we use the following -naming convention: - -* 0-based indexing variable names end with the suffix `Idx` -* 1-based indexing variable names end with the suffix `Num` - -Similarly, variables that represent a total number of items are named with the prefix `num`. - -Examples: - -* `logEventNum` for a 1-based indexing variable. -* `arrayIndexIdx` for a 0-based indexing variable. -* `numEvents` for the total number of events. - \ No newline at end of file diff --git a/docs/requirements.txt b/docs/requirements.txt new file mode 100644 index 00000000..d12d2429 --- /dev/null +++ b/docs/requirements.txt @@ -0,0 +1,6 @@ +myst-parser>=4.0.0 +pydata-sphinx-theme>=0.16.0 +sphinx_design>=0.6.1 +sphinx-copybutton>=0.5.2 +sphinx>=8.1.3 +sphinxcontrib-mermaid>=1.0.0 diff --git a/docs/src/_static/custom.css b/docs/src/_static/custom.css new file mode 100644 index 00000000..f25a6694 --- /dev/null +++ b/docs/src/_static/custom.css @@ -0,0 +1,32 @@ +html[data-theme="dark"], html[data-theme="light"] { + --pst-color-primary: #3399ff; + --pst-color-secondary: #9580ff; +} + +a { + text-decoration: none; +} + +a:hover { + text-decoration: underline; +} + +/* +Use the bottom border that's used for indicating the current page as the hover style (for a more +cohesive look). +NOTE: This selector matches the one in pydata-sphinx-theme +pydata/pydata-sphinx-theme@v0.14.4/src/pydata_sphinx_theme/assets/styles/sections/_header.scss#L86 +*/ +.bd-header .navbar-nav li a.nav-link:hover { + border-bottom: max(3px,.1875rem,.12em) solid var(--pst-color-secondary); + text-decoration: none; +} + +/* +Remove margin from sidebar-primary-items__end so that we don't have an unnecessary scrollbar. We're +not using the end items currently. +*/ +.bd-sidebar-primary .sidebar-primary-items__end { + margin-top: 0; + margin-bottom: 0; +} diff --git a/docs/src/dev-guide/building-getting-started.md b/docs/src/dev-guide/building-getting-started.md new file mode 100644 index 00000000..e75a8269 --- /dev/null +++ b/docs/src/dev-guide/building-getting-started.md @@ -0,0 +1,41 @@ +# Getting started + +## Requirements + +Node.js via [prebuilt installers][nodejs-prebuilt-installer] / [nvm][nvm] / +[nvm-windows][nvm-windows]. + +## Setup + +Install the project's dependencies: + +```shell +npm install +``` + +You may want to specify `--include=dev` if you are running in an environment where the environment +variable `NODE_ENV=production` is set. + +## Running in development + +You can build and serve the viewer in debug mode using: + +```shell +npm start +``` + +The viewer should then be available at [http://localhost:3010](http://localhost:3010). + +## Building a distribution + +To create a build, run: + +```shell +npm run build +``` + +The build should then be available in the `dist` directory. + +[nodejs-prebuilt-installer]: https://nodejs.org/en/download/prebuilt-installer +[nvm]: https://github.com/nvm-sh/nvm +[nvm-windows]: https://github.com/coreybutler/nvm-windows diff --git a/docs/src/dev-guide/coding-guidelines.md b/docs/src/dev-guide/coding-guidelines.md new file mode 100644 index 00000000..9ff07e15 --- /dev/null +++ b/docs/src/dev-guide/coding-guidelines.md @@ -0,0 +1,33 @@ +# Coding guidelines + +This project adheres to YScope's [contribution guidelines][yscope-guidelines] as well as the +project-specific guidelines below. + +# Naming + +## Index variables + +To differentiate variables that use different starting indexes (0 vs. 1), use the following naming +convention: + +* 0-based indexing variable names should end with the suffix `Idx`. +* 1-based indexing variable names should end with the suffix `Num`. + +Similarly, variables that represent a total number of items should be named with the prefix `num`. + +Examples: + +* `logEventNum` for a 1-based indexing variable. +* `arrayIndexIdx` for a 0-based indexing variable. +* `numEvents` for the total number of events. + +# React + +## Omitting state variables from React Hooks + +To avoid including a state variable in a React Hook's dependency array, you can use a reference +(mirror) to hold the current value of the state variable. The reference should use the same name as +the state variable with an additional `Ref` suffix. E.g., `logEventNumRef` is the reference variable +that corresponds to the `logEventNum` state variable. + +[yscope-guidelines]: https://docs.yscope.com/dev-guide/contrib-guides-overview.html diff --git a/docs/src/dev-guide/contributing-getting-started.md b/docs/src/dev-guide/contributing-getting-started.md new file mode 100644 index 00000000..f2882033 --- /dev/null +++ b/docs/src/dev-guide/contributing-getting-started.md @@ -0,0 +1,22 @@ +# Getting started + +Have an issue you want to fix or a feature you'd like to implement? We'd love to see it! + +The general process is as follows: + +* Submit a [bug report][bug-report] or [feature request][feature-req] describing the motivation for + the change and your proposed implementation if any. + * A repo maintainer will give you feedback and guidance about how to proceed. +* [Fork][yscope-log-viewer-fork] the repo. +* Refer to the [coding guidelines](coding-guidelines) to ensure your code adheres to the project's + conventions. +* Implement your change, adding unit/integration tests where possible. +* Validate your change following the [validation](contributing-validation) guide. +* Apply the [linters](contributing-linting). +* Submit a pull-request for your change. + * One or more repo developers will review your change. +* Respond to the review(s). + +[bug-report]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=bug&template=bug-report.yml +[feature-req]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=enhancement&template=feature-request.yml +[yscope-log-viewer-fork]: https://github.com/y-scope/yscope-log-viewer/fork diff --git a/docs/src/dev-guide/contributing-linting.md b/docs/src/dev-guide/contributing-linting.md new file mode 100644 index 00000000..c30c7836 --- /dev/null +++ b/docs/src/dev-guide/contributing-linting.md @@ -0,0 +1,26 @@ +# Linting + +Before submitting a pull request, ensure you've run our linting tools and either fixed any +violations or suppressed the warning. If you can't run the linting workflows locally, you can enable +and run the [lint][gh-workflow-lint] GitHub workflow in your fork. + +## Requirements + +To run the linting tools, you'll need to install the dependencies as specified in +[Building](building-getting-started). + +## Running the linters + +To run all linting checks: + +```shell +npm run lint:check +``` + +To run all linting checks AND automatically fix any fixable issues: + +```shell +npm run lint:fix +``` + +[gh-workflow-lint]: https://github.com/y-scope/yscope-log-viewer/blob/main/.github/workflows/lint.yaml diff --git a/docs/src/dev-guide/contributing-validation.md b/docs/src/dev-guide/contributing-validation.md new file mode 100644 index 00000000..3af91571 --- /dev/null +++ b/docs/src/dev-guide/contributing-validation.md @@ -0,0 +1,43 @@ +# Validation + +## Automated testing + +You can validate the project either by running tests locally or using the [test][gh-workflow-test] +GitHub Workflow in your fork. + +### Running locally + +```shell +npm run test +``` + +This will: + +* Find and run any tests in the `test` directory. +* Enforce line and function coverage thresholds at directory levels. +* Output detailed results, including any coverage issues. + +### Running using the GitHub Workflow + +Once you push your to a branch in your fork, the [test][gh-workflow-test] GitHub workflow will +perform the same steps listed in the [Running locally](#running-locally) section, except results +and issues will be reported as GitHub Annotations. + +## Manual testing + +Since our automated testing only covers some aspects of the project, the following features should +still be tested manually: + +* Perform a build and verify that all features are functional. +* In particular, verify that the following features work: + * Changing display themes + * Changing the number of events per page + * Navigating to the first/last/next/previous page + * Loading a log file using the open file dialog and dragging & dropping + * Copying a link to a log event + * Changing the log level filter + * Exporting all logs to a file + * Toggling tabbed panels in the sidebar + * Using keyboard shortcuts + +[gh-workflow-test]: https://github.com/y-scope/yscope-log-viewer/blob/main/.github/workflows/test.yaml diff --git a/docs/src/dev-guide/design-overview.md b/docs/src/dev-guide/design-overview.md new file mode 100644 index 00000000..cdb501ef --- /dev/null +++ b/docs/src/dev-guide/design-overview.md @@ -0,0 +1,15 @@ +# Overview + +The log viewer is written using the ReactJS framework and uses the open source [monaco-editor] and +[clp-ffi-js]. + +The viewer spawns a worker to assist with computationally intensive tasks such as: + +* Deserializing the file and creating an index of logs events. +* Paginating the indexed logs. +* Decoding the deserialized log events into plain text. + +Tasks are passed to the worker as needed and changes are rendered in the UI. + +[clp-ffi-js]: https://github.com/y-scope/clp-ffi-js +[monaco-editor]: https://microsoft.github.io/monaco-editor/ diff --git a/docs/src/dev-guide/index.md b/docs/src/dev-guide/index.md new file mode 100644 index 00000000..d6d5c184 --- /dev/null +++ b/docs/src/dev-guide/index.md @@ -0,0 +1,55 @@ +# Developer guide + +This guide contains docs useful for developing the YScope Log Viewer. Use the left sidebar (if it's +hidden, click the {fas}`bars` icon) to navigate to specific docs. + +The sections are as follows: + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: building-getting-started +Building +^^^ +Docs about building the viewer. +::: + +:::{grid-item-card} +:link: contributing-getting-started +Contributing +^^^ +Docs about contributing to the viewer. +::: + +:::{grid-item-card} +:link: design-overview +Design +^^^ +Docs about the viewer's design. +::: +:::: + +:::{toctree} +:caption: Building +:hidden: + +building-getting-started +::: + +:::{toctree} +:caption: Contributing +:hidden: + +contributing-getting-started +coding-guidelines +contributing-validation +contributing-linting +::: + +:::{toctree} +:caption: Design +:hidden: + +design-overview +::: diff --git a/docs/src/index.md b/docs/src/index.md new file mode 100644 index 00000000..0160a3bc --- /dev/null +++ b/docs/src/index.md @@ -0,0 +1,85 @@ +The YScope Log Viewer is a tool that can be used to view log files. It currently supports +[CLP][clp-repo]'s compressed log files (IR streams) and JSON log files. The viewer can be used to +navigate the log file, filter by log level, view the logs with syntax highlighting, and generate +direct links to specific log events. + +See the [demo](#demo) section to try the log viewer on a sample log file. Or you can generate your +own compressed log files using one of our [libraries](#generating-ir-stream-logs). + +Want to report a bug or request a feature? Check out the [feedback](#providing-feedback) section. +A list of [features in development](#features-in-development) is below. + +# Getting started + +Check out the relevant guide below, based on whether you'd like to use or develop the YScope Log +Viewer. + +::::{grid} 1 1 2 2 +:gutter: 2 + +:::{grid-item-card} +:link: user-guide/index +User guide +^^^ +Docs for those interested in using the YScope Log Viewer. +::: + +:::{grid-item-card} +:link: dev-guide/index +Developer guide +^^^ +Docs for those interested in developing and contributing to the YScope Log Viewer. +::: +:::: + +# Demo + +A demo of the viewer is available [here][online-demo]. + +* The demo loads a Hadoop YARN log file from the [hive-24hrs] log dataset. + * More info on the dataset and other datasets can be found [here][datasets]. +* To open an IR stream, drag and drop it onto the log viewer or use the open file dialog. + +# Generating IR stream logs + +IR stream log files can currently be generated using these libraries: + +* [Log4j Logging Library][log4j1-appenders] +* [Logback Logging Library][logback-appenders] +* [Python Logging Library][clp-loglib-py] +* Golang Logging Library (in development) + +# Providing feedback + +You can use GitHub issues to [report a bug][bug-report] or [request a feature][feature-req]. + +Join us on [Zulip][zulip] to chat with developers and other community members. + +# Features in development + +* Pretty printing to enhance the readability of structured data in the logs. +* A dashboard to visualize the distribution of log types and log levels. +* Support for plain text, archived log files, and other requested formats. +* Searching within a file, multiple files, or within provided time ranges. +* Infinite scrolling instead of pagination. +* Log correlation with sync by timestamp across multiple editors. +* Automatic conversion of text log files to IR stream format in the browser. +* Deployment of components via NPM. + +:::{toctree} +:hidden: + +user-guide/index +dev-guide/index +::: + +[bug-report]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=bug&template=bug-report.yml +[clp-loglib-py]: https://github.com/y-scope/clp-loglib-py +[clp-repo]: https://github.com/y-scope/clp +[datasets]: https://docs.yscope.com/clp/main/user-guide/resources-datasets +[feature-req]: https://github.com/y-scope/yscope-log-viewer/issues/new?labels=enhancement&template=feature-request.yml +[hive-24hrs]: https://zenodo.org/records/7094921#.Y5JbH33MKHs +[log4j1-appenders]: https://github.com/y-scope/log4j1-appenders +[logback-appenders]: https://github.com/y-scope/logback-appenders +[online-demo]: https://y-scope.github.io/yscope-log-viewer/?filePath=https://yscope.s3.us-east-2.amazonaws.com/sample-logs/yarn-ubuntu-resourcemanager-ip-172-31-17-135.log.1.clp.zst +[zulip]: https://yscope-clp.zulipchat.com/ diff --git a/docs/src/user-guide/index.md b/docs/src/user-guide/index.md new file mode 100644 index 00000000..c79208d3 --- /dev/null +++ b/docs/src/user-guide/index.md @@ -0,0 +1,6 @@ +# User guide + +This guide documents how to use and operate the YScope Log Viewer. Use the left sidebar (if it's +hidden, click the {fas}`bars` icon) to navigate to specific docs. + +More details will be added soon. diff --git a/docs/tasks.yaml b/docs/tasks.yaml new file mode 100644 index 00000000..251641c6 --- /dev/null +++ b/docs/tasks.yaml @@ -0,0 +1,116 @@ +version: "3" + +vars: + # Paths + G_DOCS_BUILD_DIR: "{{.G_BUILD_DIR}}/docs/html" + G_DOCS_VENV_DIR: "{{.G_BUILD_DIR}}/docs-venv" + G_NODE_DEPS_DIR: "{{.G_BUILD_DIR}}/docs-node" + + # Target checksum files + G_DOCS_VENV_CHECKSUM_FILE: "{{.G_BUILD_DIR}}/docs#docs-venv.md5" + +tasks: + clean: + cmds: + - "rm -rf '{{.G_DOCS_BUILD_DIR}}'" + + serve: + deps: + - "http-server" + - "site" + cmds: + - "npm --prefix '{{.G_NODE_DEPS_DIR}}' exec http-server '{{.G_DOCS_BUILD_DIR}}' -c-1" + + site: + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5" + OUTPUT_DIR: "{{.G_DOCS_BUILD_DIR}}" + dir: "{{.TASKFILE_DIR}}" + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + - "docs-venv" + cmds: + # Call `clean` before building since `sphinx-build --write-all --fresh-env` isn't always + # equivalent to building from scratch. + - task: "clean" + - "python3 '{{.ROOT_DIR}}/tools/scripts/find-broken-docs-links.py'" + - |- + . "{{.G_DOCS_VENV_DIR}}/bin/activate" + sphinx-build \ + --write-all \ + --fresh-env \ + --conf-dir conf \ + --nitpicky \ + --fail-on-warning \ + --keep-going \ + --builder html \ + src "{{.OUTPUT_DIR}}" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + sources: + - "{{.G_DOCS_VENV_CHECKSUM_FILE}}" + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + - "conf/**/*" + - "src/**/*" + generates: ["{{.CHECKSUM_FILE}}"] + + docs-venv: + internal: true + vars: + CHECKSUM_FILE: "{{.G_DOCS_VENV_CHECKSUM_FILE}}" + OUTPUT_DIR: "{{.G_DOCS_VENV_DIR}}" + REQUIREMENTS_FILE: "docs/requirements.txt" + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + cmds: + - task: ":utils:create-venv" + vars: + LABEL: "docs" + OUTPUT_DIR: "{{.OUTPUT_DIR}}" + REQUIREMENTS_FILE: "{{.REQUIREMENTS_FILE}}" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + sources: + - "{{.REQUIREMENTS_FILE}}" + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] + + http-server: + internal: true + vars: + CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK | replace \":\" \"#\"}}.md5" + OUTPUT_DIR: "{{.G_NODE_DEPS_DIR}}" + deps: + - ":init" + - task: ":utils:validate-checksum" + vars: + CHECKSUM_FILE: "{{.CHECKSUM_FILE}}" + DATA_DIR: "{{.OUTPUT_DIR}}" + cmds: + - "rm -rf '{{.OUTPUT_DIR}}'" + - "npm --prefix '{{.OUTPUT_DIR}}' install http-server" + # This command must be last + - task: ":utils:compute-checksum" + vars: + DATA_DIR: "{{.OUTPUT_DIR}}" + OUTPUT_FILE: "{{.CHECKSUM_FILE}}" + sources: + - "{{.ROOT_DIR}}/taskfile.yaml" + - "{{.TASKFILE}}" + generates: ["{{.CHECKSUM_FILE}}"] diff --git a/taskfile.yaml b/taskfile.yaml new file mode 100644 index 00000000..f889e2f6 --- /dev/null +++ b/taskfile.yaml @@ -0,0 +1,15 @@ +version: "3" + +includes: + docs: "docs/tasks.yaml" + utils: "tools/yscope-dev-utils/taskfiles/utils.yml" + +vars: + G_BUILD_DIR: "{{.ROOT_DIR}}/build" + +tasks: + init: + internal: true + silent: true + run: "once" + cmd: "mkdir -p '{{.G_BUILD_DIR}}'" diff --git a/tools/scripts/find-broken-docs-links.py b/tools/scripts/find-broken-docs-links.py new file mode 100644 index 00000000..1a1eb742 --- /dev/null +++ b/tools/scripts/find-broken-docs-links.py @@ -0,0 +1,105 @@ +import os +import subprocess +import sys +from pathlib import Path + + +def main(argv): + repo_root = _get_repo_root() + + found_violation = False + + # Check for docs.yscope.com links with ".md" suffixes + if _check_tracked_files( + r"docs\.yscope\.com/.+\.md", + repo_root, + repo_root, + 'docs.yscope.com links cannot have ".md" suffixes.', + ): + found_violation = True + + # Check for sphinx :link: attributes that have ".md" suffixes + if _check_tracked_files( + r":link:[[:space:]]*.+\.md", + repo_root, + repo_root / "docs", + 'sphinx :link: attributes cannot have ".md" suffixes', + ): + found_violation = True + + if found_violation: + return 1 + + return 0 + + +def _get_repo_root() -> Path: + path_str = subprocess.check_output( + ["git", "rev-parse", "--show-toplevel"], cwd=Path(__file__).parent, text=True + ) + return Path(path_str.strip()) + + +def _check_tracked_files( + pattern: str, repo_root: Path, dir_to_search: Path, error_msg: str +) -> bool: + """ + Check for a pattern in all tracked files in the repo (except this script). + :param pattern: The pattern to search for. + :param repo_root: The root of the repository. + :param dir_to_search: The directory to search in. + :param error_msg: Error message if the pattern is found. + :return: Whether the pattern was found in any file. + """ + found_matches = False + + # NOTE: "-z" ensures the paths won't be quoted (while delimiting them using '\0') + for path_str in subprocess.check_output( + [ + "git", + "ls-files", + "--cached", + "--exclude-standard", + "-z", + str(dir_to_search.relative_to(repo_root)), + ], + cwd=repo_root, + text=True, + ).split("\0"): + path = Path(path_str) + + # Skip directories and this script + if path == __file__ or (repo_root / path).is_dir(): + continue + + try: + for match in subprocess.check_output( + ["grep", "--extended-regexp", "--line-number", "--with-filename", pattern, path], + cwd=repo_root, + text=True, + ).splitlines(): + _parse_and_print_match(match, error_msg) + found_matches = True + except subprocess.CalledProcessError: + pass + + return found_matches + + +def _parse_and_print_match(match: str, error_msg: str): + """ + Parses and prints grep matches in a format relevant to the current environment. + :param match: The match to parse and print. + :param error_msg: Error message if the pattern is found. + """ + if os.getenv("GITHUB_ACTIONS") == "true": + # Print a GitHub Actions error annotation + file, line, _ = match.split(":", 2) + print(f"::error file={file},line={line}::{error_msg}") + else: + print(error_msg, file=sys.stderr) + print(match, file=sys.stderr) + + +if "__main__" == __name__: + sys.exit(main(sys.argv)) diff --git a/tools/yscope-dev-utils b/tools/yscope-dev-utils new file mode 160000 index 00000000..ad576e43 --- /dev/null +++ b/tools/yscope-dev-utils @@ -0,0 +1 @@ +Subproject commit ad576e43c1a43d7a6afde79fc9c3c952b7bf28bd