Skip to content
CI

Making output of commands in test more verbose to troubleshoot issue with CI #10805

Sign in to view logs

Making output of commands in test more verbose to troubleshoot issue with CI

Making output of commands in test more verbose to troubleshoot issue with CI #10805

Workflow file for this run

.github/workflows/ci.yml at efb450f
# GitHub Actions CI definition for TensorBoard.
#
# YAML schema for GitHub Actions:
# https://help.github.com/en/actions/automating-your-workflow-with-github-actions/workflow-syntax-for-github-actions
#
# Helpful YAML parser to clarify YAML syntax:
# https://yaml-online-parser.appspot.com/
name: CI
on:
push:
branches:
- master
- '[0-9]+.*'
- 'ci-*'
tags:
- '*'
pull_request: {}
workflow_call: {}
env:
# Keep this Bazel version in sync with the `versions.check` directive
# in our WORKSPACE file.
BAZEL_VERSION: '4.2.2'
BAZEL_SHA256SUM: '11dea6c7cfd866ed520af19a6bb1d952f3e9f4ee60ffe84e63c0825d95cb5859'
BUILDTOOLS_VERSION: '3.0.0'
BUILDIFIER_SHA256SUM: 'e92a6793c7134c5431c58fbc34700664f101e5c9b1c1fcd93b97978e8b7f88db'
BUILDOZER_SHA256SUM: '3d58a0b6972e4535718cdd6c12778170ea7382de7c75bc3728f5719437ffb84d'
TENSORFLOW_VERSION: 'tf-nightly'
jobs:
build:
runs-on: ubuntu-22.04
needs: lint-python-flake8 # fail fast in case of "undefined variable" errors
strategy:
fail-fast: false
matrix:
tf_version_id: ['tf', 'notf']
python_version: ['3.9']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: ${{ matrix.python_version }}
architecture: 'x64'
- name: 'Set up Bazel'
run: |
ci/download_bazel.sh "${BAZEL_VERSION}" "${BAZEL_SHA256SUM}" ~/bazel
sudo mv ~/bazel /usr/local/bin/bazel
sudo chmod +x /usr/local/bin/bazel
cp ./ci/bazelrc ~/.bazelrc
- name: 'Configure build cache write credentials'
env:
CREDS: ${{ secrets.BAZEL_CACHE_SERVICE_ACCOUNT_CREDS }}
EVENT_TYPE: ${{ github.event_name }}
run: |
if [ -z "${CREDS}" ]; then
printf 'Using read-only cache (no credentials)\n'
exit
fi
if [ "${EVENT_TYPE}" = pull_request ]; then
printf 'Using read-only cache (PR build)\n'
exit
fi
printf 'Using writable cache\n'
creds_file=/tmp/service_account_creds.json
printf '%s\n' "${CREDS}" >"${creds_file}"
printf '%s\n' >>~/.bazelrc \
"common --google_credentials=${creds_file}" \
"common --remote_upload_local_results=true" \
;
- name: 'Install TensorFlow'
run: |
python -m pip install -U pip
pip install "${TENSORFLOW_VERSION}"
if: matrix.tf_version_id != 'notf'
- name: 'Install Python dependencies'
run: |
python -m pip install -U pip
pip install \
-r ./tensorboard/pip_package/requirements.txt \
-r ./tensorboard/pip_package/requirements_dev.txt \
;
- name: 'Check Pip state'
run: pip freeze --all
- name: 'Bazel: fetch'
run: bazel fetch //tensorboard/...
- name: 'Bazel: build'
# Note we suppress a flood of warnings from the proto compiler.
# Googlers see b/222706811 & b/182876485 discussion and preconditions
# for removing this kludge.
run: |
( set -o pipefail;
bazel build //tensorboard/... |&\
grep -v 'external/com_google_protobuf/python: warning: directory' |&\
grep -v 'INFO: From ProtoCompile ' )
- name: 'Bazel: test (with TensorFlow support)'
run: bazel test //tensorboard/...
if: matrix.tf_version_id != 'notf'
- name: 'Bazel: test (non-TensorFlow only)'
run: bazel test //tensorboard/... --test_tag_filters="support_notf"
if: matrix.tf_version_id == 'notf'
- name: 'Bazel: run Pip package test (with TensorFlow support)'
run: |
bazel run //tensorboard/pip_package:test_pip_package -- \
--tf-version "${TENSORFLOW_VERSION}"
if: matrix.tf_version_id != 'notf'
- name: 'Bazel: run Pip package test (non-TensorFlow only)'
run: |
bazel run //tensorboard/pip_package:test_pip_package -- \
--tf-version notf
if: matrix.tf_version_id == 'notf'
- name: 'Bazel: run manual tests'
run: |
bazel test //tensorboard/compat/tensorflow_stub:gfile_s3_test &&
bazel test //tensorboard/summary/writer:event_file_writer_s3_test &&
bazel test //tensorboard/compat/tensorflow_stub:gfile_fsspec_test &&
bazel test //tensorboard/summary/writer:event_file_writer_fsspec_test
- name: 'Bazel: build Pip package (master branch TF build only)'
# This pip package is only needed if we will upload it, so run this step
# only under the same conditions as the next step (see comment below).
if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
run: |
./tensorboard/tools/set_nightly_version.sh
rm -rf /tmp/tb_nightly_pip_package && mkdir /tmp/tb_nightly_pip_package
bazel run //tensorboard/pip_package:build_pip_package -- /tmp/tb_nightly_pip_package
- name: 'Upload Pip package as an artifact (master branch TF build only)'
# Prevent uploads when running on forks or non-master branch.
if: matrix.tf_version_id == 'tf' && github.repository == 'tensorflow/tensorboard' && github.ref == 'refs/heads/master'
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: tb-nightly
path: /tmp/tb_nightly_pip_package/*
build-data-server-pip:
runs-on: ${{ matrix.platform }}
strategy:
fail-fast: false
matrix:
mode: ['native']
platform: ['ubuntu-22.04', 'macos-12']
rust_version: ['1.65.0']
include:
- mode: 'universal'
platform: 'ubuntu-22.04'
rust_version: '1.65.0'
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: '3.9'
architecture: 'x64'
- name: 'Cache Cargo artifacts'
if: matrix.mode == 'native'
uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2
with:
path: |
tensorboard/data/server/target/
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
# Needed if we `cargo install` binaries (at time of writing, this
# job doesn't but the files are tiny, and this will mitigate
# confusion if we need to `cargo install` later)
~/.cargo/.crates.toml
~/.cargo/.crates2.json
key: build-data-server-pip-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
- name: 'Install Rust toolchain'
if: matrix.mode == 'native'
uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
with:
toolchain: ${{ matrix.rust_version }}
components: rustfmt
- name: 'Install Python packaging deps'
run: |
python -m pip install -U pip
pip install \
-c ./tensorboard/pip_package/requirements.txt \
-c ./tensorboard/pip_package/requirements_dev.txt \
setuptools wheel
- name: 'Build'
if: matrix.mode == 'native'
run: cd tensorboard/data/server/ && cargo build --release
- name: 'Test'
if: matrix.mode == 'native'
run: cd tensorboard/data/server/ && cargo test --release
- name: 'Package (native)'
if: matrix.mode == 'native'
run: |
rm -rf /tmp/pip_package && mkdir /tmp/pip_package
python tensorboard/data/server/pip_package/build.py \
--server-binary tensorboard/data/server/target/release/rustboard \
--out-dir /tmp/pip_package \
;
- name: 'Package (universal)'
if: matrix.mode == 'universal'
run: |
rm -rf /tmp/pip_package && mkdir /tmp/pip_package
python tensorboard/data/server/pip_package/build.py \
--universal \
--out-dir /tmp/pip_package \
;
- name: 'Upload'
uses: actions/upload-artifact@83fd05a356d7e2593de66fc9913b3002723633cb # v3.1.1
with:
name: tensorboard-data-server
path: /tmp/pip_package/*
lint-python-flake8:
runs-on: ubuntu-22.04
strategy:
fail-fast: false
matrix:
# flake8 should run on each Python version that we target,
# because the errors and warnings can differ due to language
# changes, and we want to catch them all.
python_version: ['3.9', '3.10', '3.11']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: ${{ matrix.python_version }}
architecture: 'x64'
- name: 'Install flake8'
run: |
python -m pip install -U pip
pip install flake8 -c ./tensorboard/pip_package/requirements_dev.txt
- run: pip freeze --all
- name: 'Lint Python code for errors with flake8'
# See: http://flake8.pycqa.org/en/3.7.8/user/error-codes.html
# Use the comment '# noqa: <error code>' to suppress.
run: flake8 . --count --select=E9,F63,F7,F82,F401 --show-source --statistics
lint-python-yaml-docs:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-python@13ae5bb136fac2878aff31522b9efb785519f984 # v4.3.0
with:
python-version: '3.10'
architecture: 'x64'
- name: 'Install black, yamllint, and the TensorFlow docs notebook tools'
run: |
python -m pip install -U pip
nbfmt_version="84bd7bea0c468667dd0ec273d426ed50f23e1a15"
pip install black yamllint -c ./tensorboard/pip_package/requirements_dev.txt
pip install -U git+https://github.com/tensorflow/docs@${nbfmt_version}
# Workaround tensorflow incompatibility with protobuf>4.
# See: https://github.com/tensorflow/tensorboard/issues/5708.
# See: https://github.com/tensorflow/tensorflow/issues/53234.
- name: 'Downgrade to compatible protobuf version'
run: pip install -U "protobuf<3.20"
- run: pip freeze --all
- name: 'Lint Python code for style with Black'
# You can run `black .` to fix all Black complaints.
run: black --check --diff .
- name: 'Lint YAML for gotchas with yamllint'
# Use '# yamllint disable-line rule:foo' to suppress.
run: yamllint -c docs/.yamllint docs docs/.yamllint
- name: 'Lint Colab notebooks for formatting with nbfmt'
run: git ls-files -z '*.ipynb' | xargs -0 python3 -m tensorflow_docs.tools.nbfmt --test
lint-rust:
runs-on: ubuntu-22.04
strategy:
matrix:
rust_version: ['1.65.0']
cargo_raze_version: ['0.16.1']
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: 'Cache Cargo artifacts'
uses: actions/cache@58c146cc91c5b9e778e71775dfe9bf1442ad9a12 # v3.2.2
with:
path: |
tensorboard/data/server/target/
# https://doc.rust-lang.org/cargo/guide/cargo-home.html#caching-the-cargo-home-in-ci
~/.cargo/bin/
~/.cargo/registry/index/
~/.cargo/registry/cache/
~/.cargo/git/db/
# Needed for installing binaries (`cargo-raze`) with cache
~/.cargo/.crates.toml
~/.cargo/.crates2.json
key: lint-rust-${{ runner.os }}-cargo-${{ matrix.rust_version }}-${{ matrix.cargo_raze_version }}-${{ hashFiles('**/Cargo.lock', '.github/workflows/ci.yml') }}
- name: 'Install Rust toolchain'
uses: dtolnay/rust-toolchain@5f2e2a7aff63d8cbdacb4832218a30fa7a37ee6e # current HEAD as of 1/19/2023
with:
toolchain: ${{ matrix.rust_version }}
components: rustfmt, clippy
- name: 'Install cargo-raze'
run: cargo install cargo-raze --version ${{ matrix.cargo_raze_version }} --locked
- name: 'Run Rustfmt'
run: (cd tensorboard/data/server/ && cargo fmt -- --check)
- name: 'Run Clippy'
# You can run `cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml --fix` to fix all Clippy complaints.
# This will only apply `MachineApplicable` fixes (https://doc.rust-lang.org/nightly/nightly-rustc/rustc_lint_defs/enum.Applicability.html), so some modifications may need to be done manually.
run: cargo clippy --all-targets --manifest-path tensorboard/data/server/Cargo.toml -- -D warnings
- name: 'Check cargo-raze freshness'
run: |
rm -rf third_party/rust/
(cd tensorboard/data/server/ && cargo fetch && cargo raze)
git add .
git diff --staged --exit-code
lint-frontend:
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- uses: actions/setup-node@64ed1c7eab4cce3362f8c340dee64e5eaeef8f7c # v3.6.0
with:
node-version: 16
- run: yarn install --ignore-engines
# You can run `yarn fix-lint` to fix all Prettier complaints.
- run: yarn lint
# Make sure no tests are skipped with "focused" tests.
- run: |
! git grep -E 'f(it|describe)\(' 'tensorboard/*_test.ts'
# Make sure no one depends on Angular material and CDK directly. Please
# import the indirection in //tensorboard/webapp/angular.
- run: |
! git grep -E '"@npm//@angular/material"|"@npm//@angular/cdk"' 'tensorboard/*/BUILD' ':!tensorboard/webapp/BUILD' ':!tensorboard/webapp/angular/BUILD'
# Cannot directly depend on d3 in webapp. Must depend on
# `//tensorboard/webapp/third_party:d3` instead.
- run: |
! git grep -E '"@npm//d3"|"@npm//@types/d3"' 'tensorboard/webapp/**/*BUILD' ':!tensorboard/webapp/third_party/**'
- run: |
! git grep -E '"@npm_angular_bazel//:index.bzl"' 'tensorboard/**/BUILD'
- run: |
! git grep -E 'mat-color|$mat-' 'tensorboard/**/*.scss'
# Make sure no one depends on the numeric library directly. Please
# import the indirection in //tensorboard/webapp/third_party.
- run: |
! git grep -E '@npm//numeric' 'tensorboard/*/BUILD' ':!tensorboard/webapp/third_party/**'
lint-misc: # build, protos, etc.
runs-on: ubuntu-22.04
steps:
- uses: actions/checkout@ac593985615ec2ede58e132d2e21d2b1cbd6127c # v3.3.0
- name: 'Set up Buildifier'
run: |
ci/download_buildifier.sh "${BUILDTOOLS_VERSION}" "${BUILDIFIER_SHA256SUM}" ~/buildifier
sudo mv ~/buildifier /usr/local/bin/buildifier
- name: 'Set up Buildozer'
run: |
ci/download_buildozer.sh "${BUILDTOOLS_VERSION}" "${BUILDOZER_SHA256SUM}" ~/buildozer
sudo mv ~/buildozer /usr/local/bin/buildozer
- name: 'Lint BUILD files'
# TODO(tensorboard-team): address all lint warnings and remove the exemption.
run:
git ls-files -z '*BUILD' third_party/js.bzl third_party/workspace.bzl WORKSPACE | xargs -0 buildifier --mode=check --lint=warn
--warnings=-native-py,-native-java
- run: ./tensorboard/tools/mirror_urls_test.sh
- name: 'Lint for no py2 BUILD targets'
# Use | to start a literal so YAML doesn't complain about the '!' character.
run: |
! git grep 'python_version = "PY2"' '*BUILD'
- name: 'No comments on licenses rule'
# Assert buildozer error code for 'success, when no changes were made'.
# https://github.com/bazelbuild/buildtools/blob/master/buildozer/README.md#error-code
run: |
buildozer '//tensorboard/...:%licenses' remove_comment && false || test $? = 3
- name: clang-format lint
uses: DoozyX/clang-format-lint-action@0138140a2adaafd3032a3ff37f66366fd7dc88e0
with:
source: ./tensorboard
# Exclude tensorboard/compat because the source of truth is TensorFlow.
exclude: ./tensorboard/compat/proto
extensions: 'proto'
clangFormatVersion: 9
- run: ./tensorboard/tools/do_not_submit_test.sh
- run: ./tensorboard/tools/license_test.sh
- run: ./tensorboard/tools/whitespace_hygiene_test.py